From 02909e4d0ad4a21fcd6d07cb59b21ec7f13eec99 Mon Sep 17 00:00:00 2001 From: Kian Jones Date: Fri, 20 Jun 2025 16:11:46 -0700 Subject: [PATCH] draft with links --- .github/scripts/{ => model-sweep}/conftest.py | 0 .../{ => model-sweep}/feature_mappings.json | 0 .../generate_model_sweep_markdown.py | 70 +- .../scripts/{ => model-sweep}/model_sweep.py | 0 .../{ => model-sweep}/model_sweep_report.json | 0 .../model-sweep/model_sweep_report.mdx | 4395 +++++++++++++++++ .../{ => model-sweep}/test_markdown.mdx | 0 .github/scripts/model-sweep/test_output.mdx | 4395 +++++++++++++++++ .../model-sweep/test_output_with_links.mdx | 4395 +++++++++++++++++ .github/workflows/model-sweep.yaml | 16 +- 10 files changed, 13262 insertions(+), 9 deletions(-) rename .github/scripts/{ => model-sweep}/conftest.py (100%) rename .github/scripts/{ => model-sweep}/feature_mappings.json (100%) rename .github/scripts/{ => model-sweep}/generate_model_sweep_markdown.py (85%) rename .github/scripts/{ => model-sweep}/model_sweep.py (100%) rename .github/scripts/{ => model-sweep}/model_sweep_report.json (100%) create mode 100644 .github/scripts/model-sweep/model_sweep_report.mdx rename .github/scripts/{ => model-sweep}/test_markdown.mdx (100%) create mode 100644 .github/scripts/model-sweep/test_output.mdx create mode 100644 .github/scripts/model-sweep/test_output_with_links.mdx diff --git a/.github/scripts/conftest.py b/.github/scripts/model-sweep/conftest.py similarity index 100% rename from .github/scripts/conftest.py rename to .github/scripts/model-sweep/conftest.py diff --git a/.github/scripts/feature_mappings.json b/.github/scripts/model-sweep/feature_mappings.json similarity index 100% rename from .github/scripts/feature_mappings.json rename to .github/scripts/model-sweep/feature_mappings.json diff --git a/.github/scripts/generate_model_sweep_markdown.py b/.github/scripts/model-sweep/generate_model_sweep_markdown.py similarity index 85% rename from .github/scripts/generate_model_sweep_markdown.py rename to .github/scripts/model-sweep/generate_model_sweep_markdown.py index 6627b724..588a205a 100644 --- a/.github/scripts/generate_model_sweep_markdown.py +++ b/.github/scripts/model-sweep/generate_model_sweep_markdown.py @@ -92,10 +92,71 @@ def calculate_provider_support_score(models_data, feature_order): total_score = sum(model['support_score'] for model in models_data) return total_score / len(models_data) +def get_test_function_line_numbers(test_file_path): + """Extract line numbers for test functions from the test file.""" + test_line_numbers = {} + + try: + with open(test_file_path, 'r') as f: + lines = f.readlines() + + for i, line in enumerate(lines, 1): + if 'def test_' in line and line.strip().startswith('def test_'): + # Extract function name + func_name = line.strip().split('def ')[1].split('(')[0] + test_line_numbers[func_name] = i + except FileNotFoundError: + print(f"Warning: Could not find test file at {test_file_path}") + + return test_line_numbers + +def get_github_repo_info(): + """Get GitHub repository information from git remote.""" + try: + # Try to get the GitHub repo URL from git remote + import subprocess + result = subprocess.run(['git', 'remote', 'get-url', 'origin'], + capture_output=True, text=True, cwd=os.path.dirname(__file__)) + if result.returncode == 0: + remote_url = result.stdout.strip() + # Parse GitHub URL + if 'github.com' in remote_url: + if remote_url.startswith('https://'): + # https://github.com/user/repo.git -> user/repo + repo_path = remote_url.replace('https://github.com/', '').replace('.git', '') + elif remote_url.startswith('git@'): + # git@github.com:user/repo.git -> user/repo + repo_path = remote_url.split(':')[1].replace('.git', '') + else: + return None + return repo_path + except: + pass + + # Default fallback + return "letta-ai/letta" + def generate_test_details(model_info, feature_mapping): """Generate detailed test results for a model.""" details = [] + # Get test function line numbers + script_dir = os.path.dirname(os.path.abspath(__file__)) + test_file_path = os.path.join(script_dir, 'model_sweep.py') + test_line_numbers = get_test_function_line_numbers(test_file_path) + + # Get GitHub repo info + repo_path = get_github_repo_info() + + # Get current commit hash for links + try: + import subprocess + result = subprocess.run(['git', 'rev-parse', 'HEAD'], + capture_output=True, text=True, cwd=script_dir) + commit_hash = result.stdout.strip() if result.returncode == 0 else 'main' + except: + commit_hash = 'main' + for feature, tests in model_info['categorized_tests'].items(): if not tests: continue @@ -110,7 +171,14 @@ def generate_test_details(model_info, feature_mapping): status = "❌" else: status = "❓" - details.append(f"- {status} `{test}`") + + # Create GitHub link if we have line number info + if test in test_line_numbers and repo_path: + line_num = test_line_numbers[test] + github_link = f"https://github.com/{repo_path}/blob/{commit_hash}/.github/scripts/model-sweep/model_sweep.py#L{line_num}" + details.append(f"- {status} [`{test}`]({github_link})") + else: + details.append(f"- {status} `{test}`") details.append("") return details diff --git a/.github/scripts/model_sweep.py b/.github/scripts/model-sweep/model_sweep.py similarity index 100% rename from .github/scripts/model_sweep.py rename to .github/scripts/model-sweep/model_sweep.py diff --git a/.github/scripts/model_sweep_report.json b/.github/scripts/model-sweep/model_sweep_report.json similarity index 100% rename from .github/scripts/model_sweep_report.json rename to .github/scripts/model-sweep/model_sweep_report.json diff --git a/.github/scripts/model-sweep/model_sweep_report.mdx b/.github/scripts/model-sweep/model_sweep_report.mdx new file mode 100644 index 00000000..6458d98b --- /dev/null +++ b/.github/scripts/model-sweep/model_sweep_report.mdx @@ -0,0 +1,4395 @@ +--- +title: Support Models +generated: 2025-06-20T16:10:08.575462 +--- + +# Supported Models + +## Overview + +Letta routinely runs automated scans against available providers and models. These are the results of the latest scan. + +Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 + + +## anthropic + +| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | +|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| +| [`claude-3-5-haiku-20241022`](#anthropic_claude_3_5_haiku_20241022_details) | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | +| [`claude-3-5-sonnet-20241022`](#anthropic_claude_3_5_sonnet_20241022_details) | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | +| [`claude-3-7-sonnet-20250219`](#anthropic_claude_3_7_sonnet_20250219_details) | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | +| [`claude-sonnet-4-20250514`](#anthropic_claude_sonnet_4_20250514_details) | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | +| [`claude-opus-4-20250514`](#anthropic_claude_opus_4_20250514_details) | ✅ | ✅ | ⚠️ | 200,000 | 2025-06-20 | +| [`claude-3-5-sonnet-20240620`](#anthropic_claude_3_5_sonnet_20240620_details) | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 | +| [`claude-3-haiku-20240307`](#anthropic_claude_3_haiku_20240307_details) | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 | +| [`claude-3-opus-20240229`](#anthropic_claude_3_opus_20240229_details) | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 | +| [`claude-3-sonnet-20240229`](#anthropic_claude_3_sonnet_20240229_details) | ❌ | ❌ | ❌ | 200,000 | 2025-06-20 | + +### Detailed Test Results + +#### claude-3-5-haiku-20241022 {#anthropic_claude_3_5_haiku_20241022_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### claude-3-5-sonnet-20241022 {#anthropic_claude_3_5_sonnet_20241022_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### claude-3-7-sonnet-20250219 {#anthropic_claude_3_7_sonnet_20250219_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### claude-sonnet-4-20250514 {#anthropic_claude_sonnet_4_20250514_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### claude-opus-4-20250514 {#anthropic_claude_opus_4_20250514_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### claude-3-5-sonnet-20240620 {#anthropic_claude_3_5_sonnet_20240620_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### claude-3-haiku-20240307 {#anthropic_claude_3_haiku_20240307_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### claude-3-opus-20240229 {#anthropic_claude_3_opus_20240229_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### claude-3-sonnet-20240229 {#anthropic_claude_3_sonnet_20240229_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + + +--- + +## openai + +| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | +|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| +| [`gpt-4.1`](#openai_gpt_4.1_details) | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 | +| [`gpt-4.1-2025-04-14`](#openai_gpt_4.1_2025_04_14_details) | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 | +| [`gpt-4.1-nano-2025-04-14`](#openai_gpt_4.1_nano_2025_04_14_details) | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 | +| [`gpt-4o`](#openai_gpt_4o_details) | ✅ | ✅ | ✅ | 128,000 | 2025-06-20 | +| [`gpt-4o-2024-05-13`](#openai_gpt_4o_2024_05_13_details) | ✅ | ✅ | ✅ | 128,000 | 2025-06-20 | +| [`gpt-4-turbo`](#openai_gpt_4_turbo_details) | ✅ | ✅ | ⚠️ | 8,192 | 2025-06-20 | +| [`gpt-4.1-mini`](#openai_gpt_4.1_mini_details) | ✅ | ✅ | ⚠️ | 1,047,576 | 2025-06-20 | +| [`gpt-4.5-preview`](#openai_gpt_4.5_preview_details) | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 | +| [`gpt-4.5-preview-2025-02-27`](#openai_gpt_4.5_preview_2025_02_27_details) | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 | +| [`gpt-4o-2024-08-06`](#openai_gpt_4o_2024_08_06_details) | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 | +| [`gpt-4-0613`](#openai_gpt_4_0613_details) | ✅ | ✅ | ❌ | 8,192 | 2025-06-20 | +| [`gpt-4-1106-preview`](#openai_gpt_4_1106_preview_details) | ✅ | ✅ | ❌ | 128,000 | 2025-06-20 | +| [`gpt-4-turbo-2024-04-09`](#openai_gpt_4_turbo_2024_04_09_details) | ✅ | ⚠️ | ✅ | 128,000 | 2025-06-20 | +| [`gpt-4.1-mini-2025-04-14`](#openai_gpt_4.1_mini_2025_04_14_details) | ⚠️ | ✅ | ✅ | 1,047,576 | 2025-06-20 | +| [`gpt-4.1-nano`](#openai_gpt_4.1_nano_details) | ⚠️ | ✅ | ✅ | 1,047,576 | 2025-06-20 | +| [`gpt-4o-2024-11-20`](#openai_gpt_4o_2024_11_20_details) | ⚠️ | ✅ | ✅ | 8,192 | 2025-06-20 | +| [`gpt-4-turbo-preview`](#openai_gpt_4_turbo_preview_details) | ✅ | ⚠️ | ❌ | 128,000 | 2025-06-20 | +| [`gpt-4-0125-preview`](#openai_gpt_4_0125_preview_details) | ⚠️ | ✅ | ❌ | 128,000 | 2025-06-20 | +| [`gpt-4o-mini`](#openai_gpt_4o_mini_details) | ⚠️ | ⚠️ | ⚠️ | 128,000 | 2025-06-20 | +| [`gpt-4o-mini-2024-07-18`](#openai_gpt_4o_mini_2024_07_18_details) | ⚠️ | ⚠️ | ❌ | 128,000 | 2025-06-20 | +| [`gpt-4`](#openai_gpt_4_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`o1`](#openai_o1_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`o1-2024-12-17`](#openai_o1_2024_12_17_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`o3`](#openai_o3_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`o3-2025-04-16`](#openai_o3_2025_04_16_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`o3-mini`](#openai_o3_mini_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`o3-mini-2025-01-31`](#openai_o3_mini_2025_01_31_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`o3-pro`](#openai_o3_pro_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`o3-pro-2025-06-10`](#openai_o3_pro_2025_06_10_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | + +### Detailed Test Results + +#### gpt-4.1 {#openai_gpt_4.1_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gpt-4.1-2025-04-14 {#openai_gpt_4.1_2025_04_14_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gpt-4.1-nano-2025-04-14 {#openai_gpt_4.1_nano_2025_04_14_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gpt-4o {#openai_gpt_4o_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gpt-4o-2024-05-13 {#openai_gpt_4o_2024_05_13_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gpt-4-turbo {#openai_gpt_4_turbo_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gpt-4.1-mini {#openai_gpt_4.1_mini_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gpt-4.5-preview {#openai_gpt_4.5_preview_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gpt-4.5-preview-2025-02-27 {#openai_gpt_4.5_preview_2025_02_27_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gpt-4o-2024-08-06 {#openai_gpt_4o_2024_08_06_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gpt-4-0613 {#openai_gpt_4_0613_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gpt-4-1106-preview {#openai_gpt_4_1106_preview_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gpt-4-turbo-2024-04-09 {#openai_gpt_4_turbo_2024_04_09_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gpt-4.1-mini-2025-04-14 {#openai_gpt_4.1_mini_2025_04_14_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gpt-4.1-nano {#openai_gpt_4.1_nano_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gpt-4o-2024-11-20 {#openai_gpt_4o_2024_11_20_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gpt-4-turbo-preview {#openai_gpt_4_turbo_preview_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gpt-4-0125-preview {#openai_gpt_4_0125_preview_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gpt-4o-mini {#openai_gpt_4o_mini_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gpt-4o-mini-2024-07-18 {#openai_gpt_4o_mini_2024_07_18_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gpt-4 {#openai_gpt_4_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### o1 {#openai_o1_details} + +### Basic + +- ❌ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### o1-2024-12-17 {#openai_o1_2024_12_17_details} + +### Basic + +- ❌ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### o3 {#openai_o3_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### o3-2025-04-16 {#openai_o3_2025_04_16_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### o3-mini {#openai_o3_mini_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### o3-mini-2025-01-31 {#openai_o3_mini_2025_01_31_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### o3-pro {#openai_o3_pro_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### o3-pro-2025-06-10 {#openai_o3_pro_2025_06_10_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + + +--- + +## google_ai + +| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | +|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| +| [`gemini-1.5-pro`](#google_ai_gemini_1.5_pro_details) | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 | +| [`gemini-1.5-pro-002`](#google_ai_gemini_1.5_pro_002_details) | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 | +| [`gemini-1.5-pro-latest`](#google_ai_gemini_1.5_pro_latest_details) | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 | +| [`gemini-2.5-flash-preview-04-17-thinking`](#google_ai_gemini_2.5_flash_preview_04_17_thinking_details) | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 | +| [`gemini-2.5-pro-preview-03-25`](#google_ai_gemini_2.5_pro_preview_03_25_details) | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 | +| [`gemini-2.5-pro-preview-05-06`](#google_ai_gemini_2.5_pro_preview_05_06_details) | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 | +| [`gemini-2.5-flash-preview-05-20`](#google_ai_gemini_2.5_flash_preview_05_20_details) | ✅ | ⚠️ | ✅ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-flash-thinking-exp`](#google_ai_gemini_2.0_flash_thinking_exp_details) | ⚠️ | ✅ | ✅ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-flash-thinking-exp-1219`](#google_ai_gemini_2.0_flash_thinking_exp_1219_details) | ⚠️ | ✅ | ✅ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-flash-thinking-exp-01-21`](#google_ai_gemini_2.0_flash_thinking_exp_01_21_details) | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 | +| [`gemini-2.5-flash-preview-04-17`](#google_ai_gemini_2.5_flash_preview_04_17_details) | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 | +| [`gemini-2.5-pro-preview-06-05`](#google_ai_gemini_2.5_pro_preview_06_05_details) | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 | +| [`gemini-1.0-pro-vision-latest`](#google_ai_gemini_1.0_pro_vision_latest_details) | ❌ | ❌ | ❌ | 12,288 | 2025-06-20 | +| [`gemini-1.5-flash`](#google_ai_gemini_1.5_flash_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | +| [`gemini-1.5-flash-002`](#google_ai_gemini_1.5_flash_002_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | +| [`gemini-1.5-flash-8b`](#google_ai_gemini_1.5_flash_8b_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | +| [`gemini-1.5-flash-8b-001`](#google_ai_gemini_1.5_flash_8b_001_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | +| [`gemini-1.5-flash-8b-latest`](#google_ai_gemini_1.5_flash_8b_latest_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | +| [`gemini-1.5-flash-latest`](#google_ai_gemini_1.5_flash_latest_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | +| [`gemini-2.0-flash`](#google_ai_gemini_2.0_flash_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-flash-001`](#google_ai_gemini_2.0_flash_001_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-flash-exp`](#google_ai_gemini_2.0_flash_exp_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-flash-exp-image-generation`](#google_ai_gemini_2.0_flash_exp_image_generation_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-flash-lite`](#google_ai_gemini_2.0_flash_lite_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-flash-lite-001`](#google_ai_gemini_2.0_flash_lite_001_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-flash-lite-preview`](#google_ai_gemini_2.0_flash_lite_preview_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-flash-lite-preview-02-05`](#google_ai_gemini_2.0_flash_lite_preview_02_05_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-flash-preview-image-generation`](#google_ai_gemini_2.0_flash_preview_image_generation_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`gemini-2.0-pro-exp`](#google_ai_gemini_2.0_pro_exp_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-pro-exp-02-05`](#google_ai_gemini_2.0_pro_exp_02_05_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-2.5-flash-preview-tts`](#google_ai_gemini_2.5_flash_preview_tts_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`gemini-2.5-pro-exp-03-25`](#google_ai_gemini_2.5_pro_exp_03_25_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-2.5-pro-preview-tts`](#google_ai_gemini_2.5_pro_preview_tts_details) | ❌ | ❌ | ❌ | 65,536 | 2025-06-20 | +| [`gemini-exp-1206`](#google_ai_gemini_exp_1206_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-pro-vision`](#google_ai_gemini_pro_vision_details) | ❌ | ❌ | ❌ | 12,288 | 2025-06-20 | + +### Detailed Test Results + +#### gemini-1.5-pro {#google_ai_gemini_1.5_pro_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemini-1.5-pro-002 {#google_ai_gemini_1.5_pro_002_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemini-1.5-pro-latest {#google_ai_gemini_1.5_pro_latest_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemini-2.5-flash-preview-04-17-thinking {#google_ai_gemini_2.5_flash_preview_04_17_thinking_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemini-2.5-pro-preview-03-25 {#google_ai_gemini_2.5_pro_preview_03_25_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemini-2.5-pro-preview-05-06 {#google_ai_gemini_2.5_pro_preview_05_06_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemini-2.5-flash-preview-05-20 {#google_ai_gemini_2.5_flash_preview_05_20_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemini-2.0-flash-thinking-exp {#google_ai_gemini_2.0_flash_thinking_exp_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemini-2.0-flash-thinking-exp-1219 {#google_ai_gemini_2.0_flash_thinking_exp_1219_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemini-2.0-flash-thinking-exp-01-21 {#google_ai_gemini_2.0_flash_thinking_exp_01_21_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemini-2.5-flash-preview-04-17 {#google_ai_gemini_2.5_flash_preview_04_17_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemini-2.5-pro-preview-06-05 {#google_ai_gemini_2.5_pro_preview_06_05_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemini-1.0-pro-vision-latest {#google_ai_gemini_1.0_pro_vision_latest_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemini-1.5-flash {#google_ai_gemini_1.5_flash_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemini-1.5-flash-002 {#google_ai_gemini_1.5_flash_002_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemini-1.5-flash-8b {#google_ai_gemini_1.5_flash_8b_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemini-1.5-flash-8b-001 {#google_ai_gemini_1.5_flash_8b_001_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemini-1.5-flash-8b-latest {#google_ai_gemini_1.5_flash_8b_latest_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemini-1.5-flash-latest {#google_ai_gemini_1.5_flash_latest_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemini-2.0-flash {#google_ai_gemini_2.0_flash_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemini-2.0-flash-001 {#google_ai_gemini_2.0_flash_001_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemini-2.0-flash-exp {#google_ai_gemini_2.0_flash_exp_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemini-2.0-flash-exp-image-generation {#google_ai_gemini_2.0_flash_exp_image_generation_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemini-2.0-flash-lite {#google_ai_gemini_2.0_flash_lite_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemini-2.0-flash-lite-001 {#google_ai_gemini_2.0_flash_lite_001_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemini-2.0-flash-lite-preview {#google_ai_gemini_2.0_flash_lite_preview_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemini-2.0-flash-lite-preview-02-05 {#google_ai_gemini_2.0_flash_lite_preview_02_05_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemini-2.0-flash-preview-image-generation {#google_ai_gemini_2.0_flash_preview_image_generation_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemini-2.0-pro-exp {#google_ai_gemini_2.0_pro_exp_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemini-2.0-pro-exp-02-05 {#google_ai_gemini_2.0_pro_exp_02_05_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemini-2.5-flash-preview-tts {#google_ai_gemini_2.5_flash_preview_tts_details} + +### Basic + +- ❌ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemini-2.5-pro-exp-03-25 {#google_ai_gemini_2.5_pro_exp_03_25_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemini-2.5-pro-preview-tts {#google_ai_gemini_2.5_pro_preview_tts_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemini-exp-1206 {#google_ai_gemini_exp_1206_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemini-pro-vision {#google_ai_gemini_pro_vision_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + + +--- + +## letta + +| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | +|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| +| [`letta-free`](#letta_letta_free_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | + +### Detailed Test Results + +#### letta-free {#letta_letta_free_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + + +--- + +## together + +| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | +|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| +| [`Qwen/Qwen2.5-72B-Instruct-Turbo`](#together_qwen_qwen2.5_72b_instruct_turbo_details) | ✅ | ✅ | ⚠️ | 131,072 | 2025-06-20 | +| [`arcee-ai/virtuoso-large`](#together_arcee_ai_virtuoso_large_details) | ⚠️ | ✅ | ✅ | 131,072 | 2025-06-20 | +| [`Qwen/QwQ-32B`](#together_qwen_qwq_32b_details) | ⚠️ | ✅ | ⚠️ | 131,072 | 2025-06-20 | +| [`Qwen/Qwen2.5-7B-Instruct-Turbo`](#together_qwen_qwen2.5_7b_instruct_turbo_details) | ⚠️ | ✅ | ⚠️ | 32,768 | 2025-06-20 | +| [`Qwen/Qwen2.5-Coder-32B-Instruct`](#together_qwen_qwen2.5_coder_32b_instruct_details) | ⚠️ | ✅ | ⚠️ | 16,384 | 2025-06-20 | +| [`arcee-ai/coder-large`](#together_arcee_ai_coder_large_details) | ⚠️ | ✅ | ⚠️ | 32,768 | 2025-06-20 | +| [`arcee_ai/arcee-spotlight`](#together_arcee_ai_arcee_spotlight_details) | ⚠️ | ✅ | ⚠️ | 131,072 | 2025-06-20 | +| [`meta-llama/Llama-3.2-3B-Instruct-Turbo`](#together_meta_llama_llama_3.2_3b_instruct_turbo_details) | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | +| [`meta-llama/Llama-3.3-70B-Instruct-Turbo`](#together_meta_llama_llama_3.3_70b_instruct_turbo_details) | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | +| [`meta-llama/Llama-3.3-70B-Instruct-Turbo-Free`](#together_meta_llama_llama_3.3_70b_instruct_turbo_free_details) | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | +| [`meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo`](#together_meta_llama_meta_llama_3.1_405b_instruct_turbo_details) | ⚠️ | ✅ | ❌ | 130,815 | 2025-06-20 | +| [`meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo`](#together_meta_llama_meta_llama_3.1_70b_instruct_turbo_details) | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | +| [`nvidia/Llama-3.1-Nemotron-70B-Instruct-HF`](#together_nvidia_llama_3.1_nemotron_70b_instruct_hf_details) | ⚠️ | ✅ | ❌ | 32,768 | 2025-06-20 | +| [`arcee-ai/virtuoso-medium-v2`](#together_arcee_ai_virtuoso_medium_v2_details) | ⚠️ | ⚠️ | ✅ | 131,072 | 2025-06-20 | +| [`meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8`](#together_meta_llama_llama_4_maverick_17b_128e_instruct_fp8_details) | ⚠️ | ❌ | ✅ | 1,048,576 | 2025-06-20 | +| [`Qwen/Qwen3-235B-A22B-fp8-tput`](#together_qwen_qwen3_235b_a22b_fp8_tput_details) | ⚠️ | ⚠️ | ❌ | 40,960 | 2025-06-20 | +| [`deepseek-ai/DeepSeek-V3`](#together_deepseek_ai_deepseek_v3_details) | ⚠️ | ⚠️ | ❌ | 131,072 | 2025-06-20 | +| [`meta-llama/Llama-4-Scout-17B-16E-Instruct`](#together_meta_llama_llama_4_scout_17b_16e_instruct_details) | ⚠️ | ⚠️ | ❌ | 1,048,576 | 2025-06-20 | +| [`meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo`](#together_meta_llama_meta_llama_3.1_8b_instruct_turbo_details) | ⚠️ | ⚠️ | ❌ | 131,072 | 2025-06-20 | +| [`mistralai/Mixtral-8x7B-Instruct-v0.1`](#together_mistralai_mixtral_8x7b_instruct_v0.1_details) | ⚠️ | ⚠️ | ❌ | 32,768 | 2025-06-20 | +| [`arcee-ai/caller`](#together_arcee_ai_caller_details) | ❌ | ⚠️ | ❌ | 32,768 | 2025-06-20 | +| [`mistralai/Mistral-Small-24B-Instruct-2501`](#together_mistralai_mistral_small_24b_instruct_2501_details) | ❌ | ⚠️ | ❌ | 32,768 | 2025-06-20 | +| [`NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO`](#together_nousresearch_nous_hermes_2_mixtral_8x7b_dpo_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`Qwen/Qwen2-72B-Instruct`](#together_qwen_qwen2_72b_instruct_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`Qwen/Qwen2-VL-72B-Instruct`](#together_qwen_qwen2_vl_72b_instruct_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`Qwen/Qwen2.5-VL-72B-Instruct`](#together_qwen_qwen2.5_vl_72b_instruct_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`arcee-ai/arcee-blitz`](#together_arcee_ai_arcee_blitz_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`arcee-ai/maestro-reasoning`](#together_arcee_ai_maestro_reasoning_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| [`deepseek-ai/DeepSeek-R1`](#together_deepseek_ai_deepseek_r1_details) | ❌ | ❌ | ❌ | 163,840 | 2025-06-20 | +| [`deepseek-ai/DeepSeek-R1-Distill-Llama-70B`](#together_deepseek_ai_deepseek_r1_distill_llama_70b_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| [`deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free`](#together_deepseek_ai_deepseek_r1_distill_llama_70b_free_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B`](#together_deepseek_ai_deepseek_r1_distill_qwen_1.5b_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| [`deepseek-ai/DeepSeek-R1-Distill-Qwen-14B`](#together_deepseek_ai_deepseek_r1_distill_qwen_14b_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| [`deepseek-ai/DeepSeek-V3-p-dp`](#together_deepseek_ai_deepseek_v3_p_dp_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| [`google/gemma-2-27b-it`](#together_google_gemma_2_27b_it_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`lgai/exaone-3-5-32b-instruct`](#together_lgai_exaone_3_5_32b_instruct_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`lgai/exaone-deep-32b`](#together_lgai_exaone_deep_32b_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`marin-community/marin-8b-instruct`](#together_marin_community_marin_8b_instruct_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| [`meta-llama/Llama-3-70b-chat-hf`](#together_meta_llama_llama_3_70b_chat_hf_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`meta-llama/Llama-3-8b-chat-hf`](#together_meta_llama_llama_3_8b_chat_hf_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo`](#together_meta_llama_llama_3.2_11b_vision_instruct_turbo_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| [`meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo`](#together_meta_llama_llama_3.2_90b_vision_instruct_turbo_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| [`meta-llama/Llama-Vision-Free`](#together_meta_llama_llama_vision_free_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| [`meta-llama/Meta-Llama-3-70B-Instruct-Turbo`](#together_meta_llama_meta_llama_3_70b_instruct_turbo_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`meta-llama/Meta-Llama-3-8B-Instruct-Lite`](#together_meta_llama_meta_llama_3_8b_instruct_lite_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`mistralai/Mistral-7B-Instruct-v0.1`](#together_mistralai_mistral_7b_instruct_v0.1_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`mistralai/Mistral-7B-Instruct-v0.2`](#together_mistralai_mistral_7b_instruct_v0.2_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`mistralai/Mistral-7B-Instruct-v0.3`](#together_mistralai_mistral_7b_instruct_v0.3_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`perplexity-ai/r1-1776`](#together_perplexity_ai_r1_1776_details) | ❌ | ❌ | ❌ | 163,840 | 2025-06-20 | +| [`scb10x/scb10x-llama3-1-typhoon2-70b-instruct`](#together_scb10x_scb10x_llama3_1_typhoon2_70b_instruct_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`scb10x/scb10x-typhoon-2-1-gemma3-12b`](#together_scb10x_scb10x_typhoon_2_1_gemma3_12b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`togethercomputer/MoA-1`](#together_togethercomputer_moa_1_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`togethercomputer/MoA-1-Turbo`](#together_togethercomputer_moa_1_turbo_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`togethercomputer/Refuel-Llm-V2`](#together_togethercomputer_refuel_llm_v2_details) | ❌ | ❌ | ❌ | 16,384 | 2025-06-20 | +| [`togethercomputer/Refuel-Llm-V2-Small`](#together_togethercomputer_refuel_llm_v2_small_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | + +### Detailed Test Results + +#### Qwen/Qwen2.5-72B-Instruct-Turbo {#together_qwen_qwen2.5_72b_instruct_turbo_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### arcee-ai/virtuoso-large {#together_arcee_ai_virtuoso_large_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### Qwen/QwQ-32B {#together_qwen_qwq_32b_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### Qwen/Qwen2.5-7B-Instruct-Turbo {#together_qwen_qwen2.5_7b_instruct_turbo_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### Qwen/Qwen2.5-Coder-32B-Instruct {#together_qwen_qwen2.5_coder_32b_instruct_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### arcee-ai/coder-large {#together_arcee_ai_coder_large_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### arcee_ai/arcee-spotlight {#together_arcee_ai_arcee_spotlight_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### meta-llama/Llama-3.2-3B-Instruct-Turbo {#together_meta_llama_llama_3.2_3b_instruct_turbo_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### meta-llama/Llama-3.3-70B-Instruct-Turbo {#together_meta_llama_llama_3.3_70b_instruct_turbo_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### meta-llama/Llama-3.3-70B-Instruct-Turbo-Free {#together_meta_llama_llama_3.3_70b_instruct_turbo_free_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo {#together_meta_llama_meta_llama_3.1_405b_instruct_turbo_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo {#together_meta_llama_meta_llama_3.1_70b_instruct_turbo_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### nvidia/Llama-3.1-Nemotron-70B-Instruct-HF {#together_nvidia_llama_3.1_nemotron_70b_instruct_hf_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ✅ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### arcee-ai/virtuoso-medium-v2 {#together_arcee_ai_virtuoso_medium_v2_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 {#together_meta_llama_llama_4_maverick_17b_128e_instruct_fp8_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ✅ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### Qwen/Qwen3-235B-A22B-fp8-tput {#together_qwen_qwen3_235b_a22b_fp8_tput_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### deepseek-ai/DeepSeek-V3 {#together_deepseek_ai_deepseek_v3_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### meta-llama/Llama-4-Scout-17B-16E-Instruct {#together_meta_llama_llama_4_scout_17b_16e_instruct_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo {#together_meta_llama_meta_llama_3.1_8b_instruct_turbo_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ✅ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### mistralai/Mixtral-8x7B-Instruct-v0.1 {#together_mistralai_mixtral_8x7b_instruct_v0.1_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ✅ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ✅ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ✅ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ✅ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ✅ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ✅ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### arcee-ai/caller {#together_arcee_ai_caller_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ✅ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### mistralai/Mistral-Small-24B-Instruct-2501 {#together_mistralai_mistral_small_24b_instruct_2501_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ✅ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO {#together_nousresearch_nous_hermes_2_mixtral_8x7b_dpo_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### Qwen/Qwen2-72B-Instruct {#together_qwen_qwen2_72b_instruct_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### Qwen/Qwen2-VL-72B-Instruct {#together_qwen_qwen2_vl_72b_instruct_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### Qwen/Qwen2.5-VL-72B-Instruct {#together_qwen_qwen2.5_vl_72b_instruct_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### arcee-ai/arcee-blitz {#together_arcee_ai_arcee_blitz_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### arcee-ai/maestro-reasoning {#together_arcee_ai_maestro_reasoning_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### deepseek-ai/DeepSeek-R1 {#together_deepseek_ai_deepseek_r1_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### deepseek-ai/DeepSeek-R1-Distill-Llama-70B {#together_deepseek_ai_deepseek_r1_distill_llama_70b_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free {#together_deepseek_ai_deepseek_r1_distill_llama_70b_free_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B {#together_deepseek_ai_deepseek_r1_distill_qwen_1.5b_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### deepseek-ai/DeepSeek-R1-Distill-Qwen-14B {#together_deepseek_ai_deepseek_r1_distill_qwen_14b_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### deepseek-ai/DeepSeek-V3-p-dp {#together_deepseek_ai_deepseek_v3_p_dp_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### google/gemma-2-27b-it {#together_google_gemma_2_27b_it_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### lgai/exaone-3-5-32b-instruct {#together_lgai_exaone_3_5_32b_instruct_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### lgai/exaone-deep-32b {#together_lgai_exaone_deep_32b_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### marin-community/marin-8b-instruct {#together_marin_community_marin_8b_instruct_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### meta-llama/Llama-3-70b-chat-hf {#together_meta_llama_llama_3_70b_chat_hf_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### meta-llama/Llama-3-8b-chat-hf {#together_meta_llama_llama_3_8b_chat_hf_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo {#together_meta_llama_llama_3.2_11b_vision_instruct_turbo_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo {#together_meta_llama_llama_3.2_90b_vision_instruct_turbo_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### meta-llama/Llama-Vision-Free {#together_meta_llama_llama_vision_free_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### meta-llama/Meta-Llama-3-70B-Instruct-Turbo {#together_meta_llama_meta_llama_3_70b_instruct_turbo_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### meta-llama/Meta-Llama-3-8B-Instruct-Lite {#together_meta_llama_meta_llama_3_8b_instruct_lite_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### mistralai/Mistral-7B-Instruct-v0.1 {#together_mistralai_mistral_7b_instruct_v0.1_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### mistralai/Mistral-7B-Instruct-v0.2 {#together_mistralai_mistral_7b_instruct_v0.2_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### mistralai/Mistral-7B-Instruct-v0.3 {#together_mistralai_mistral_7b_instruct_v0.3_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### perplexity-ai/r1-1776 {#together_perplexity_ai_r1_1776_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### scb10x/scb10x-llama3-1-typhoon2-70b-instruct {#together_scb10x_scb10x_llama3_1_typhoon2_70b_instruct_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### scb10x/scb10x-typhoon-2-1-gemma3-12b {#together_scb10x_scb10x_typhoon_2_1_gemma3_12b_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### togethercomputer/MoA-1 {#together_togethercomputer_moa_1_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### togethercomputer/MoA-1-Turbo {#together_togethercomputer_moa_1_turbo_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### togethercomputer/Refuel-Llm-V2 {#together_togethercomputer_refuel_llm_v2_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### togethercomputer/Refuel-Llm-V2-Small {#together_togethercomputer_refuel_llm_v2_small_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + + +--- + +## deepseek + +| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | +|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| +| [`deepseek-chat`](#deepseek_deepseek_chat_details) | ❌ | ❌ | ❌ | 64,000 | 2025-06-20 | +| [`deepseek-reasoner`](#deepseek_deepseek_reasoner_details) | ❌ | ❌ | ❌ | 64,000 | 2025-06-20 | + +### Detailed Test Results + +#### deepseek-chat {#deepseek_deepseek_chat_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### deepseek-reasoner {#deepseek_deepseek_reasoner_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + + +--- + +## groq + +| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | +|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| +| [`allam-2-7b`](#groq_allam_2_7b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`compound-beta`](#groq_compound_beta_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`compound-beta-mini`](#groq_compound_beta_mini_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`deepseek-r1-distill-llama-70b`](#groq_deepseek_r1_distill_llama_70b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`distil-whisper-large-v3-en`](#groq_distil_whisper_large_v3_en_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`gemma2-9b-it`](#groq_gemma2_9b_it_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`llama-3.1-8b-instant`](#groq_llama_3.1_8b_instant_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`llama-3.3-70b-versatile`](#groq_llama_3.3_70b_versatile_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`llama-guard-3-8b`](#groq_llama_guard_3_8b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`llama3-70b-8192`](#groq_llama3_70b_8192_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`llama3-8b-8192`](#groq_llama3_8b_8192_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`meta-llama/llama-4-maverick-17b-128e-instruct`](#groq_meta_llama_llama_4_maverick_17b_128e_instruct_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`meta-llama/llama-4-scout-17b-16e-instruct`](#groq_meta_llama_llama_4_scout_17b_16e_instruct_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`meta-llama/llama-guard-4-12b`](#groq_meta_llama_llama_guard_4_12b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`meta-llama/llama-prompt-guard-2-22m`](#groq_meta_llama_llama_prompt_guard_2_22m_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`meta-llama/llama-prompt-guard-2-86m`](#groq_meta_llama_llama_prompt_guard_2_86m_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`mistral-saba-24b`](#groq_mistral_saba_24b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`playai-tts`](#groq_playai_tts_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`playai-tts-arabic`](#groq_playai_tts_arabic_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`qwen-qwq-32b`](#groq_qwen_qwq_32b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`qwen/qwen3-32b`](#groq_qwen_qwen3_32b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`whisper-large-v3`](#groq_whisper_large_v3_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`whisper-large-v3-turbo`](#groq_whisper_large_v3_turbo_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | + +### Detailed Test Results + +#### allam-2-7b {#groq_allam_2_7b_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### compound-beta {#groq_compound_beta_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### compound-beta-mini {#groq_compound_beta_mini_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### deepseek-r1-distill-llama-70b {#groq_deepseek_r1_distill_llama_70b_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### distil-whisper-large-v3-en {#groq_distil_whisper_large_v3_en_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### gemma2-9b-it {#groq_gemma2_9b_it_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### llama-3.1-8b-instant {#groq_llama_3.1_8b_instant_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### llama-3.3-70b-versatile {#groq_llama_3.3_70b_versatile_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### llama-guard-3-8b {#groq_llama_guard_3_8b_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### llama3-70b-8192 {#groq_llama3_70b_8192_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### llama3-8b-8192 {#groq_llama3_8b_8192_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### meta-llama/llama-4-maverick-17b-128e-instruct {#groq_meta_llama_llama_4_maverick_17b_128e_instruct_details} + +### Basic + +- ❌ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### meta-llama/llama-4-scout-17b-16e-instruct {#groq_meta_llama_llama_4_scout_17b_16e_instruct_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### meta-llama/llama-guard-4-12b {#groq_meta_llama_llama_guard_4_12b_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### meta-llama/llama-prompt-guard-2-22m {#groq_meta_llama_llama_prompt_guard_2_22m_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### meta-llama/llama-prompt-guard-2-86m {#groq_meta_llama_llama_prompt_guard_2_86m_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### mistral-saba-24b {#groq_mistral_saba_24b_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### playai-tts {#groq_playai_tts_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### playai-tts-arabic {#groq_playai_tts_arabic_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### qwen-qwq-32b {#groq_qwen_qwq_32b_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### qwen/qwen3-32b {#groq_qwen_qwen3_32b_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### whisper-large-v3 {#groq_whisper_large_v3_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + +#### whisper-large-v3-turbo {#groq_whisper_large_v3_turbo_details} + +### Basic + +- ✅ [`test_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L463) +- ❌ [`test_async_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L704) +- ❌ [`test_auto_summarize`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L731) +- ❌ [`test_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L345) +- ❌ [`test_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L368) +- ✅ [`test_step_stream_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L566) +- ❌ [`test_step_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L488) +- ❌ [`test_step_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L513) +- ❌ [`test_step_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L539) +- ❌ [`test_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L392) + +### Token Streaming + +- ✅ [`test_token_streaming_agent_loop_error`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L674) +- ❌ [`test_token_streaming_greeting_with_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L593) +- ❌ [`test_token_streaming_greeting_without_assistant_message`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L619) +- ❌ [`test_token_streaming_tool_call`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L646) + +### Multimodal + +- ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L440) +- ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/4b8c690fda9120641be656b11751f1c7fe232aee/.github/scripts/model-sweep/model_sweep.py#L417) + + +--- diff --git a/.github/scripts/test_markdown.mdx b/.github/scripts/model-sweep/test_markdown.mdx similarity index 100% rename from .github/scripts/test_markdown.mdx rename to .github/scripts/model-sweep/test_markdown.mdx diff --git a/.github/scripts/model-sweep/test_output.mdx b/.github/scripts/model-sweep/test_output.mdx new file mode 100644 index 00000000..c5fcd41e --- /dev/null +++ b/.github/scripts/model-sweep/test_output.mdx @@ -0,0 +1,4395 @@ +--- +title: Support Models +generated: 2025-06-20T15:57:32.626938 +--- + +# Supported Models + +## Overview + +Letta routinely runs automated scans against available providers and models. These are the results of the latest scan. + +Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 + + +## anthropic + +| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | +|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| +| [`claude-3-5-haiku-20241022`](#anthropic_claude_3_5_haiku_20241022_details) | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | +| [`claude-3-5-sonnet-20241022`](#anthropic_claude_3_5_sonnet_20241022_details) | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | +| [`claude-3-7-sonnet-20250219`](#anthropic_claude_3_7_sonnet_20250219_details) | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | +| [`claude-sonnet-4-20250514`](#anthropic_claude_sonnet_4_20250514_details) | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | +| [`claude-opus-4-20250514`](#anthropic_claude_opus_4_20250514_details) | ✅ | ✅ | ⚠️ | 200,000 | 2025-06-20 | +| [`claude-3-5-sonnet-20240620`](#anthropic_claude_3_5_sonnet_20240620_details) | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 | +| [`claude-3-haiku-20240307`](#anthropic_claude_3_haiku_20240307_details) | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 | +| [`claude-3-opus-20240229`](#anthropic_claude_3_opus_20240229_details) | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 | +| [`claude-3-sonnet-20240229`](#anthropic_claude_3_sonnet_20240229_details) | ❌ | ❌ | ❌ | 200,000 | 2025-06-20 | + +### Detailed Test Results + +#### claude-3-5-haiku-20241022 {#anthropic_claude_3_5_haiku_20241022_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### claude-3-5-sonnet-20241022 {#anthropic_claude_3_5_sonnet_20241022_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### claude-3-7-sonnet-20250219 {#anthropic_claude_3_7_sonnet_20250219_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### claude-sonnet-4-20250514 {#anthropic_claude_sonnet_4_20250514_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### claude-opus-4-20250514 {#anthropic_claude_opus_4_20250514_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### claude-3-5-sonnet-20240620 {#anthropic_claude_3_5_sonnet_20240620_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### claude-3-haiku-20240307 {#anthropic_claude_3_haiku_20240307_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### claude-3-opus-20240229 {#anthropic_claude_3_opus_20240229_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### claude-3-sonnet-20240229 {#anthropic_claude_3_sonnet_20240229_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + + +--- + +## openai + +| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | +|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| +| [`gpt-4.1`](#openai_gpt_4.1_details) | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 | +| [`gpt-4.1-2025-04-14`](#openai_gpt_4.1_2025_04_14_details) | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 | +| [`gpt-4.1-nano-2025-04-14`](#openai_gpt_4.1_nano_2025_04_14_details) | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 | +| [`gpt-4o`](#openai_gpt_4o_details) | ✅ | ✅ | ✅ | 128,000 | 2025-06-20 | +| [`gpt-4o-2024-05-13`](#openai_gpt_4o_2024_05_13_details) | ✅ | ✅ | ✅ | 128,000 | 2025-06-20 | +| [`gpt-4-turbo`](#openai_gpt_4_turbo_details) | ✅ | ✅ | ⚠️ | 8,192 | 2025-06-20 | +| [`gpt-4.1-mini`](#openai_gpt_4.1_mini_details) | ✅ | ✅ | ⚠️ | 1,047,576 | 2025-06-20 | +| [`gpt-4.5-preview`](#openai_gpt_4.5_preview_details) | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 | +| [`gpt-4.5-preview-2025-02-27`](#openai_gpt_4.5_preview_2025_02_27_details) | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 | +| [`gpt-4o-2024-08-06`](#openai_gpt_4o_2024_08_06_details) | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 | +| [`gpt-4-0613`](#openai_gpt_4_0613_details) | ✅ | ✅ | ❌ | 8,192 | 2025-06-20 | +| [`gpt-4-1106-preview`](#openai_gpt_4_1106_preview_details) | ✅ | ✅ | ❌ | 128,000 | 2025-06-20 | +| [`gpt-4-turbo-2024-04-09`](#openai_gpt_4_turbo_2024_04_09_details) | ✅ | ⚠️ | ✅ | 128,000 | 2025-06-20 | +| [`gpt-4.1-mini-2025-04-14`](#openai_gpt_4.1_mini_2025_04_14_details) | ⚠️ | ✅ | ✅ | 1,047,576 | 2025-06-20 | +| [`gpt-4.1-nano`](#openai_gpt_4.1_nano_details) | ⚠️ | ✅ | ✅ | 1,047,576 | 2025-06-20 | +| [`gpt-4o-2024-11-20`](#openai_gpt_4o_2024_11_20_details) | ⚠️ | ✅ | ✅ | 8,192 | 2025-06-20 | +| [`gpt-4-turbo-preview`](#openai_gpt_4_turbo_preview_details) | ✅ | ⚠️ | ❌ | 128,000 | 2025-06-20 | +| [`gpt-4-0125-preview`](#openai_gpt_4_0125_preview_details) | ⚠️ | ✅ | ❌ | 128,000 | 2025-06-20 | +| [`gpt-4o-mini`](#openai_gpt_4o_mini_details) | ⚠️ | ⚠️ | ⚠️ | 128,000 | 2025-06-20 | +| [`gpt-4o-mini-2024-07-18`](#openai_gpt_4o_mini_2024_07_18_details) | ⚠️ | ⚠️ | ❌ | 128,000 | 2025-06-20 | +| [`gpt-4`](#openai_gpt_4_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`o1`](#openai_o1_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`o1-2024-12-17`](#openai_o1_2024_12_17_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`o3`](#openai_o3_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`o3-2025-04-16`](#openai_o3_2025_04_16_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`o3-mini`](#openai_o3_mini_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`o3-mini-2025-01-31`](#openai_o3_mini_2025_01_31_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`o3-pro`](#openai_o3_pro_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`o3-pro-2025-06-10`](#openai_o3_pro_2025_06_10_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | + +### Detailed Test Results + +#### gpt-4.1 {#openai_gpt_4.1_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gpt-4.1-2025-04-14 {#openai_gpt_4.1_2025_04_14_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gpt-4.1-nano-2025-04-14 {#openai_gpt_4.1_nano_2025_04_14_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gpt-4o {#openai_gpt_4o_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gpt-4o-2024-05-13 {#openai_gpt_4o_2024_05_13_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gpt-4-turbo {#openai_gpt_4_turbo_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gpt-4.1-mini {#openai_gpt_4.1_mini_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gpt-4.5-preview {#openai_gpt_4.5_preview_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gpt-4.5-preview-2025-02-27 {#openai_gpt_4.5_preview_2025_02_27_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gpt-4o-2024-08-06 {#openai_gpt_4o_2024_08_06_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gpt-4-0613 {#openai_gpt_4_0613_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gpt-4-1106-preview {#openai_gpt_4_1106_preview_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gpt-4-turbo-2024-04-09 {#openai_gpt_4_turbo_2024_04_09_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gpt-4.1-mini-2025-04-14 {#openai_gpt_4.1_mini_2025_04_14_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gpt-4.1-nano {#openai_gpt_4.1_nano_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gpt-4o-2024-11-20 {#openai_gpt_4o_2024_11_20_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gpt-4-turbo-preview {#openai_gpt_4_turbo_preview_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gpt-4-0125-preview {#openai_gpt_4_0125_preview_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gpt-4o-mini {#openai_gpt_4o_mini_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gpt-4o-mini-2024-07-18 {#openai_gpt_4o_mini_2024_07_18_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gpt-4 {#openai_gpt_4_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### o1 {#openai_o1_details} + +### Basic + +- ❌ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### o1-2024-12-17 {#openai_o1_2024_12_17_details} + +### Basic + +- ❌ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### o3 {#openai_o3_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### o3-2025-04-16 {#openai_o3_2025_04_16_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### o3-mini {#openai_o3_mini_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### o3-mini-2025-01-31 {#openai_o3_mini_2025_01_31_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### o3-pro {#openai_o3_pro_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### o3-pro-2025-06-10 {#openai_o3_pro_2025_06_10_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + + +--- + +## google_ai + +| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | +|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| +| [`gemini-1.5-pro`](#google_ai_gemini_1.5_pro_details) | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 | +| [`gemini-1.5-pro-002`](#google_ai_gemini_1.5_pro_002_details) | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 | +| [`gemini-1.5-pro-latest`](#google_ai_gemini_1.5_pro_latest_details) | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 | +| [`gemini-2.5-flash-preview-04-17-thinking`](#google_ai_gemini_2.5_flash_preview_04_17_thinking_details) | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 | +| [`gemini-2.5-pro-preview-03-25`](#google_ai_gemini_2.5_pro_preview_03_25_details) | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 | +| [`gemini-2.5-pro-preview-05-06`](#google_ai_gemini_2.5_pro_preview_05_06_details) | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 | +| [`gemini-2.5-flash-preview-05-20`](#google_ai_gemini_2.5_flash_preview_05_20_details) | ✅ | ⚠️ | ✅ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-flash-thinking-exp`](#google_ai_gemini_2.0_flash_thinking_exp_details) | ⚠️ | ✅ | ✅ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-flash-thinking-exp-1219`](#google_ai_gemini_2.0_flash_thinking_exp_1219_details) | ⚠️ | ✅ | ✅ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-flash-thinking-exp-01-21`](#google_ai_gemini_2.0_flash_thinking_exp_01_21_details) | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 | +| [`gemini-2.5-flash-preview-04-17`](#google_ai_gemini_2.5_flash_preview_04_17_details) | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 | +| [`gemini-2.5-pro-preview-06-05`](#google_ai_gemini_2.5_pro_preview_06_05_details) | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 | +| [`gemini-1.0-pro-vision-latest`](#google_ai_gemini_1.0_pro_vision_latest_details) | ❌ | ❌ | ❌ | 12,288 | 2025-06-20 | +| [`gemini-1.5-flash`](#google_ai_gemini_1.5_flash_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | +| [`gemini-1.5-flash-002`](#google_ai_gemini_1.5_flash_002_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | +| [`gemini-1.5-flash-8b`](#google_ai_gemini_1.5_flash_8b_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | +| [`gemini-1.5-flash-8b-001`](#google_ai_gemini_1.5_flash_8b_001_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | +| [`gemini-1.5-flash-8b-latest`](#google_ai_gemini_1.5_flash_8b_latest_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | +| [`gemini-1.5-flash-latest`](#google_ai_gemini_1.5_flash_latest_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | +| [`gemini-2.0-flash`](#google_ai_gemini_2.0_flash_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-flash-001`](#google_ai_gemini_2.0_flash_001_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-flash-exp`](#google_ai_gemini_2.0_flash_exp_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-flash-exp-image-generation`](#google_ai_gemini_2.0_flash_exp_image_generation_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-flash-lite`](#google_ai_gemini_2.0_flash_lite_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-flash-lite-001`](#google_ai_gemini_2.0_flash_lite_001_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-flash-lite-preview`](#google_ai_gemini_2.0_flash_lite_preview_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-flash-lite-preview-02-05`](#google_ai_gemini_2.0_flash_lite_preview_02_05_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-flash-preview-image-generation`](#google_ai_gemini_2.0_flash_preview_image_generation_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`gemini-2.0-pro-exp`](#google_ai_gemini_2.0_pro_exp_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-pro-exp-02-05`](#google_ai_gemini_2.0_pro_exp_02_05_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-2.5-flash-preview-tts`](#google_ai_gemini_2.5_flash_preview_tts_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`gemini-2.5-pro-exp-03-25`](#google_ai_gemini_2.5_pro_exp_03_25_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-2.5-pro-preview-tts`](#google_ai_gemini_2.5_pro_preview_tts_details) | ❌ | ❌ | ❌ | 65,536 | 2025-06-20 | +| [`gemini-exp-1206`](#google_ai_gemini_exp_1206_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-pro-vision`](#google_ai_gemini_pro_vision_details) | ❌ | ❌ | ❌ | 12,288 | 2025-06-20 | + +### Detailed Test Results + +#### gemini-1.5-pro {#google_ai_gemini_1.5_pro_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gemini-1.5-pro-002 {#google_ai_gemini_1.5_pro_002_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gemini-1.5-pro-latest {#google_ai_gemini_1.5_pro_latest_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gemini-2.5-flash-preview-04-17-thinking {#google_ai_gemini_2.5_flash_preview_04_17_thinking_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gemini-2.5-pro-preview-03-25 {#google_ai_gemini_2.5_pro_preview_03_25_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gemini-2.5-pro-preview-05-06 {#google_ai_gemini_2.5_pro_preview_05_06_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gemini-2.5-flash-preview-05-20 {#google_ai_gemini_2.5_flash_preview_05_20_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gemini-2.0-flash-thinking-exp {#google_ai_gemini_2.0_flash_thinking_exp_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gemini-2.0-flash-thinking-exp-1219 {#google_ai_gemini_2.0_flash_thinking_exp_1219_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gemini-2.0-flash-thinking-exp-01-21 {#google_ai_gemini_2.0_flash_thinking_exp_01_21_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-2.5-flash-preview-04-17 {#google_ai_gemini_2.5_flash_preview_04_17_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-2.5-pro-preview-06-05 {#google_ai_gemini_2.5_pro_preview_06_05_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gemini-1.0-pro-vision-latest {#google_ai_gemini_1.0_pro_vision_latest_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-1.5-flash {#google_ai_gemini_1.5_flash_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-1.5-flash-002 {#google_ai_gemini_1.5_flash_002_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-1.5-flash-8b {#google_ai_gemini_1.5_flash_8b_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-1.5-flash-8b-001 {#google_ai_gemini_1.5_flash_8b_001_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-1.5-flash-8b-latest {#google_ai_gemini_1.5_flash_8b_latest_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-1.5-flash-latest {#google_ai_gemini_1.5_flash_latest_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-2.0-flash {#google_ai_gemini_2.0_flash_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-2.0-flash-001 {#google_ai_gemini_2.0_flash_001_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-2.0-flash-exp {#google_ai_gemini_2.0_flash_exp_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-2.0-flash-exp-image-generation {#google_ai_gemini_2.0_flash_exp_image_generation_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-2.0-flash-lite {#google_ai_gemini_2.0_flash_lite_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-2.0-flash-lite-001 {#google_ai_gemini_2.0_flash_lite_001_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-2.0-flash-lite-preview {#google_ai_gemini_2.0_flash_lite_preview_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-2.0-flash-lite-preview-02-05 {#google_ai_gemini_2.0_flash_lite_preview_02_05_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-2.0-flash-preview-image-generation {#google_ai_gemini_2.0_flash_preview_image_generation_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-2.0-pro-exp {#google_ai_gemini_2.0_pro_exp_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-2.0-pro-exp-02-05 {#google_ai_gemini_2.0_pro_exp_02_05_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-2.5-flash-preview-tts {#google_ai_gemini_2.5_flash_preview_tts_details} + +### Basic + +- ❌ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-2.5-pro-exp-03-25 {#google_ai_gemini_2.5_pro_exp_03_25_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-2.5-pro-preview-tts {#google_ai_gemini_2.5_pro_preview_tts_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-exp-1206 {#google_ai_gemini_exp_1206_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-pro-vision {#google_ai_gemini_pro_vision_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + + +--- + +## letta + +| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | +|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| +| [`letta-free`](#letta_letta_free_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | + +### Detailed Test Results + +#### letta-free {#letta_letta_free_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + + +--- + +## together + +| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | +|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| +| [`Qwen/Qwen2.5-72B-Instruct-Turbo`](#together_qwen_qwen2.5_72b_instruct_turbo_details) | ✅ | ✅ | ⚠️ | 131,072 | 2025-06-20 | +| [`arcee-ai/virtuoso-large`](#together_arcee_ai_virtuoso_large_details) | ⚠️ | ✅ | ✅ | 131,072 | 2025-06-20 | +| [`Qwen/QwQ-32B`](#together_qwen_qwq_32b_details) | ⚠️ | ✅ | ⚠️ | 131,072 | 2025-06-20 | +| [`Qwen/Qwen2.5-7B-Instruct-Turbo`](#together_qwen_qwen2.5_7b_instruct_turbo_details) | ⚠️ | ✅ | ⚠️ | 32,768 | 2025-06-20 | +| [`Qwen/Qwen2.5-Coder-32B-Instruct`](#together_qwen_qwen2.5_coder_32b_instruct_details) | ⚠️ | ✅ | ⚠️ | 16,384 | 2025-06-20 | +| [`arcee-ai/coder-large`](#together_arcee_ai_coder_large_details) | ⚠️ | ✅ | ⚠️ | 32,768 | 2025-06-20 | +| [`arcee_ai/arcee-spotlight`](#together_arcee_ai_arcee_spotlight_details) | ⚠️ | ✅ | ⚠️ | 131,072 | 2025-06-20 | +| [`meta-llama/Llama-3.2-3B-Instruct-Turbo`](#together_meta_llama_llama_3.2_3b_instruct_turbo_details) | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | +| [`meta-llama/Llama-3.3-70B-Instruct-Turbo`](#together_meta_llama_llama_3.3_70b_instruct_turbo_details) | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | +| [`meta-llama/Llama-3.3-70B-Instruct-Turbo-Free`](#together_meta_llama_llama_3.3_70b_instruct_turbo_free_details) | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | +| [`meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo`](#together_meta_llama_meta_llama_3.1_405b_instruct_turbo_details) | ⚠️ | ✅ | ❌ | 130,815 | 2025-06-20 | +| [`meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo`](#together_meta_llama_meta_llama_3.1_70b_instruct_turbo_details) | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | +| [`nvidia/Llama-3.1-Nemotron-70B-Instruct-HF`](#together_nvidia_llama_3.1_nemotron_70b_instruct_hf_details) | ⚠️ | ✅ | ❌ | 32,768 | 2025-06-20 | +| [`arcee-ai/virtuoso-medium-v2`](#together_arcee_ai_virtuoso_medium_v2_details) | ⚠️ | ⚠️ | ✅ | 131,072 | 2025-06-20 | +| [`meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8`](#together_meta_llama_llama_4_maverick_17b_128e_instruct_fp8_details) | ⚠️ | ❌ | ✅ | 1,048,576 | 2025-06-20 | +| [`Qwen/Qwen3-235B-A22B-fp8-tput`](#together_qwen_qwen3_235b_a22b_fp8_tput_details) | ⚠️ | ⚠️ | ❌ | 40,960 | 2025-06-20 | +| [`deepseek-ai/DeepSeek-V3`](#together_deepseek_ai_deepseek_v3_details) | ⚠️ | ⚠️ | ❌ | 131,072 | 2025-06-20 | +| [`meta-llama/Llama-4-Scout-17B-16E-Instruct`](#together_meta_llama_llama_4_scout_17b_16e_instruct_details) | ⚠️ | ⚠️ | ❌ | 1,048,576 | 2025-06-20 | +| [`meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo`](#together_meta_llama_meta_llama_3.1_8b_instruct_turbo_details) | ⚠️ | ⚠️ | ❌ | 131,072 | 2025-06-20 | +| [`mistralai/Mixtral-8x7B-Instruct-v0.1`](#together_mistralai_mixtral_8x7b_instruct_v0.1_details) | ⚠️ | ⚠️ | ❌ | 32,768 | 2025-06-20 | +| [`arcee-ai/caller`](#together_arcee_ai_caller_details) | ❌ | ⚠️ | ❌ | 32,768 | 2025-06-20 | +| [`mistralai/Mistral-Small-24B-Instruct-2501`](#together_mistralai_mistral_small_24b_instruct_2501_details) | ❌ | ⚠️ | ❌ | 32,768 | 2025-06-20 | +| [`NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO`](#together_nousresearch_nous_hermes_2_mixtral_8x7b_dpo_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`Qwen/Qwen2-72B-Instruct`](#together_qwen_qwen2_72b_instruct_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`Qwen/Qwen2-VL-72B-Instruct`](#together_qwen_qwen2_vl_72b_instruct_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`Qwen/Qwen2.5-VL-72B-Instruct`](#together_qwen_qwen2.5_vl_72b_instruct_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`arcee-ai/arcee-blitz`](#together_arcee_ai_arcee_blitz_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`arcee-ai/maestro-reasoning`](#together_arcee_ai_maestro_reasoning_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| [`deepseek-ai/DeepSeek-R1`](#together_deepseek_ai_deepseek_r1_details) | ❌ | ❌ | ❌ | 163,840 | 2025-06-20 | +| [`deepseek-ai/DeepSeek-R1-Distill-Llama-70B`](#together_deepseek_ai_deepseek_r1_distill_llama_70b_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| [`deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free`](#together_deepseek_ai_deepseek_r1_distill_llama_70b_free_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B`](#together_deepseek_ai_deepseek_r1_distill_qwen_1.5b_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| [`deepseek-ai/DeepSeek-R1-Distill-Qwen-14B`](#together_deepseek_ai_deepseek_r1_distill_qwen_14b_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| [`deepseek-ai/DeepSeek-V3-p-dp`](#together_deepseek_ai_deepseek_v3_p_dp_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| [`google/gemma-2-27b-it`](#together_google_gemma_2_27b_it_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`lgai/exaone-3-5-32b-instruct`](#together_lgai_exaone_3_5_32b_instruct_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`lgai/exaone-deep-32b`](#together_lgai_exaone_deep_32b_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`marin-community/marin-8b-instruct`](#together_marin_community_marin_8b_instruct_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| [`meta-llama/Llama-3-70b-chat-hf`](#together_meta_llama_llama_3_70b_chat_hf_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`meta-llama/Llama-3-8b-chat-hf`](#together_meta_llama_llama_3_8b_chat_hf_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo`](#together_meta_llama_llama_3.2_11b_vision_instruct_turbo_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| [`meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo`](#together_meta_llama_llama_3.2_90b_vision_instruct_turbo_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| [`meta-llama/Llama-Vision-Free`](#together_meta_llama_llama_vision_free_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| [`meta-llama/Meta-Llama-3-70B-Instruct-Turbo`](#together_meta_llama_meta_llama_3_70b_instruct_turbo_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`meta-llama/Meta-Llama-3-8B-Instruct-Lite`](#together_meta_llama_meta_llama_3_8b_instruct_lite_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`mistralai/Mistral-7B-Instruct-v0.1`](#together_mistralai_mistral_7b_instruct_v0.1_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`mistralai/Mistral-7B-Instruct-v0.2`](#together_mistralai_mistral_7b_instruct_v0.2_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`mistralai/Mistral-7B-Instruct-v0.3`](#together_mistralai_mistral_7b_instruct_v0.3_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`perplexity-ai/r1-1776`](#together_perplexity_ai_r1_1776_details) | ❌ | ❌ | ❌ | 163,840 | 2025-06-20 | +| [`scb10x/scb10x-llama3-1-typhoon2-70b-instruct`](#together_scb10x_scb10x_llama3_1_typhoon2_70b_instruct_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`scb10x/scb10x-typhoon-2-1-gemma3-12b`](#together_scb10x_scb10x_typhoon_2_1_gemma3_12b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`togethercomputer/MoA-1`](#together_togethercomputer_moa_1_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`togethercomputer/MoA-1-Turbo`](#together_togethercomputer_moa_1_turbo_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`togethercomputer/Refuel-Llm-V2`](#together_togethercomputer_refuel_llm_v2_details) | ❌ | ❌ | ❌ | 16,384 | 2025-06-20 | +| [`togethercomputer/Refuel-Llm-V2-Small`](#together_togethercomputer_refuel_llm_v2_small_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | + +### Detailed Test Results + +#### Qwen/Qwen2.5-72B-Instruct-Turbo {#together_qwen_qwen2.5_72b_instruct_turbo_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### arcee-ai/virtuoso-large {#together_arcee_ai_virtuoso_large_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### Qwen/QwQ-32B {#together_qwen_qwq_32b_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### Qwen/Qwen2.5-7B-Instruct-Turbo {#together_qwen_qwen2.5_7b_instruct_turbo_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### Qwen/Qwen2.5-Coder-32B-Instruct {#together_qwen_qwen2.5_coder_32b_instruct_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### arcee-ai/coder-large {#together_arcee_ai_coder_large_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### arcee_ai/arcee-spotlight {#together_arcee_ai_arcee_spotlight_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/Llama-3.2-3B-Instruct-Turbo {#together_meta_llama_llama_3.2_3b_instruct_turbo_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/Llama-3.3-70B-Instruct-Turbo {#together_meta_llama_llama_3.3_70b_instruct_turbo_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/Llama-3.3-70B-Instruct-Turbo-Free {#together_meta_llama_llama_3.3_70b_instruct_turbo_free_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo {#together_meta_llama_meta_llama_3.1_405b_instruct_turbo_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo {#together_meta_llama_meta_llama_3.1_70b_instruct_turbo_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### nvidia/Llama-3.1-Nemotron-70B-Instruct-HF {#together_nvidia_llama_3.1_nemotron_70b_instruct_hf_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### arcee-ai/virtuoso-medium-v2 {#together_arcee_ai_virtuoso_medium_v2_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 {#together_meta_llama_llama_4_maverick_17b_128e_instruct_fp8_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### Qwen/Qwen3-235B-A22B-fp8-tput {#together_qwen_qwen3_235b_a22b_fp8_tput_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### deepseek-ai/DeepSeek-V3 {#together_deepseek_ai_deepseek_v3_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/Llama-4-Scout-17B-16E-Instruct {#together_meta_llama_llama_4_scout_17b_16e_instruct_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo {#together_meta_llama_meta_llama_3.1_8b_instruct_turbo_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### mistralai/Mixtral-8x7B-Instruct-v0.1 {#together_mistralai_mixtral_8x7b_instruct_v0.1_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### arcee-ai/caller {#together_arcee_ai_caller_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### mistralai/Mistral-Small-24B-Instruct-2501 {#together_mistralai_mistral_small_24b_instruct_2501_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO {#together_nousresearch_nous_hermes_2_mixtral_8x7b_dpo_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### Qwen/Qwen2-72B-Instruct {#together_qwen_qwen2_72b_instruct_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### Qwen/Qwen2-VL-72B-Instruct {#together_qwen_qwen2_vl_72b_instruct_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### Qwen/Qwen2.5-VL-72B-Instruct {#together_qwen_qwen2.5_vl_72b_instruct_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### arcee-ai/arcee-blitz {#together_arcee_ai_arcee_blitz_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### arcee-ai/maestro-reasoning {#together_arcee_ai_maestro_reasoning_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### deepseek-ai/DeepSeek-R1 {#together_deepseek_ai_deepseek_r1_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### deepseek-ai/DeepSeek-R1-Distill-Llama-70B {#together_deepseek_ai_deepseek_r1_distill_llama_70b_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free {#together_deepseek_ai_deepseek_r1_distill_llama_70b_free_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B {#together_deepseek_ai_deepseek_r1_distill_qwen_1.5b_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### deepseek-ai/DeepSeek-R1-Distill-Qwen-14B {#together_deepseek_ai_deepseek_r1_distill_qwen_14b_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### deepseek-ai/DeepSeek-V3-p-dp {#together_deepseek_ai_deepseek_v3_p_dp_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### google/gemma-2-27b-it {#together_google_gemma_2_27b_it_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### lgai/exaone-3-5-32b-instruct {#together_lgai_exaone_3_5_32b_instruct_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### lgai/exaone-deep-32b {#together_lgai_exaone_deep_32b_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### marin-community/marin-8b-instruct {#together_marin_community_marin_8b_instruct_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/Llama-3-70b-chat-hf {#together_meta_llama_llama_3_70b_chat_hf_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/Llama-3-8b-chat-hf {#together_meta_llama_llama_3_8b_chat_hf_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo {#together_meta_llama_llama_3.2_11b_vision_instruct_turbo_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo {#together_meta_llama_llama_3.2_90b_vision_instruct_turbo_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/Llama-Vision-Free {#together_meta_llama_llama_vision_free_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/Meta-Llama-3-70B-Instruct-Turbo {#together_meta_llama_meta_llama_3_70b_instruct_turbo_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/Meta-Llama-3-8B-Instruct-Lite {#together_meta_llama_meta_llama_3_8b_instruct_lite_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### mistralai/Mistral-7B-Instruct-v0.1 {#together_mistralai_mistral_7b_instruct_v0.1_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### mistralai/Mistral-7B-Instruct-v0.2 {#together_mistralai_mistral_7b_instruct_v0.2_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### mistralai/Mistral-7B-Instruct-v0.3 {#together_mistralai_mistral_7b_instruct_v0.3_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### perplexity-ai/r1-1776 {#together_perplexity_ai_r1_1776_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### scb10x/scb10x-llama3-1-typhoon2-70b-instruct {#together_scb10x_scb10x_llama3_1_typhoon2_70b_instruct_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### scb10x/scb10x-typhoon-2-1-gemma3-12b {#together_scb10x_scb10x_typhoon_2_1_gemma3_12b_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### togethercomputer/MoA-1 {#together_togethercomputer_moa_1_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### togethercomputer/MoA-1-Turbo {#together_togethercomputer_moa_1_turbo_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### togethercomputer/Refuel-Llm-V2 {#together_togethercomputer_refuel_llm_v2_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### togethercomputer/Refuel-Llm-V2-Small {#together_togethercomputer_refuel_llm_v2_small_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + + +--- + +## deepseek + +| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | +|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| +| [`deepseek-chat`](#deepseek_deepseek_chat_details) | ❌ | ❌ | ❌ | 64,000 | 2025-06-20 | +| [`deepseek-reasoner`](#deepseek_deepseek_reasoner_details) | ❌ | ❌ | ❌ | 64,000 | 2025-06-20 | + +### Detailed Test Results + +#### deepseek-chat {#deepseek_deepseek_chat_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### deepseek-reasoner {#deepseek_deepseek_reasoner_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + + +--- + +## groq + +| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | +|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| +| [`allam-2-7b`](#groq_allam_2_7b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`compound-beta`](#groq_compound_beta_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`compound-beta-mini`](#groq_compound_beta_mini_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`deepseek-r1-distill-llama-70b`](#groq_deepseek_r1_distill_llama_70b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`distil-whisper-large-v3-en`](#groq_distil_whisper_large_v3_en_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`gemma2-9b-it`](#groq_gemma2_9b_it_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`llama-3.1-8b-instant`](#groq_llama_3.1_8b_instant_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`llama-3.3-70b-versatile`](#groq_llama_3.3_70b_versatile_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`llama-guard-3-8b`](#groq_llama_guard_3_8b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`llama3-70b-8192`](#groq_llama3_70b_8192_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`llama3-8b-8192`](#groq_llama3_8b_8192_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`meta-llama/llama-4-maverick-17b-128e-instruct`](#groq_meta_llama_llama_4_maverick_17b_128e_instruct_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`meta-llama/llama-4-scout-17b-16e-instruct`](#groq_meta_llama_llama_4_scout_17b_16e_instruct_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`meta-llama/llama-guard-4-12b`](#groq_meta_llama_llama_guard_4_12b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`meta-llama/llama-prompt-guard-2-22m`](#groq_meta_llama_llama_prompt_guard_2_22m_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`meta-llama/llama-prompt-guard-2-86m`](#groq_meta_llama_llama_prompt_guard_2_86m_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`mistral-saba-24b`](#groq_mistral_saba_24b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`playai-tts`](#groq_playai_tts_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`playai-tts-arabic`](#groq_playai_tts_arabic_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`qwen-qwq-32b`](#groq_qwen_qwq_32b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`qwen/qwen3-32b`](#groq_qwen_qwen3_32b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`whisper-large-v3`](#groq_whisper_large_v3_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`whisper-large-v3-turbo`](#groq_whisper_large_v3_turbo_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | + +### Detailed Test Results + +#### allam-2-7b {#groq_allam_2_7b_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### compound-beta {#groq_compound_beta_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### compound-beta-mini {#groq_compound_beta_mini_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### deepseek-r1-distill-llama-70b {#groq_deepseek_r1_distill_llama_70b_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### distil-whisper-large-v3-en {#groq_distil_whisper_large_v3_en_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemma2-9b-it {#groq_gemma2_9b_it_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### llama-3.1-8b-instant {#groq_llama_3.1_8b_instant_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### llama-3.3-70b-versatile {#groq_llama_3.3_70b_versatile_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### llama-guard-3-8b {#groq_llama_guard_3_8b_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### llama3-70b-8192 {#groq_llama3_70b_8192_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### llama3-8b-8192 {#groq_llama3_8b_8192_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/llama-4-maverick-17b-128e-instruct {#groq_meta_llama_llama_4_maverick_17b_128e_instruct_details} + +### Basic + +- ❌ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/llama-4-scout-17b-16e-instruct {#groq_meta_llama_llama_4_scout_17b_16e_instruct_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/llama-guard-4-12b {#groq_meta_llama_llama_guard_4_12b_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/llama-prompt-guard-2-22m {#groq_meta_llama_llama_prompt_guard_2_22m_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/llama-prompt-guard-2-86m {#groq_meta_llama_llama_prompt_guard_2_86m_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### mistral-saba-24b {#groq_mistral_saba_24b_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### playai-tts {#groq_playai_tts_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### playai-tts-arabic {#groq_playai_tts_arabic_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### qwen-qwq-32b {#groq_qwen_qwq_32b_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### qwen/qwen3-32b {#groq_qwen_qwen3_32b_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### whisper-large-v3 {#groq_whisper_large_v3_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### whisper-large-v3-turbo {#groq_whisper_large_v3_turbo_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + + +--- diff --git a/.github/scripts/model-sweep/test_output_with_links.mdx b/.github/scripts/model-sweep/test_output_with_links.mdx new file mode 100644 index 00000000..64a14ea7 --- /dev/null +++ b/.github/scripts/model-sweep/test_output_with_links.mdx @@ -0,0 +1,4395 @@ +--- +title: Support Models +generated: 2025-06-20T16:06:14.319292 +--- + +# Supported Models + +## Overview + +Letta routinely runs automated scans against available providers and models. These are the results of the latest scan. + +Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 + + +## anthropic + +| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | +|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| +| [`claude-3-5-haiku-20241022`](#anthropic_claude_3_5_haiku_20241022_details) | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | +| [`claude-3-5-sonnet-20241022`](#anthropic_claude_3_5_sonnet_20241022_details) | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | +| [`claude-3-7-sonnet-20250219`](#anthropic_claude_3_7_sonnet_20250219_details) | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | +| [`claude-sonnet-4-20250514`](#anthropic_claude_sonnet_4_20250514_details) | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | +| [`claude-opus-4-20250514`](#anthropic_claude_opus_4_20250514_details) | ✅ | ✅ | ⚠️ | 200,000 | 2025-06-20 | +| [`claude-3-5-sonnet-20240620`](#anthropic_claude_3_5_sonnet_20240620_details) | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 | +| [`claude-3-haiku-20240307`](#anthropic_claude_3_haiku_20240307_details) | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 | +| [`claude-3-opus-20240229`](#anthropic_claude_3_opus_20240229_details) | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 | +| [`claude-3-sonnet-20240229`](#anthropic_claude_3_sonnet_20240229_details) | ❌ | ❌ | ❌ | 200,000 | 2025-06-20 | + +### Detailed Test Results + +#### claude-3-5-haiku-20241022 {#anthropic_claude_3_5_haiku_20241022_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### claude-3-5-sonnet-20241022 {#anthropic_claude_3_5_sonnet_20241022_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### claude-3-7-sonnet-20250219 {#anthropic_claude_3_7_sonnet_20250219_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### claude-sonnet-4-20250514 {#anthropic_claude_sonnet_4_20250514_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### claude-opus-4-20250514 {#anthropic_claude_opus_4_20250514_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### claude-3-5-sonnet-20240620 {#anthropic_claude_3_5_sonnet_20240620_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### claude-3-haiku-20240307 {#anthropic_claude_3_haiku_20240307_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### claude-3-opus-20240229 {#anthropic_claude_3_opus_20240229_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### claude-3-sonnet-20240229 {#anthropic_claude_3_sonnet_20240229_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + + +--- + +## openai + +| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | +|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| +| [`gpt-4.1`](#openai_gpt_4.1_details) | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 | +| [`gpt-4.1-2025-04-14`](#openai_gpt_4.1_2025_04_14_details) | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 | +| [`gpt-4.1-nano-2025-04-14`](#openai_gpt_4.1_nano_2025_04_14_details) | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 | +| [`gpt-4o`](#openai_gpt_4o_details) | ✅ | ✅ | ✅ | 128,000 | 2025-06-20 | +| [`gpt-4o-2024-05-13`](#openai_gpt_4o_2024_05_13_details) | ✅ | ✅ | ✅ | 128,000 | 2025-06-20 | +| [`gpt-4-turbo`](#openai_gpt_4_turbo_details) | ✅ | ✅ | ⚠️ | 8,192 | 2025-06-20 | +| [`gpt-4.1-mini`](#openai_gpt_4.1_mini_details) | ✅ | ✅ | ⚠️ | 1,047,576 | 2025-06-20 | +| [`gpt-4.5-preview`](#openai_gpt_4.5_preview_details) | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 | +| [`gpt-4.5-preview-2025-02-27`](#openai_gpt_4.5_preview_2025_02_27_details) | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 | +| [`gpt-4o-2024-08-06`](#openai_gpt_4o_2024_08_06_details) | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 | +| [`gpt-4-0613`](#openai_gpt_4_0613_details) | ✅ | ✅ | ❌ | 8,192 | 2025-06-20 | +| [`gpt-4-1106-preview`](#openai_gpt_4_1106_preview_details) | ✅ | ✅ | ❌ | 128,000 | 2025-06-20 | +| [`gpt-4-turbo-2024-04-09`](#openai_gpt_4_turbo_2024_04_09_details) | ✅ | ⚠️ | ✅ | 128,000 | 2025-06-20 | +| [`gpt-4.1-mini-2025-04-14`](#openai_gpt_4.1_mini_2025_04_14_details) | ⚠️ | ✅ | ✅ | 1,047,576 | 2025-06-20 | +| [`gpt-4.1-nano`](#openai_gpt_4.1_nano_details) | ⚠️ | ✅ | ✅ | 1,047,576 | 2025-06-20 | +| [`gpt-4o-2024-11-20`](#openai_gpt_4o_2024_11_20_details) | ⚠️ | ✅ | ✅ | 8,192 | 2025-06-20 | +| [`gpt-4-turbo-preview`](#openai_gpt_4_turbo_preview_details) | ✅ | ⚠️ | ❌ | 128,000 | 2025-06-20 | +| [`gpt-4-0125-preview`](#openai_gpt_4_0125_preview_details) | ⚠️ | ✅ | ❌ | 128,000 | 2025-06-20 | +| [`gpt-4o-mini`](#openai_gpt_4o_mini_details) | ⚠️ | ⚠️ | ⚠️ | 128,000 | 2025-06-20 | +| [`gpt-4o-mini-2024-07-18`](#openai_gpt_4o_mini_2024_07_18_details) | ⚠️ | ⚠️ | ❌ | 128,000 | 2025-06-20 | +| [`gpt-4`](#openai_gpt_4_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`o1`](#openai_o1_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`o1-2024-12-17`](#openai_o1_2024_12_17_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`o3`](#openai_o3_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`o3-2025-04-16`](#openai_o3_2025_04_16_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`o3-mini`](#openai_o3_mini_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`o3-mini-2025-01-31`](#openai_o3_mini_2025_01_31_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`o3-pro`](#openai_o3_pro_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`o3-pro-2025-06-10`](#openai_o3_pro_2025_06_10_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | + +### Detailed Test Results + +#### gpt-4.1 {#openai_gpt_4.1_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gpt-4.1-2025-04-14 {#openai_gpt_4.1_2025_04_14_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gpt-4.1-nano-2025-04-14 {#openai_gpt_4.1_nano_2025_04_14_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gpt-4o {#openai_gpt_4o_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gpt-4o-2024-05-13 {#openai_gpt_4o_2024_05_13_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gpt-4-turbo {#openai_gpt_4_turbo_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gpt-4.1-mini {#openai_gpt_4.1_mini_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gpt-4.5-preview {#openai_gpt_4.5_preview_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gpt-4.5-preview-2025-02-27 {#openai_gpt_4.5_preview_2025_02_27_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gpt-4o-2024-08-06 {#openai_gpt_4o_2024_08_06_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gpt-4-0613 {#openai_gpt_4_0613_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gpt-4-1106-preview {#openai_gpt_4_1106_preview_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gpt-4-turbo-2024-04-09 {#openai_gpt_4_turbo_2024_04_09_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gpt-4.1-mini-2025-04-14 {#openai_gpt_4.1_mini_2025_04_14_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gpt-4.1-nano {#openai_gpt_4.1_nano_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gpt-4o-2024-11-20 {#openai_gpt_4o_2024_11_20_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gpt-4-turbo-preview {#openai_gpt_4_turbo_preview_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gpt-4-0125-preview {#openai_gpt_4_0125_preview_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gpt-4o-mini {#openai_gpt_4o_mini_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gpt-4o-mini-2024-07-18 {#openai_gpt_4o_mini_2024_07_18_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gpt-4 {#openai_gpt_4_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### o1 {#openai_o1_details} + +### Basic + +- ❌ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### o1-2024-12-17 {#openai_o1_2024_12_17_details} + +### Basic + +- ❌ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### o3 {#openai_o3_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### o3-2025-04-16 {#openai_o3_2025_04_16_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### o3-mini {#openai_o3_mini_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### o3-mini-2025-01-31 {#openai_o3_mini_2025_01_31_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### o3-pro {#openai_o3_pro_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### o3-pro-2025-06-10 {#openai_o3_pro_2025_06_10_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + + +--- + +## google_ai + +| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | +|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| +| [`gemini-1.5-pro`](#google_ai_gemini_1.5_pro_details) | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 | +| [`gemini-1.5-pro-002`](#google_ai_gemini_1.5_pro_002_details) | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 | +| [`gemini-1.5-pro-latest`](#google_ai_gemini_1.5_pro_latest_details) | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 | +| [`gemini-2.5-flash-preview-04-17-thinking`](#google_ai_gemini_2.5_flash_preview_04_17_thinking_details) | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 | +| [`gemini-2.5-pro-preview-03-25`](#google_ai_gemini_2.5_pro_preview_03_25_details) | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 | +| [`gemini-2.5-pro-preview-05-06`](#google_ai_gemini_2.5_pro_preview_05_06_details) | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 | +| [`gemini-2.5-flash-preview-05-20`](#google_ai_gemini_2.5_flash_preview_05_20_details) | ✅ | ⚠️ | ✅ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-flash-thinking-exp`](#google_ai_gemini_2.0_flash_thinking_exp_details) | ⚠️ | ✅ | ✅ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-flash-thinking-exp-1219`](#google_ai_gemini_2.0_flash_thinking_exp_1219_details) | ⚠️ | ✅ | ✅ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-flash-thinking-exp-01-21`](#google_ai_gemini_2.0_flash_thinking_exp_01_21_details) | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 | +| [`gemini-2.5-flash-preview-04-17`](#google_ai_gemini_2.5_flash_preview_04_17_details) | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 | +| [`gemini-2.5-pro-preview-06-05`](#google_ai_gemini_2.5_pro_preview_06_05_details) | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 | +| [`gemini-1.0-pro-vision-latest`](#google_ai_gemini_1.0_pro_vision_latest_details) | ❌ | ❌ | ❌ | 12,288 | 2025-06-20 | +| [`gemini-1.5-flash`](#google_ai_gemini_1.5_flash_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | +| [`gemini-1.5-flash-002`](#google_ai_gemini_1.5_flash_002_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | +| [`gemini-1.5-flash-8b`](#google_ai_gemini_1.5_flash_8b_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | +| [`gemini-1.5-flash-8b-001`](#google_ai_gemini_1.5_flash_8b_001_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | +| [`gemini-1.5-flash-8b-latest`](#google_ai_gemini_1.5_flash_8b_latest_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | +| [`gemini-1.5-flash-latest`](#google_ai_gemini_1.5_flash_latest_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | +| [`gemini-2.0-flash`](#google_ai_gemini_2.0_flash_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-flash-001`](#google_ai_gemini_2.0_flash_001_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-flash-exp`](#google_ai_gemini_2.0_flash_exp_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-flash-exp-image-generation`](#google_ai_gemini_2.0_flash_exp_image_generation_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-flash-lite`](#google_ai_gemini_2.0_flash_lite_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-flash-lite-001`](#google_ai_gemini_2.0_flash_lite_001_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-flash-lite-preview`](#google_ai_gemini_2.0_flash_lite_preview_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-flash-lite-preview-02-05`](#google_ai_gemini_2.0_flash_lite_preview_02_05_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-flash-preview-image-generation`](#google_ai_gemini_2.0_flash_preview_image_generation_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`gemini-2.0-pro-exp`](#google_ai_gemini_2.0_pro_exp_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-2.0-pro-exp-02-05`](#google_ai_gemini_2.0_pro_exp_02_05_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-2.5-flash-preview-tts`](#google_ai_gemini_2.5_flash_preview_tts_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`gemini-2.5-pro-exp-03-25`](#google_ai_gemini_2.5_pro_exp_03_25_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-2.5-pro-preview-tts`](#google_ai_gemini_2.5_pro_preview_tts_details) | ❌ | ❌ | ❌ | 65,536 | 2025-06-20 | +| [`gemini-exp-1206`](#google_ai_gemini_exp_1206_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| [`gemini-pro-vision`](#google_ai_gemini_pro_vision_details) | ❌ | ❌ | ❌ | 12,288 | 2025-06-20 | + +### Detailed Test Results + +#### gemini-1.5-pro {#google_ai_gemini_1.5_pro_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gemini-1.5-pro-002 {#google_ai_gemini_1.5_pro_002_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gemini-1.5-pro-latest {#google_ai_gemini_1.5_pro_latest_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gemini-2.5-flash-preview-04-17-thinking {#google_ai_gemini_2.5_flash_preview_04_17_thinking_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gemini-2.5-pro-preview-03-25 {#google_ai_gemini_2.5_pro_preview_03_25_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gemini-2.5-pro-preview-05-06 {#google_ai_gemini_2.5_pro_preview_05_06_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gemini-2.5-flash-preview-05-20 {#google_ai_gemini_2.5_flash_preview_05_20_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gemini-2.0-flash-thinking-exp {#google_ai_gemini_2.0_flash_thinking_exp_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gemini-2.0-flash-thinking-exp-1219 {#google_ai_gemini_2.0_flash_thinking_exp_1219_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gemini-2.0-flash-thinking-exp-01-21 {#google_ai_gemini_2.0_flash_thinking_exp_01_21_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-2.5-flash-preview-04-17 {#google_ai_gemini_2.5_flash_preview_04_17_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-2.5-pro-preview-06-05 {#google_ai_gemini_2.5_pro_preview_06_05_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### gemini-1.0-pro-vision-latest {#google_ai_gemini_1.0_pro_vision_latest_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-1.5-flash {#google_ai_gemini_1.5_flash_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-1.5-flash-002 {#google_ai_gemini_1.5_flash_002_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-1.5-flash-8b {#google_ai_gemini_1.5_flash_8b_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-1.5-flash-8b-001 {#google_ai_gemini_1.5_flash_8b_001_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-1.5-flash-8b-latest {#google_ai_gemini_1.5_flash_8b_latest_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-1.5-flash-latest {#google_ai_gemini_1.5_flash_latest_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-2.0-flash {#google_ai_gemini_2.0_flash_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-2.0-flash-001 {#google_ai_gemini_2.0_flash_001_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-2.0-flash-exp {#google_ai_gemini_2.0_flash_exp_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-2.0-flash-exp-image-generation {#google_ai_gemini_2.0_flash_exp_image_generation_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-2.0-flash-lite {#google_ai_gemini_2.0_flash_lite_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-2.0-flash-lite-001 {#google_ai_gemini_2.0_flash_lite_001_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-2.0-flash-lite-preview {#google_ai_gemini_2.0_flash_lite_preview_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-2.0-flash-lite-preview-02-05 {#google_ai_gemini_2.0_flash_lite_preview_02_05_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-2.0-flash-preview-image-generation {#google_ai_gemini_2.0_flash_preview_image_generation_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-2.0-pro-exp {#google_ai_gemini_2.0_pro_exp_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-2.0-pro-exp-02-05 {#google_ai_gemini_2.0_pro_exp_02_05_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-2.5-flash-preview-tts {#google_ai_gemini_2.5_flash_preview_tts_details} + +### Basic + +- ❌ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-2.5-pro-exp-03-25 {#google_ai_gemini_2.5_pro_exp_03_25_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-2.5-pro-preview-tts {#google_ai_gemini_2.5_pro_preview_tts_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-exp-1206 {#google_ai_gemini_exp_1206_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemini-pro-vision {#google_ai_gemini_pro_vision_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + + +--- + +## letta + +| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | +|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| +| [`letta-free`](#letta_letta_free_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | + +### Detailed Test Results + +#### letta-free {#letta_letta_free_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + + +--- + +## together + +| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | +|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| +| [`Qwen/Qwen2.5-72B-Instruct-Turbo`](#together_qwen_qwen2.5_72b_instruct_turbo_details) | ✅ | ✅ | ⚠️ | 131,072 | 2025-06-20 | +| [`arcee-ai/virtuoso-large`](#together_arcee_ai_virtuoso_large_details) | ⚠️ | ✅ | ✅ | 131,072 | 2025-06-20 | +| [`Qwen/QwQ-32B`](#together_qwen_qwq_32b_details) | ⚠️ | ✅ | ⚠️ | 131,072 | 2025-06-20 | +| [`Qwen/Qwen2.5-7B-Instruct-Turbo`](#together_qwen_qwen2.5_7b_instruct_turbo_details) | ⚠️ | ✅ | ⚠️ | 32,768 | 2025-06-20 | +| [`Qwen/Qwen2.5-Coder-32B-Instruct`](#together_qwen_qwen2.5_coder_32b_instruct_details) | ⚠️ | ✅ | ⚠️ | 16,384 | 2025-06-20 | +| [`arcee-ai/coder-large`](#together_arcee_ai_coder_large_details) | ⚠️ | ✅ | ⚠️ | 32,768 | 2025-06-20 | +| [`arcee_ai/arcee-spotlight`](#together_arcee_ai_arcee_spotlight_details) | ⚠️ | ✅ | ⚠️ | 131,072 | 2025-06-20 | +| [`meta-llama/Llama-3.2-3B-Instruct-Turbo`](#together_meta_llama_llama_3.2_3b_instruct_turbo_details) | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | +| [`meta-llama/Llama-3.3-70B-Instruct-Turbo`](#together_meta_llama_llama_3.3_70b_instruct_turbo_details) | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | +| [`meta-llama/Llama-3.3-70B-Instruct-Turbo-Free`](#together_meta_llama_llama_3.3_70b_instruct_turbo_free_details) | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | +| [`meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo`](#together_meta_llama_meta_llama_3.1_405b_instruct_turbo_details) | ⚠️ | ✅ | ❌ | 130,815 | 2025-06-20 | +| [`meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo`](#together_meta_llama_meta_llama_3.1_70b_instruct_turbo_details) | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | +| [`nvidia/Llama-3.1-Nemotron-70B-Instruct-HF`](#together_nvidia_llama_3.1_nemotron_70b_instruct_hf_details) | ⚠️ | ✅ | ❌ | 32,768 | 2025-06-20 | +| [`arcee-ai/virtuoso-medium-v2`](#together_arcee_ai_virtuoso_medium_v2_details) | ⚠️ | ⚠️ | ✅ | 131,072 | 2025-06-20 | +| [`meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8`](#together_meta_llama_llama_4_maverick_17b_128e_instruct_fp8_details) | ⚠️ | ❌ | ✅ | 1,048,576 | 2025-06-20 | +| [`Qwen/Qwen3-235B-A22B-fp8-tput`](#together_qwen_qwen3_235b_a22b_fp8_tput_details) | ⚠️ | ⚠️ | ❌ | 40,960 | 2025-06-20 | +| [`deepseek-ai/DeepSeek-V3`](#together_deepseek_ai_deepseek_v3_details) | ⚠️ | ⚠️ | ❌ | 131,072 | 2025-06-20 | +| [`meta-llama/Llama-4-Scout-17B-16E-Instruct`](#together_meta_llama_llama_4_scout_17b_16e_instruct_details) | ⚠️ | ⚠️ | ❌ | 1,048,576 | 2025-06-20 | +| [`meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo`](#together_meta_llama_meta_llama_3.1_8b_instruct_turbo_details) | ⚠️ | ⚠️ | ❌ | 131,072 | 2025-06-20 | +| [`mistralai/Mixtral-8x7B-Instruct-v0.1`](#together_mistralai_mixtral_8x7b_instruct_v0.1_details) | ⚠️ | ⚠️ | ❌ | 32,768 | 2025-06-20 | +| [`arcee-ai/caller`](#together_arcee_ai_caller_details) | ❌ | ⚠️ | ❌ | 32,768 | 2025-06-20 | +| [`mistralai/Mistral-Small-24B-Instruct-2501`](#together_mistralai_mistral_small_24b_instruct_2501_details) | ❌ | ⚠️ | ❌ | 32,768 | 2025-06-20 | +| [`NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO`](#together_nousresearch_nous_hermes_2_mixtral_8x7b_dpo_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`Qwen/Qwen2-72B-Instruct`](#together_qwen_qwen2_72b_instruct_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`Qwen/Qwen2-VL-72B-Instruct`](#together_qwen_qwen2_vl_72b_instruct_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`Qwen/Qwen2.5-VL-72B-Instruct`](#together_qwen_qwen2.5_vl_72b_instruct_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`arcee-ai/arcee-blitz`](#together_arcee_ai_arcee_blitz_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`arcee-ai/maestro-reasoning`](#together_arcee_ai_maestro_reasoning_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| [`deepseek-ai/DeepSeek-R1`](#together_deepseek_ai_deepseek_r1_details) | ❌ | ❌ | ❌ | 163,840 | 2025-06-20 | +| [`deepseek-ai/DeepSeek-R1-Distill-Llama-70B`](#together_deepseek_ai_deepseek_r1_distill_llama_70b_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| [`deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free`](#together_deepseek_ai_deepseek_r1_distill_llama_70b_free_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B`](#together_deepseek_ai_deepseek_r1_distill_qwen_1.5b_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| [`deepseek-ai/DeepSeek-R1-Distill-Qwen-14B`](#together_deepseek_ai_deepseek_r1_distill_qwen_14b_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| [`deepseek-ai/DeepSeek-V3-p-dp`](#together_deepseek_ai_deepseek_v3_p_dp_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| [`google/gemma-2-27b-it`](#together_google_gemma_2_27b_it_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`lgai/exaone-3-5-32b-instruct`](#together_lgai_exaone_3_5_32b_instruct_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`lgai/exaone-deep-32b`](#together_lgai_exaone_deep_32b_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`marin-community/marin-8b-instruct`](#together_marin_community_marin_8b_instruct_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| [`meta-llama/Llama-3-70b-chat-hf`](#together_meta_llama_llama_3_70b_chat_hf_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`meta-llama/Llama-3-8b-chat-hf`](#together_meta_llama_llama_3_8b_chat_hf_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo`](#together_meta_llama_llama_3.2_11b_vision_instruct_turbo_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| [`meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo`](#together_meta_llama_llama_3.2_90b_vision_instruct_turbo_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| [`meta-llama/Llama-Vision-Free`](#together_meta_llama_llama_vision_free_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| [`meta-llama/Meta-Llama-3-70B-Instruct-Turbo`](#together_meta_llama_meta_llama_3_70b_instruct_turbo_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`meta-llama/Meta-Llama-3-8B-Instruct-Lite`](#together_meta_llama_meta_llama_3_8b_instruct_lite_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`mistralai/Mistral-7B-Instruct-v0.1`](#together_mistralai_mistral_7b_instruct_v0.1_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`mistralai/Mistral-7B-Instruct-v0.2`](#together_mistralai_mistral_7b_instruct_v0.2_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`mistralai/Mistral-7B-Instruct-v0.3`](#together_mistralai_mistral_7b_instruct_v0.3_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`perplexity-ai/r1-1776`](#together_perplexity_ai_r1_1776_details) | ❌ | ❌ | ❌ | 163,840 | 2025-06-20 | +| [`scb10x/scb10x-llama3-1-typhoon2-70b-instruct`](#together_scb10x_scb10x_llama3_1_typhoon2_70b_instruct_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`scb10x/scb10x-typhoon-2-1-gemma3-12b`](#together_scb10x_scb10x_typhoon_2_1_gemma3_12b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`togethercomputer/MoA-1`](#together_togethercomputer_moa_1_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`togethercomputer/MoA-1-Turbo`](#together_togethercomputer_moa_1_turbo_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| [`togethercomputer/Refuel-Llm-V2`](#together_togethercomputer_refuel_llm_v2_details) | ❌ | ❌ | ❌ | 16,384 | 2025-06-20 | +| [`togethercomputer/Refuel-Llm-V2-Small`](#together_togethercomputer_refuel_llm_v2_small_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | + +### Detailed Test Results + +#### Qwen/Qwen2.5-72B-Instruct-Turbo {#together_qwen_qwen2.5_72b_instruct_turbo_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### arcee-ai/virtuoso-large {#together_arcee_ai_virtuoso_large_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### Qwen/QwQ-32B {#together_qwen_qwq_32b_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### Qwen/Qwen2.5-7B-Instruct-Turbo {#together_qwen_qwen2.5_7b_instruct_turbo_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### Qwen/Qwen2.5-Coder-32B-Instruct {#together_qwen_qwen2.5_coder_32b_instruct_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### arcee-ai/coder-large {#together_arcee_ai_coder_large_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### arcee_ai/arcee-spotlight {#together_arcee_ai_arcee_spotlight_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/Llama-3.2-3B-Instruct-Turbo {#together_meta_llama_llama_3.2_3b_instruct_turbo_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/Llama-3.3-70B-Instruct-Turbo {#together_meta_llama_llama_3.3_70b_instruct_turbo_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/Llama-3.3-70B-Instruct-Turbo-Free {#together_meta_llama_llama_3.3_70b_instruct_turbo_free_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo {#together_meta_llama_meta_llama_3.1_405b_instruct_turbo_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo {#together_meta_llama_meta_llama_3.1_70b_instruct_turbo_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### nvidia/Llama-3.1-Nemotron-70B-Instruct-HF {#together_nvidia_llama_3.1_nemotron_70b_instruct_hf_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ✅ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### arcee-ai/virtuoso-medium-v2 {#together_arcee_ai_virtuoso_medium_v2_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 {#together_meta_llama_llama_4_maverick_17b_128e_instruct_fp8_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ✅ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ✅ `test_base64_image_input` +- ✅ `test_url_image_input` + +#### Qwen/Qwen3-235B-A22B-fp8-tput {#together_qwen_qwen3_235b_a22b_fp8_tput_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### deepseek-ai/DeepSeek-V3 {#together_deepseek_ai_deepseek_v3_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/Llama-4-Scout-17B-16E-Instruct {#together_meta_llama_llama_4_scout_17b_16e_instruct_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo {#together_meta_llama_meta_llama_3.1_8b_instruct_turbo_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ✅ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### mistralai/Mixtral-8x7B-Instruct-v0.1 {#together_mistralai_mixtral_8x7b_instruct_v0.1_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ✅ `test_auto_summarize` +- ✅ `test_greeting_with_assistant_message` +- ✅ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ✅ `test_step_streaming_greeting_with_assistant_message` +- ✅ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ✅ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### arcee-ai/caller {#together_arcee_ai_caller_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ✅ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### mistralai/Mistral-Small-24B-Instruct-2501 {#together_mistralai_mistral_small_24b_instruct_2501_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ✅ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO {#together_nousresearch_nous_hermes_2_mixtral_8x7b_dpo_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### Qwen/Qwen2-72B-Instruct {#together_qwen_qwen2_72b_instruct_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### Qwen/Qwen2-VL-72B-Instruct {#together_qwen_qwen2_vl_72b_instruct_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### Qwen/Qwen2.5-VL-72B-Instruct {#together_qwen_qwen2.5_vl_72b_instruct_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### arcee-ai/arcee-blitz {#together_arcee_ai_arcee_blitz_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### arcee-ai/maestro-reasoning {#together_arcee_ai_maestro_reasoning_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### deepseek-ai/DeepSeek-R1 {#together_deepseek_ai_deepseek_r1_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### deepseek-ai/DeepSeek-R1-Distill-Llama-70B {#together_deepseek_ai_deepseek_r1_distill_llama_70b_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free {#together_deepseek_ai_deepseek_r1_distill_llama_70b_free_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B {#together_deepseek_ai_deepseek_r1_distill_qwen_1.5b_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### deepseek-ai/DeepSeek-R1-Distill-Qwen-14B {#together_deepseek_ai_deepseek_r1_distill_qwen_14b_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### deepseek-ai/DeepSeek-V3-p-dp {#together_deepseek_ai_deepseek_v3_p_dp_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### google/gemma-2-27b-it {#together_google_gemma_2_27b_it_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### lgai/exaone-3-5-32b-instruct {#together_lgai_exaone_3_5_32b_instruct_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### lgai/exaone-deep-32b {#together_lgai_exaone_deep_32b_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### marin-community/marin-8b-instruct {#together_marin_community_marin_8b_instruct_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/Llama-3-70b-chat-hf {#together_meta_llama_llama_3_70b_chat_hf_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/Llama-3-8b-chat-hf {#together_meta_llama_llama_3_8b_chat_hf_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo {#together_meta_llama_llama_3.2_11b_vision_instruct_turbo_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo {#together_meta_llama_llama_3.2_90b_vision_instruct_turbo_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/Llama-Vision-Free {#together_meta_llama_llama_vision_free_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/Meta-Llama-3-70B-Instruct-Turbo {#together_meta_llama_meta_llama_3_70b_instruct_turbo_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/Meta-Llama-3-8B-Instruct-Lite {#together_meta_llama_meta_llama_3_8b_instruct_lite_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### mistralai/Mistral-7B-Instruct-v0.1 {#together_mistralai_mistral_7b_instruct_v0.1_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### mistralai/Mistral-7B-Instruct-v0.2 {#together_mistralai_mistral_7b_instruct_v0.2_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### mistralai/Mistral-7B-Instruct-v0.3 {#together_mistralai_mistral_7b_instruct_v0.3_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### perplexity-ai/r1-1776 {#together_perplexity_ai_r1_1776_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### scb10x/scb10x-llama3-1-typhoon2-70b-instruct {#together_scb10x_scb10x_llama3_1_typhoon2_70b_instruct_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### scb10x/scb10x-typhoon-2-1-gemma3-12b {#together_scb10x_scb10x_typhoon_2_1_gemma3_12b_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### togethercomputer/MoA-1 {#together_togethercomputer_moa_1_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### togethercomputer/MoA-1-Turbo {#together_togethercomputer_moa_1_turbo_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### togethercomputer/Refuel-Llm-V2 {#together_togethercomputer_refuel_llm_v2_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### togethercomputer/Refuel-Llm-V2-Small {#together_togethercomputer_refuel_llm_v2_small_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + + +--- + +## deepseek + +| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | +|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| +| [`deepseek-chat`](#deepseek_deepseek_chat_details) | ❌ | ❌ | ❌ | 64,000 | 2025-06-20 | +| [`deepseek-reasoner`](#deepseek_deepseek_reasoner_details) | ❌ | ❌ | ❌ | 64,000 | 2025-06-20 | + +### Detailed Test Results + +#### deepseek-chat {#deepseek_deepseek_chat_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### deepseek-reasoner {#deepseek_deepseek_reasoner_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + + +--- + +## groq + +| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | +|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| +| [`allam-2-7b`](#groq_allam_2_7b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`compound-beta`](#groq_compound_beta_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`compound-beta-mini`](#groq_compound_beta_mini_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`deepseek-r1-distill-llama-70b`](#groq_deepseek_r1_distill_llama_70b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`distil-whisper-large-v3-en`](#groq_distil_whisper_large_v3_en_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`gemma2-9b-it`](#groq_gemma2_9b_it_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`llama-3.1-8b-instant`](#groq_llama_3.1_8b_instant_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`llama-3.3-70b-versatile`](#groq_llama_3.3_70b_versatile_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`llama-guard-3-8b`](#groq_llama_guard_3_8b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`llama3-70b-8192`](#groq_llama3_70b_8192_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`llama3-8b-8192`](#groq_llama3_8b_8192_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`meta-llama/llama-4-maverick-17b-128e-instruct`](#groq_meta_llama_llama_4_maverick_17b_128e_instruct_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`meta-llama/llama-4-scout-17b-16e-instruct`](#groq_meta_llama_llama_4_scout_17b_16e_instruct_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`meta-llama/llama-guard-4-12b`](#groq_meta_llama_llama_guard_4_12b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`meta-llama/llama-prompt-guard-2-22m`](#groq_meta_llama_llama_prompt_guard_2_22m_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`meta-llama/llama-prompt-guard-2-86m`](#groq_meta_llama_llama_prompt_guard_2_86m_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`mistral-saba-24b`](#groq_mistral_saba_24b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`playai-tts`](#groq_playai_tts_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`playai-tts-arabic`](#groq_playai_tts_arabic_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`qwen-qwq-32b`](#groq_qwen_qwq_32b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`qwen/qwen3-32b`](#groq_qwen_qwen3_32b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`whisper-large-v3`](#groq_whisper_large_v3_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| [`whisper-large-v3-turbo`](#groq_whisper_large_v3_turbo_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | + +### Detailed Test Results + +#### allam-2-7b {#groq_allam_2_7b_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### compound-beta {#groq_compound_beta_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### compound-beta-mini {#groq_compound_beta_mini_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### deepseek-r1-distill-llama-70b {#groq_deepseek_r1_distill_llama_70b_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### distil-whisper-large-v3-en {#groq_distil_whisper_large_v3_en_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### gemma2-9b-it {#groq_gemma2_9b_it_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### llama-3.1-8b-instant {#groq_llama_3.1_8b_instant_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### llama-3.3-70b-versatile {#groq_llama_3.3_70b_versatile_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### llama-guard-3-8b {#groq_llama_guard_3_8b_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### llama3-70b-8192 {#groq_llama3_70b_8192_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### llama3-8b-8192 {#groq_llama3_8b_8192_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/llama-4-maverick-17b-128e-instruct {#groq_meta_llama_llama_4_maverick_17b_128e_instruct_details} + +### Basic + +- ❌ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/llama-4-scout-17b-16e-instruct {#groq_meta_llama_llama_4_scout_17b_16e_instruct_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/llama-guard-4-12b {#groq_meta_llama_llama_guard_4_12b_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/llama-prompt-guard-2-22m {#groq_meta_llama_llama_prompt_guard_2_22m_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### meta-llama/llama-prompt-guard-2-86m {#groq_meta_llama_llama_prompt_guard_2_86m_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### mistral-saba-24b {#groq_mistral_saba_24b_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### playai-tts {#groq_playai_tts_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### playai-tts-arabic {#groq_playai_tts_arabic_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### qwen-qwq-32b {#groq_qwen_qwq_32b_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### qwen/qwen3-32b {#groq_qwen_qwen3_32b_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### whisper-large-v3 {#groq_whisper_large_v3_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + +#### whisper-large-v3-turbo {#groq_whisper_large_v3_turbo_details} + +### Basic + +- ✅ `test_agent_loop_error` +- ❌ `test_async_greeting_with_assistant_message` +- ❌ `test_auto_summarize` +- ❌ `test_greeting_with_assistant_message` +- ❌ `test_greeting_without_assistant_message` +- ✅ `test_step_stream_agent_loop_error` +- ❌ `test_step_streaming_greeting_with_assistant_message` +- ❌ `test_step_streaming_greeting_without_assistant_message` +- ❌ `test_step_streaming_tool_call` +- ❌ `test_tool_call` + +### Token Streaming + +- ✅ `test_token_streaming_agent_loop_error` +- ❌ `test_token_streaming_greeting_with_assistant_message` +- ❌ `test_token_streaming_greeting_without_assistant_message` +- ❌ `test_token_streaming_tool_call` + +### Multimodal + +- ❌ `test_base64_image_input` +- ❌ `test_url_image_input` + + +--- diff --git a/.github/workflows/model-sweep.yaml b/.github/workflows/model-sweep.yaml index 1cf8650c..1b8d9ec7 100644 --- a/.github/workflows/model-sweep.yaml +++ b/.github/workflows/model-sweep.yaml @@ -101,18 +101,18 @@ jobs: run: | poetry run pytest \ -s -vv \ - .github/scripts/model_sweep.py \ - --json-report --json-report-file=.github/scripts/model_sweep_report.json --json-report-indent=4 + .github/scripts/model-sweep/model_sweep.py \ + --json-report --json-report-file=.github/scripts/model-sweep/model_sweep_report.json --json-report-indent=4 - name: Convert report to markdown continue-on-error: true # file path args to generate_model_sweep_markdown.py are relative to the script run: | poetry run python \ - .github/scripts/generate_model_sweep_markdown.py \ - .github/scripts/model_sweep_report.json \ - .github/scripts/supported-models.mdx - echo "Model sweep report saved to .github/scripts/supported-models.mdx" + .github/scripts/model-sweep/generate_model_sweep_markdown.py \ + .github/scripts/model-sweep/model_sweep_report.json \ + .github/scripts/model-sweep/supported-models.mdx + echo "Model sweep report saved to .github/scripts/model-sweep/supported-models.mdx" - id: date run: echo "date=$(date +%Y-%m-%d)" >> $GITHUB_OUTPUT @@ -126,7 +126,7 @@ jobs: git config --global user.name "github-actions[bot]" git config --global user.email "github-actions[bot]@users.noreply.github.com" git checkout -b $BRANCH_NAME - git add .github/scripts/supported-models.mdx + git add .github/scripts/model-sweep/supported-models.mdx git commit -m "Update model sweep report" # only push if changes were made if git diff main --quiet; then @@ -146,4 +146,4 @@ jobs: uses: actions/upload-artifact@v4 with: name: model-sweep-report - path: .github/scripts/model_sweep_report.json + path: .github/scripts/model-sweep/model_sweep_report.json