From ebe6aa68ce01d729fa7beea0816892ff64f8137a Mon Sep 17 00:00:00 2001 From: Kian Jones Date: Fri, 20 Jun 2025 16:24:29 -0700 Subject: [PATCH] tweak --- .../generate_model_sweep_markdown.py | 4 +- .../model-sweep/model_sweep_report.mdx | 618 +-- .github/scripts/model-sweep/test_markdown.mdx | 216 - .github/scripts/model-sweep/test_output.mdx | 4395 ----------------- .../model-sweep/test_output_with_links.mdx | 4395 ----------------- 5 files changed, 311 insertions(+), 9317 deletions(-) delete mode 100644 .github/scripts/model-sweep/test_markdown.mdx delete mode 100644 .github/scripts/model-sweep/test_output.mdx delete mode 100644 .github/scripts/model-sweep/test_output_with_links.mdx diff --git a/.github/scripts/model-sweep/generate_model_sweep_markdown.py b/.github/scripts/model-sweep/generate_model_sweep_markdown.py index 7382939d..09aafa7e 100644 --- a/.github/scripts/model-sweep/generate_model_sweep_markdown.py +++ b/.github/scripts/model-sweep/generate_model_sweep_markdown.py @@ -406,7 +406,7 @@ def process_model_sweep_report(input_file, output_file, config_file=None, debug= model_anchor = model_info['name'].replace('/', '_').replace(':', '_').replace('-', '_').lower() # Build row with left-aligned first column, centered others - row_parts = [f"[`{model_info['name']}`](#{provider_name.lower().replace(' ', '_')}_{model_anchor}_details)".ljust(column_widths['model'] + 20)] # Add extra space for link syntax + row_parts = [f"`{model_info['name']}`".ljust(column_widths['model'])] for feature in feature_names: row_parts.append(f"{model_info['feature_support'][feature]:^{column_widths[feature]}}") row_parts.extend([ @@ -421,7 +421,7 @@ def process_model_sweep_report(input_file, output_file, config_file=None, debug= for model_info in model_data: model_anchor = model_info['name'].replace('/', '_').replace(':', '_').replace('-', '_').lower() - mdx_lines.append(f"#### {model_info['name']} {{#{provider_name.lower().replace(' ', '_')}_{model_anchor}_details}}") + mdx_lines.append(f"#### {model_info['name']}") mdx_lines.append("") # Add test details diff --git a/.github/scripts/model-sweep/model_sweep_report.mdx b/.github/scripts/model-sweep/model_sweep_report.mdx index 350ce7e5..8990e440 100644 --- a/.github/scripts/model-sweep/model_sweep_report.mdx +++ b/.github/scripts/model-sweep/model_sweep_report.mdx @@ -1,6 +1,6 @@ --- title: Support Models -generated: 2025-06-20T16:18:18.561937 +generated: 2025-06-20T16:24:07.581430 --- # Supported Models @@ -16,19 +16,19 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 | Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | |---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| -| [`claude-3-5-haiku-20241022`](#anthropic_claude_3_5_haiku_20241022_details) | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | -| [`claude-3-5-sonnet-20241022`](#anthropic_claude_3_5_sonnet_20241022_details) | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | -| [`claude-3-7-sonnet-20250219`](#anthropic_claude_3_7_sonnet_20250219_details) | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | -| [`claude-sonnet-4-20250514`](#anthropic_claude_sonnet_4_20250514_details) | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | -| [`claude-opus-4-20250514`](#anthropic_claude_opus_4_20250514_details) | ✅ | ✅ | ⚠️ | 200,000 | 2025-06-20 | -| [`claude-3-5-sonnet-20240620`](#anthropic_claude_3_5_sonnet_20240620_details) | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 | -| [`claude-3-haiku-20240307`](#anthropic_claude_3_haiku_20240307_details) | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 | -| [`claude-3-opus-20240229`](#anthropic_claude_3_opus_20240229_details) | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 | -| [`claude-3-sonnet-20240229`](#anthropic_claude_3_sonnet_20240229_details) | ❌ | ❌ | ❌ | 200,000 | 2025-06-20 | +| `claude-3-5-haiku-20241022` | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | +| `claude-3-5-sonnet-20241022` | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | +| `claude-3-7-sonnet-20250219` | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | +| `claude-sonnet-4-20250514` | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | +| `claude-opus-4-20250514` | ✅ | ✅ | ⚠️ | 200,000 | 2025-06-20 | +| `claude-3-5-sonnet-20240620` | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 | +| `claude-3-haiku-20240307` | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 | +| `claude-3-opus-20240229` | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 | +| `claude-3-sonnet-20240229` | ❌ | ❌ | ❌ | 200,000 | 2025-06-20 | ### Detailed Test Results -#### claude-3-5-haiku-20241022 {#anthropic_claude_3_5_haiku_20241022_details} +#### claude-3-5-haiku-20241022 ### Basic @@ -55,7 +55,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### claude-3-5-sonnet-20241022 {#anthropic_claude_3_5_sonnet_20241022_details} +#### claude-3-5-sonnet-20241022 ### Basic @@ -82,7 +82,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### claude-3-7-sonnet-20250219 {#anthropic_claude_3_7_sonnet_20250219_details} +#### claude-3-7-sonnet-20250219 ### Basic @@ -109,7 +109,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### claude-sonnet-4-20250514 {#anthropic_claude_sonnet_4_20250514_details} +#### claude-sonnet-4-20250514 ### Basic @@ -136,7 +136,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### claude-opus-4-20250514 {#anthropic_claude_opus_4_20250514_details} +#### claude-opus-4-20250514 ### Basic @@ -163,7 +163,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### claude-3-5-sonnet-20240620 {#anthropic_claude_3_5_sonnet_20240620_details} +#### claude-3-5-sonnet-20240620 ### Basic @@ -190,7 +190,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### claude-3-haiku-20240307 {#anthropic_claude_3_haiku_20240307_details} +#### claude-3-haiku-20240307 ### Basic @@ -217,7 +217,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### claude-3-opus-20240229 {#anthropic_claude_3_opus_20240229_details} +#### claude-3-opus-20240229 ### Basic @@ -244,7 +244,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### claude-3-sonnet-20240229 {#anthropic_claude_3_sonnet_20240229_details} +#### claude-3-sonnet-20240229 ### Basic @@ -278,39 +278,39 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 | Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | |---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| -| [`gpt-4.1`](#openai_gpt_4.1_details) | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 | -| [`gpt-4.1-2025-04-14`](#openai_gpt_4.1_2025_04_14_details) | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 | -| [`gpt-4.1-nano-2025-04-14`](#openai_gpt_4.1_nano_2025_04_14_details) | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 | -| [`gpt-4o`](#openai_gpt_4o_details) | ✅ | ✅ | ✅ | 128,000 | 2025-06-20 | -| [`gpt-4o-2024-05-13`](#openai_gpt_4o_2024_05_13_details) | ✅ | ✅ | ✅ | 128,000 | 2025-06-20 | -| [`gpt-4-turbo`](#openai_gpt_4_turbo_details) | ✅ | ✅ | ⚠️ | 8,192 | 2025-06-20 | -| [`gpt-4.1-mini`](#openai_gpt_4.1_mini_details) | ✅ | ✅ | ⚠️ | 1,047,576 | 2025-06-20 | -| [`gpt-4.5-preview`](#openai_gpt_4.5_preview_details) | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 | -| [`gpt-4.5-preview-2025-02-27`](#openai_gpt_4.5_preview_2025_02_27_details) | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 | -| [`gpt-4o-2024-08-06`](#openai_gpt_4o_2024_08_06_details) | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 | -| [`gpt-4-0613`](#openai_gpt_4_0613_details) | ✅ | ✅ | ❌ | 8,192 | 2025-06-20 | -| [`gpt-4-1106-preview`](#openai_gpt_4_1106_preview_details) | ✅ | ✅ | ❌ | 128,000 | 2025-06-20 | -| [`gpt-4-turbo-2024-04-09`](#openai_gpt_4_turbo_2024_04_09_details) | ✅ | ⚠️ | ✅ | 128,000 | 2025-06-20 | -| [`gpt-4.1-mini-2025-04-14`](#openai_gpt_4.1_mini_2025_04_14_details) | ⚠️ | ✅ | ✅ | 1,047,576 | 2025-06-20 | -| [`gpt-4.1-nano`](#openai_gpt_4.1_nano_details) | ⚠️ | ✅ | ✅ | 1,047,576 | 2025-06-20 | -| [`gpt-4o-2024-11-20`](#openai_gpt_4o_2024_11_20_details) | ⚠️ | ✅ | ✅ | 8,192 | 2025-06-20 | -| [`gpt-4-turbo-preview`](#openai_gpt_4_turbo_preview_details) | ✅ | ⚠️ | ❌ | 128,000 | 2025-06-20 | -| [`gpt-4-0125-preview`](#openai_gpt_4_0125_preview_details) | ⚠️ | ✅ | ❌ | 128,000 | 2025-06-20 | -| [`gpt-4o-mini`](#openai_gpt_4o_mini_details) | ⚠️ | ⚠️ | ⚠️ | 128,000 | 2025-06-20 | -| [`gpt-4o-mini-2024-07-18`](#openai_gpt_4o_mini_2024_07_18_details) | ⚠️ | ⚠️ | ❌ | 128,000 | 2025-06-20 | -| [`gpt-4`](#openai_gpt_4_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`o1`](#openai_o1_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`o1-2024-12-17`](#openai_o1_2024_12_17_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`o3`](#openai_o3_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`o3-2025-04-16`](#openai_o3_2025_04_16_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`o3-mini`](#openai_o3_mini_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`o3-mini-2025-01-31`](#openai_o3_mini_2025_01_31_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`o3-pro`](#openai_o3_pro_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`o3-pro-2025-06-10`](#openai_o3_pro_2025_06_10_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `gpt-4.1` | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 | +| `gpt-4.1-2025-04-14` | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 | +| `gpt-4.1-nano-2025-04-14` | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 | +| `gpt-4o` | ✅ | ✅ | ✅ | 128,000 | 2025-06-20 | +| `gpt-4o-2024-05-13` | ✅ | ✅ | ✅ | 128,000 | 2025-06-20 | +| `gpt-4-turbo` | ✅ | ✅ | ⚠️ | 8,192 | 2025-06-20 | +| `gpt-4.1-mini` | ✅ | ✅ | ⚠️ | 1,047,576 | 2025-06-20 | +| `gpt-4.5-preview` | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 | +| `gpt-4.5-preview-2025-02-27` | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 | +| `gpt-4o-2024-08-06` | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 | +| `gpt-4-0613` | ✅ | ✅ | ❌ | 8,192 | 2025-06-20 | +| `gpt-4-1106-preview` | ✅ | ✅ | ❌ | 128,000 | 2025-06-20 | +| `gpt-4-turbo-2024-04-09` | ✅ | ⚠️ | ✅ | 128,000 | 2025-06-20 | +| `gpt-4.1-mini-2025-04-14` | ⚠️ | ✅ | ✅ | 1,047,576 | 2025-06-20 | +| `gpt-4.1-nano` | ⚠️ | ✅ | ✅ | 1,047,576 | 2025-06-20 | +| `gpt-4o-2024-11-20` | ⚠️ | ✅ | ✅ | 8,192 | 2025-06-20 | +| `gpt-4-turbo-preview` | ✅ | ⚠️ | ❌ | 128,000 | 2025-06-20 | +| `gpt-4-0125-preview` | ⚠️ | ✅ | ❌ | 128,000 | 2025-06-20 | +| `gpt-4o-mini` | ⚠️ | ⚠️ | ⚠️ | 128,000 | 2025-06-20 | +| `gpt-4o-mini-2024-07-18` | ⚠️ | ⚠️ | ❌ | 128,000 | 2025-06-20 | +| `gpt-4` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `o1` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `o1-2024-12-17` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `o3` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `o3-2025-04-16` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `o3-mini` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `o3-mini-2025-01-31` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `o3-pro` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `o3-pro-2025-06-10` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | ### Detailed Test Results -#### gpt-4.1 {#openai_gpt_4.1_details} +#### gpt-4.1 ### Basic @@ -337,7 +337,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gpt-4.1-2025-04-14 {#openai_gpt_4.1_2025_04_14_details} +#### gpt-4.1-2025-04-14 ### Basic @@ -364,7 +364,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gpt-4.1-nano-2025-04-14 {#openai_gpt_4.1_nano_2025_04_14_details} +#### gpt-4.1-nano-2025-04-14 ### Basic @@ -391,7 +391,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gpt-4o {#openai_gpt_4o_details} +#### gpt-4o ### Basic @@ -418,7 +418,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gpt-4o-2024-05-13 {#openai_gpt_4o_2024_05_13_details} +#### gpt-4o-2024-05-13 ### Basic @@ -445,7 +445,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gpt-4-turbo {#openai_gpt_4_turbo_details} +#### gpt-4-turbo ### Basic @@ -472,7 +472,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gpt-4.1-mini {#openai_gpt_4.1_mini_details} +#### gpt-4.1-mini ### Basic @@ -499,7 +499,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gpt-4.5-preview {#openai_gpt_4.5_preview_details} +#### gpt-4.5-preview ### Basic @@ -526,7 +526,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gpt-4.5-preview-2025-02-27 {#openai_gpt_4.5_preview_2025_02_27_details} +#### gpt-4.5-preview-2025-02-27 ### Basic @@ -553,7 +553,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gpt-4o-2024-08-06 {#openai_gpt_4o_2024_08_06_details} +#### gpt-4o-2024-08-06 ### Basic @@ -580,7 +580,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gpt-4-0613 {#openai_gpt_4_0613_details} +#### gpt-4-0613 ### Basic @@ -607,7 +607,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gpt-4-1106-preview {#openai_gpt_4_1106_preview_details} +#### gpt-4-1106-preview ### Basic @@ -634,7 +634,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gpt-4-turbo-2024-04-09 {#openai_gpt_4_turbo_2024_04_09_details} +#### gpt-4-turbo-2024-04-09 ### Basic @@ -661,7 +661,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gpt-4.1-mini-2025-04-14 {#openai_gpt_4.1_mini_2025_04_14_details} +#### gpt-4.1-mini-2025-04-14 ### Basic @@ -688,7 +688,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gpt-4.1-nano {#openai_gpt_4.1_nano_details} +#### gpt-4.1-nano ### Basic @@ -715,7 +715,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gpt-4o-2024-11-20 {#openai_gpt_4o_2024_11_20_details} +#### gpt-4o-2024-11-20 ### Basic @@ -742,7 +742,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gpt-4-turbo-preview {#openai_gpt_4_turbo_preview_details} +#### gpt-4-turbo-preview ### Basic @@ -769,7 +769,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gpt-4-0125-preview {#openai_gpt_4_0125_preview_details} +#### gpt-4-0125-preview ### Basic @@ -796,7 +796,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gpt-4o-mini {#openai_gpt_4o_mini_details} +#### gpt-4o-mini ### Basic @@ -823,7 +823,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gpt-4o-mini-2024-07-18 {#openai_gpt_4o_mini_2024_07_18_details} +#### gpt-4o-mini-2024-07-18 ### Basic @@ -850,7 +850,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gpt-4 {#openai_gpt_4_details} +#### gpt-4 ### Basic @@ -877,7 +877,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### o1 {#openai_o1_details} +#### o1 ### Basic @@ -904,7 +904,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### o1-2024-12-17 {#openai_o1_2024_12_17_details} +#### o1-2024-12-17 ### Basic @@ -931,7 +931,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### o3 {#openai_o3_details} +#### o3 ### Basic @@ -958,7 +958,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### o3-2025-04-16 {#openai_o3_2025_04_16_details} +#### o3-2025-04-16 ### Basic @@ -985,7 +985,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### o3-mini {#openai_o3_mini_details} +#### o3-mini ### Basic @@ -1012,7 +1012,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### o3-mini-2025-01-31 {#openai_o3_mini_2025_01_31_details} +#### o3-mini-2025-01-31 ### Basic @@ -1039,7 +1039,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### o3-pro {#openai_o3_pro_details} +#### o3-pro ### Basic @@ -1066,7 +1066,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### o3-pro-2025-06-10 {#openai_o3_pro_2025_06_10_details} +#### o3-pro-2025-06-10 ### Basic @@ -1100,45 +1100,45 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 | Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | |---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| -| [`gemini-1.5-pro`](#google_ai_gemini_1.5_pro_details) | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 | -| [`gemini-1.5-pro-002`](#google_ai_gemini_1.5_pro_002_details) | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 | -| [`gemini-1.5-pro-latest`](#google_ai_gemini_1.5_pro_latest_details) | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 | -| [`gemini-2.5-flash-preview-04-17-thinking`](#google_ai_gemini_2.5_flash_preview_04_17_thinking_details) | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 | -| [`gemini-2.5-pro-preview-03-25`](#google_ai_gemini_2.5_pro_preview_03_25_details) | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 | -| [`gemini-2.5-pro-preview-05-06`](#google_ai_gemini_2.5_pro_preview_05_06_details) | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 | -| [`gemini-2.5-flash-preview-05-20`](#google_ai_gemini_2.5_flash_preview_05_20_details) | ✅ | ⚠️ | ✅ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-flash-thinking-exp`](#google_ai_gemini_2.0_flash_thinking_exp_details) | ⚠️ | ✅ | ✅ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-flash-thinking-exp-1219`](#google_ai_gemini_2.0_flash_thinking_exp_1219_details) | ⚠️ | ✅ | ✅ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-flash-thinking-exp-01-21`](#google_ai_gemini_2.0_flash_thinking_exp_01_21_details) | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 | -| [`gemini-2.5-flash-preview-04-17`](#google_ai_gemini_2.5_flash_preview_04_17_details) | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 | -| [`gemini-2.5-pro-preview-06-05`](#google_ai_gemini_2.5_pro_preview_06_05_details) | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 | -| [`gemini-1.0-pro-vision-latest`](#google_ai_gemini_1.0_pro_vision_latest_details) | ❌ | ❌ | ❌ | 12,288 | 2025-06-20 | -| [`gemini-1.5-flash`](#google_ai_gemini_1.5_flash_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | -| [`gemini-1.5-flash-002`](#google_ai_gemini_1.5_flash_002_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | -| [`gemini-1.5-flash-8b`](#google_ai_gemini_1.5_flash_8b_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | -| [`gemini-1.5-flash-8b-001`](#google_ai_gemini_1.5_flash_8b_001_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | -| [`gemini-1.5-flash-8b-latest`](#google_ai_gemini_1.5_flash_8b_latest_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | -| [`gemini-1.5-flash-latest`](#google_ai_gemini_1.5_flash_latest_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | -| [`gemini-2.0-flash`](#google_ai_gemini_2.0_flash_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-flash-001`](#google_ai_gemini_2.0_flash_001_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-flash-exp`](#google_ai_gemini_2.0_flash_exp_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-flash-exp-image-generation`](#google_ai_gemini_2.0_flash_exp_image_generation_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-flash-lite`](#google_ai_gemini_2.0_flash_lite_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-flash-lite-001`](#google_ai_gemini_2.0_flash_lite_001_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-flash-lite-preview`](#google_ai_gemini_2.0_flash_lite_preview_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-flash-lite-preview-02-05`](#google_ai_gemini_2.0_flash_lite_preview_02_05_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-flash-preview-image-generation`](#google_ai_gemini_2.0_flash_preview_image_generation_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`gemini-2.0-pro-exp`](#google_ai_gemini_2.0_pro_exp_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-pro-exp-02-05`](#google_ai_gemini_2.0_pro_exp_02_05_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-2.5-flash-preview-tts`](#google_ai_gemini_2.5_flash_preview_tts_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`gemini-2.5-pro-exp-03-25`](#google_ai_gemini_2.5_pro_exp_03_25_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-2.5-pro-preview-tts`](#google_ai_gemini_2.5_pro_preview_tts_details) | ❌ | ❌ | ❌ | 65,536 | 2025-06-20 | -| [`gemini-exp-1206`](#google_ai_gemini_exp_1206_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-pro-vision`](#google_ai_gemini_pro_vision_details) | ❌ | ❌ | ❌ | 12,288 | 2025-06-20 | +| `gemini-1.5-pro` | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 | +| `gemini-1.5-pro-002` | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 | +| `gemini-1.5-pro-latest` | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 | +| `gemini-2.5-flash-preview-04-17-thinking` | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 | +| `gemini-2.5-pro-preview-03-25` | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 | +| `gemini-2.5-pro-preview-05-06` | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 | +| `gemini-2.5-flash-preview-05-20` | ✅ | ⚠️ | ✅ | 1,048,576 | 2025-06-20 | +| `gemini-2.0-flash-thinking-exp` | ⚠️ | ✅ | ✅ | 1,048,576 | 2025-06-20 | +| `gemini-2.0-flash-thinking-exp-1219` | ⚠️ | ✅ | ✅ | 1,048,576 | 2025-06-20 | +| `gemini-2.0-flash-thinking-exp-01-21` | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 | +| `gemini-2.5-flash-preview-04-17` | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 | +| `gemini-2.5-pro-preview-06-05` | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 | +| `gemini-1.0-pro-vision-latest` | ❌ | ❌ | ❌ | 12,288 | 2025-06-20 | +| `gemini-1.5-flash` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | +| `gemini-1.5-flash-002` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | +| `gemini-1.5-flash-8b` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | +| `gemini-1.5-flash-8b-001` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | +| `gemini-1.5-flash-8b-latest` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | +| `gemini-1.5-flash-latest` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | +| `gemini-2.0-flash` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| `gemini-2.0-flash-001` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| `gemini-2.0-flash-exp` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| `gemini-2.0-flash-exp-image-generation` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| `gemini-2.0-flash-lite` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| `gemini-2.0-flash-lite-001` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| `gemini-2.0-flash-lite-preview` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| `gemini-2.0-flash-lite-preview-02-05` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| `gemini-2.0-flash-preview-image-generation` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| `gemini-2.0-pro-exp` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| `gemini-2.0-pro-exp-02-05` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| `gemini-2.5-flash-preview-tts` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| `gemini-2.5-pro-exp-03-25` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| `gemini-2.5-pro-preview-tts` | ❌ | ❌ | ❌ | 65,536 | 2025-06-20 | +| `gemini-exp-1206` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | +| `gemini-pro-vision` | ❌ | ❌ | ❌ | 12,288 | 2025-06-20 | ### Detailed Test Results -#### gemini-1.5-pro {#google_ai_gemini_1.5_pro_details} +#### gemini-1.5-pro ### Basic @@ -1165,7 +1165,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemini-1.5-pro-002 {#google_ai_gemini_1.5_pro_002_details} +#### gemini-1.5-pro-002 ### Basic @@ -1192,7 +1192,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemini-1.5-pro-latest {#google_ai_gemini_1.5_pro_latest_details} +#### gemini-1.5-pro-latest ### Basic @@ -1219,7 +1219,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemini-2.5-flash-preview-04-17-thinking {#google_ai_gemini_2.5_flash_preview_04_17_thinking_details} +#### gemini-2.5-flash-preview-04-17-thinking ### Basic @@ -1246,7 +1246,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemini-2.5-pro-preview-03-25 {#google_ai_gemini_2.5_pro_preview_03_25_details} +#### gemini-2.5-pro-preview-03-25 ### Basic @@ -1273,7 +1273,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemini-2.5-pro-preview-05-06 {#google_ai_gemini_2.5_pro_preview_05_06_details} +#### gemini-2.5-pro-preview-05-06 ### Basic @@ -1300,7 +1300,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemini-2.5-flash-preview-05-20 {#google_ai_gemini_2.5_flash_preview_05_20_details} +#### gemini-2.5-flash-preview-05-20 ### Basic @@ -1327,7 +1327,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemini-2.0-flash-thinking-exp {#google_ai_gemini_2.0_flash_thinking_exp_details} +#### gemini-2.0-flash-thinking-exp ### Basic @@ -1354,7 +1354,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemini-2.0-flash-thinking-exp-1219 {#google_ai_gemini_2.0_flash_thinking_exp_1219_details} +#### gemini-2.0-flash-thinking-exp-1219 ### Basic @@ -1381,7 +1381,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemini-2.0-flash-thinking-exp-01-21 {#google_ai_gemini_2.0_flash_thinking_exp_01_21_details} +#### gemini-2.0-flash-thinking-exp-01-21 ### Basic @@ -1408,7 +1408,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemini-2.5-flash-preview-04-17 {#google_ai_gemini_2.5_flash_preview_04_17_details} +#### gemini-2.5-flash-preview-04-17 ### Basic @@ -1435,7 +1435,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemini-2.5-pro-preview-06-05 {#google_ai_gemini_2.5_pro_preview_06_05_details} +#### gemini-2.5-pro-preview-06-05 ### Basic @@ -1462,7 +1462,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemini-1.0-pro-vision-latest {#google_ai_gemini_1.0_pro_vision_latest_details} +#### gemini-1.0-pro-vision-latest ### Basic @@ -1489,7 +1489,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemini-1.5-flash {#google_ai_gemini_1.5_flash_details} +#### gemini-1.5-flash ### Basic @@ -1516,7 +1516,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemini-1.5-flash-002 {#google_ai_gemini_1.5_flash_002_details} +#### gemini-1.5-flash-002 ### Basic @@ -1543,7 +1543,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemini-1.5-flash-8b {#google_ai_gemini_1.5_flash_8b_details} +#### gemini-1.5-flash-8b ### Basic @@ -1570,7 +1570,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemini-1.5-flash-8b-001 {#google_ai_gemini_1.5_flash_8b_001_details} +#### gemini-1.5-flash-8b-001 ### Basic @@ -1597,7 +1597,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemini-1.5-flash-8b-latest {#google_ai_gemini_1.5_flash_8b_latest_details} +#### gemini-1.5-flash-8b-latest ### Basic @@ -1624,7 +1624,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemini-1.5-flash-latest {#google_ai_gemini_1.5_flash_latest_details} +#### gemini-1.5-flash-latest ### Basic @@ -1651,7 +1651,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemini-2.0-flash {#google_ai_gemini_2.0_flash_details} +#### gemini-2.0-flash ### Basic @@ -1678,7 +1678,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemini-2.0-flash-001 {#google_ai_gemini_2.0_flash_001_details} +#### gemini-2.0-flash-001 ### Basic @@ -1705,7 +1705,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemini-2.0-flash-exp {#google_ai_gemini_2.0_flash_exp_details} +#### gemini-2.0-flash-exp ### Basic @@ -1732,7 +1732,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemini-2.0-flash-exp-image-generation {#google_ai_gemini_2.0_flash_exp_image_generation_details} +#### gemini-2.0-flash-exp-image-generation ### Basic @@ -1759,7 +1759,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemini-2.0-flash-lite {#google_ai_gemini_2.0_flash_lite_details} +#### gemini-2.0-flash-lite ### Basic @@ -1786,7 +1786,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemini-2.0-flash-lite-001 {#google_ai_gemini_2.0_flash_lite_001_details} +#### gemini-2.0-flash-lite-001 ### Basic @@ -1813,7 +1813,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemini-2.0-flash-lite-preview {#google_ai_gemini_2.0_flash_lite_preview_details} +#### gemini-2.0-flash-lite-preview ### Basic @@ -1840,7 +1840,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemini-2.0-flash-lite-preview-02-05 {#google_ai_gemini_2.0_flash_lite_preview_02_05_details} +#### gemini-2.0-flash-lite-preview-02-05 ### Basic @@ -1867,7 +1867,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemini-2.0-flash-preview-image-generation {#google_ai_gemini_2.0_flash_preview_image_generation_details} +#### gemini-2.0-flash-preview-image-generation ### Basic @@ -1894,7 +1894,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemini-2.0-pro-exp {#google_ai_gemini_2.0_pro_exp_details} +#### gemini-2.0-pro-exp ### Basic @@ -1921,7 +1921,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemini-2.0-pro-exp-02-05 {#google_ai_gemini_2.0_pro_exp_02_05_details} +#### gemini-2.0-pro-exp-02-05 ### Basic @@ -1948,7 +1948,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemini-2.5-flash-preview-tts {#google_ai_gemini_2.5_flash_preview_tts_details} +#### gemini-2.5-flash-preview-tts ### Basic @@ -1975,7 +1975,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemini-2.5-pro-exp-03-25 {#google_ai_gemini_2.5_pro_exp_03_25_details} +#### gemini-2.5-pro-exp-03-25 ### Basic @@ -2002,7 +2002,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemini-2.5-pro-preview-tts {#google_ai_gemini_2.5_pro_preview_tts_details} +#### gemini-2.5-pro-preview-tts ### Basic @@ -2029,7 +2029,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemini-exp-1206 {#google_ai_gemini_exp_1206_details} +#### gemini-exp-1206 ### Basic @@ -2056,7 +2056,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemini-pro-vision {#google_ai_gemini_pro_vision_details} +#### gemini-pro-vision ### Basic @@ -2090,11 +2090,11 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 | Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | |---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| -| [`letta-free`](#letta_letta_free_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `letta-free` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | ### Detailed Test Results -#### letta-free {#letta_letta_free_details} +#### letta-free ### Basic @@ -2128,65 +2128,65 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 | Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | |---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| -| [`Qwen/Qwen2.5-72B-Instruct-Turbo`](#together_qwen_qwen2.5_72b_instruct_turbo_details) | ✅ | ✅ | ⚠️ | 131,072 | 2025-06-20 | -| [`arcee-ai/virtuoso-large`](#together_arcee_ai_virtuoso_large_details) | ⚠️ | ✅ | ✅ | 131,072 | 2025-06-20 | -| [`Qwen/QwQ-32B`](#together_qwen_qwq_32b_details) | ⚠️ | ✅ | ⚠️ | 131,072 | 2025-06-20 | -| [`Qwen/Qwen2.5-7B-Instruct-Turbo`](#together_qwen_qwen2.5_7b_instruct_turbo_details) | ⚠️ | ✅ | ⚠️ | 32,768 | 2025-06-20 | -| [`Qwen/Qwen2.5-Coder-32B-Instruct`](#together_qwen_qwen2.5_coder_32b_instruct_details) | ⚠️ | ✅ | ⚠️ | 16,384 | 2025-06-20 | -| [`arcee-ai/coder-large`](#together_arcee_ai_coder_large_details) | ⚠️ | ✅ | ⚠️ | 32,768 | 2025-06-20 | -| [`arcee_ai/arcee-spotlight`](#together_arcee_ai_arcee_spotlight_details) | ⚠️ | ✅ | ⚠️ | 131,072 | 2025-06-20 | -| [`meta-llama/Llama-3.2-3B-Instruct-Turbo`](#together_meta_llama_llama_3.2_3b_instruct_turbo_details) | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | -| [`meta-llama/Llama-3.3-70B-Instruct-Turbo`](#together_meta_llama_llama_3.3_70b_instruct_turbo_details) | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | -| [`meta-llama/Llama-3.3-70B-Instruct-Turbo-Free`](#together_meta_llama_llama_3.3_70b_instruct_turbo_free_details) | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | -| [`meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo`](#together_meta_llama_meta_llama_3.1_405b_instruct_turbo_details) | ⚠️ | ✅ | ❌ | 130,815 | 2025-06-20 | -| [`meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo`](#together_meta_llama_meta_llama_3.1_70b_instruct_turbo_details) | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | -| [`nvidia/Llama-3.1-Nemotron-70B-Instruct-HF`](#together_nvidia_llama_3.1_nemotron_70b_instruct_hf_details) | ⚠️ | ✅ | ❌ | 32,768 | 2025-06-20 | -| [`arcee-ai/virtuoso-medium-v2`](#together_arcee_ai_virtuoso_medium_v2_details) | ⚠️ | ⚠️ | ✅ | 131,072 | 2025-06-20 | -| [`meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8`](#together_meta_llama_llama_4_maverick_17b_128e_instruct_fp8_details) | ⚠️ | ❌ | ✅ | 1,048,576 | 2025-06-20 | -| [`Qwen/Qwen3-235B-A22B-fp8-tput`](#together_qwen_qwen3_235b_a22b_fp8_tput_details) | ⚠️ | ⚠️ | ❌ | 40,960 | 2025-06-20 | -| [`deepseek-ai/DeepSeek-V3`](#together_deepseek_ai_deepseek_v3_details) | ⚠️ | ⚠️ | ❌ | 131,072 | 2025-06-20 | -| [`meta-llama/Llama-4-Scout-17B-16E-Instruct`](#together_meta_llama_llama_4_scout_17b_16e_instruct_details) | ⚠️ | ⚠️ | ❌ | 1,048,576 | 2025-06-20 | -| [`meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo`](#together_meta_llama_meta_llama_3.1_8b_instruct_turbo_details) | ⚠️ | ⚠️ | ❌ | 131,072 | 2025-06-20 | -| [`mistralai/Mixtral-8x7B-Instruct-v0.1`](#together_mistralai_mixtral_8x7b_instruct_v0.1_details) | ⚠️ | ⚠️ | ❌ | 32,768 | 2025-06-20 | -| [`arcee-ai/caller`](#together_arcee_ai_caller_details) | ❌ | ⚠️ | ❌ | 32,768 | 2025-06-20 | -| [`mistralai/Mistral-Small-24B-Instruct-2501`](#together_mistralai_mistral_small_24b_instruct_2501_details) | ❌ | ⚠️ | ❌ | 32,768 | 2025-06-20 | -| [`NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO`](#together_nousresearch_nous_hermes_2_mixtral_8x7b_dpo_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`Qwen/Qwen2-72B-Instruct`](#together_qwen_qwen2_72b_instruct_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`Qwen/Qwen2-VL-72B-Instruct`](#together_qwen_qwen2_vl_72b_instruct_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`Qwen/Qwen2.5-VL-72B-Instruct`](#together_qwen_qwen2.5_vl_72b_instruct_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`arcee-ai/arcee-blitz`](#together_arcee_ai_arcee_blitz_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`arcee-ai/maestro-reasoning`](#together_arcee_ai_maestro_reasoning_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | -| [`deepseek-ai/DeepSeek-R1`](#together_deepseek_ai_deepseek_r1_details) | ❌ | ❌ | ❌ | 163,840 | 2025-06-20 | -| [`deepseek-ai/DeepSeek-R1-Distill-Llama-70B`](#together_deepseek_ai_deepseek_r1_distill_llama_70b_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | -| [`deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free`](#together_deepseek_ai_deepseek_r1_distill_llama_70b_free_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B`](#together_deepseek_ai_deepseek_r1_distill_qwen_1.5b_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | -| [`deepseek-ai/DeepSeek-R1-Distill-Qwen-14B`](#together_deepseek_ai_deepseek_r1_distill_qwen_14b_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | -| [`deepseek-ai/DeepSeek-V3-p-dp`](#together_deepseek_ai_deepseek_v3_p_dp_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | -| [`google/gemma-2-27b-it`](#together_google_gemma_2_27b_it_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`lgai/exaone-3-5-32b-instruct`](#together_lgai_exaone_3_5_32b_instruct_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`lgai/exaone-deep-32b`](#together_lgai_exaone_deep_32b_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`marin-community/marin-8b-instruct`](#together_marin_community_marin_8b_instruct_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | -| [`meta-llama/Llama-3-70b-chat-hf`](#together_meta_llama_llama_3_70b_chat_hf_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`meta-llama/Llama-3-8b-chat-hf`](#together_meta_llama_llama_3_8b_chat_hf_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo`](#together_meta_llama_llama_3.2_11b_vision_instruct_turbo_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | -| [`meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo`](#together_meta_llama_llama_3.2_90b_vision_instruct_turbo_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | -| [`meta-llama/Llama-Vision-Free`](#together_meta_llama_llama_vision_free_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | -| [`meta-llama/Meta-Llama-3-70B-Instruct-Turbo`](#together_meta_llama_meta_llama_3_70b_instruct_turbo_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`meta-llama/Meta-Llama-3-8B-Instruct-Lite`](#together_meta_llama_meta_llama_3_8b_instruct_lite_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`mistralai/Mistral-7B-Instruct-v0.1`](#together_mistralai_mistral_7b_instruct_v0.1_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`mistralai/Mistral-7B-Instruct-v0.2`](#together_mistralai_mistral_7b_instruct_v0.2_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`mistralai/Mistral-7B-Instruct-v0.3`](#together_mistralai_mistral_7b_instruct_v0.3_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`perplexity-ai/r1-1776`](#together_perplexity_ai_r1_1776_details) | ❌ | ❌ | ❌ | 163,840 | 2025-06-20 | -| [`scb10x/scb10x-llama3-1-typhoon2-70b-instruct`](#together_scb10x_scb10x_llama3_1_typhoon2_70b_instruct_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`scb10x/scb10x-typhoon-2-1-gemma3-12b`](#together_scb10x_scb10x_typhoon_2_1_gemma3_12b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`togethercomputer/MoA-1`](#together_togethercomputer_moa_1_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`togethercomputer/MoA-1-Turbo`](#together_togethercomputer_moa_1_turbo_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`togethercomputer/Refuel-Llm-V2`](#together_togethercomputer_refuel_llm_v2_details) | ❌ | ❌ | ❌ | 16,384 | 2025-06-20 | -| [`togethercomputer/Refuel-Llm-V2-Small`](#together_togethercomputer_refuel_llm_v2_small_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `Qwen/Qwen2.5-72B-Instruct-Turbo` | ✅ | ✅ | ⚠️ | 131,072 | 2025-06-20 | +| `arcee-ai/virtuoso-large` | ⚠️ | ✅ | ✅ | 131,072 | 2025-06-20 | +| `Qwen/QwQ-32B` | ⚠️ | ✅ | ⚠️ | 131,072 | 2025-06-20 | +| `Qwen/Qwen2.5-7B-Instruct-Turbo` | ⚠️ | ✅ | ⚠️ | 32,768 | 2025-06-20 | +| `Qwen/Qwen2.5-Coder-32B-Instruct` | ⚠️ | ✅ | ⚠️ | 16,384 | 2025-06-20 | +| `arcee-ai/coder-large` | ⚠️ | ✅ | ⚠️ | 32,768 | 2025-06-20 | +| `arcee_ai/arcee-spotlight` | ⚠️ | ✅ | ⚠️ | 131,072 | 2025-06-20 | +| `meta-llama/Llama-3.2-3B-Instruct-Turbo` | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | +| `meta-llama/Llama-3.3-70B-Instruct-Turbo` | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | +| `meta-llama/Llama-3.3-70B-Instruct-Turbo-Free` | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | +| `meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo` | ⚠️ | ✅ | ❌ | 130,815 | 2025-06-20 | +| `meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo` | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | +| `nvidia/Llama-3.1-Nemotron-70B-Instruct-HF` | ⚠️ | ✅ | ❌ | 32,768 | 2025-06-20 | +| `arcee-ai/virtuoso-medium-v2` | ⚠️ | ⚠️ | ✅ | 131,072 | 2025-06-20 | +| `meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8` | ⚠️ | ❌ | ✅ | 1,048,576 | 2025-06-20 | +| `Qwen/Qwen3-235B-A22B-fp8-tput` | ⚠️ | ⚠️ | ❌ | 40,960 | 2025-06-20 | +| `deepseek-ai/DeepSeek-V3` | ⚠️ | ⚠️ | ❌ | 131,072 | 2025-06-20 | +| `meta-llama/Llama-4-Scout-17B-16E-Instruct` | ⚠️ | ⚠️ | ❌ | 1,048,576 | 2025-06-20 | +| `meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo` | ⚠️ | ⚠️ | ❌ | 131,072 | 2025-06-20 | +| `mistralai/Mixtral-8x7B-Instruct-v0.1` | ⚠️ | ⚠️ | ❌ | 32,768 | 2025-06-20 | +| `arcee-ai/caller` | ❌ | ⚠️ | ❌ | 32,768 | 2025-06-20 | +| `mistralai/Mistral-Small-24B-Instruct-2501` | ❌ | ⚠️ | ❌ | 32,768 | 2025-06-20 | +| `NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| `Qwen/Qwen2-72B-Instruct` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| `Qwen/Qwen2-VL-72B-Instruct` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| `Qwen/Qwen2.5-VL-72B-Instruct` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| `arcee-ai/arcee-blitz` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| `arcee-ai/maestro-reasoning` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| `deepseek-ai/DeepSeek-R1` | ❌ | ❌ | ❌ | 163,840 | 2025-06-20 | +| `deepseek-ai/DeepSeek-R1-Distill-Llama-70B` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| `deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| `deepseek-ai/DeepSeek-R1-Distill-Qwen-14B` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| `deepseek-ai/DeepSeek-V3-p-dp` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| `google/gemma-2-27b-it` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `lgai/exaone-3-5-32b-instruct` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| `lgai/exaone-deep-32b` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| `marin-community/marin-8b-instruct` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| `meta-llama/Llama-3-70b-chat-hf` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `meta-llama/Llama-3-8b-chat-hf` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| `meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| `meta-llama/Llama-Vision-Free` | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | +| `meta-llama/Meta-Llama-3-70B-Instruct-Turbo` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `meta-llama/Meta-Llama-3-8B-Instruct-Lite` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `mistralai/Mistral-7B-Instruct-v0.1` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| `mistralai/Mistral-7B-Instruct-v0.2` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| `mistralai/Mistral-7B-Instruct-v0.3` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| `perplexity-ai/r1-1776` | ❌ | ❌ | ❌ | 163,840 | 2025-06-20 | +| `scb10x/scb10x-llama3-1-typhoon2-70b-instruct` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `scb10x/scb10x-typhoon-2-1-gemma3-12b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `togethercomputer/MoA-1` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| `togethercomputer/MoA-1-Turbo` | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | +| `togethercomputer/Refuel-Llm-V2` | ❌ | ❌ | ❌ | 16,384 | 2025-06-20 | +| `togethercomputer/Refuel-Llm-V2-Small` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | ### Detailed Test Results -#### Qwen/Qwen2.5-72B-Instruct-Turbo {#together_qwen_qwen2.5_72b_instruct_turbo_details} +#### Qwen/Qwen2.5-72B-Instruct-Turbo ### Basic @@ -2213,7 +2213,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### arcee-ai/virtuoso-large {#together_arcee_ai_virtuoso_large_details} +#### arcee-ai/virtuoso-large ### Basic @@ -2240,7 +2240,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### Qwen/QwQ-32B {#together_qwen_qwq_32b_details} +#### Qwen/QwQ-32B ### Basic @@ -2267,7 +2267,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### Qwen/Qwen2.5-7B-Instruct-Turbo {#together_qwen_qwen2.5_7b_instruct_turbo_details} +#### Qwen/Qwen2.5-7B-Instruct-Turbo ### Basic @@ -2294,7 +2294,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### Qwen/Qwen2.5-Coder-32B-Instruct {#together_qwen_qwen2.5_coder_32b_instruct_details} +#### Qwen/Qwen2.5-Coder-32B-Instruct ### Basic @@ -2321,7 +2321,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### arcee-ai/coder-large {#together_arcee_ai_coder_large_details} +#### arcee-ai/coder-large ### Basic @@ -2348,7 +2348,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### arcee_ai/arcee-spotlight {#together_arcee_ai_arcee_spotlight_details} +#### arcee_ai/arcee-spotlight ### Basic @@ -2375,7 +2375,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### meta-llama/Llama-3.2-3B-Instruct-Turbo {#together_meta_llama_llama_3.2_3b_instruct_turbo_details} +#### meta-llama/Llama-3.2-3B-Instruct-Turbo ### Basic @@ -2402,7 +2402,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### meta-llama/Llama-3.3-70B-Instruct-Turbo {#together_meta_llama_llama_3.3_70b_instruct_turbo_details} +#### meta-llama/Llama-3.3-70B-Instruct-Turbo ### Basic @@ -2429,7 +2429,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### meta-llama/Llama-3.3-70B-Instruct-Turbo-Free {#together_meta_llama_llama_3.3_70b_instruct_turbo_free_details} +#### meta-llama/Llama-3.3-70B-Instruct-Turbo-Free ### Basic @@ -2456,7 +2456,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo {#together_meta_llama_meta_llama_3.1_405b_instruct_turbo_details} +#### meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo ### Basic @@ -2483,7 +2483,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo {#together_meta_llama_meta_llama_3.1_70b_instruct_turbo_details} +#### meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo ### Basic @@ -2510,7 +2510,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### nvidia/Llama-3.1-Nemotron-70B-Instruct-HF {#together_nvidia_llama_3.1_nemotron_70b_instruct_hf_details} +#### nvidia/Llama-3.1-Nemotron-70B-Instruct-HF ### Basic @@ -2537,7 +2537,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### arcee-ai/virtuoso-medium-v2 {#together_arcee_ai_virtuoso_medium_v2_details} +#### arcee-ai/virtuoso-medium-v2 ### Basic @@ -2564,7 +2564,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 {#together_meta_llama_llama_4_maverick_17b_128e_instruct_fp8_details} +#### meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 ### Basic @@ -2591,7 +2591,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ✅ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ✅ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### Qwen/Qwen3-235B-A22B-fp8-tput {#together_qwen_qwen3_235b_a22b_fp8_tput_details} +#### Qwen/Qwen3-235B-A22B-fp8-tput ### Basic @@ -2618,7 +2618,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### deepseek-ai/DeepSeek-V3 {#together_deepseek_ai_deepseek_v3_details} +#### deepseek-ai/DeepSeek-V3 ### Basic @@ -2645,7 +2645,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### meta-llama/Llama-4-Scout-17B-16E-Instruct {#together_meta_llama_llama_4_scout_17b_16e_instruct_details} +#### meta-llama/Llama-4-Scout-17B-16E-Instruct ### Basic @@ -2672,7 +2672,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo {#together_meta_llama_meta_llama_3.1_8b_instruct_turbo_details} +#### meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo ### Basic @@ -2699,7 +2699,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### mistralai/Mixtral-8x7B-Instruct-v0.1 {#together_mistralai_mixtral_8x7b_instruct_v0.1_details} +#### mistralai/Mixtral-8x7B-Instruct-v0.1 ### Basic @@ -2726,7 +2726,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### arcee-ai/caller {#together_arcee_ai_caller_details} +#### arcee-ai/caller ### Basic @@ -2753,7 +2753,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### mistralai/Mistral-Small-24B-Instruct-2501 {#together_mistralai_mistral_small_24b_instruct_2501_details} +#### mistralai/Mistral-Small-24B-Instruct-2501 ### Basic @@ -2780,7 +2780,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO {#together_nousresearch_nous_hermes_2_mixtral_8x7b_dpo_details} +#### NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO ### Basic @@ -2807,7 +2807,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### Qwen/Qwen2-72B-Instruct {#together_qwen_qwen2_72b_instruct_details} +#### Qwen/Qwen2-72B-Instruct ### Basic @@ -2834,7 +2834,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### Qwen/Qwen2-VL-72B-Instruct {#together_qwen_qwen2_vl_72b_instruct_details} +#### Qwen/Qwen2-VL-72B-Instruct ### Basic @@ -2861,7 +2861,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### Qwen/Qwen2.5-VL-72B-Instruct {#together_qwen_qwen2.5_vl_72b_instruct_details} +#### Qwen/Qwen2.5-VL-72B-Instruct ### Basic @@ -2888,7 +2888,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### arcee-ai/arcee-blitz {#together_arcee_ai_arcee_blitz_details} +#### arcee-ai/arcee-blitz ### Basic @@ -2915,7 +2915,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### arcee-ai/maestro-reasoning {#together_arcee_ai_maestro_reasoning_details} +#### arcee-ai/maestro-reasoning ### Basic @@ -2942,7 +2942,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### deepseek-ai/DeepSeek-R1 {#together_deepseek_ai_deepseek_r1_details} +#### deepseek-ai/DeepSeek-R1 ### Basic @@ -2969,7 +2969,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### deepseek-ai/DeepSeek-R1-Distill-Llama-70B {#together_deepseek_ai_deepseek_r1_distill_llama_70b_details} +#### deepseek-ai/DeepSeek-R1-Distill-Llama-70B ### Basic @@ -2996,7 +2996,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free {#together_deepseek_ai_deepseek_r1_distill_llama_70b_free_details} +#### deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free ### Basic @@ -3023,7 +3023,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B {#together_deepseek_ai_deepseek_r1_distill_qwen_1.5b_details} +#### deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B ### Basic @@ -3050,7 +3050,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### deepseek-ai/DeepSeek-R1-Distill-Qwen-14B {#together_deepseek_ai_deepseek_r1_distill_qwen_14b_details} +#### deepseek-ai/DeepSeek-R1-Distill-Qwen-14B ### Basic @@ -3077,7 +3077,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### deepseek-ai/DeepSeek-V3-p-dp {#together_deepseek_ai_deepseek_v3_p_dp_details} +#### deepseek-ai/DeepSeek-V3-p-dp ### Basic @@ -3104,7 +3104,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### google/gemma-2-27b-it {#together_google_gemma_2_27b_it_details} +#### google/gemma-2-27b-it ### Basic @@ -3131,7 +3131,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### lgai/exaone-3-5-32b-instruct {#together_lgai_exaone_3_5_32b_instruct_details} +#### lgai/exaone-3-5-32b-instruct ### Basic @@ -3158,7 +3158,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### lgai/exaone-deep-32b {#together_lgai_exaone_deep_32b_details} +#### lgai/exaone-deep-32b ### Basic @@ -3185,7 +3185,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### marin-community/marin-8b-instruct {#together_marin_community_marin_8b_instruct_details} +#### marin-community/marin-8b-instruct ### Basic @@ -3212,7 +3212,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### meta-llama/Llama-3-70b-chat-hf {#together_meta_llama_llama_3_70b_chat_hf_details} +#### meta-llama/Llama-3-70b-chat-hf ### Basic @@ -3239,7 +3239,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### meta-llama/Llama-3-8b-chat-hf {#together_meta_llama_llama_3_8b_chat_hf_details} +#### meta-llama/Llama-3-8b-chat-hf ### Basic @@ -3266,7 +3266,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo {#together_meta_llama_llama_3.2_11b_vision_instruct_turbo_details} +#### meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo ### Basic @@ -3293,7 +3293,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo {#together_meta_llama_llama_3.2_90b_vision_instruct_turbo_details} +#### meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo ### Basic @@ -3320,7 +3320,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### meta-llama/Llama-Vision-Free {#together_meta_llama_llama_vision_free_details} +#### meta-llama/Llama-Vision-Free ### Basic @@ -3347,7 +3347,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### meta-llama/Meta-Llama-3-70B-Instruct-Turbo {#together_meta_llama_meta_llama_3_70b_instruct_turbo_details} +#### meta-llama/Meta-Llama-3-70B-Instruct-Turbo ### Basic @@ -3374,7 +3374,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### meta-llama/Meta-Llama-3-8B-Instruct-Lite {#together_meta_llama_meta_llama_3_8b_instruct_lite_details} +#### meta-llama/Meta-Llama-3-8B-Instruct-Lite ### Basic @@ -3401,7 +3401,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### mistralai/Mistral-7B-Instruct-v0.1 {#together_mistralai_mistral_7b_instruct_v0.1_details} +#### mistralai/Mistral-7B-Instruct-v0.1 ### Basic @@ -3428,7 +3428,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### mistralai/Mistral-7B-Instruct-v0.2 {#together_mistralai_mistral_7b_instruct_v0.2_details} +#### mistralai/Mistral-7B-Instruct-v0.2 ### Basic @@ -3455,7 +3455,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### mistralai/Mistral-7B-Instruct-v0.3 {#together_mistralai_mistral_7b_instruct_v0.3_details} +#### mistralai/Mistral-7B-Instruct-v0.3 ### Basic @@ -3482,7 +3482,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### perplexity-ai/r1-1776 {#together_perplexity_ai_r1_1776_details} +#### perplexity-ai/r1-1776 ### Basic @@ -3509,7 +3509,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### scb10x/scb10x-llama3-1-typhoon2-70b-instruct {#together_scb10x_scb10x_llama3_1_typhoon2_70b_instruct_details} +#### scb10x/scb10x-llama3-1-typhoon2-70b-instruct ### Basic @@ -3536,7 +3536,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### scb10x/scb10x-typhoon-2-1-gemma3-12b {#together_scb10x_scb10x_typhoon_2_1_gemma3_12b_details} +#### scb10x/scb10x-typhoon-2-1-gemma3-12b ### Basic @@ -3563,7 +3563,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### togethercomputer/MoA-1 {#together_togethercomputer_moa_1_details} +#### togethercomputer/MoA-1 ### Basic @@ -3590,7 +3590,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### togethercomputer/MoA-1-Turbo {#together_togethercomputer_moa_1_turbo_details} +#### togethercomputer/MoA-1-Turbo ### Basic @@ -3617,7 +3617,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### togethercomputer/Refuel-Llm-V2 {#together_togethercomputer_refuel_llm_v2_details} +#### togethercomputer/Refuel-Llm-V2 ### Basic @@ -3644,7 +3644,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### togethercomputer/Refuel-Llm-V2-Small {#together_togethercomputer_refuel_llm_v2_small_details} +#### togethercomputer/Refuel-Llm-V2-Small ### Basic @@ -3678,12 +3678,12 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 | Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | |---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| -| [`deepseek-chat`](#deepseek_deepseek_chat_details) | ❌ | ❌ | ❌ | 64,000 | 2025-06-20 | -| [`deepseek-reasoner`](#deepseek_deepseek_reasoner_details) | ❌ | ❌ | ❌ | 64,000 | 2025-06-20 | +| `deepseek-chat` | ❌ | ❌ | ❌ | 64,000 | 2025-06-20 | +| `deepseek-reasoner` | ❌ | ❌ | ❌ | 64,000 | 2025-06-20 | ### Detailed Test Results -#### deepseek-chat {#deepseek_deepseek_chat_details} +#### deepseek-chat ### Basic @@ -3710,7 +3710,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### deepseek-reasoner {#deepseek_deepseek_reasoner_details} +#### deepseek-reasoner ### Basic @@ -3744,33 +3744,33 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 | Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | |---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| -| [`allam-2-7b`](#groq_allam_2_7b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`compound-beta`](#groq_compound_beta_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`compound-beta-mini`](#groq_compound_beta_mini_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`deepseek-r1-distill-llama-70b`](#groq_deepseek_r1_distill_llama_70b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`distil-whisper-large-v3-en`](#groq_distil_whisper_large_v3_en_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`gemma2-9b-it`](#groq_gemma2_9b_it_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`llama-3.1-8b-instant`](#groq_llama_3.1_8b_instant_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`llama-3.3-70b-versatile`](#groq_llama_3.3_70b_versatile_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`llama-guard-3-8b`](#groq_llama_guard_3_8b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`llama3-70b-8192`](#groq_llama3_70b_8192_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`llama3-8b-8192`](#groq_llama3_8b_8192_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`meta-llama/llama-4-maverick-17b-128e-instruct`](#groq_meta_llama_llama_4_maverick_17b_128e_instruct_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`meta-llama/llama-4-scout-17b-16e-instruct`](#groq_meta_llama_llama_4_scout_17b_16e_instruct_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`meta-llama/llama-guard-4-12b`](#groq_meta_llama_llama_guard_4_12b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`meta-llama/llama-prompt-guard-2-22m`](#groq_meta_llama_llama_prompt_guard_2_22m_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`meta-llama/llama-prompt-guard-2-86m`](#groq_meta_llama_llama_prompt_guard_2_86m_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`mistral-saba-24b`](#groq_mistral_saba_24b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`playai-tts`](#groq_playai_tts_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`playai-tts-arabic`](#groq_playai_tts_arabic_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`qwen-qwq-32b`](#groq_qwen_qwq_32b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`qwen/qwen3-32b`](#groq_qwen_qwen3_32b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`whisper-large-v3`](#groq_whisper_large_v3_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`whisper-large-v3-turbo`](#groq_whisper_large_v3_turbo_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `allam-2-7b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `compound-beta` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `compound-beta-mini` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `deepseek-r1-distill-llama-70b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `distil-whisper-large-v3-en` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `gemma2-9b-it` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `llama-3.1-8b-instant` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `llama-3.3-70b-versatile` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `llama-guard-3-8b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `llama3-70b-8192` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `llama3-8b-8192` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `meta-llama/llama-4-maverick-17b-128e-instruct` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `meta-llama/llama-4-scout-17b-16e-instruct` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `meta-llama/llama-guard-4-12b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `meta-llama/llama-prompt-guard-2-22m` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `meta-llama/llama-prompt-guard-2-86m` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `mistral-saba-24b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `playai-tts` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `playai-tts-arabic` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `qwen-qwq-32b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `qwen/qwen3-32b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `whisper-large-v3` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | +| `whisper-large-v3-turbo` | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | ### Detailed Test Results -#### allam-2-7b {#groq_allam_2_7b_details} +#### allam-2-7b ### Basic @@ -3797,7 +3797,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### compound-beta {#groq_compound_beta_details} +#### compound-beta ### Basic @@ -3824,7 +3824,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### compound-beta-mini {#groq_compound_beta_mini_details} +#### compound-beta-mini ### Basic @@ -3851,7 +3851,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### deepseek-r1-distill-llama-70b {#groq_deepseek_r1_distill_llama_70b_details} +#### deepseek-r1-distill-llama-70b ### Basic @@ -3878,7 +3878,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### distil-whisper-large-v3-en {#groq_distil_whisper_large_v3_en_details} +#### distil-whisper-large-v3-en ### Basic @@ -3905,7 +3905,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### gemma2-9b-it {#groq_gemma2_9b_it_details} +#### gemma2-9b-it ### Basic @@ -3932,7 +3932,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### llama-3.1-8b-instant {#groq_llama_3.1_8b_instant_details} +#### llama-3.1-8b-instant ### Basic @@ -3959,7 +3959,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### llama-3.3-70b-versatile {#groq_llama_3.3_70b_versatile_details} +#### llama-3.3-70b-versatile ### Basic @@ -3986,7 +3986,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### llama-guard-3-8b {#groq_llama_guard_3_8b_details} +#### llama-guard-3-8b ### Basic @@ -4013,7 +4013,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### llama3-70b-8192 {#groq_llama3_70b_8192_details} +#### llama3-70b-8192 ### Basic @@ -4040,7 +4040,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### llama3-8b-8192 {#groq_llama3_8b_8192_details} +#### llama3-8b-8192 ### Basic @@ -4067,7 +4067,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### meta-llama/llama-4-maverick-17b-128e-instruct {#groq_meta_llama_llama_4_maverick_17b_128e_instruct_details} +#### meta-llama/llama-4-maverick-17b-128e-instruct ### Basic @@ -4094,7 +4094,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### meta-llama/llama-4-scout-17b-16e-instruct {#groq_meta_llama_llama_4_scout_17b_16e_instruct_details} +#### meta-llama/llama-4-scout-17b-16e-instruct ### Basic @@ -4121,7 +4121,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### meta-llama/llama-guard-4-12b {#groq_meta_llama_llama_guard_4_12b_details} +#### meta-llama/llama-guard-4-12b ### Basic @@ -4148,7 +4148,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### meta-llama/llama-prompt-guard-2-22m {#groq_meta_llama_llama_prompt_guard_2_22m_details} +#### meta-llama/llama-prompt-guard-2-22m ### Basic @@ -4175,7 +4175,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### meta-llama/llama-prompt-guard-2-86m {#groq_meta_llama_llama_prompt_guard_2_86m_details} +#### meta-llama/llama-prompt-guard-2-86m ### Basic @@ -4202,7 +4202,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### mistral-saba-24b {#groq_mistral_saba_24b_details} +#### mistral-saba-24b ### Basic @@ -4229,7 +4229,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### playai-tts {#groq_playai_tts_details} +#### playai-tts ### Basic @@ -4256,7 +4256,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### playai-tts-arabic {#groq_playai_tts_arabic_details} +#### playai-tts-arabic ### Basic @@ -4283,7 +4283,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### qwen-qwq-32b {#groq_qwen_qwq_32b_details} +#### qwen-qwq-32b ### Basic @@ -4310,7 +4310,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### qwen/qwen3-32b {#groq_qwen_qwen3_32b_details} +#### qwen/qwen3-32b ### Basic @@ -4337,7 +4337,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### whisper-large-v3 {#groq_whisper_large_v3_details} +#### whisper-large-v3 ### Basic @@ -4364,7 +4364,7 @@ Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - ❌ [`test_base64_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L440) - ❌ [`test_url_image_input`](https://github.com/letta-ai/letta/blob/kian/add_model_sweep/.github/scripts/model-sweep/model_sweep.py#L417) -#### whisper-large-v3-turbo {#groq_whisper_large_v3_turbo_details} +#### whisper-large-v3-turbo ### Basic diff --git a/.github/scripts/model-sweep/test_markdown.mdx b/.github/scripts/model-sweep/test_markdown.mdx deleted file mode 100644 index 5a25816d..00000000 --- a/.github/scripts/model-sweep/test_markdown.mdx +++ /dev/null @@ -1,216 +0,0 @@ ---- -title: Support Models -generated: 2025-06-16T17:37:43.167565 ---- - -# Supported Models - -## Overview - -Letta routinely runs automated scans against available providers and models. These are the results of the latest scan. - -Ran 2464 tests against 154 models across 7 providers on June 16th, 2025 - - -## anthropic - -| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | -|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| -| `claude-3-5-haiku-20241022` | ✅ | ✅ | ✅ | 200,000 | 2025-06-16 | -| `claude-3-5-sonnet-20241022` | ✅ | ✅ | ✅ | 200,000 | 2025-06-16 | -| `claude-3-7-sonnet-20250219` | ✅ | ✅ | ✅ | 200,000 | 2025-06-16 | -| `claude-sonnet-4-20250514` | ✅ | ✅ | ✅ | 200,000 | 2025-06-16 | -| `claude-opus-4-20250514` | ✅ | ✅ | ⚠️ | 200,000 | 2025-06-16 | -| `claude-3-5-sonnet-20240620` | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-16 | -| `claude-3-haiku-20240307` | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-16 | -| `claude-3-opus-20240229` | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-16 | -| `claude-3-sonnet-20240229` | ❌ | ❌ | ❌ | 200,000 | 2025-06-16 | - ---- - -## openai - -| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | -|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| -| `gpt-4.1` | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-16 | -| `gpt-4.1-2025-04-14` | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-16 | -| `gpt-4.1-nano-2025-04-14` | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-16 | -| `gpt-4o` | ✅ | ✅ | ✅ | 128,000 | 2025-06-16 | -| `gpt-4o-2024-05-13` | ✅ | ✅ | ✅ | 128,000 | 2025-06-16 | -| `gpt-4-turbo` | ✅ | ✅ | ⚠️ | 8,192 | 2025-06-16 | -| `gpt-4.1-mini` | ✅ | ✅ | ⚠️ | 1,047,576 | 2025-06-16 | -| `gpt-4.5-preview` | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-16 | -| `gpt-4.5-preview-2025-02-27` | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-16 | -| `gpt-4o-2024-08-06` | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-16 | -| `gpt-4-0613` | ✅ | ✅ | ❌ | 8,192 | 2025-06-16 | -| `gpt-4-1106-preview` | ✅ | ✅ | ❌ | 128,000 | 2025-06-16 | -| `gpt-4-turbo-2024-04-09` | ✅ | ⚠️ | ✅ | 128,000 | 2025-06-16 | -| `gpt-4.1-mini-2025-04-14` | ⚠️ | ✅ | ✅ | 1,047,576 | 2025-06-16 | -| `gpt-4.1-nano` | ⚠️ | ✅ | ✅ | 1,047,576 | 2025-06-16 | -| `gpt-4o-2024-11-20` | ⚠️ | ✅ | ✅ | 8,192 | 2025-06-16 | -| `gpt-4-turbo-preview` | ✅ | ⚠️ | ❌ | 128,000 | 2025-06-16 | -| `gpt-4-0125-preview` | ⚠️ | ✅ | ❌ | 128,000 | 2025-06-16 | -| `gpt-4o-mini` | ⚠️ | ⚠️ | ⚠️ | 128,000 | 2025-06-16 | -| `gpt-4o-mini-2024-07-18` | ⚠️ | ⚠️ | ❌ | 128,000 | 2025-06-16 | -| `gpt-4` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `o1` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `o1-2024-12-17` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `o3` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `o3-2025-04-16` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `o3-mini` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `o3-mini-2025-01-31` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `o3-pro` | ❌ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `o3-pro-2025-06-10` | ❌ | ❌ | ❌ | 8,192 | 2025-06-16 | - ---- - -## google_ai - -| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | -|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| -| `gemini-1.5-pro` | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-16 | -| `gemini-1.5-pro-002` | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-16 | -| `gemini-1.5-pro-latest` | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-16 | -| `gemini-2.5-flash-preview-04-17-thinking` | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-16 | -| `gemini-2.5-pro-preview-03-25` | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-16 | -| `gemini-2.5-pro-preview-05-06` | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-16 | -| `gemini-2.5-flash-preview-05-20` | ✅ | ⚠️ | ✅ | 1,048,576 | 2025-06-16 | -| `gemini-2.0-flash-thinking-exp` | ⚠️ | ✅ | ✅ | 1,048,576 | 2025-06-16 | -| `gemini-2.0-flash-thinking-exp-1219` | ⚠️ | ✅ | ✅ | 1,048,576 | 2025-06-16 | -| `gemini-2.0-flash-thinking-exp-01-21` | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-16 | -| `gemini-2.5-flash-preview-04-17` | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-16 | -| `gemini-2.5-pro-preview-06-05` | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-16 | -| `gemini-1.0-pro-vision-latest` | ❌ | ❌ | ❌ | 12,288 | 2025-06-16 | -| `gemini-1.5-flash` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-16 | -| `gemini-1.5-flash-002` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-16 | -| `gemini-1.5-flash-8b` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-16 | -| `gemini-1.5-flash-8b-001` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-16 | -| `gemini-1.5-flash-8b-latest` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-16 | -| `gemini-1.5-flash-latest` | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-16 | -| `gemini-2.0-flash` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-16 | -| `gemini-2.0-flash-001` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-16 | -| `gemini-2.0-flash-exp` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-16 | -| `gemini-2.0-flash-exp-image-generation` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-16 | -| `gemini-2.0-flash-lite` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-16 | -| `gemini-2.0-flash-lite-001` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-16 | -| `gemini-2.0-flash-lite-preview` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-16 | -| `gemini-2.0-flash-lite-preview-02-05` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-16 | -| `gemini-2.0-flash-preview-image-generation` | ❌ | ❌ | ❌ | 32,768 | 2025-06-16 | -| `gemini-2.0-pro-exp` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-16 | -| `gemini-2.0-pro-exp-02-05` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-16 | -| `gemini-2.5-flash-preview-tts` | ❌ | ❌ | ❌ | 32,768 | 2025-06-16 | -| `gemini-2.5-pro-exp-03-25` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-16 | -| `gemini-2.5-pro-preview-tts` | ❌ | ❌ | ❌ | 65,536 | 2025-06-16 | -| `gemini-exp-1206` | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-16 | -| `gemini-pro-vision` | ❌ | ❌ | ❌ | 12,288 | 2025-06-16 | - ---- - -## letta - -| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | -|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| -| `letta-free` | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-16 | - ---- - -## together - -| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | -|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| -| `Qwen/Qwen2.5-72B-Instruct-Turbo` | ✅ | ✅ | ⚠️ | 131,072 | 2025-06-16 | -| `arcee-ai/virtuoso-large` | ⚠️ | ✅ | ✅ | 131,072 | 2025-06-16 | -| `Qwen/QwQ-32B` | ⚠️ | ✅ | ⚠️ | 131,072 | 2025-06-16 | -| `Qwen/Qwen2.5-7B-Instruct-Turbo` | ⚠️ | ✅ | ⚠️ | 32,768 | 2025-06-16 | -| `Qwen/Qwen2.5-Coder-32B-Instruct` | ⚠️ | ✅ | ⚠️ | 16,384 | 2025-06-16 | -| `arcee-ai/coder-large` | ⚠️ | ✅ | ⚠️ | 32,768 | 2025-06-16 | -| `arcee_ai/arcee-spotlight` | ⚠️ | ✅ | ⚠️ | 131,072 | 2025-06-16 | -| `meta-llama/Llama-3.2-3B-Instruct-Turbo` | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-16 | -| `meta-llama/Llama-3.3-70B-Instruct-Turbo` | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-16 | -| `meta-llama/Llama-3.3-70B-Instruct-Turbo-Free` | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-16 | -| `meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo` | ⚠️ | ✅ | ❌ | 130,815 | 2025-06-16 | -| `meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo` | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-16 | -| `nvidia/Llama-3.1-Nemotron-70B-Instruct-HF` | ⚠️ | ✅ | ❌ | 32,768 | 2025-06-16 | -| `arcee-ai/virtuoso-medium-v2` | ⚠️ | ⚠️ | ✅ | 131,072 | 2025-06-16 | -| `meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8` | ⚠️ | ❌ | ✅ | 1,048,576 | 2025-06-16 | -| `Qwen/Qwen3-235B-A22B-fp8-tput` | ⚠️ | ⚠️ | ❌ | 40,960 | 2025-06-16 | -| `deepseek-ai/DeepSeek-V3` | ⚠️ | ⚠️ | ❌ | 131,072 | 2025-06-16 | -| `meta-llama/Llama-4-Scout-17B-16E-Instruct` | ⚠️ | ⚠️ | ❌ | 1,048,576 | 2025-06-16 | -| `meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo` | ⚠️ | ⚠️ | ❌ | 131,072 | 2025-06-16 | -| `mistralai/Mixtral-8x7B-Instruct-v0.1` | ⚠️ | ⚠️ | ❌ | 32,768 | 2025-06-16 | -| `arcee-ai/caller` | ❌ | ⚠️ | ❌ | 32,768 | 2025-06-16 | -| `mistralai/Mistral-Small-24B-Instruct-2501` | ❌ | ⚠️ | ❌ | 32,768 | 2025-06-16 | -| `NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO` | ❌ | ❌ | ❌ | 32,768 | 2025-06-16 | -| `Qwen/Qwen2-72B-Instruct` | ❌ | ❌ | ❌ | 32,768 | 2025-06-16 | -| `Qwen/Qwen2-VL-72B-Instruct` | ❌ | ❌ | ❌ | 32,768 | 2025-06-16 | -| `Qwen/Qwen2.5-VL-72B-Instruct` | ❌ | ❌ | ❌ | 32,768 | 2025-06-16 | -| `arcee-ai/arcee-blitz` | ❌ | ❌ | ❌ | 32,768 | 2025-06-16 | -| `arcee-ai/maestro-reasoning` | ❌ | ❌ | ❌ | 131,072 | 2025-06-16 | -| `deepseek-ai/DeepSeek-R1` | ❌ | ❌ | ❌ | 163,840 | 2025-06-16 | -| `deepseek-ai/DeepSeek-R1-Distill-Llama-70B` | ❌ | ❌ | ❌ | 131,072 | 2025-06-16 | -| `deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free` | ❌ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B` | ❌ | ❌ | ❌ | 131,072 | 2025-06-16 | -| `deepseek-ai/DeepSeek-R1-Distill-Qwen-14B` | ❌ | ❌ | ❌ | 131,072 | 2025-06-16 | -| `deepseek-ai/DeepSeek-V3-p-dp` | ❌ | ❌ | ❌ | 131,072 | 2025-06-16 | -| `google/gemma-2-27b-it` | ❌ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `lgai/exaone-3-5-32b-instruct` | ❌ | ❌ | ❌ | 32,768 | 2025-06-16 | -| `lgai/exaone-deep-32b` | ❌ | ❌ | ❌ | 32,768 | 2025-06-16 | -| `marin-community/marin-8b-instruct` | ❌ | ❌ | ❌ | 131,072 | 2025-06-16 | -| `meta-llama/Llama-3-70b-chat-hf` | ❌ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `meta-llama/Llama-3-8b-chat-hf` | ❌ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo` | ❌ | ❌ | ❌ | 131,072 | 2025-06-16 | -| `meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo` | ❌ | ❌ | ❌ | 131,072 | 2025-06-16 | -| `meta-llama/Llama-Vision-Free` | ❌ | ❌ | ❌ | 131,072 | 2025-06-16 | -| `meta-llama/Meta-Llama-3-70B-Instruct-Turbo` | ❌ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `meta-llama/Meta-Llama-3-8B-Instruct-Lite` | ❌ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `mistralai/Mistral-7B-Instruct-v0.1` | ❌ | ❌ | ❌ | 32,768 | 2025-06-16 | -| `mistralai/Mistral-7B-Instruct-v0.2` | ❌ | ❌ | ❌ | 32,768 | 2025-06-16 | -| `mistralai/Mistral-7B-Instruct-v0.3` | ❌ | ❌ | ❌ | 32,768 | 2025-06-16 | -| `perplexity-ai/r1-1776` | ❌ | ❌ | ❌ | 163,840 | 2025-06-16 | -| `scb10x/scb10x-llama3-1-typhoon2-70b-instruct` | ❌ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `scb10x/scb10x-typhoon-2-1-gemma3-12b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `togethercomputer/MoA-1` | ❌ | ❌ | ❌ | 32,768 | 2025-06-16 | -| `togethercomputer/MoA-1-Turbo` | ❌ | ❌ | ❌ | 32,768 | 2025-06-16 | -| `togethercomputer/Refuel-Llm-V2` | ❌ | ❌ | ❌ | 16,384 | 2025-06-16 | -| `togethercomputer/Refuel-Llm-V2-Small` | ❌ | ❌ | ❌ | 8,192 | 2025-06-16 | - ---- - -## deepseek - -| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | -|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| -| `deepseek-chat` | ❌ | ❌ | ❌ | 64,000 | 2025-06-16 | -| `deepseek-reasoner` | ❌ | ❌ | ❌ | 64,000 | 2025-06-16 | - ---- - -## groq - -| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | -|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| -| `allam-2-7b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `compound-beta` | ❌ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `compound-beta-mini` | ❌ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `deepseek-r1-distill-llama-70b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `distil-whisper-large-v3-en` | ❌ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `gemma2-9b-it` | ❌ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `llama-3.1-8b-instant` | ❌ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `llama-3.3-70b-versatile` | ❌ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `llama-guard-3-8b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `llama3-70b-8192` | ❌ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `llama3-8b-8192` | ❌ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `meta-llama/llama-4-maverick-17b-128e-instruct` | ❌ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `meta-llama/llama-4-scout-17b-16e-instruct` | ❌ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `meta-llama/llama-guard-4-12b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `meta-llama/llama-prompt-guard-2-22m` | ❌ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `meta-llama/llama-prompt-guard-2-86m` | ❌ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `mistral-saba-24b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `playai-tts` | ❌ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `playai-tts-arabic` | ❌ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `qwen-qwq-32b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `qwen/qwen3-32b` | ❌ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `whisper-large-v3` | ❌ | ❌ | ❌ | 8,192 | 2025-06-16 | -| `whisper-large-v3-turbo` | ❌ | ❌ | ❌ | 8,192 | 2025-06-16 | - ---- diff --git a/.github/scripts/model-sweep/test_output.mdx b/.github/scripts/model-sweep/test_output.mdx deleted file mode 100644 index c5fcd41e..00000000 --- a/.github/scripts/model-sweep/test_output.mdx +++ /dev/null @@ -1,4395 +0,0 @@ ---- -title: Support Models -generated: 2025-06-20T15:57:32.626938 ---- - -# Supported Models - -## Overview - -Letta routinely runs automated scans against available providers and models. These are the results of the latest scan. - -Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - - -## anthropic - -| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | -|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| -| [`claude-3-5-haiku-20241022`](#anthropic_claude_3_5_haiku_20241022_details) | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | -| [`claude-3-5-sonnet-20241022`](#anthropic_claude_3_5_sonnet_20241022_details) | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | -| [`claude-3-7-sonnet-20250219`](#anthropic_claude_3_7_sonnet_20250219_details) | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | -| [`claude-sonnet-4-20250514`](#anthropic_claude_sonnet_4_20250514_details) | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | -| [`claude-opus-4-20250514`](#anthropic_claude_opus_4_20250514_details) | ✅ | ✅ | ⚠️ | 200,000 | 2025-06-20 | -| [`claude-3-5-sonnet-20240620`](#anthropic_claude_3_5_sonnet_20240620_details) | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 | -| [`claude-3-haiku-20240307`](#anthropic_claude_3_haiku_20240307_details) | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 | -| [`claude-3-opus-20240229`](#anthropic_claude_3_opus_20240229_details) | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 | -| [`claude-3-sonnet-20240229`](#anthropic_claude_3_sonnet_20240229_details) | ❌ | ❌ | ❌ | 200,000 | 2025-06-20 | - -### Detailed Test Results - -#### claude-3-5-haiku-20241022 {#anthropic_claude_3_5_haiku_20241022_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### claude-3-5-sonnet-20241022 {#anthropic_claude_3_5_sonnet_20241022_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### claude-3-7-sonnet-20250219 {#anthropic_claude_3_7_sonnet_20250219_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### claude-sonnet-4-20250514 {#anthropic_claude_sonnet_4_20250514_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### claude-opus-4-20250514 {#anthropic_claude_opus_4_20250514_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### claude-3-5-sonnet-20240620 {#anthropic_claude_3_5_sonnet_20240620_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### claude-3-haiku-20240307 {#anthropic_claude_3_haiku_20240307_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### claude-3-opus-20240229 {#anthropic_claude_3_opus_20240229_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### claude-3-sonnet-20240229 {#anthropic_claude_3_sonnet_20240229_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - - ---- - -## openai - -| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | -|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| -| [`gpt-4.1`](#openai_gpt_4.1_details) | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 | -| [`gpt-4.1-2025-04-14`](#openai_gpt_4.1_2025_04_14_details) | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 | -| [`gpt-4.1-nano-2025-04-14`](#openai_gpt_4.1_nano_2025_04_14_details) | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 | -| [`gpt-4o`](#openai_gpt_4o_details) | ✅ | ✅ | ✅ | 128,000 | 2025-06-20 | -| [`gpt-4o-2024-05-13`](#openai_gpt_4o_2024_05_13_details) | ✅ | ✅ | ✅ | 128,000 | 2025-06-20 | -| [`gpt-4-turbo`](#openai_gpt_4_turbo_details) | ✅ | ✅ | ⚠️ | 8,192 | 2025-06-20 | -| [`gpt-4.1-mini`](#openai_gpt_4.1_mini_details) | ✅ | ✅ | ⚠️ | 1,047,576 | 2025-06-20 | -| [`gpt-4.5-preview`](#openai_gpt_4.5_preview_details) | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 | -| [`gpt-4.5-preview-2025-02-27`](#openai_gpt_4.5_preview_2025_02_27_details) | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 | -| [`gpt-4o-2024-08-06`](#openai_gpt_4o_2024_08_06_details) | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 | -| [`gpt-4-0613`](#openai_gpt_4_0613_details) | ✅ | ✅ | ❌ | 8,192 | 2025-06-20 | -| [`gpt-4-1106-preview`](#openai_gpt_4_1106_preview_details) | ✅ | ✅ | ❌ | 128,000 | 2025-06-20 | -| [`gpt-4-turbo-2024-04-09`](#openai_gpt_4_turbo_2024_04_09_details) | ✅ | ⚠️ | ✅ | 128,000 | 2025-06-20 | -| [`gpt-4.1-mini-2025-04-14`](#openai_gpt_4.1_mini_2025_04_14_details) | ⚠️ | ✅ | ✅ | 1,047,576 | 2025-06-20 | -| [`gpt-4.1-nano`](#openai_gpt_4.1_nano_details) | ⚠️ | ✅ | ✅ | 1,047,576 | 2025-06-20 | -| [`gpt-4o-2024-11-20`](#openai_gpt_4o_2024_11_20_details) | ⚠️ | ✅ | ✅ | 8,192 | 2025-06-20 | -| [`gpt-4-turbo-preview`](#openai_gpt_4_turbo_preview_details) | ✅ | ⚠️ | ❌ | 128,000 | 2025-06-20 | -| [`gpt-4-0125-preview`](#openai_gpt_4_0125_preview_details) | ⚠️ | ✅ | ❌ | 128,000 | 2025-06-20 | -| [`gpt-4o-mini`](#openai_gpt_4o_mini_details) | ⚠️ | ⚠️ | ⚠️ | 128,000 | 2025-06-20 | -| [`gpt-4o-mini-2024-07-18`](#openai_gpt_4o_mini_2024_07_18_details) | ⚠️ | ⚠️ | ❌ | 128,000 | 2025-06-20 | -| [`gpt-4`](#openai_gpt_4_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`o1`](#openai_o1_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`o1-2024-12-17`](#openai_o1_2024_12_17_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`o3`](#openai_o3_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`o3-2025-04-16`](#openai_o3_2025_04_16_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`o3-mini`](#openai_o3_mini_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`o3-mini-2025-01-31`](#openai_o3_mini_2025_01_31_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`o3-pro`](#openai_o3_pro_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`o3-pro-2025-06-10`](#openai_o3_pro_2025_06_10_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | - -### Detailed Test Results - -#### gpt-4.1 {#openai_gpt_4.1_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gpt-4.1-2025-04-14 {#openai_gpt_4.1_2025_04_14_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gpt-4.1-nano-2025-04-14 {#openai_gpt_4.1_nano_2025_04_14_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gpt-4o {#openai_gpt_4o_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gpt-4o-2024-05-13 {#openai_gpt_4o_2024_05_13_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gpt-4-turbo {#openai_gpt_4_turbo_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gpt-4.1-mini {#openai_gpt_4.1_mini_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gpt-4.5-preview {#openai_gpt_4.5_preview_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gpt-4.5-preview-2025-02-27 {#openai_gpt_4.5_preview_2025_02_27_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gpt-4o-2024-08-06 {#openai_gpt_4o_2024_08_06_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gpt-4-0613 {#openai_gpt_4_0613_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gpt-4-1106-preview {#openai_gpt_4_1106_preview_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gpt-4-turbo-2024-04-09 {#openai_gpt_4_turbo_2024_04_09_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gpt-4.1-mini-2025-04-14 {#openai_gpt_4.1_mini_2025_04_14_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gpt-4.1-nano {#openai_gpt_4.1_nano_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gpt-4o-2024-11-20 {#openai_gpt_4o_2024_11_20_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gpt-4-turbo-preview {#openai_gpt_4_turbo_preview_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gpt-4-0125-preview {#openai_gpt_4_0125_preview_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gpt-4o-mini {#openai_gpt_4o_mini_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gpt-4o-mini-2024-07-18 {#openai_gpt_4o_mini_2024_07_18_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gpt-4 {#openai_gpt_4_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### o1 {#openai_o1_details} - -### Basic - -- ❌ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### o1-2024-12-17 {#openai_o1_2024_12_17_details} - -### Basic - -- ❌ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### o3 {#openai_o3_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### o3-2025-04-16 {#openai_o3_2025_04_16_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### o3-mini {#openai_o3_mini_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### o3-mini-2025-01-31 {#openai_o3_mini_2025_01_31_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### o3-pro {#openai_o3_pro_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### o3-pro-2025-06-10 {#openai_o3_pro_2025_06_10_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - - ---- - -## google_ai - -| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | -|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| -| [`gemini-1.5-pro`](#google_ai_gemini_1.5_pro_details) | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 | -| [`gemini-1.5-pro-002`](#google_ai_gemini_1.5_pro_002_details) | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 | -| [`gemini-1.5-pro-latest`](#google_ai_gemini_1.5_pro_latest_details) | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 | -| [`gemini-2.5-flash-preview-04-17-thinking`](#google_ai_gemini_2.5_flash_preview_04_17_thinking_details) | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 | -| [`gemini-2.5-pro-preview-03-25`](#google_ai_gemini_2.5_pro_preview_03_25_details) | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 | -| [`gemini-2.5-pro-preview-05-06`](#google_ai_gemini_2.5_pro_preview_05_06_details) | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 | -| [`gemini-2.5-flash-preview-05-20`](#google_ai_gemini_2.5_flash_preview_05_20_details) | ✅ | ⚠️ | ✅ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-flash-thinking-exp`](#google_ai_gemini_2.0_flash_thinking_exp_details) | ⚠️ | ✅ | ✅ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-flash-thinking-exp-1219`](#google_ai_gemini_2.0_flash_thinking_exp_1219_details) | ⚠️ | ✅ | ✅ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-flash-thinking-exp-01-21`](#google_ai_gemini_2.0_flash_thinking_exp_01_21_details) | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 | -| [`gemini-2.5-flash-preview-04-17`](#google_ai_gemini_2.5_flash_preview_04_17_details) | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 | -| [`gemini-2.5-pro-preview-06-05`](#google_ai_gemini_2.5_pro_preview_06_05_details) | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 | -| [`gemini-1.0-pro-vision-latest`](#google_ai_gemini_1.0_pro_vision_latest_details) | ❌ | ❌ | ❌ | 12,288 | 2025-06-20 | -| [`gemini-1.5-flash`](#google_ai_gemini_1.5_flash_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | -| [`gemini-1.5-flash-002`](#google_ai_gemini_1.5_flash_002_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | -| [`gemini-1.5-flash-8b`](#google_ai_gemini_1.5_flash_8b_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | -| [`gemini-1.5-flash-8b-001`](#google_ai_gemini_1.5_flash_8b_001_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | -| [`gemini-1.5-flash-8b-latest`](#google_ai_gemini_1.5_flash_8b_latest_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | -| [`gemini-1.5-flash-latest`](#google_ai_gemini_1.5_flash_latest_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | -| [`gemini-2.0-flash`](#google_ai_gemini_2.0_flash_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-flash-001`](#google_ai_gemini_2.0_flash_001_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-flash-exp`](#google_ai_gemini_2.0_flash_exp_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-flash-exp-image-generation`](#google_ai_gemini_2.0_flash_exp_image_generation_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-flash-lite`](#google_ai_gemini_2.0_flash_lite_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-flash-lite-001`](#google_ai_gemini_2.0_flash_lite_001_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-flash-lite-preview`](#google_ai_gemini_2.0_flash_lite_preview_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-flash-lite-preview-02-05`](#google_ai_gemini_2.0_flash_lite_preview_02_05_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-flash-preview-image-generation`](#google_ai_gemini_2.0_flash_preview_image_generation_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`gemini-2.0-pro-exp`](#google_ai_gemini_2.0_pro_exp_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-pro-exp-02-05`](#google_ai_gemini_2.0_pro_exp_02_05_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-2.5-flash-preview-tts`](#google_ai_gemini_2.5_flash_preview_tts_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`gemini-2.5-pro-exp-03-25`](#google_ai_gemini_2.5_pro_exp_03_25_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-2.5-pro-preview-tts`](#google_ai_gemini_2.5_pro_preview_tts_details) | ❌ | ❌ | ❌ | 65,536 | 2025-06-20 | -| [`gemini-exp-1206`](#google_ai_gemini_exp_1206_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-pro-vision`](#google_ai_gemini_pro_vision_details) | ❌ | ❌ | ❌ | 12,288 | 2025-06-20 | - -### Detailed Test Results - -#### gemini-1.5-pro {#google_ai_gemini_1.5_pro_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gemini-1.5-pro-002 {#google_ai_gemini_1.5_pro_002_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gemini-1.5-pro-latest {#google_ai_gemini_1.5_pro_latest_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gemini-2.5-flash-preview-04-17-thinking {#google_ai_gemini_2.5_flash_preview_04_17_thinking_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gemini-2.5-pro-preview-03-25 {#google_ai_gemini_2.5_pro_preview_03_25_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gemini-2.5-pro-preview-05-06 {#google_ai_gemini_2.5_pro_preview_05_06_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gemini-2.5-flash-preview-05-20 {#google_ai_gemini_2.5_flash_preview_05_20_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gemini-2.0-flash-thinking-exp {#google_ai_gemini_2.0_flash_thinking_exp_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gemini-2.0-flash-thinking-exp-1219 {#google_ai_gemini_2.0_flash_thinking_exp_1219_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gemini-2.0-flash-thinking-exp-01-21 {#google_ai_gemini_2.0_flash_thinking_exp_01_21_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-2.5-flash-preview-04-17 {#google_ai_gemini_2.5_flash_preview_04_17_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-2.5-pro-preview-06-05 {#google_ai_gemini_2.5_pro_preview_06_05_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gemini-1.0-pro-vision-latest {#google_ai_gemini_1.0_pro_vision_latest_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-1.5-flash {#google_ai_gemini_1.5_flash_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-1.5-flash-002 {#google_ai_gemini_1.5_flash_002_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-1.5-flash-8b {#google_ai_gemini_1.5_flash_8b_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-1.5-flash-8b-001 {#google_ai_gemini_1.5_flash_8b_001_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-1.5-flash-8b-latest {#google_ai_gemini_1.5_flash_8b_latest_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-1.5-flash-latest {#google_ai_gemini_1.5_flash_latest_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-2.0-flash {#google_ai_gemini_2.0_flash_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-2.0-flash-001 {#google_ai_gemini_2.0_flash_001_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-2.0-flash-exp {#google_ai_gemini_2.0_flash_exp_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-2.0-flash-exp-image-generation {#google_ai_gemini_2.0_flash_exp_image_generation_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-2.0-flash-lite {#google_ai_gemini_2.0_flash_lite_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-2.0-flash-lite-001 {#google_ai_gemini_2.0_flash_lite_001_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-2.0-flash-lite-preview {#google_ai_gemini_2.0_flash_lite_preview_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-2.0-flash-lite-preview-02-05 {#google_ai_gemini_2.0_flash_lite_preview_02_05_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-2.0-flash-preview-image-generation {#google_ai_gemini_2.0_flash_preview_image_generation_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-2.0-pro-exp {#google_ai_gemini_2.0_pro_exp_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-2.0-pro-exp-02-05 {#google_ai_gemini_2.0_pro_exp_02_05_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-2.5-flash-preview-tts {#google_ai_gemini_2.5_flash_preview_tts_details} - -### Basic - -- ❌ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-2.5-pro-exp-03-25 {#google_ai_gemini_2.5_pro_exp_03_25_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-2.5-pro-preview-tts {#google_ai_gemini_2.5_pro_preview_tts_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-exp-1206 {#google_ai_gemini_exp_1206_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-pro-vision {#google_ai_gemini_pro_vision_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - - ---- - -## letta - -| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | -|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| -| [`letta-free`](#letta_letta_free_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | - -### Detailed Test Results - -#### letta-free {#letta_letta_free_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - - ---- - -## together - -| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | -|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| -| [`Qwen/Qwen2.5-72B-Instruct-Turbo`](#together_qwen_qwen2.5_72b_instruct_turbo_details) | ✅ | ✅ | ⚠️ | 131,072 | 2025-06-20 | -| [`arcee-ai/virtuoso-large`](#together_arcee_ai_virtuoso_large_details) | ⚠️ | ✅ | ✅ | 131,072 | 2025-06-20 | -| [`Qwen/QwQ-32B`](#together_qwen_qwq_32b_details) | ⚠️ | ✅ | ⚠️ | 131,072 | 2025-06-20 | -| [`Qwen/Qwen2.5-7B-Instruct-Turbo`](#together_qwen_qwen2.5_7b_instruct_turbo_details) | ⚠️ | ✅ | ⚠️ | 32,768 | 2025-06-20 | -| [`Qwen/Qwen2.5-Coder-32B-Instruct`](#together_qwen_qwen2.5_coder_32b_instruct_details) | ⚠️ | ✅ | ⚠️ | 16,384 | 2025-06-20 | -| [`arcee-ai/coder-large`](#together_arcee_ai_coder_large_details) | ⚠️ | ✅ | ⚠️ | 32,768 | 2025-06-20 | -| [`arcee_ai/arcee-spotlight`](#together_arcee_ai_arcee_spotlight_details) | ⚠️ | ✅ | ⚠️ | 131,072 | 2025-06-20 | -| [`meta-llama/Llama-3.2-3B-Instruct-Turbo`](#together_meta_llama_llama_3.2_3b_instruct_turbo_details) | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | -| [`meta-llama/Llama-3.3-70B-Instruct-Turbo`](#together_meta_llama_llama_3.3_70b_instruct_turbo_details) | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | -| [`meta-llama/Llama-3.3-70B-Instruct-Turbo-Free`](#together_meta_llama_llama_3.3_70b_instruct_turbo_free_details) | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | -| [`meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo`](#together_meta_llama_meta_llama_3.1_405b_instruct_turbo_details) | ⚠️ | ✅ | ❌ | 130,815 | 2025-06-20 | -| [`meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo`](#together_meta_llama_meta_llama_3.1_70b_instruct_turbo_details) | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | -| [`nvidia/Llama-3.1-Nemotron-70B-Instruct-HF`](#together_nvidia_llama_3.1_nemotron_70b_instruct_hf_details) | ⚠️ | ✅ | ❌ | 32,768 | 2025-06-20 | -| [`arcee-ai/virtuoso-medium-v2`](#together_arcee_ai_virtuoso_medium_v2_details) | ⚠️ | ⚠️ | ✅ | 131,072 | 2025-06-20 | -| [`meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8`](#together_meta_llama_llama_4_maverick_17b_128e_instruct_fp8_details) | ⚠️ | ❌ | ✅ | 1,048,576 | 2025-06-20 | -| [`Qwen/Qwen3-235B-A22B-fp8-tput`](#together_qwen_qwen3_235b_a22b_fp8_tput_details) | ⚠️ | ⚠️ | ❌ | 40,960 | 2025-06-20 | -| [`deepseek-ai/DeepSeek-V3`](#together_deepseek_ai_deepseek_v3_details) | ⚠️ | ⚠️ | ❌ | 131,072 | 2025-06-20 | -| [`meta-llama/Llama-4-Scout-17B-16E-Instruct`](#together_meta_llama_llama_4_scout_17b_16e_instruct_details) | ⚠️ | ⚠️ | ❌ | 1,048,576 | 2025-06-20 | -| [`meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo`](#together_meta_llama_meta_llama_3.1_8b_instruct_turbo_details) | ⚠️ | ⚠️ | ❌ | 131,072 | 2025-06-20 | -| [`mistralai/Mixtral-8x7B-Instruct-v0.1`](#together_mistralai_mixtral_8x7b_instruct_v0.1_details) | ⚠️ | ⚠️ | ❌ | 32,768 | 2025-06-20 | -| [`arcee-ai/caller`](#together_arcee_ai_caller_details) | ❌ | ⚠️ | ❌ | 32,768 | 2025-06-20 | -| [`mistralai/Mistral-Small-24B-Instruct-2501`](#together_mistralai_mistral_small_24b_instruct_2501_details) | ❌ | ⚠️ | ❌ | 32,768 | 2025-06-20 | -| [`NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO`](#together_nousresearch_nous_hermes_2_mixtral_8x7b_dpo_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`Qwen/Qwen2-72B-Instruct`](#together_qwen_qwen2_72b_instruct_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`Qwen/Qwen2-VL-72B-Instruct`](#together_qwen_qwen2_vl_72b_instruct_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`Qwen/Qwen2.5-VL-72B-Instruct`](#together_qwen_qwen2.5_vl_72b_instruct_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`arcee-ai/arcee-blitz`](#together_arcee_ai_arcee_blitz_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`arcee-ai/maestro-reasoning`](#together_arcee_ai_maestro_reasoning_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | -| [`deepseek-ai/DeepSeek-R1`](#together_deepseek_ai_deepseek_r1_details) | ❌ | ❌ | ❌ | 163,840 | 2025-06-20 | -| [`deepseek-ai/DeepSeek-R1-Distill-Llama-70B`](#together_deepseek_ai_deepseek_r1_distill_llama_70b_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | -| [`deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free`](#together_deepseek_ai_deepseek_r1_distill_llama_70b_free_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B`](#together_deepseek_ai_deepseek_r1_distill_qwen_1.5b_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | -| [`deepseek-ai/DeepSeek-R1-Distill-Qwen-14B`](#together_deepseek_ai_deepseek_r1_distill_qwen_14b_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | -| [`deepseek-ai/DeepSeek-V3-p-dp`](#together_deepseek_ai_deepseek_v3_p_dp_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | -| [`google/gemma-2-27b-it`](#together_google_gemma_2_27b_it_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`lgai/exaone-3-5-32b-instruct`](#together_lgai_exaone_3_5_32b_instruct_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`lgai/exaone-deep-32b`](#together_lgai_exaone_deep_32b_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`marin-community/marin-8b-instruct`](#together_marin_community_marin_8b_instruct_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | -| [`meta-llama/Llama-3-70b-chat-hf`](#together_meta_llama_llama_3_70b_chat_hf_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`meta-llama/Llama-3-8b-chat-hf`](#together_meta_llama_llama_3_8b_chat_hf_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo`](#together_meta_llama_llama_3.2_11b_vision_instruct_turbo_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | -| [`meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo`](#together_meta_llama_llama_3.2_90b_vision_instruct_turbo_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | -| [`meta-llama/Llama-Vision-Free`](#together_meta_llama_llama_vision_free_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | -| [`meta-llama/Meta-Llama-3-70B-Instruct-Turbo`](#together_meta_llama_meta_llama_3_70b_instruct_turbo_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`meta-llama/Meta-Llama-3-8B-Instruct-Lite`](#together_meta_llama_meta_llama_3_8b_instruct_lite_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`mistralai/Mistral-7B-Instruct-v0.1`](#together_mistralai_mistral_7b_instruct_v0.1_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`mistralai/Mistral-7B-Instruct-v0.2`](#together_mistralai_mistral_7b_instruct_v0.2_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`mistralai/Mistral-7B-Instruct-v0.3`](#together_mistralai_mistral_7b_instruct_v0.3_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`perplexity-ai/r1-1776`](#together_perplexity_ai_r1_1776_details) | ❌ | ❌ | ❌ | 163,840 | 2025-06-20 | -| [`scb10x/scb10x-llama3-1-typhoon2-70b-instruct`](#together_scb10x_scb10x_llama3_1_typhoon2_70b_instruct_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`scb10x/scb10x-typhoon-2-1-gemma3-12b`](#together_scb10x_scb10x_typhoon_2_1_gemma3_12b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`togethercomputer/MoA-1`](#together_togethercomputer_moa_1_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`togethercomputer/MoA-1-Turbo`](#together_togethercomputer_moa_1_turbo_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`togethercomputer/Refuel-Llm-V2`](#together_togethercomputer_refuel_llm_v2_details) | ❌ | ❌ | ❌ | 16,384 | 2025-06-20 | -| [`togethercomputer/Refuel-Llm-V2-Small`](#together_togethercomputer_refuel_llm_v2_small_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | - -### Detailed Test Results - -#### Qwen/Qwen2.5-72B-Instruct-Turbo {#together_qwen_qwen2.5_72b_instruct_turbo_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### arcee-ai/virtuoso-large {#together_arcee_ai_virtuoso_large_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### Qwen/QwQ-32B {#together_qwen_qwq_32b_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### Qwen/Qwen2.5-7B-Instruct-Turbo {#together_qwen_qwen2.5_7b_instruct_turbo_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### Qwen/Qwen2.5-Coder-32B-Instruct {#together_qwen_qwen2.5_coder_32b_instruct_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### arcee-ai/coder-large {#together_arcee_ai_coder_large_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### arcee_ai/arcee-spotlight {#together_arcee_ai_arcee_spotlight_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/Llama-3.2-3B-Instruct-Turbo {#together_meta_llama_llama_3.2_3b_instruct_turbo_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/Llama-3.3-70B-Instruct-Turbo {#together_meta_llama_llama_3.3_70b_instruct_turbo_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/Llama-3.3-70B-Instruct-Turbo-Free {#together_meta_llama_llama_3.3_70b_instruct_turbo_free_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo {#together_meta_llama_meta_llama_3.1_405b_instruct_turbo_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo {#together_meta_llama_meta_llama_3.1_70b_instruct_turbo_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### nvidia/Llama-3.1-Nemotron-70B-Instruct-HF {#together_nvidia_llama_3.1_nemotron_70b_instruct_hf_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### arcee-ai/virtuoso-medium-v2 {#together_arcee_ai_virtuoso_medium_v2_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 {#together_meta_llama_llama_4_maverick_17b_128e_instruct_fp8_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### Qwen/Qwen3-235B-A22B-fp8-tput {#together_qwen_qwen3_235b_a22b_fp8_tput_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### deepseek-ai/DeepSeek-V3 {#together_deepseek_ai_deepseek_v3_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/Llama-4-Scout-17B-16E-Instruct {#together_meta_llama_llama_4_scout_17b_16e_instruct_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo {#together_meta_llama_meta_llama_3.1_8b_instruct_turbo_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### mistralai/Mixtral-8x7B-Instruct-v0.1 {#together_mistralai_mixtral_8x7b_instruct_v0.1_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### arcee-ai/caller {#together_arcee_ai_caller_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### mistralai/Mistral-Small-24B-Instruct-2501 {#together_mistralai_mistral_small_24b_instruct_2501_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO {#together_nousresearch_nous_hermes_2_mixtral_8x7b_dpo_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### Qwen/Qwen2-72B-Instruct {#together_qwen_qwen2_72b_instruct_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### Qwen/Qwen2-VL-72B-Instruct {#together_qwen_qwen2_vl_72b_instruct_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### Qwen/Qwen2.5-VL-72B-Instruct {#together_qwen_qwen2.5_vl_72b_instruct_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### arcee-ai/arcee-blitz {#together_arcee_ai_arcee_blitz_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### arcee-ai/maestro-reasoning {#together_arcee_ai_maestro_reasoning_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### deepseek-ai/DeepSeek-R1 {#together_deepseek_ai_deepseek_r1_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### deepseek-ai/DeepSeek-R1-Distill-Llama-70B {#together_deepseek_ai_deepseek_r1_distill_llama_70b_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free {#together_deepseek_ai_deepseek_r1_distill_llama_70b_free_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B {#together_deepseek_ai_deepseek_r1_distill_qwen_1.5b_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### deepseek-ai/DeepSeek-R1-Distill-Qwen-14B {#together_deepseek_ai_deepseek_r1_distill_qwen_14b_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### deepseek-ai/DeepSeek-V3-p-dp {#together_deepseek_ai_deepseek_v3_p_dp_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### google/gemma-2-27b-it {#together_google_gemma_2_27b_it_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### lgai/exaone-3-5-32b-instruct {#together_lgai_exaone_3_5_32b_instruct_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### lgai/exaone-deep-32b {#together_lgai_exaone_deep_32b_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### marin-community/marin-8b-instruct {#together_marin_community_marin_8b_instruct_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/Llama-3-70b-chat-hf {#together_meta_llama_llama_3_70b_chat_hf_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/Llama-3-8b-chat-hf {#together_meta_llama_llama_3_8b_chat_hf_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo {#together_meta_llama_llama_3.2_11b_vision_instruct_turbo_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo {#together_meta_llama_llama_3.2_90b_vision_instruct_turbo_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/Llama-Vision-Free {#together_meta_llama_llama_vision_free_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/Meta-Llama-3-70B-Instruct-Turbo {#together_meta_llama_meta_llama_3_70b_instruct_turbo_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/Meta-Llama-3-8B-Instruct-Lite {#together_meta_llama_meta_llama_3_8b_instruct_lite_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### mistralai/Mistral-7B-Instruct-v0.1 {#together_mistralai_mistral_7b_instruct_v0.1_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### mistralai/Mistral-7B-Instruct-v0.2 {#together_mistralai_mistral_7b_instruct_v0.2_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### mistralai/Mistral-7B-Instruct-v0.3 {#together_mistralai_mistral_7b_instruct_v0.3_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### perplexity-ai/r1-1776 {#together_perplexity_ai_r1_1776_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### scb10x/scb10x-llama3-1-typhoon2-70b-instruct {#together_scb10x_scb10x_llama3_1_typhoon2_70b_instruct_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### scb10x/scb10x-typhoon-2-1-gemma3-12b {#together_scb10x_scb10x_typhoon_2_1_gemma3_12b_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### togethercomputer/MoA-1 {#together_togethercomputer_moa_1_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### togethercomputer/MoA-1-Turbo {#together_togethercomputer_moa_1_turbo_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### togethercomputer/Refuel-Llm-V2 {#together_togethercomputer_refuel_llm_v2_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### togethercomputer/Refuel-Llm-V2-Small {#together_togethercomputer_refuel_llm_v2_small_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - - ---- - -## deepseek - -| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | -|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| -| [`deepseek-chat`](#deepseek_deepseek_chat_details) | ❌ | ❌ | ❌ | 64,000 | 2025-06-20 | -| [`deepseek-reasoner`](#deepseek_deepseek_reasoner_details) | ❌ | ❌ | ❌ | 64,000 | 2025-06-20 | - -### Detailed Test Results - -#### deepseek-chat {#deepseek_deepseek_chat_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### deepseek-reasoner {#deepseek_deepseek_reasoner_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - - ---- - -## groq - -| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | -|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| -| [`allam-2-7b`](#groq_allam_2_7b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`compound-beta`](#groq_compound_beta_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`compound-beta-mini`](#groq_compound_beta_mini_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`deepseek-r1-distill-llama-70b`](#groq_deepseek_r1_distill_llama_70b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`distil-whisper-large-v3-en`](#groq_distil_whisper_large_v3_en_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`gemma2-9b-it`](#groq_gemma2_9b_it_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`llama-3.1-8b-instant`](#groq_llama_3.1_8b_instant_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`llama-3.3-70b-versatile`](#groq_llama_3.3_70b_versatile_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`llama-guard-3-8b`](#groq_llama_guard_3_8b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`llama3-70b-8192`](#groq_llama3_70b_8192_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`llama3-8b-8192`](#groq_llama3_8b_8192_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`meta-llama/llama-4-maverick-17b-128e-instruct`](#groq_meta_llama_llama_4_maverick_17b_128e_instruct_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`meta-llama/llama-4-scout-17b-16e-instruct`](#groq_meta_llama_llama_4_scout_17b_16e_instruct_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`meta-llama/llama-guard-4-12b`](#groq_meta_llama_llama_guard_4_12b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`meta-llama/llama-prompt-guard-2-22m`](#groq_meta_llama_llama_prompt_guard_2_22m_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`meta-llama/llama-prompt-guard-2-86m`](#groq_meta_llama_llama_prompt_guard_2_86m_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`mistral-saba-24b`](#groq_mistral_saba_24b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`playai-tts`](#groq_playai_tts_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`playai-tts-arabic`](#groq_playai_tts_arabic_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`qwen-qwq-32b`](#groq_qwen_qwq_32b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`qwen/qwen3-32b`](#groq_qwen_qwen3_32b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`whisper-large-v3`](#groq_whisper_large_v3_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`whisper-large-v3-turbo`](#groq_whisper_large_v3_turbo_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | - -### Detailed Test Results - -#### allam-2-7b {#groq_allam_2_7b_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### compound-beta {#groq_compound_beta_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### compound-beta-mini {#groq_compound_beta_mini_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### deepseek-r1-distill-llama-70b {#groq_deepseek_r1_distill_llama_70b_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### distil-whisper-large-v3-en {#groq_distil_whisper_large_v3_en_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemma2-9b-it {#groq_gemma2_9b_it_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### llama-3.1-8b-instant {#groq_llama_3.1_8b_instant_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### llama-3.3-70b-versatile {#groq_llama_3.3_70b_versatile_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### llama-guard-3-8b {#groq_llama_guard_3_8b_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### llama3-70b-8192 {#groq_llama3_70b_8192_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### llama3-8b-8192 {#groq_llama3_8b_8192_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/llama-4-maverick-17b-128e-instruct {#groq_meta_llama_llama_4_maverick_17b_128e_instruct_details} - -### Basic - -- ❌ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/llama-4-scout-17b-16e-instruct {#groq_meta_llama_llama_4_scout_17b_16e_instruct_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/llama-guard-4-12b {#groq_meta_llama_llama_guard_4_12b_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/llama-prompt-guard-2-22m {#groq_meta_llama_llama_prompt_guard_2_22m_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/llama-prompt-guard-2-86m {#groq_meta_llama_llama_prompt_guard_2_86m_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### mistral-saba-24b {#groq_mistral_saba_24b_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### playai-tts {#groq_playai_tts_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### playai-tts-arabic {#groq_playai_tts_arabic_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### qwen-qwq-32b {#groq_qwen_qwq_32b_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### qwen/qwen3-32b {#groq_qwen_qwen3_32b_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### whisper-large-v3 {#groq_whisper_large_v3_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### whisper-large-v3-turbo {#groq_whisper_large_v3_turbo_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - - ---- diff --git a/.github/scripts/model-sweep/test_output_with_links.mdx b/.github/scripts/model-sweep/test_output_with_links.mdx deleted file mode 100644 index 64a14ea7..00000000 --- a/.github/scripts/model-sweep/test_output_with_links.mdx +++ /dev/null @@ -1,4395 +0,0 @@ ---- -title: Support Models -generated: 2025-06-20T16:06:14.319292 ---- - -# Supported Models - -## Overview - -Letta routinely runs automated scans against available providers and models. These are the results of the latest scan. - -Ran 2464 tests against 154 models across 7 providers on June 20th, 2025 - - -## anthropic - -| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | -|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| -| [`claude-3-5-haiku-20241022`](#anthropic_claude_3_5_haiku_20241022_details) | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | -| [`claude-3-5-sonnet-20241022`](#anthropic_claude_3_5_sonnet_20241022_details) | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | -| [`claude-3-7-sonnet-20250219`](#anthropic_claude_3_7_sonnet_20250219_details) | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | -| [`claude-sonnet-4-20250514`](#anthropic_claude_sonnet_4_20250514_details) | ✅ | ✅ | ✅ | 200,000 | 2025-06-20 | -| [`claude-opus-4-20250514`](#anthropic_claude_opus_4_20250514_details) | ✅ | ✅ | ⚠️ | 200,000 | 2025-06-20 | -| [`claude-3-5-sonnet-20240620`](#anthropic_claude_3_5_sonnet_20240620_details) | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 | -| [`claude-3-haiku-20240307`](#anthropic_claude_3_haiku_20240307_details) | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 | -| [`claude-3-opus-20240229`](#anthropic_claude_3_opus_20240229_details) | ⚠️ | ❌ | ✅ | 200,000 | 2025-06-20 | -| [`claude-3-sonnet-20240229`](#anthropic_claude_3_sonnet_20240229_details) | ❌ | ❌ | ❌ | 200,000 | 2025-06-20 | - -### Detailed Test Results - -#### claude-3-5-haiku-20241022 {#anthropic_claude_3_5_haiku_20241022_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### claude-3-5-sonnet-20241022 {#anthropic_claude_3_5_sonnet_20241022_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### claude-3-7-sonnet-20250219 {#anthropic_claude_3_7_sonnet_20250219_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### claude-sonnet-4-20250514 {#anthropic_claude_sonnet_4_20250514_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### claude-opus-4-20250514 {#anthropic_claude_opus_4_20250514_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### claude-3-5-sonnet-20240620 {#anthropic_claude_3_5_sonnet_20240620_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### claude-3-haiku-20240307 {#anthropic_claude_3_haiku_20240307_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### claude-3-opus-20240229 {#anthropic_claude_3_opus_20240229_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### claude-3-sonnet-20240229 {#anthropic_claude_3_sonnet_20240229_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - - ---- - -## openai - -| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | -|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| -| [`gpt-4.1`](#openai_gpt_4.1_details) | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 | -| [`gpt-4.1-2025-04-14`](#openai_gpt_4.1_2025_04_14_details) | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 | -| [`gpt-4.1-nano-2025-04-14`](#openai_gpt_4.1_nano_2025_04_14_details) | ✅ | ✅ | ✅ | 1,047,576 | 2025-06-20 | -| [`gpt-4o`](#openai_gpt_4o_details) | ✅ | ✅ | ✅ | 128,000 | 2025-06-20 | -| [`gpt-4o-2024-05-13`](#openai_gpt_4o_2024_05_13_details) | ✅ | ✅ | ✅ | 128,000 | 2025-06-20 | -| [`gpt-4-turbo`](#openai_gpt_4_turbo_details) | ✅ | ✅ | ⚠️ | 8,192 | 2025-06-20 | -| [`gpt-4.1-mini`](#openai_gpt_4.1_mini_details) | ✅ | ✅ | ⚠️ | 1,047,576 | 2025-06-20 | -| [`gpt-4.5-preview`](#openai_gpt_4.5_preview_details) | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 | -| [`gpt-4.5-preview-2025-02-27`](#openai_gpt_4.5_preview_2025_02_27_details) | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 | -| [`gpt-4o-2024-08-06`](#openai_gpt_4o_2024_08_06_details) | ✅ | ✅ | ⚠️ | 128,000 | 2025-06-20 | -| [`gpt-4-0613`](#openai_gpt_4_0613_details) | ✅ | ✅ | ❌ | 8,192 | 2025-06-20 | -| [`gpt-4-1106-preview`](#openai_gpt_4_1106_preview_details) | ✅ | ✅ | ❌ | 128,000 | 2025-06-20 | -| [`gpt-4-turbo-2024-04-09`](#openai_gpt_4_turbo_2024_04_09_details) | ✅ | ⚠️ | ✅ | 128,000 | 2025-06-20 | -| [`gpt-4.1-mini-2025-04-14`](#openai_gpt_4.1_mini_2025_04_14_details) | ⚠️ | ✅ | ✅ | 1,047,576 | 2025-06-20 | -| [`gpt-4.1-nano`](#openai_gpt_4.1_nano_details) | ⚠️ | ✅ | ✅ | 1,047,576 | 2025-06-20 | -| [`gpt-4o-2024-11-20`](#openai_gpt_4o_2024_11_20_details) | ⚠️ | ✅ | ✅ | 8,192 | 2025-06-20 | -| [`gpt-4-turbo-preview`](#openai_gpt_4_turbo_preview_details) | ✅ | ⚠️ | ❌ | 128,000 | 2025-06-20 | -| [`gpt-4-0125-preview`](#openai_gpt_4_0125_preview_details) | ⚠️ | ✅ | ❌ | 128,000 | 2025-06-20 | -| [`gpt-4o-mini`](#openai_gpt_4o_mini_details) | ⚠️ | ⚠️ | ⚠️ | 128,000 | 2025-06-20 | -| [`gpt-4o-mini-2024-07-18`](#openai_gpt_4o_mini_2024_07_18_details) | ⚠️ | ⚠️ | ❌ | 128,000 | 2025-06-20 | -| [`gpt-4`](#openai_gpt_4_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`o1`](#openai_o1_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`o1-2024-12-17`](#openai_o1_2024_12_17_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`o3`](#openai_o3_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`o3-2025-04-16`](#openai_o3_2025_04_16_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`o3-mini`](#openai_o3_mini_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`o3-mini-2025-01-31`](#openai_o3_mini_2025_01_31_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`o3-pro`](#openai_o3_pro_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`o3-pro-2025-06-10`](#openai_o3_pro_2025_06_10_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | - -### Detailed Test Results - -#### gpt-4.1 {#openai_gpt_4.1_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gpt-4.1-2025-04-14 {#openai_gpt_4.1_2025_04_14_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gpt-4.1-nano-2025-04-14 {#openai_gpt_4.1_nano_2025_04_14_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gpt-4o {#openai_gpt_4o_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gpt-4o-2024-05-13 {#openai_gpt_4o_2024_05_13_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gpt-4-turbo {#openai_gpt_4_turbo_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gpt-4.1-mini {#openai_gpt_4.1_mini_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gpt-4.5-preview {#openai_gpt_4.5_preview_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gpt-4.5-preview-2025-02-27 {#openai_gpt_4.5_preview_2025_02_27_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gpt-4o-2024-08-06 {#openai_gpt_4o_2024_08_06_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gpt-4-0613 {#openai_gpt_4_0613_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gpt-4-1106-preview {#openai_gpt_4_1106_preview_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gpt-4-turbo-2024-04-09 {#openai_gpt_4_turbo_2024_04_09_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gpt-4.1-mini-2025-04-14 {#openai_gpt_4.1_mini_2025_04_14_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gpt-4.1-nano {#openai_gpt_4.1_nano_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gpt-4o-2024-11-20 {#openai_gpt_4o_2024_11_20_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gpt-4-turbo-preview {#openai_gpt_4_turbo_preview_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gpt-4-0125-preview {#openai_gpt_4_0125_preview_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gpt-4o-mini {#openai_gpt_4o_mini_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gpt-4o-mini-2024-07-18 {#openai_gpt_4o_mini_2024_07_18_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gpt-4 {#openai_gpt_4_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### o1 {#openai_o1_details} - -### Basic - -- ❌ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### o1-2024-12-17 {#openai_o1_2024_12_17_details} - -### Basic - -- ❌ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### o3 {#openai_o3_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### o3-2025-04-16 {#openai_o3_2025_04_16_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### o3-mini {#openai_o3_mini_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### o3-mini-2025-01-31 {#openai_o3_mini_2025_01_31_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### o3-pro {#openai_o3_pro_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### o3-pro-2025-06-10 {#openai_o3_pro_2025_06_10_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - - ---- - -## google_ai - -| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | -|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| -| [`gemini-1.5-pro`](#google_ai_gemini_1.5_pro_details) | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 | -| [`gemini-1.5-pro-002`](#google_ai_gemini_1.5_pro_002_details) | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 | -| [`gemini-1.5-pro-latest`](#google_ai_gemini_1.5_pro_latest_details) | ✅ | ✅ | ✅ | 2,000,000 | 2025-06-20 | -| [`gemini-2.5-flash-preview-04-17-thinking`](#google_ai_gemini_2.5_flash_preview_04_17_thinking_details) | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 | -| [`gemini-2.5-pro-preview-03-25`](#google_ai_gemini_2.5_pro_preview_03_25_details) | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 | -| [`gemini-2.5-pro-preview-05-06`](#google_ai_gemini_2.5_pro_preview_05_06_details) | ✅ | ✅ | ✅ | 1,048,576 | 2025-06-20 | -| [`gemini-2.5-flash-preview-05-20`](#google_ai_gemini_2.5_flash_preview_05_20_details) | ✅ | ⚠️ | ✅ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-flash-thinking-exp`](#google_ai_gemini_2.0_flash_thinking_exp_details) | ⚠️ | ✅ | ✅ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-flash-thinking-exp-1219`](#google_ai_gemini_2.0_flash_thinking_exp_1219_details) | ⚠️ | ✅ | ✅ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-flash-thinking-exp-01-21`](#google_ai_gemini_2.0_flash_thinking_exp_01_21_details) | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 | -| [`gemini-2.5-flash-preview-04-17`](#google_ai_gemini_2.5_flash_preview_04_17_details) | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 | -| [`gemini-2.5-pro-preview-06-05`](#google_ai_gemini_2.5_pro_preview_06_05_details) | ⚠️ | ✅ | ⚠️ | 1,048,576 | 2025-06-20 | -| [`gemini-1.0-pro-vision-latest`](#google_ai_gemini_1.0_pro_vision_latest_details) | ❌ | ❌ | ❌ | 12,288 | 2025-06-20 | -| [`gemini-1.5-flash`](#google_ai_gemini_1.5_flash_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | -| [`gemini-1.5-flash-002`](#google_ai_gemini_1.5_flash_002_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | -| [`gemini-1.5-flash-8b`](#google_ai_gemini_1.5_flash_8b_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | -| [`gemini-1.5-flash-8b-001`](#google_ai_gemini_1.5_flash_8b_001_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | -| [`gemini-1.5-flash-8b-latest`](#google_ai_gemini_1.5_flash_8b_latest_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | -| [`gemini-1.5-flash-latest`](#google_ai_gemini_1.5_flash_latest_details) | ❌ | ❌ | ❌ | 1,000,000 | 2025-06-20 | -| [`gemini-2.0-flash`](#google_ai_gemini_2.0_flash_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-flash-001`](#google_ai_gemini_2.0_flash_001_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-flash-exp`](#google_ai_gemini_2.0_flash_exp_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-flash-exp-image-generation`](#google_ai_gemini_2.0_flash_exp_image_generation_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-flash-lite`](#google_ai_gemini_2.0_flash_lite_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-flash-lite-001`](#google_ai_gemini_2.0_flash_lite_001_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-flash-lite-preview`](#google_ai_gemini_2.0_flash_lite_preview_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-flash-lite-preview-02-05`](#google_ai_gemini_2.0_flash_lite_preview_02_05_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-flash-preview-image-generation`](#google_ai_gemini_2.0_flash_preview_image_generation_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`gemini-2.0-pro-exp`](#google_ai_gemini_2.0_pro_exp_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-2.0-pro-exp-02-05`](#google_ai_gemini_2.0_pro_exp_02_05_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-2.5-flash-preview-tts`](#google_ai_gemini_2.5_flash_preview_tts_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`gemini-2.5-pro-exp-03-25`](#google_ai_gemini_2.5_pro_exp_03_25_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-2.5-pro-preview-tts`](#google_ai_gemini_2.5_pro_preview_tts_details) | ❌ | ❌ | ❌ | 65,536 | 2025-06-20 | -| [`gemini-exp-1206`](#google_ai_gemini_exp_1206_details) | ❌ | ❌ | ❌ | 1,048,576 | 2025-06-20 | -| [`gemini-pro-vision`](#google_ai_gemini_pro_vision_details) | ❌ | ❌ | ❌ | 12,288 | 2025-06-20 | - -### Detailed Test Results - -#### gemini-1.5-pro {#google_ai_gemini_1.5_pro_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gemini-1.5-pro-002 {#google_ai_gemini_1.5_pro_002_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gemini-1.5-pro-latest {#google_ai_gemini_1.5_pro_latest_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gemini-2.5-flash-preview-04-17-thinking {#google_ai_gemini_2.5_flash_preview_04_17_thinking_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gemini-2.5-pro-preview-03-25 {#google_ai_gemini_2.5_pro_preview_03_25_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gemini-2.5-pro-preview-05-06 {#google_ai_gemini_2.5_pro_preview_05_06_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gemini-2.5-flash-preview-05-20 {#google_ai_gemini_2.5_flash_preview_05_20_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gemini-2.0-flash-thinking-exp {#google_ai_gemini_2.0_flash_thinking_exp_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gemini-2.0-flash-thinking-exp-1219 {#google_ai_gemini_2.0_flash_thinking_exp_1219_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gemini-2.0-flash-thinking-exp-01-21 {#google_ai_gemini_2.0_flash_thinking_exp_01_21_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-2.5-flash-preview-04-17 {#google_ai_gemini_2.5_flash_preview_04_17_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-2.5-pro-preview-06-05 {#google_ai_gemini_2.5_pro_preview_06_05_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### gemini-1.0-pro-vision-latest {#google_ai_gemini_1.0_pro_vision_latest_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-1.5-flash {#google_ai_gemini_1.5_flash_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-1.5-flash-002 {#google_ai_gemini_1.5_flash_002_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-1.5-flash-8b {#google_ai_gemini_1.5_flash_8b_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-1.5-flash-8b-001 {#google_ai_gemini_1.5_flash_8b_001_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-1.5-flash-8b-latest {#google_ai_gemini_1.5_flash_8b_latest_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-1.5-flash-latest {#google_ai_gemini_1.5_flash_latest_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-2.0-flash {#google_ai_gemini_2.0_flash_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-2.0-flash-001 {#google_ai_gemini_2.0_flash_001_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-2.0-flash-exp {#google_ai_gemini_2.0_flash_exp_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-2.0-flash-exp-image-generation {#google_ai_gemini_2.0_flash_exp_image_generation_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-2.0-flash-lite {#google_ai_gemini_2.0_flash_lite_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-2.0-flash-lite-001 {#google_ai_gemini_2.0_flash_lite_001_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-2.0-flash-lite-preview {#google_ai_gemini_2.0_flash_lite_preview_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-2.0-flash-lite-preview-02-05 {#google_ai_gemini_2.0_flash_lite_preview_02_05_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-2.0-flash-preview-image-generation {#google_ai_gemini_2.0_flash_preview_image_generation_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-2.0-pro-exp {#google_ai_gemini_2.0_pro_exp_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-2.0-pro-exp-02-05 {#google_ai_gemini_2.0_pro_exp_02_05_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-2.5-flash-preview-tts {#google_ai_gemini_2.5_flash_preview_tts_details} - -### Basic - -- ❌ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-2.5-pro-exp-03-25 {#google_ai_gemini_2.5_pro_exp_03_25_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-2.5-pro-preview-tts {#google_ai_gemini_2.5_pro_preview_tts_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-exp-1206 {#google_ai_gemini_exp_1206_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemini-pro-vision {#google_ai_gemini_pro_vision_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - - ---- - -## letta - -| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | -|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| -| [`letta-free`](#letta_letta_free_details) | ⚠️ | ❌ | ❌ | 8,192 | 2025-06-20 | - -### Detailed Test Results - -#### letta-free {#letta_letta_free_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - - ---- - -## together - -| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | -|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| -| [`Qwen/Qwen2.5-72B-Instruct-Turbo`](#together_qwen_qwen2.5_72b_instruct_turbo_details) | ✅ | ✅ | ⚠️ | 131,072 | 2025-06-20 | -| [`arcee-ai/virtuoso-large`](#together_arcee_ai_virtuoso_large_details) | ⚠️ | ✅ | ✅ | 131,072 | 2025-06-20 | -| [`Qwen/QwQ-32B`](#together_qwen_qwq_32b_details) | ⚠️ | ✅ | ⚠️ | 131,072 | 2025-06-20 | -| [`Qwen/Qwen2.5-7B-Instruct-Turbo`](#together_qwen_qwen2.5_7b_instruct_turbo_details) | ⚠️ | ✅ | ⚠️ | 32,768 | 2025-06-20 | -| [`Qwen/Qwen2.5-Coder-32B-Instruct`](#together_qwen_qwen2.5_coder_32b_instruct_details) | ⚠️ | ✅ | ⚠️ | 16,384 | 2025-06-20 | -| [`arcee-ai/coder-large`](#together_arcee_ai_coder_large_details) | ⚠️ | ✅ | ⚠️ | 32,768 | 2025-06-20 | -| [`arcee_ai/arcee-spotlight`](#together_arcee_ai_arcee_spotlight_details) | ⚠️ | ✅ | ⚠️ | 131,072 | 2025-06-20 | -| [`meta-llama/Llama-3.2-3B-Instruct-Turbo`](#together_meta_llama_llama_3.2_3b_instruct_turbo_details) | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | -| [`meta-llama/Llama-3.3-70B-Instruct-Turbo`](#together_meta_llama_llama_3.3_70b_instruct_turbo_details) | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | -| [`meta-llama/Llama-3.3-70B-Instruct-Turbo-Free`](#together_meta_llama_llama_3.3_70b_instruct_turbo_free_details) | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | -| [`meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo`](#together_meta_llama_meta_llama_3.1_405b_instruct_turbo_details) | ⚠️ | ✅ | ❌ | 130,815 | 2025-06-20 | -| [`meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo`](#together_meta_llama_meta_llama_3.1_70b_instruct_turbo_details) | ⚠️ | ✅ | ❌ | 131,072 | 2025-06-20 | -| [`nvidia/Llama-3.1-Nemotron-70B-Instruct-HF`](#together_nvidia_llama_3.1_nemotron_70b_instruct_hf_details) | ⚠️ | ✅ | ❌ | 32,768 | 2025-06-20 | -| [`arcee-ai/virtuoso-medium-v2`](#together_arcee_ai_virtuoso_medium_v2_details) | ⚠️ | ⚠️ | ✅ | 131,072 | 2025-06-20 | -| [`meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8`](#together_meta_llama_llama_4_maverick_17b_128e_instruct_fp8_details) | ⚠️ | ❌ | ✅ | 1,048,576 | 2025-06-20 | -| [`Qwen/Qwen3-235B-A22B-fp8-tput`](#together_qwen_qwen3_235b_a22b_fp8_tput_details) | ⚠️ | ⚠️ | ❌ | 40,960 | 2025-06-20 | -| [`deepseek-ai/DeepSeek-V3`](#together_deepseek_ai_deepseek_v3_details) | ⚠️ | ⚠️ | ❌ | 131,072 | 2025-06-20 | -| [`meta-llama/Llama-4-Scout-17B-16E-Instruct`](#together_meta_llama_llama_4_scout_17b_16e_instruct_details) | ⚠️ | ⚠️ | ❌ | 1,048,576 | 2025-06-20 | -| [`meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo`](#together_meta_llama_meta_llama_3.1_8b_instruct_turbo_details) | ⚠️ | ⚠️ | ❌ | 131,072 | 2025-06-20 | -| [`mistralai/Mixtral-8x7B-Instruct-v0.1`](#together_mistralai_mixtral_8x7b_instruct_v0.1_details) | ⚠️ | ⚠️ | ❌ | 32,768 | 2025-06-20 | -| [`arcee-ai/caller`](#together_arcee_ai_caller_details) | ❌ | ⚠️ | ❌ | 32,768 | 2025-06-20 | -| [`mistralai/Mistral-Small-24B-Instruct-2501`](#together_mistralai_mistral_small_24b_instruct_2501_details) | ❌ | ⚠️ | ❌ | 32,768 | 2025-06-20 | -| [`NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO`](#together_nousresearch_nous_hermes_2_mixtral_8x7b_dpo_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`Qwen/Qwen2-72B-Instruct`](#together_qwen_qwen2_72b_instruct_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`Qwen/Qwen2-VL-72B-Instruct`](#together_qwen_qwen2_vl_72b_instruct_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`Qwen/Qwen2.5-VL-72B-Instruct`](#together_qwen_qwen2.5_vl_72b_instruct_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`arcee-ai/arcee-blitz`](#together_arcee_ai_arcee_blitz_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`arcee-ai/maestro-reasoning`](#together_arcee_ai_maestro_reasoning_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | -| [`deepseek-ai/DeepSeek-R1`](#together_deepseek_ai_deepseek_r1_details) | ❌ | ❌ | ❌ | 163,840 | 2025-06-20 | -| [`deepseek-ai/DeepSeek-R1-Distill-Llama-70B`](#together_deepseek_ai_deepseek_r1_distill_llama_70b_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | -| [`deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free`](#together_deepseek_ai_deepseek_r1_distill_llama_70b_free_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B`](#together_deepseek_ai_deepseek_r1_distill_qwen_1.5b_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | -| [`deepseek-ai/DeepSeek-R1-Distill-Qwen-14B`](#together_deepseek_ai_deepseek_r1_distill_qwen_14b_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | -| [`deepseek-ai/DeepSeek-V3-p-dp`](#together_deepseek_ai_deepseek_v3_p_dp_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | -| [`google/gemma-2-27b-it`](#together_google_gemma_2_27b_it_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`lgai/exaone-3-5-32b-instruct`](#together_lgai_exaone_3_5_32b_instruct_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`lgai/exaone-deep-32b`](#together_lgai_exaone_deep_32b_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`marin-community/marin-8b-instruct`](#together_marin_community_marin_8b_instruct_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | -| [`meta-llama/Llama-3-70b-chat-hf`](#together_meta_llama_llama_3_70b_chat_hf_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`meta-llama/Llama-3-8b-chat-hf`](#together_meta_llama_llama_3_8b_chat_hf_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo`](#together_meta_llama_llama_3.2_11b_vision_instruct_turbo_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | -| [`meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo`](#together_meta_llama_llama_3.2_90b_vision_instruct_turbo_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | -| [`meta-llama/Llama-Vision-Free`](#together_meta_llama_llama_vision_free_details) | ❌ | ❌ | ❌ | 131,072 | 2025-06-20 | -| [`meta-llama/Meta-Llama-3-70B-Instruct-Turbo`](#together_meta_llama_meta_llama_3_70b_instruct_turbo_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`meta-llama/Meta-Llama-3-8B-Instruct-Lite`](#together_meta_llama_meta_llama_3_8b_instruct_lite_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`mistralai/Mistral-7B-Instruct-v0.1`](#together_mistralai_mistral_7b_instruct_v0.1_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`mistralai/Mistral-7B-Instruct-v0.2`](#together_mistralai_mistral_7b_instruct_v0.2_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`mistralai/Mistral-7B-Instruct-v0.3`](#together_mistralai_mistral_7b_instruct_v0.3_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`perplexity-ai/r1-1776`](#together_perplexity_ai_r1_1776_details) | ❌ | ❌ | ❌ | 163,840 | 2025-06-20 | -| [`scb10x/scb10x-llama3-1-typhoon2-70b-instruct`](#together_scb10x_scb10x_llama3_1_typhoon2_70b_instruct_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`scb10x/scb10x-typhoon-2-1-gemma3-12b`](#together_scb10x_scb10x_typhoon_2_1_gemma3_12b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`togethercomputer/MoA-1`](#together_togethercomputer_moa_1_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`togethercomputer/MoA-1-Turbo`](#together_togethercomputer_moa_1_turbo_details) | ❌ | ❌ | ❌ | 32,768 | 2025-06-20 | -| [`togethercomputer/Refuel-Llm-V2`](#together_togethercomputer_refuel_llm_v2_details) | ❌ | ❌ | ❌ | 16,384 | 2025-06-20 | -| [`togethercomputer/Refuel-Llm-V2-Small`](#together_togethercomputer_refuel_llm_v2_small_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | - -### Detailed Test Results - -#### Qwen/Qwen2.5-72B-Instruct-Turbo {#together_qwen_qwen2.5_72b_instruct_turbo_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### arcee-ai/virtuoso-large {#together_arcee_ai_virtuoso_large_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### Qwen/QwQ-32B {#together_qwen_qwq_32b_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### Qwen/Qwen2.5-7B-Instruct-Turbo {#together_qwen_qwen2.5_7b_instruct_turbo_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### Qwen/Qwen2.5-Coder-32B-Instruct {#together_qwen_qwen2.5_coder_32b_instruct_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### arcee-ai/coder-large {#together_arcee_ai_coder_large_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### arcee_ai/arcee-spotlight {#together_arcee_ai_arcee_spotlight_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/Llama-3.2-3B-Instruct-Turbo {#together_meta_llama_llama_3.2_3b_instruct_turbo_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/Llama-3.3-70B-Instruct-Turbo {#together_meta_llama_llama_3.3_70b_instruct_turbo_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/Llama-3.3-70B-Instruct-Turbo-Free {#together_meta_llama_llama_3.3_70b_instruct_turbo_free_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo {#together_meta_llama_meta_llama_3.1_405b_instruct_turbo_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo {#together_meta_llama_meta_llama_3.1_70b_instruct_turbo_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### nvidia/Llama-3.1-Nemotron-70B-Instruct-HF {#together_nvidia_llama_3.1_nemotron_70b_instruct_hf_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ✅ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### arcee-ai/virtuoso-medium-v2 {#together_arcee_ai_virtuoso_medium_v2_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 {#together_meta_llama_llama_4_maverick_17b_128e_instruct_fp8_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ✅ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ✅ `test_base64_image_input` -- ✅ `test_url_image_input` - -#### Qwen/Qwen3-235B-A22B-fp8-tput {#together_qwen_qwen3_235b_a22b_fp8_tput_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### deepseek-ai/DeepSeek-V3 {#together_deepseek_ai_deepseek_v3_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/Llama-4-Scout-17B-16E-Instruct {#together_meta_llama_llama_4_scout_17b_16e_instruct_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo {#together_meta_llama_meta_llama_3.1_8b_instruct_turbo_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ✅ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### mistralai/Mixtral-8x7B-Instruct-v0.1 {#together_mistralai_mixtral_8x7b_instruct_v0.1_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ✅ `test_auto_summarize` -- ✅ `test_greeting_with_assistant_message` -- ✅ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ✅ `test_step_streaming_greeting_with_assistant_message` -- ✅ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ✅ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### arcee-ai/caller {#together_arcee_ai_caller_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ✅ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### mistralai/Mistral-Small-24B-Instruct-2501 {#together_mistralai_mistral_small_24b_instruct_2501_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ✅ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO {#together_nousresearch_nous_hermes_2_mixtral_8x7b_dpo_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### Qwen/Qwen2-72B-Instruct {#together_qwen_qwen2_72b_instruct_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### Qwen/Qwen2-VL-72B-Instruct {#together_qwen_qwen2_vl_72b_instruct_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### Qwen/Qwen2.5-VL-72B-Instruct {#together_qwen_qwen2.5_vl_72b_instruct_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### arcee-ai/arcee-blitz {#together_arcee_ai_arcee_blitz_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### arcee-ai/maestro-reasoning {#together_arcee_ai_maestro_reasoning_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### deepseek-ai/DeepSeek-R1 {#together_deepseek_ai_deepseek_r1_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### deepseek-ai/DeepSeek-R1-Distill-Llama-70B {#together_deepseek_ai_deepseek_r1_distill_llama_70b_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free {#together_deepseek_ai_deepseek_r1_distill_llama_70b_free_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B {#together_deepseek_ai_deepseek_r1_distill_qwen_1.5b_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### deepseek-ai/DeepSeek-R1-Distill-Qwen-14B {#together_deepseek_ai_deepseek_r1_distill_qwen_14b_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### deepseek-ai/DeepSeek-V3-p-dp {#together_deepseek_ai_deepseek_v3_p_dp_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### google/gemma-2-27b-it {#together_google_gemma_2_27b_it_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### lgai/exaone-3-5-32b-instruct {#together_lgai_exaone_3_5_32b_instruct_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### lgai/exaone-deep-32b {#together_lgai_exaone_deep_32b_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### marin-community/marin-8b-instruct {#together_marin_community_marin_8b_instruct_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/Llama-3-70b-chat-hf {#together_meta_llama_llama_3_70b_chat_hf_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/Llama-3-8b-chat-hf {#together_meta_llama_llama_3_8b_chat_hf_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo {#together_meta_llama_llama_3.2_11b_vision_instruct_turbo_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo {#together_meta_llama_llama_3.2_90b_vision_instruct_turbo_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/Llama-Vision-Free {#together_meta_llama_llama_vision_free_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/Meta-Llama-3-70B-Instruct-Turbo {#together_meta_llama_meta_llama_3_70b_instruct_turbo_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/Meta-Llama-3-8B-Instruct-Lite {#together_meta_llama_meta_llama_3_8b_instruct_lite_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### mistralai/Mistral-7B-Instruct-v0.1 {#together_mistralai_mistral_7b_instruct_v0.1_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### mistralai/Mistral-7B-Instruct-v0.2 {#together_mistralai_mistral_7b_instruct_v0.2_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### mistralai/Mistral-7B-Instruct-v0.3 {#together_mistralai_mistral_7b_instruct_v0.3_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### perplexity-ai/r1-1776 {#together_perplexity_ai_r1_1776_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### scb10x/scb10x-llama3-1-typhoon2-70b-instruct {#together_scb10x_scb10x_llama3_1_typhoon2_70b_instruct_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### scb10x/scb10x-typhoon-2-1-gemma3-12b {#together_scb10x_scb10x_typhoon_2_1_gemma3_12b_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### togethercomputer/MoA-1 {#together_togethercomputer_moa_1_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### togethercomputer/MoA-1-Turbo {#together_togethercomputer_moa_1_turbo_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### togethercomputer/Refuel-Llm-V2 {#together_togethercomputer_refuel_llm_v2_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### togethercomputer/Refuel-Llm-V2-Small {#together_togethercomputer_refuel_llm_v2_small_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - - ---- - -## deepseek - -| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | -|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| -| [`deepseek-chat`](#deepseek_deepseek_chat_details) | ❌ | ❌ | ❌ | 64,000 | 2025-06-20 | -| [`deepseek-reasoner`](#deepseek_deepseek_reasoner_details) | ❌ | ❌ | ❌ | 64,000 | 2025-06-20 | - -### Detailed Test Results - -#### deepseek-chat {#deepseek_deepseek_chat_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### deepseek-reasoner {#deepseek_deepseek_reasoner_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - - ---- - -## groq - -| Model | Basic | Token Streaming | Multimodal | Context Window | Last Scanned | -|---------------------------------------------------|:---:|:-------------:|:--------:|:------------:|:----------:| -| [`allam-2-7b`](#groq_allam_2_7b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`compound-beta`](#groq_compound_beta_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`compound-beta-mini`](#groq_compound_beta_mini_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`deepseek-r1-distill-llama-70b`](#groq_deepseek_r1_distill_llama_70b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`distil-whisper-large-v3-en`](#groq_distil_whisper_large_v3_en_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`gemma2-9b-it`](#groq_gemma2_9b_it_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`llama-3.1-8b-instant`](#groq_llama_3.1_8b_instant_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`llama-3.3-70b-versatile`](#groq_llama_3.3_70b_versatile_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`llama-guard-3-8b`](#groq_llama_guard_3_8b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`llama3-70b-8192`](#groq_llama3_70b_8192_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`llama3-8b-8192`](#groq_llama3_8b_8192_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`meta-llama/llama-4-maverick-17b-128e-instruct`](#groq_meta_llama_llama_4_maverick_17b_128e_instruct_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`meta-llama/llama-4-scout-17b-16e-instruct`](#groq_meta_llama_llama_4_scout_17b_16e_instruct_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`meta-llama/llama-guard-4-12b`](#groq_meta_llama_llama_guard_4_12b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`meta-llama/llama-prompt-guard-2-22m`](#groq_meta_llama_llama_prompt_guard_2_22m_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`meta-llama/llama-prompt-guard-2-86m`](#groq_meta_llama_llama_prompt_guard_2_86m_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`mistral-saba-24b`](#groq_mistral_saba_24b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`playai-tts`](#groq_playai_tts_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`playai-tts-arabic`](#groq_playai_tts_arabic_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`qwen-qwq-32b`](#groq_qwen_qwq_32b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`qwen/qwen3-32b`](#groq_qwen_qwen3_32b_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`whisper-large-v3`](#groq_whisper_large_v3_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | -| [`whisper-large-v3-turbo`](#groq_whisper_large_v3_turbo_details) | ❌ | ❌ | ❌ | 8,192 | 2025-06-20 | - -### Detailed Test Results - -#### allam-2-7b {#groq_allam_2_7b_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### compound-beta {#groq_compound_beta_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### compound-beta-mini {#groq_compound_beta_mini_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### deepseek-r1-distill-llama-70b {#groq_deepseek_r1_distill_llama_70b_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### distil-whisper-large-v3-en {#groq_distil_whisper_large_v3_en_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### gemma2-9b-it {#groq_gemma2_9b_it_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### llama-3.1-8b-instant {#groq_llama_3.1_8b_instant_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### llama-3.3-70b-versatile {#groq_llama_3.3_70b_versatile_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### llama-guard-3-8b {#groq_llama_guard_3_8b_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### llama3-70b-8192 {#groq_llama3_70b_8192_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### llama3-8b-8192 {#groq_llama3_8b_8192_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/llama-4-maverick-17b-128e-instruct {#groq_meta_llama_llama_4_maverick_17b_128e_instruct_details} - -### Basic - -- ❌ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/llama-4-scout-17b-16e-instruct {#groq_meta_llama_llama_4_scout_17b_16e_instruct_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/llama-guard-4-12b {#groq_meta_llama_llama_guard_4_12b_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/llama-prompt-guard-2-22m {#groq_meta_llama_llama_prompt_guard_2_22m_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### meta-llama/llama-prompt-guard-2-86m {#groq_meta_llama_llama_prompt_guard_2_86m_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### mistral-saba-24b {#groq_mistral_saba_24b_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### playai-tts {#groq_playai_tts_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### playai-tts-arabic {#groq_playai_tts_arabic_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### qwen-qwq-32b {#groq_qwen_qwq_32b_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### qwen/qwen3-32b {#groq_qwen_qwen3_32b_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### whisper-large-v3 {#groq_whisper_large_v3_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - -#### whisper-large-v3-turbo {#groq_whisper_large_v3_turbo_details} - -### Basic - -- ✅ `test_agent_loop_error` -- ❌ `test_async_greeting_with_assistant_message` -- ❌ `test_auto_summarize` -- ❌ `test_greeting_with_assistant_message` -- ❌ `test_greeting_without_assistant_message` -- ✅ `test_step_stream_agent_loop_error` -- ❌ `test_step_streaming_greeting_with_assistant_message` -- ❌ `test_step_streaming_greeting_without_assistant_message` -- ❌ `test_step_streaming_tool_call` -- ❌ `test_tool_call` - -### Token Streaming - -- ✅ `test_token_streaming_agent_loop_error` -- ❌ `test_token_streaming_greeting_with_assistant_message` -- ❌ `test_token_streaming_greeting_without_assistant_message` -- ❌ `test_token_streaming_tool_call` - -### Multimodal - -- ❌ `test_base64_image_input` -- ❌ `test_url_image_input` - - ----