From a328cbd60760c1aea273a0d0c7f46a7467976af4 Mon Sep 17 00:00:00 2001 From: cthomas Date: Tue, 12 Aug 2025 13:44:01 -0700 Subject: [PATCH] feat: add together byok support (#3877) --- letta/llm_api/together_client.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/letta/llm_api/together_client.py b/letta/llm_api/together_client.py index 83ccd435..949cb6b4 100644 --- a/letta/llm_api/together_client.py +++ b/letta/llm_api/together_client.py @@ -7,6 +7,7 @@ from openai.types.chat.chat_completion import ChatCompletion from letta.llm_api.openai_client import OpenAIClient from letta.otel.tracing import trace_method from letta.schemas.embedding_config import EmbeddingConfig +from letta.schemas.enums import ProviderCategory from letta.schemas.llm_config import LLMConfig from letta.settings import model_settings @@ -21,7 +22,14 @@ class TogetherClient(OpenAIClient): """ Performs underlying synchronous request to OpenAI API and returns raw response dict. """ - api_key = model_settings.together_api_key or os.environ.get("TOGETHER_API_KEY") + api_key = None + if llm_config.provider_category == ProviderCategory.byok: + from letta.services.provider_manager import ProviderManager + + api_key = ProviderManager().get_override_key(llm_config.provider_name, actor=self.actor) + + if not api_key: + api_key = model_settings.together_api_key or os.environ.get("TOGETHER_API_KEY") client = OpenAI(api_key=api_key, base_url=llm_config.model_endpoint) response: ChatCompletion = client.chat.completions.create(**request_data) @@ -32,7 +40,14 @@ class TogetherClient(OpenAIClient): """ Performs underlying asynchronous request to OpenAI API and returns raw response dict. """ - api_key = model_settings.together_api_key or os.environ.get("TOGETHER_API_KEY") + api_key = None + if llm_config.provider_category == ProviderCategory.byok: + from letta.services.provider_manager import ProviderManager + + api_key = ProviderManager().get_override_key(llm_config.provider_name, actor=self.actor) + + if not api_key: + api_key = model_settings.together_api_key or os.environ.get("TOGETHER_API_KEY") client = AsyncOpenAI(api_key=api_key, base_url=llm_config.model_endpoint) response: ChatCompletion = await client.chat.completions.create(**request_data)