Fimeg/letta-server
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat: add together byok support (#3877)

Browse Source
This commit is contained in:
cthomas
2025-08-12 13:44:01 -07:00
committed by GitHub
parent 2e19f2a801
commit a328cbd607
1 changed files with 17 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
letta/llm_api/together_client.py
View File

@@ -7,6 +7,7 @@ from openai.types.chat.chat_completion import ChatCompletion
from letta.llm_api.openai_client import OpenAIClient
from letta.otel.tracing import trace_method
from letta.schemas.embedding_config import EmbeddingConfig
from letta.schemas.enums import ProviderCategory
from letta.schemas.llm_config import LLMConfig
from letta.settings import model_settings
@@ -21,7 +22,14 @@ class TogetherClient(OpenAIClient):
"""
Performs underlying synchronous request to OpenAI API and returns raw response dict.
"""
api_key = model_settings.together_api_key or os.environ.get("TOGETHER_API_KEY")
api_key = None
if llm_config.provider_category == ProviderCategory.byok:
from letta.services.provider_manager import ProviderManager
api_key = ProviderManager().get_override_key(llm_config.provider_name, actor=self.actor)
if not api_key:
api_key = model_settings.together_api_key or os.environ.get("TOGETHER_API_KEY")
client = OpenAI(api_key=api_key, base_url=llm_config.model_endpoint)
response: ChatCompletion = client.chat.completions.create(**request_data)
@@ -32,7 +40,14 @@ class TogetherClient(OpenAIClient):
"""
Performs underlying asynchronous request to OpenAI API and returns raw response dict.
"""
api_key = model_settings.together_api_key or os.environ.get("TOGETHER_API_KEY")
api_key = None
if llm_config.provider_category == ProviderCategory.byok:
from letta.services.provider_manager import ProviderManager
api_key = ProviderManager().get_override_key(llm_config.provider_name, actor=self.actor)
if not api_key:
api_key = model_settings.together_api_key or os.environ.get("TOGETHER_API_KEY")
client = AsyncOpenAI(api_key=api_key, base_url=llm_config.model_endpoint)
response: ChatCompletion = await client.chat.completions.create(**request_data)