104 lines
4.0 KiB
Python
104 lines
4.0 KiB
Python
"""
|
|
Note: this supports completions (deprecated by openai) and chat completions via the OpenAI API.
|
|
"""
|
|
|
|
from typing import Literal, Optional
|
|
|
|
from letta.log import get_logger
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
from pydantic import Field
|
|
|
|
from letta.constants import MIN_CONTEXT_WINDOW
|
|
from letta.errors import ErrorCode, LLMAuthenticationError
|
|
from letta.schemas.embedding_config import EmbeddingConfig
|
|
from letta.schemas.enums import ProviderCategory, ProviderType
|
|
from letta.schemas.llm_config import LLMConfig
|
|
from letta.schemas.providers.openai import OpenAIProvider
|
|
|
|
|
|
class TogetherProvider(OpenAIProvider):
|
|
provider_type: Literal[ProviderType.together] = Field(ProviderType.together, description="The type of the provider.")
|
|
provider_category: ProviderCategory = Field(ProviderCategory.base, description="The category of the provider (base or byok)")
|
|
base_url: str = "https://api.together.xyz/v1"
|
|
api_key: str | None = Field(None, description="API key for the Together API.", deprecated=True)
|
|
default_prompt_formatter: Optional[str] = Field(
|
|
None, description="Default prompt formatter (aka model wrapper) to use on vLLM /completions API."
|
|
)
|
|
|
|
async def list_llm_models_async(self) -> list[LLMConfig]:
|
|
from letta.llm_api.openai import openai_get_model_list_async
|
|
|
|
api_key = await self.api_key_enc.get_plaintext_async() if self.api_key_enc else None
|
|
models = await openai_get_model_list_async(self.base_url, api_key=api_key)
|
|
return self._list_llm_models(models)
|
|
|
|
async def list_embedding_models_async(self) -> list[EmbeddingConfig]:
|
|
import warnings
|
|
|
|
logger.warning(
|
|
"Letta does not currently support listing embedding models for Together. Please "
|
|
"contact support or reach out via GitHub or Discord to get support."
|
|
)
|
|
return []
|
|
|
|
# TODO (cliandy): verify this with openai
|
|
def _list_llm_models(self, models) -> list[LLMConfig]:
|
|
pass
|
|
|
|
# TogetherAI's response is missing the 'data' field
|
|
# assert "data" in response, f"OpenAI model query response missing 'data' field: {response}"
|
|
if "data" in models:
|
|
data = models["data"]
|
|
else:
|
|
data = models
|
|
|
|
configs = []
|
|
for model in data:
|
|
assert "id" in model, f"TogetherAI model missing 'id' field: {model}"
|
|
model_name = model["id"]
|
|
|
|
if "context_length" in model:
|
|
# Context length is returned in OpenRouter as "context_length"
|
|
context_window_size = model["context_length"]
|
|
else:
|
|
context_window_size = self.get_model_context_window_size(model_name)
|
|
|
|
# We need the context length for embeddings too
|
|
if not context_window_size:
|
|
continue
|
|
|
|
# Skip models that are too small for Letta
|
|
if context_window_size <= MIN_CONTEXT_WINDOW:
|
|
continue
|
|
|
|
# TogetherAI includes the type, which we can use to filter for embedding models
|
|
if "type" in model and model["type"] not in ["chat", "language"]:
|
|
continue
|
|
|
|
configs.append(
|
|
LLMConfig(
|
|
model=model_name,
|
|
model_endpoint_type="together",
|
|
model_endpoint=self.base_url,
|
|
model_wrapper=self.default_prompt_formatter,
|
|
context_window=context_window_size,
|
|
handle=self.get_handle(model_name),
|
|
provider_name=self.name,
|
|
provider_category=self.provider_category,
|
|
)
|
|
)
|
|
|
|
return configs
|
|
|
|
async def check_api_key(self):
|
|
api_key = await self.api_key_enc.get_plaintext_async() if self.api_key_enc else None
|
|
if not api_key:
|
|
raise ValueError("No API key provided")
|
|
|
|
try:
|
|
await self.list_llm_models_async()
|
|
except Exception as e:
|
|
raise LLMAuthenticationError(message=f"Failed to authenticate with Together: {e}", code=ErrorCode.UNAUTHENTICATED)
|