diff --git a/letta/llm_api/openai_client.py b/letta/llm_api/openai_client.py index c9561cc2..84675f6c 100644 --- a/letta/llm_api/openai_client.py +++ b/letta/llm_api/openai_client.py @@ -101,6 +101,8 @@ def requires_auto_tool_choice(llm_config: LLMConfig) -> bool: return True if llm_config.handle and "vllm" in llm_config.handle: return True + if llm_config.compatibility_type == "mlx": + return True return False diff --git a/letta/schemas/llm_config.py b/letta/schemas/llm_config.py index cccf93b0..e2677334 100644 --- a/letta/schemas/llm_config.py +++ b/letta/schemas/llm_config.py @@ -81,6 +81,7 @@ class LLMConfig(BaseModel): None, # Can also deafult to 0.0? description="Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim. From OpenAI: Number between -2.0 and 2.0.", ) + compatibility_type: Optional[Literal["gguf", "mlx"]] = Field(None, description="The framework compatibility type for the model.") # FIXME hack to silence pydantic protected namespace warning model_config = ConfigDict(protected_namespaces=()) diff --git a/letta/schemas/providers/lmstudio.py b/letta/schemas/providers/lmstudio.py index 6e4a639a..910e37da 100644 --- a/letta/schemas/providers/lmstudio.py +++ b/letta/schemas/providers/lmstudio.py @@ -45,6 +45,12 @@ class LMStudioOpenAIProvider(OpenAIProvider): continue model_name, context_window_size = check + if "compatibility_type" in model: + compatibility_type = model["compatibility_type"] + else: + warnings.warn(f"LMStudio OpenAI model missing 'compatibility_type' field: {model}") + continue + configs.append( LLMConfig( model=model_name, @@ -52,6 +58,7 @@ class LMStudioOpenAIProvider(OpenAIProvider): model_endpoint=self.base_url, context_window=context_window_size, handle=self.get_handle(model_name), + compatibility_type=compatibility_type, provider_name=self.name, provider_category=self.provider_category, ) diff --git a/letta/services/helpers/agent_manager_helper.py b/letta/services/helpers/agent_manager_helper.py index ae13435f..ccebf35b 100644 --- a/letta/services/helpers/agent_manager_helper.py +++ b/letta/services/helpers/agent_manager_helper.py @@ -1,4 +1,5 @@ import os +import uuid from datetime import datetime from typing import List, Literal, Optional, Set @@ -351,21 +352,40 @@ def initialize_message_sequence( first_user_message = get_login_event(agent_state.timezone) # event letting Letta know the user just logged in if include_initial_boot_message: + llm_config = agent_state.llm_config + uuid_str = str(uuid.uuid4()) + + # Some LMStudio models (e.g. ministral) require the tool call ID to be 9 alphanumeric characters + tool_call_id = uuid_str[:9] if llm_config.provider_name == "lmstudio_openai" else uuid_str + if agent_state.agent_type == AgentType.sleeptime_agent: initial_boot_messages = [] - elif agent_state.llm_config.model is not None and "gpt-3.5" in agent_state.llm_config.model: - initial_boot_messages = get_initial_boot_messages("startup_with_send_message_gpt35", agent_state.timezone) + elif llm_config.model is not None and "gpt-3.5" in llm_config.model: + initial_boot_messages = get_initial_boot_messages("startup_with_send_message_gpt35", agent_state.timezone, tool_call_id) else: - initial_boot_messages = get_initial_boot_messages("startup_with_send_message", agent_state.timezone) - messages = ( - [ - {"role": "system", "content": full_system_message}, - ] - + initial_boot_messages - + [ - {"role": "user", "content": first_user_message}, - ] - ) + initial_boot_messages = get_initial_boot_messages("startup_with_send_message", agent_state.timezone, tool_call_id) + + # Some LMStudio models (e.g. meta-llama-3.1) require the user message before any tool calls + if llm_config.provider_name == "lmstudio_openai": + messages = ( + [ + {"role": "system", "content": full_system_message}, + ] + + [ + {"role": "user", "content": first_user_message}, + ] + + initial_boot_messages + ) + else: + messages = ( + [ + {"role": "system", "content": full_system_message}, + ] + + initial_boot_messages + + [ + {"role": "user", "content": first_user_message}, + ] + ) else: messages = [ diff --git a/letta/system.py b/letta/system.py index f64cc9d0..a44f5f7b 100644 --- a/letta/system.py +++ b/letta/system.py @@ -1,5 +1,4 @@ import json -import uuid import warnings from typing import Optional @@ -13,7 +12,7 @@ from .helpers.datetime_helpers import get_local_time from .helpers.json_helpers import json_dumps -def get_initial_boot_messages(version, timezone): +def get_initial_boot_messages(version, timezone, tool_call_id): if version == "startup": initial_boot_message = INITIAL_BOOT_MESSAGE messages = [ @@ -21,7 +20,6 @@ def get_initial_boot_messages(version, timezone): ] elif version == "startup_with_send_message": - tool_call_id = str(uuid.uuid4()) messages = [ # first message includes both inner monologue and function call to send_message { @@ -53,7 +51,6 @@ def get_initial_boot_messages(version, timezone): ] elif version == "startup_with_send_message_gpt35": - tool_call_id = str(uuid.uuid4()) messages = [ # first message includes both inner monologue and function call to send_message {