From c4f603d7b64f8a5b662698829044ff5a17bd375e Mon Sep 17 00:00:00 2001 From: cthomas Date: Wed, 30 Apr 2025 23:23:01 -0700 Subject: [PATCH] feat: always add user id to openai requests (#1969) --- letta/agent.py | 1 + letta/agents/letta_agent.py | 5 ++++- letta/agents/letta_agent_batch.py | 2 ++ letta/llm_api/llm_client.py | 5 +++++ letta/llm_api/llm_client_base.py | 2 ++ letta/llm_api/openai.py | 14 ++++++++++---- letta/llm_api/openai_client.py | 14 ++++++++++---- letta/memory.py | 1 + tests/helpers/endpoints_helper.py | 1 + 9 files changed, 36 insertions(+), 9 deletions(-) diff --git a/letta/agent.py b/letta/agent.py index a9f3a1ee..4e76bf12 100644 --- a/letta/agent.py +++ b/letta/agent.py @@ -334,6 +334,7 @@ class Agent(BaseAgent): provider_name=self.agent_state.llm_config.provider_name, provider_type=self.agent_state.llm_config.model_endpoint_type, put_inner_thoughts_first=put_inner_thoughts_first, + actor_id=self.user.id, ) if llm_client and not stream: diff --git a/letta/agents/letta_agent.py b/letta/agents/letta_agent.py index efc81331..5d859b34 100644 --- a/letta/agents/letta_agent.py +++ b/letta/agents/letta_agent.py @@ -70,6 +70,7 @@ class LettaAgent(BaseAgent): provider_name=agent_state.llm_config.provider_name, provider_type=agent_state.llm_config.model_endpoint_type, put_inner_thoughts_first=True, + actor_id=self.actor.id, ) for step in range(max_steps): response = await self._get_ai_reply( @@ -110,8 +111,10 @@ class LettaAgent(BaseAgent): ) tool_rules_solver = ToolRulesSolver(agent_state.tool_rules) llm_client = LLMClient.create( - provider=agent_state.llm_config.model_endpoint_type, + provider_name=agent_state.llm_config.provider_name, + provider_type=agent_state.llm_config.model_endpoint_type, put_inner_thoughts_first=True, + actor_id=self.actor.id, ) for step in range(max_steps): diff --git a/letta/agents/letta_agent_batch.py b/letta/agents/letta_agent_batch.py index b832968e..3610bf2e 100644 --- a/letta/agents/letta_agent_batch.py +++ b/letta/agents/letta_agent_batch.py @@ -159,6 +159,7 @@ class LettaAgentBatch: provider_name=agent_states[0].llm_config.provider_name, provider_type=agent_states[0].llm_config.model_endpoint_type, put_inner_thoughts_first=True, + actor_id=self.actor.id, ) agent_llm_config_mapping = {s.id: s.llm_config for s in agent_states} @@ -277,6 +278,7 @@ class LettaAgentBatch: provider_name=item.llm_config.provider_name, provider_type=item.llm_config.model_endpoint_type, put_inner_thoughts_first=True, + actor_id=self.actor.id, ) tool_call = ( llm_client.convert_response_to_chat_completion( diff --git a/letta/llm_api/llm_client.py b/letta/llm_api/llm_client.py index 9028dd04..a63913a4 100644 --- a/letta/llm_api/llm_client.py +++ b/letta/llm_api/llm_client.py @@ -12,6 +12,7 @@ class LLMClient: provider_type: ProviderType, provider_name: Optional[str] = None, put_inner_thoughts_first: bool = True, + actor_id: Optional[str] = None, ) -> Optional[LLMClientBase]: """ Create an LLM client based on the model endpoint type. @@ -33,6 +34,7 @@ class LLMClient: return GoogleAIClient( provider_name=provider_name, put_inner_thoughts_first=put_inner_thoughts_first, + actor_id=actor_id, ) case ProviderType.google_vertex: from letta.llm_api.google_vertex_client import GoogleVertexClient @@ -40,6 +42,7 @@ class LLMClient: return GoogleVertexClient( provider_name=provider_name, put_inner_thoughts_first=put_inner_thoughts_first, + actor_id=actor_id, ) case ProviderType.anthropic: from letta.llm_api.anthropic_client import AnthropicClient @@ -47,6 +50,7 @@ class LLMClient: return AnthropicClient( provider_name=provider_name, put_inner_thoughts_first=put_inner_thoughts_first, + actor_id=actor_id, ) case ProviderType.openai: from letta.llm_api.openai_client import OpenAIClient @@ -54,6 +58,7 @@ class LLMClient: return OpenAIClient( provider_name=provider_name, put_inner_thoughts_first=put_inner_thoughts_first, + actor_id=actor_id, ) case _: return None diff --git a/letta/llm_api/llm_client_base.py b/letta/llm_api/llm_client_base.py index 2b251289..223921f9 100644 --- a/letta/llm_api/llm_client_base.py +++ b/letta/llm_api/llm_client_base.py @@ -23,7 +23,9 @@ class LLMClientBase: provider_name: Optional[str] = None, put_inner_thoughts_first: Optional[bool] = True, use_tool_naming: bool = True, + actor_id: Optional[str] = None, ): + self.actor_id = actor_id self.provider_name = provider_name self.put_inner_thoughts_first = put_inner_thoughts_first self.use_tool_naming = use_tool_naming diff --git a/letta/llm_api/openai.py b/letta/llm_api/openai.py index 42ad06ec..d72fb259 100644 --- a/letta/llm_api/openai.py +++ b/letta/llm_api/openai.py @@ -157,11 +157,17 @@ def build_openai_chat_completions_request( # if "gpt-4o" in llm_config.model or "gpt-4-turbo" in llm_config.model or "gpt-3.5-turbo" in llm_config.model: # data.response_format = {"type": "json_object"} - if llm_config.model_endpoint == LETTA_MODEL_ENDPOINT: - # override user id for inference.letta.com - import uuid + # always set user id for openai requests + if user_id: + data.user = str(user_id) + + if llm_config.model_endpoint == LETTA_MODEL_ENDPOINT: + if not user_id: + # override user id for inference.letta.com + import uuid + + data.user = str(uuid.UUID(int=0)) - data.user = str(uuid.UUID(int=0)) data.model = "memgpt-openai" if use_structured_output and data.tools is not None and len(data.tools) > 0: diff --git a/letta/llm_api/openai_client.py b/letta/llm_api/openai_client.py index 2c205717..c5e512d0 100644 --- a/letta/llm_api/openai_client.py +++ b/letta/llm_api/openai_client.py @@ -143,11 +143,17 @@ class OpenAIClient(LLMClientBase): temperature=llm_config.temperature if supports_temperature_param(model) else None, ) - if llm_config.model_endpoint == LETTA_MODEL_ENDPOINT: - # override user id for inference.letta.com - import uuid + # always set user id for openai requests + if self.actor_id: + data.user = self.actor_id + + if llm_config.model_endpoint == LETTA_MODEL_ENDPOINT: + if not self.actor_id: + # override user id for inference.letta.com + import uuid + + data.user = str(uuid.UUID(int=0)) - data.user = str(uuid.UUID(int=0)) data.model = "memgpt-openai" if data.tools is not None and len(data.tools) > 0: diff --git a/letta/memory.py b/letta/memory.py index 83b169a0..100d3966 100644 --- a/letta/memory.py +++ b/letta/memory.py @@ -82,6 +82,7 @@ def summarize_messages( provider_name=llm_config_no_inner_thoughts.provider_name, provider_type=llm_config_no_inner_thoughts.model_endpoint_type, put_inner_thoughts_first=False, + actor_id=agent_state.created_by_id, ) # try to use new client, otherwise fallback to old flow # TODO: we can just directly call the LLM here? diff --git a/tests/helpers/endpoints_helper.py b/tests/helpers/endpoints_helper.py index 51bf3320..b0cb2802 100644 --- a/tests/helpers/endpoints_helper.py +++ b/tests/helpers/endpoints_helper.py @@ -107,6 +107,7 @@ def check_first_response_is_valid_for_llm_endpoint(filename: str, validate_inner llm_client = LLMClient.create( provider_name=agent_state.llm_config.provider_name, provider_type=agent_state.llm_config.model_endpoint_type, + actor_id=client.user.id, ) if llm_client: response = llm_client.send_llm_request(