Fimeg/letta-server
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fix: azure regression due to openai client changes (#2863)

Browse Source
This commit is contained in:
cthomas
2025-09-12 16:45:46 -07:00
committed by GitHub
parent 2899171e2c
commit 16efa1b868
2 changed files with 10 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
letta/llm_api/azure_client.py
View File

@@ -54,9 +54,12 @@ class AzureClient(OpenAIClient):
api_key = model_settings.azure_api_key or os.environ.get("AZURE_API_KEY")
base_url = model_settings.azure_base_url or os.environ.get("AZURE_BASE_URL")
api_version = model_settings.azure_api_version or os.environ.get("AZURE_API_VERSION")
try:
client = AsyncAzureOpenAI(api_key=api_key, azure_endpoint=base_url, api_version=api_version)
response: ChatCompletion = await client.chat.completions.create(**request_data)
except Exception as e:
raise self.handle_llm_error(e)
client = AsyncAzureOpenAI(api_key=api_key, azure_endpoint=base_url, api_version=api_version)
response: ChatCompletion = await client.chat.completions.create(**request_data)
return response.model_dump()
@trace_method

8
letta/llm_api/openai_client.py
View File

@@ -99,7 +99,7 @@ def supports_structured_output(llm_config: LLMConfig) -> bool:
# FIXME pretty hacky - turn off for providers we know users will use,
# but also don't support structured output
if "nebius.com" in llm_config.model_endpoint:
if llm_config.model_endpoint and "nebius.com" in llm_config.model_endpoint:
return False
else:
return True
@@ -108,7 +108,7 @@ def supports_structured_output(llm_config: LLMConfig) -> bool:
# TODO move into LLMConfig as a field?
def requires_auto_tool_choice(llm_config: LLMConfig) -> bool:
"""Certain providers require the tool choice to be set to 'auto'."""
if "nebius.com" in llm_config.model_endpoint:
if llm_config.model_endpoint and "nebius.com" in llm_config.model_endpoint:
return True
if llm_config.handle and "vllm" in llm_config.handle:
return True
@@ -168,7 +168,9 @@ class OpenAIClient(LLMClientBase):
# Special case for LM Studio backend since it needs extra guidance to force out the thoughts first
# TODO(fix)
inner_thoughts_desc = (
INNER_THOUGHTS_KWARG_DESCRIPTION_GO_FIRST if ":1234" in llm_config.model_endpoint else INNER_THOUGHTS_KWARG_DESCRIPTION
INNER_THOUGHTS_KWARG_DESCRIPTION_GO_FIRST
if llm_config.model_endpoint and ":1234" in llm_config.model_endpoint
else INNER_THOUGHTS_KWARG_DESCRIPTION
)
tools = add_inner_thoughts_to_functions(
functions=tools,