Fimeg/letta-server
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat: remove ssl allocation from startup (#6127)

Browse Source
This commit is contained in:
cthomas
2025-11-12 07:56:55 -08:00
committed by Caren Thomas
parent ff87673963
commit 0c06dbf047
2 changed files with 14 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
letta/server/rest_api/routers/v1/messages.py
View File

@@ -191,6 +191,8 @@ async def cancel_batch(
# TODO: Extend to providers beyond anthropic
# TODO: For now, we only support anthropic
# Cancel the job
if server.anthropic_async_client is None:
raise HTTPException(status_code=501, detail="Batch job cancellation is not enabled")
anthropic_batch_id = llm_batch_job.create_batch_response.id
await server.anthropic_async_client.messages.batches.cancel(anthropic_batch_id)

19
letta/server/server.py
View File

@@ -184,10 +184,18 @@ class SyncServer(object):
message_manager=self.message_manager,
)
# A resusable httpx client
timeout = httpx.Timeout(connect=10.0, read=20.0, write=10.0, pool=10.0)
limits = httpx.Limits(max_connections=100, max_keepalive_connections=80, keepalive_expiry=300)
self.httpx_client = httpx.AsyncClient(timeout=timeout, follow_redirects=True, limits=limits)
if settings.enable_batch_job_polling:
# A resusable httpx client
timeout = httpx.Timeout(connect=10.0, read=20.0, write=10.0, pool=10.0)
limits = httpx.Limits(max_connections=100, max_keepalive_connections=80, keepalive_expiry=300)
self.httpx_client = httpx.AsyncClient(timeout=timeout, follow_redirects=True, limits=limits)
# TODO: Replace this with the Anthropic client we have in house
# Reuse the shared httpx client to prevent duplicate SSL contexts and connection pools
self.anthropic_async_client = AsyncAnthropic(http_client=self.httpx_client)
else:
self.httpx_client = None
self.anthropic_async_client = None
# For MCP
# TODO: remove this
@@ -198,9 +206,6 @@ class SyncServer(object):
self._llm_config_cache = {}
self._embedding_config_cache = {}
# TODO: Replace this with the Anthropic client we have in house
self.anthropic_async_client = AsyncAnthropic()
# collect providers (always has Letta as a default)
self._enabled_providers: List[Provider] = [LettaProvider(name="letta")]
if model_settings.openai_api_key: