Fimeg/letta-server
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat: support for uvloop and granian (#2542)

Browse Source
This commit is contained in:
Andy Li
2025-06-03 15:42:45 -07:00
committed by GitHub
parent a57d4e6c19
commit 2e0bc916a1
4 changed files with 781 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

90
letta/server/rest_api/app.py
View File

@@ -1,3 +1,4 @@
import importlib.util
import json
import logging
import os
@@ -315,19 +316,53 @@ def start_server(
# Add the handler to the logger
server_logger.addHandler(stream_handler)
# Experimental UV Loop Support
try:
if importlib.util.find_spec("uvloop") is not None and settings.use_uvloop:
print("Running server on uvloop...")
import asyncio
import uvloop
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
except:
pass
if (os.getenv("LOCAL_HTTPS") == "true") or "--localhttps" in sys.argv:
print(f"▶ Server running at: https://{host or 'localhost'}:{port or REST_DEFAULT_PORT}")
print(f"▶ View using ADE at: https://app.letta.com/development-servers/local/dashboard\n")
uvicorn.run(
"letta.server.rest_api.app:app",
host=host or "localhost",
port=port or REST_DEFAULT_PORT,
workers=settings.uvicorn_workers,
reload=reload or settings.uvicorn_reload,
timeout_keep_alive=settings.uvicorn_timeout_keep_alive,
ssl_keyfile="certs/localhost-key.pem",
ssl_certfile="certs/localhost.pem",
)
if importlib.util.find_spec("granian") is not None and settings.use_uvloop:
from granian import Granian
# Experimental Granian engine
Granian(
target="letta.server.rest_api.app:app",
# factory=True,
interface="asgi",
address=host or "localhost",
port=port or REST_DEFAULT_PORT,
workers=settings.uvicorn_workers,
# threads=
reload=reload or settings.uvicorn_reload,
reload_ignore_patterns=["openapi_letta.json"],
reload_ignore_worker_failure=True,
reload_tick=100,
# log_level="info"
ssl_keyfile="certs/localhost-key.pem",
ssl_cert="certs/localhost.pem",
).serve()
else:
uvicorn.run(
"letta.server.rest_api.app:app",
host=host or "localhost",
port=port or REST_DEFAULT_PORT,
workers=settings.uvicorn_workers,
reload=reload or settings.uvicorn_reload,
timeout_keep_alive=settings.uvicorn_timeout_keep_alive,
ssl_keyfile="certs/localhost-key.pem",
ssl_certfile="certs/localhost.pem",
)
else:
if is_windows:
# Windows doesn't those the fancy unicode characters
@@ -337,11 +372,30 @@ def start_server(
print(f"▶ Server running at: http://{host or 'localhost'}:{port or REST_DEFAULT_PORT}")
print(f"▶ View using ADE at: https://app.letta.com/development-servers/local/dashboard\n")
uvicorn.run(
"letta.server.rest_api.app:app",
host=host or "localhost",
port=port or REST_DEFAULT_PORT,
workers=settings.uvicorn_workers,
reload=reload or settings.uvicorn_reload,
timeout_keep_alive=settings.uvicorn_timeout_keep_alive,
)
if importlib.util.find_spec("granian") is not None and settings.use_granian:
# Experimental Granian engine
from granian import Granian
Granian(
target="letta.server.rest_api.app:app",
# factory=True,
interface="asgi",
address=host or "localhost",
port=port or REST_DEFAULT_PORT,
workers=settings.uvicorn_workers,
# threads=
reload=reload or settings.uvicorn_reload,
reload_ignore_patterns=["openapi_letta.json"],
reload_ignore_worker_failure=True,
reload_tick=100,
# log_level="info"
).serve()
else:
uvicorn.run(
"letta.server.rest_api.app:app",
host=host or "localhost",
port=port or REST_DEFAULT_PORT,
workers=settings.uvicorn_workers,
reload=reload or settings.uvicorn_reload,
timeout_keep_alive=settings.uvicorn_timeout_keep_alive,
)

6
letta/settings.py
View File

@@ -208,15 +208,15 @@ class Settings(BaseSettings):
uvicorn_reload: bool = False
uvicorn_timeout_keep_alive: int = 5
use_uvloop: bool = False
use_granian: bool = False
# event loop parallelism
event_loop_threadpool_max_workers: int = 43
# experimental toggle
use_experimental: bool = False
use_vertex_structured_outputs_experimental: bool = False
use_vertex_async_loop_experimental: bool = False
experimental_enable_async_db_engine: bool = False
experimental_skip_rebuild_memory: bool = False
# LLM provider client settings
httpx_max_retries: int = 5

759
poetry.lock generated
View File

File diff suppressed because it is too large Load Diff

5
pyproject.toml
View File

@@ -94,11 +94,14 @@ asyncpg = {version = "^0.30.0", optional = true}
tavily-python = "^0.7.2"
async-lru = "^2.0.5"
mistralai = "^1.8.1"
uvloop = {version = "^0.21.0", optional = true}
granian = {version = "^2.3.2", extras = ["uvloop", "reload"], optional = true}
[tool.poetry.extras]
postgres = ["pgvector", "pg8000", "psycopg2-binary", "psycopg2", "asyncpg"]
dev = ["pytest", "pytest-asyncio", "pexpect", "black", "pre-commit", "pyright", "pytest-order", "autoflake", "isort", "locust"]
experimental = ["uvloop", "granian"]
server = ["websockets", "fastapi", "uvicorn"]
qdrant = ["qdrant-client"]
cloud-tool-sandbox = ["e2b-code-interpreter"]
@@ -107,7 +110,7 @@ tests = ["wikipedia"]
bedrock = ["boto3"]
google = ["google-genai"]
desktop = ["pgvector", "pg8000", "psycopg2-binary", "psycopg2", "pyright", "websockets", "fastapi", "uvicorn", "docker", "langchain", "wikipedia", "langchain-community", "locust"]
all = ["pgvector", "pg8000", "psycopg2-binary", "psycopg2", "pytest", "pytest-asyncio", "pexpect", "black", "pre-commit", "pyright", "pytest-order", "autoflake", "isort", "websockets", "fastapi", "uvicorn", "docker", "langchain", "wikipedia", "langchain-community", "locust"]
all = ["pgvector", "pg8000", "psycopg2-binary", "psycopg2", "pytest", "pytest-asyncio", "pexpect", "black", "pre-commit", "pyright", "pytest-order", "autoflake", "isort", "websockets", "fastapi", "uvicorn", "docker", "langchain", "wikipedia", "langchain-community", "locust", "uvloop", "granian"]
[tool.poetry.group.dev.dependencies]
black = "^24.4.2"