Fimeg/letta-server
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat: dump thread state on event loop hang (#8388)

Browse Source
This commit is contained in:
cthomas
2026-01-07 11:59:21 -08:00
committed by Caren Thomas
parent a775bbfdf2
commit 1d360cc239
1 changed files with 3 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
letta/monitoring/event_loop_watchdog.py
View File

@@ -139,7 +139,8 @@ class EventLoopWatchdog:
# Only dump stack traces with 60s cooldown to avoid spam # Only dump stack traces with 60s cooldown to avoid spam
if (now - self._last_dump_time) > 60.0: if (now - self._last_dump_time) > 60.0:
self._dump_asyncio_tasks() self._dump_asyncio_tasks() # Dump async tasks
self._dump_state() # Dump thread stacks
self._last_dump_time = now self._last_dump_time = now
else: else:
# Reset saturation tracking when recovered # Reset saturation tracking when recovered
@@ -156,8 +157,8 @@ class EventLoopWatchdog:
) )
# Dump both thread state and asyncio tasks # Dump both thread state and asyncio tasks
self._dump_state()
self._dump_asyncio_tasks() self._dump_asyncio_tasks()
self._dump_state()
if consecutive_hangs >= 2: if consecutive_hangs >= 2:
logger.critical(f"Event loop appears frozen ({consecutive_hangs} consecutive hangs), tasks={task_count}") logger.critical(f"Event loop appears frozen ({consecutive_hangs} consecutive hangs), tasks={task_count}")