fix: recover default-conversation approval deadlocks without conversation reset (#541)
Co-authored-by: Letta Code <noreply@letta.com>
This commit is contained in:
@@ -15,7 +15,7 @@ import { formatApiErrorForUser } from './errors.js';
|
||||
import { formatToolCallDisplay, formatReasoningDisplay, formatQuestionsForChannel } from './display.js';
|
||||
import type { AgentSession } from './interfaces.js';
|
||||
import { Store } from './store.js';
|
||||
import { getPendingApprovals, rejectApproval, cancelRuns, cancelConversation, recoverOrphanedConversationApproval, getLatestRunError, getAgentModel, updateAgentModel } from '../tools/letta-api.js';
|
||||
import { getPendingApprovals, rejectApproval, cancelRuns, cancelConversation, recoverOrphanedConversationApproval, getLatestRunError, getAgentModel, updateAgentModel, isRecoverableConversationId, recoverPendingApprovalsForAgent } from '../tools/letta-api.js';
|
||||
import { getAgentSkillExecutableDirs, isVoiceMemoConfigured } from '../skills/loader.js';
|
||||
import { formatMessageEnvelope, formatGroupBatchEnvelope, type SessionContextOptions } from './formatter.js';
|
||||
import type { GroupBatcher } from './group-batcher.js';
|
||||
@@ -831,7 +831,7 @@ export class LettaBot implements AgentSession {
|
||||
);
|
||||
|
||||
if (pendingApprovals.length === 0) {
|
||||
if (this.store.conversationId) {
|
||||
if (isRecoverableConversationId(this.store.conversationId)) {
|
||||
const convResult = await recoverOrphanedConversationApproval(
|
||||
this.store.agentId!,
|
||||
this.store.conversationId
|
||||
@@ -1609,9 +1609,15 @@ export class LettaBot implements AgentSession {
|
||||
const retryConvIdFromStore = (retryConvKey === 'shared'
|
||||
? this.store.conversationId
|
||||
: this.store.getConversationId(retryConvKey)) ?? undefined;
|
||||
const retryConvId = (typeof streamMsg.conversationId === 'string' && streamMsg.conversationId.length > 0)
|
||||
const retryConvIdRaw = (typeof streamMsg.conversationId === 'string' && streamMsg.conversationId.length > 0)
|
||||
? streamMsg.conversationId
|
||||
: retryConvIdFromStore;
|
||||
const retryConvId = isRecoverableConversationId(retryConvIdRaw)
|
||||
? retryConvIdRaw
|
||||
: undefined;
|
||||
if (!retryConvId && retryConvIdRaw) {
|
||||
log.info(`Skipping approval recovery for non-recoverable conversation id: ${retryConvIdRaw}`);
|
||||
}
|
||||
const initialRetryDecision = this.buildResultRetryDecision(
|
||||
streamMsg,
|
||||
resultText,
|
||||
@@ -1664,6 +1670,19 @@ export class LettaBot implements AgentSession {
|
||||
log.warn(`Approval recovery failed: ${convResult.details}`);
|
||||
log.info('Retrying once with a fresh session after approval conflict...');
|
||||
return this.processMessage(msg, adapter, true);
|
||||
} else {
|
||||
log.info('Approval conflict detected in default/alias conversation -- attempting agent-level recovery...');
|
||||
this.sessionManager.invalidateSession(retryConvKey);
|
||||
session = null;
|
||||
clearInterval(typingInterval);
|
||||
const agentResult = await recoverPendingApprovalsForAgent(this.store.agentId);
|
||||
if (agentResult.recovered) {
|
||||
log.info(`Agent-level recovery succeeded (${agentResult.details}), retrying message...`);
|
||||
return this.processMessage(msg, adapter, true);
|
||||
}
|
||||
log.warn(`Agent-level recovery failed: ${agentResult.details}`);
|
||||
log.info('Retrying once with a fresh session after approval conflict...');
|
||||
return this.processMessage(msg, adapter, true);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1697,6 +1716,8 @@ export class LettaBot implements AgentSession {
|
||||
log.info('Retrying once after terminal error (no orphaned approvals detected)...');
|
||||
return this.processMessage(msg, adapter, true);
|
||||
}
|
||||
} else if (!retried && retryDecision.shouldRetryForErrorResult && !retryConvId) {
|
||||
log.warn('Skipping terminal-error retry because no recoverable conversation id is available.');
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1934,6 +1955,14 @@ export class LettaBot implements AgentSession {
|
||||
|| ((lastErrorDetail?.message?.toLowerCase().includes('conflict') || false)
|
||||
&& (lastErrorDetail?.message?.toLowerCase().includes('waiting for approval') || false));
|
||||
if (isApprovalIssue && !retried) {
|
||||
if (this.store.agentId) {
|
||||
const recovery = await recoverPendingApprovalsForAgent(this.store.agentId);
|
||||
if (recovery.recovered) {
|
||||
log.info(`sendToAgent: agent-level approval recovery succeeded (${recovery.details})`);
|
||||
} else {
|
||||
log.warn(`sendToAgent: agent-level approval recovery did not resolve approvals (${recovery.details})`);
|
||||
}
|
||||
}
|
||||
log.info('sendToAgent: approval issue detected -- retrying once with fresh session...');
|
||||
this.sessionManager.invalidateSession(convKey);
|
||||
retried = true;
|
||||
|
||||
@@ -17,6 +17,12 @@ vi.mock('../tools/letta-api.js', () => ({
|
||||
rejectApproval: vi.fn(),
|
||||
cancelRuns: vi.fn(),
|
||||
recoverOrphanedConversationApproval: vi.fn(),
|
||||
recoverPendingApprovalsForAgent: vi.fn(),
|
||||
isRecoverableConversationId: vi.fn((conversationId?: string | null) => (
|
||||
typeof conversationId === 'string' && conversationId.length > 0
|
||||
&& conversationId !== 'default'
|
||||
&& conversationId !== 'shared'
|
||||
)),
|
||||
getLatestRunError: vi.fn().mockResolvedValue(null),
|
||||
}));
|
||||
|
||||
@@ -36,7 +42,7 @@ vi.mock('./system-prompt.js', () => ({
|
||||
}));
|
||||
|
||||
import { createAgent, createSession, resumeSession } from '@letta-ai/letta-code-sdk';
|
||||
import { getLatestRunError, recoverOrphanedConversationApproval } from '../tools/letta-api.js';
|
||||
import { getLatestRunError, recoverOrphanedConversationApproval, recoverPendingApprovalsForAgent } from '../tools/letta-api.js';
|
||||
import { LettaBot } from './bot.js';
|
||||
import { SessionManager } from './session-manager.js';
|
||||
import { Store } from './store.js';
|
||||
@@ -71,6 +77,10 @@ describe('SDK session contract', () => {
|
||||
delete process.env.LETTA_SESSION_TIMEOUT_MS;
|
||||
|
||||
vi.clearAllMocks();
|
||||
vi.mocked(recoverPendingApprovalsForAgent).mockResolvedValue({
|
||||
recovered: false,
|
||||
details: 'No pending approvals found on agent',
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
@@ -402,6 +412,59 @@ describe('SDK session contract', () => {
|
||||
expect(vi.mocked(resumeSession)).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('uses agent-level proactive recovery when bootstrap conversation id is default alias', async () => {
|
||||
const initialSession = {
|
||||
initialize: vi.fn(async () => undefined),
|
||||
bootstrapState: vi.fn(async () => ({ hasPendingApproval: true, conversationId: 'default' })),
|
||||
send: vi.fn(async (_message: unknown) => undefined),
|
||||
stream: vi.fn(() =>
|
||||
(async function* () {
|
||||
yield { type: 'result', success: true };
|
||||
})()
|
||||
),
|
||||
close: vi.fn(() => undefined),
|
||||
agentId: 'agent-contract-test',
|
||||
conversationId: 'default',
|
||||
};
|
||||
|
||||
const recoveredSession = {
|
||||
initialize: vi.fn(async () => undefined),
|
||||
bootstrapState: vi.fn(async () => ({ hasPendingApproval: false, conversationId: 'default' })),
|
||||
send: vi.fn(async (_message: unknown) => undefined),
|
||||
stream: vi.fn(() =>
|
||||
(async function* () {
|
||||
yield { type: 'assistant', content: 'ok' };
|
||||
yield { type: 'result', success: true };
|
||||
})()
|
||||
),
|
||||
close: vi.fn(() => undefined),
|
||||
agentId: 'agent-contract-test',
|
||||
conversationId: 'default',
|
||||
};
|
||||
|
||||
vi.mocked(recoverPendingApprovalsForAgent).mockResolvedValueOnce({
|
||||
recovered: true,
|
||||
details: 'Rejected 1 pending approval(s) and cancelled 1 run(s)',
|
||||
});
|
||||
|
||||
vi.mocked(resumeSession)
|
||||
.mockReturnValueOnce(initialSession as never)
|
||||
.mockReturnValueOnce(recoveredSession as never);
|
||||
|
||||
const bot = new LettaBot({
|
||||
workingDir: join(dataDir, 'working'),
|
||||
allowedTools: [],
|
||||
});
|
||||
|
||||
const response = await bot.sendToAgent('hello');
|
||||
|
||||
expect(response).toBe('ok');
|
||||
expect(vi.mocked(resumeSession)).toHaveBeenCalledTimes(2);
|
||||
expect(recoverOrphanedConversationApproval).not.toHaveBeenCalled();
|
||||
expect(recoverPendingApprovalsForAgent).toHaveBeenCalledWith('agent-contract-test');
|
||||
expect(initialSession.close).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('passes memfs: true to resumeSession when config sets memfs true', async () => {
|
||||
const mockSession = {
|
||||
initialize: vi.fn(async () => undefined),
|
||||
@@ -880,6 +943,69 @@ describe('SDK session contract', () => {
|
||||
expect(sentTexts).toContain('after retry');
|
||||
});
|
||||
|
||||
it('uses agent-level recovery for default conversation alias on terminal approval conflict', async () => {
|
||||
const bot = new LettaBot({
|
||||
workingDir: join(dataDir, 'working'),
|
||||
allowedTools: [],
|
||||
});
|
||||
|
||||
let runCall = 0;
|
||||
(bot as any).sessionManager.runSession = vi.fn(async () => ({
|
||||
session: { abort: vi.fn(async () => undefined) },
|
||||
stream: async function* () {
|
||||
if (runCall++ === 0) {
|
||||
yield { type: 'result', success: false, error: 'error', conversationId: 'default' };
|
||||
return;
|
||||
}
|
||||
yield { type: 'assistant', content: 'after default recovery' };
|
||||
yield { type: 'result', success: true, result: 'after default recovery', conversationId: 'default' };
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mocked(getLatestRunError).mockResolvedValueOnce({
|
||||
message: 'CONFLICT: Cannot send a new message: The agent is waiting for approval on a tool call.',
|
||||
stopReason: 'error',
|
||||
isApprovalError: true,
|
||||
});
|
||||
vi.mocked(recoverPendingApprovalsForAgent).mockResolvedValueOnce({
|
||||
recovered: true,
|
||||
details: 'Rejected 1 pending approval(s) and cancelled 1 run(s)',
|
||||
});
|
||||
|
||||
const adapter = {
|
||||
id: 'mock',
|
||||
name: 'Mock',
|
||||
start: vi.fn(async () => {}),
|
||||
stop: vi.fn(async () => {}),
|
||||
isRunning: vi.fn(() => true),
|
||||
sendMessage: vi.fn(async (_payload: unknown) => ({ messageId: 'msg-1' })),
|
||||
editMessage: vi.fn(async () => {}),
|
||||
sendTypingIndicator: vi.fn(async () => {}),
|
||||
stopTypingIndicator: vi.fn(async () => {}),
|
||||
supportsEditing: vi.fn(() => false),
|
||||
sendFile: vi.fn(async () => ({ messageId: 'file-1' })),
|
||||
};
|
||||
|
||||
const msg = {
|
||||
channel: 'discord',
|
||||
chatId: 'chat-1',
|
||||
userId: 'user-1',
|
||||
text: 'hello',
|
||||
timestamp: new Date(),
|
||||
};
|
||||
|
||||
await (bot as any).processMessage(msg, adapter);
|
||||
|
||||
expect((bot as any).sessionManager.runSession).toHaveBeenCalledTimes(2);
|
||||
expect(recoverOrphanedConversationApproval).not.toHaveBeenCalled();
|
||||
expect(recoverPendingApprovalsForAgent).toHaveBeenCalledWith('agent-contract-test');
|
||||
const sentTexts = adapter.sendMessage.mock.calls.map((call) => {
|
||||
const payload = call[0] as { text?: string };
|
||||
return payload.text;
|
||||
});
|
||||
expect(sentTexts).toContain('after default recovery');
|
||||
});
|
||||
|
||||
it('passes tags: [origin:lettabot] to createAgent when creating a new agent', async () => {
|
||||
delete process.env.LETTA_AGENT_ID;
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ import { createAgent, createSession, resumeSession, type Session, type SendMessa
|
||||
import type { BotConfig, StreamMsg } from './types.js';
|
||||
import { isApprovalConflictError, isConversationMissingError, isAgentMissingFromInitError } from './errors.js';
|
||||
import { Store } from './store.js';
|
||||
import { updateAgentName, recoverOrphanedConversationApproval } from '../tools/letta-api.js';
|
||||
import { updateAgentName, recoverOrphanedConversationApproval, isRecoverableConversationId, recoverPendingApprovalsForAgent } from '../tools/letta-api.js';
|
||||
import { installSkillsToAgent, prependSkillDirsToPath } from '../skills/loader.js';
|
||||
import { loadMemoryBlocks } from './memory.js';
|
||||
import { SYSTEM_PROMPT } from './system-prompt.js';
|
||||
@@ -331,9 +331,22 @@ export class SessionManager {
|
||||
);
|
||||
if (bootstrap.hasPendingApproval) {
|
||||
const convId = bootstrap.conversationId || session.conversationId;
|
||||
log.warn(`Pending approval detected at session startup (key=${key}, conv=${convId}), recovering...`);
|
||||
session.close();
|
||||
if (convId) {
|
||||
if (!isRecoverableConversationId(convId)) {
|
||||
log.warn(
|
||||
`Pending approval detected at session startup (key=${key}, conv=${convId}) ` +
|
||||
'using agent-level recovery fallback.'
|
||||
);
|
||||
session.close();
|
||||
const result = await recoverPendingApprovalsForAgent(this.store.agentId);
|
||||
if (result.recovered) {
|
||||
log.info(`Proactive agent-level recovery succeeded: ${result.details}`);
|
||||
} else {
|
||||
log.warn(`Proactive agent-level recovery did not resolve approvals: ${result.details}`);
|
||||
}
|
||||
return this._createSessionForKey(key, true, generation);
|
||||
} else {
|
||||
log.warn(`Pending approval detected at session startup (key=${key}, conv=${convId}), recovering...`);
|
||||
session.close();
|
||||
const result = await recoverOrphanedConversationApproval(
|
||||
this.store.agentId,
|
||||
convId,
|
||||
@@ -344,8 +357,8 @@ export class SessionManager {
|
||||
} else {
|
||||
log.warn(`Proactive approval recovery did not find resolvable approvals: ${result.details}`);
|
||||
}
|
||||
return this._createSessionForKey(key, true, generation);
|
||||
}
|
||||
return this._createSessionForKey(key, true, generation);
|
||||
}
|
||||
} catch (err) {
|
||||
// bootstrapState failure is non-fatal -- the reactive 409 handler in
|
||||
@@ -521,13 +534,12 @@ export class SessionManager {
|
||||
await this.withSessionTimeout(session.send(message), `Session send (key=${convKey})`);
|
||||
} catch (error) {
|
||||
// 409 CONFLICT from orphaned approval
|
||||
if (!retried && isApprovalConflictError(error) && this.store.agentId && convId) {
|
||||
if (!retried && isApprovalConflictError(error) && this.store.agentId) {
|
||||
log.info('CONFLICT detected - attempting orphaned approval recovery...');
|
||||
this.invalidateSession(convKey);
|
||||
const result = await recoverOrphanedConversationApproval(
|
||||
this.store.agentId,
|
||||
convId
|
||||
);
|
||||
const result = isRecoverableConversationId(convId)
|
||||
? await recoverOrphanedConversationApproval(this.store.agentId, convId)
|
||||
: await recoverPendingApprovalsForAgent(this.store.agentId);
|
||||
if (result.recovered) {
|
||||
log.info(`Recovery succeeded (${result.details}), retrying...`);
|
||||
return this.runSession(message, { retried: true, canUseTool, convKey });
|
||||
|
||||
@@ -6,6 +6,8 @@ const mockConversationsMessagesCreate = vi.fn();
|
||||
const mockRunsRetrieve = vi.fn();
|
||||
const mockRunsList = vi.fn();
|
||||
const mockAgentsMessagesCancel = vi.fn();
|
||||
const mockAgentsRetrieve = vi.fn();
|
||||
const mockAgentsMessagesList = vi.fn();
|
||||
|
||||
vi.mock('@letta-ai/letta-client', () => {
|
||||
return {
|
||||
@@ -20,12 +22,73 @@ vi.mock('@letta-ai/letta-client', () => {
|
||||
retrieve: mockRunsRetrieve,
|
||||
list: mockRunsList,
|
||||
};
|
||||
agents = { messages: { cancel: mockAgentsMessagesCancel } };
|
||||
agents = {
|
||||
retrieve: mockAgentsRetrieve,
|
||||
messages: {
|
||||
cancel: mockAgentsMessagesCancel,
|
||||
list: mockAgentsMessagesList,
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
import { getLatestRunError, recoverOrphanedConversationApproval } from './letta-api.js';
|
||||
describe('recoverPendingApprovalsForAgent', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mockAgentsRetrieve.mockResolvedValue({ pending_approval: null });
|
||||
mockAgentsMessagesList.mockReturnValue(mockPageIterator([]));
|
||||
mockAgentsMessagesCancel.mockResolvedValue(undefined);
|
||||
});
|
||||
|
||||
it('cancels approval-blocked runs when pending approval payload is unavailable', async () => {
|
||||
// First runs.list call: getPendingApprovals run scan (no tool calls resolved)
|
||||
mockRunsList
|
||||
.mockReturnValueOnce(mockPageIterator([
|
||||
{ id: 'run-stuck', status: 'created', stop_reason: 'requires_approval' },
|
||||
]))
|
||||
// Second runs.list call: listAgentApprovalRunIds fallback
|
||||
.mockReturnValueOnce(mockPageIterator([
|
||||
{ id: 'run-stuck', status: 'created', stop_reason: 'requires_approval' },
|
||||
]));
|
||||
|
||||
const result = await recoverPendingApprovalsForAgent('agent-1');
|
||||
|
||||
expect(result.recovered).toBe(true);
|
||||
expect(result.details).toContain('Cancelled 1 approval-blocked run(s)');
|
||||
expect(mockAgentsMessagesCancel).toHaveBeenCalledWith('agent-1', {
|
||||
run_ids: ['run-stuck'],
|
||||
});
|
||||
});
|
||||
|
||||
it('returns false when no pending approvals and no approval-blocked runs are found', async () => {
|
||||
mockRunsList
|
||||
.mockReturnValueOnce(mockPageIterator([]))
|
||||
.mockReturnValueOnce(mockPageIterator([]));
|
||||
|
||||
const result = await recoverPendingApprovalsForAgent('agent-1');
|
||||
|
||||
expect(result.recovered).toBe(false);
|
||||
expect(result.details).toBe('No pending approvals found on agent');
|
||||
expect(mockAgentsMessagesCancel).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
import { getLatestRunError, recoverOrphanedConversationApproval, isRecoverableConversationId, recoverPendingApprovalsForAgent } from './letta-api.js';
|
||||
|
||||
describe('isRecoverableConversationId', () => {
|
||||
it('returns false for aliases and empty values', () => {
|
||||
expect(isRecoverableConversationId(undefined)).toBe(false);
|
||||
expect(isRecoverableConversationId(null)).toBe(false);
|
||||
expect(isRecoverableConversationId('')).toBe(false);
|
||||
expect(isRecoverableConversationId('default')).toBe(false);
|
||||
expect(isRecoverableConversationId('shared')).toBe(false);
|
||||
});
|
||||
|
||||
it('returns true for materialized conversation ids', () => {
|
||||
expect(isRecoverableConversationId('conv-123')).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// Helper to create a mock async iterable from an array (Letta client returns paginated iterators)
|
||||
function mockPageIterator<T>(items: T[]) {
|
||||
@@ -40,6 +103,8 @@ describe('recoverOrphanedConversationApproval', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mockRunsList.mockReturnValue(mockPageIterator([]));
|
||||
mockAgentsRetrieve.mockResolvedValue({ pending_approval: null });
|
||||
mockAgentsMessagesList.mockReturnValue(mockPageIterator([]));
|
||||
vi.useFakeTimers();
|
||||
});
|
||||
|
||||
@@ -56,6 +121,14 @@ describe('recoverOrphanedConversationApproval', () => {
|
||||
expect(result.details).toBe('No messages in conversation');
|
||||
});
|
||||
|
||||
it('skips non-recoverable conversation ids like default', async () => {
|
||||
const result = await recoverOrphanedConversationApproval('agent-1', 'default');
|
||||
|
||||
expect(result.recovered).toBe(false);
|
||||
expect(result.details).toContain('Conversation is not recoverable: default');
|
||||
expect(mockConversationsMessagesList).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('returns false when no unresolved approval requests', async () => {
|
||||
mockConversationsMessagesList.mockReturnValue(mockPageIterator([
|
||||
{ message_type: 'assistant_message', content: 'hello' },
|
||||
|
||||
@@ -21,6 +21,31 @@ function getClient(): Letta {
|
||||
});
|
||||
}
|
||||
|
||||
async function listAgentApprovalRunIds(agentId: string, limit = 10): Promise<string[]> {
|
||||
try {
|
||||
const client = getClient();
|
||||
const runsPage = await client.runs.list({
|
||||
agent_id: agentId,
|
||||
stop_reason: 'requires_approval',
|
||||
limit,
|
||||
});
|
||||
|
||||
const runIds: string[] = [];
|
||||
for await (const run of runsPage) {
|
||||
if (run.stop_reason !== 'requires_approval') continue;
|
||||
const id = (run as { id?: unknown }).id;
|
||||
if (typeof id === 'string' && id.length > 0) {
|
||||
runIds.push(id);
|
||||
}
|
||||
if (runIds.length >= limit) break;
|
||||
}
|
||||
return runIds;
|
||||
} catch (e) {
|
||||
log.warn('Failed to list approval-blocked runs:', e instanceof Error ? e.message : e);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Test connection to Letta server (silent, no error logging)
|
||||
*/
|
||||
@@ -35,6 +60,83 @@ export async function testConnection(): Promise<boolean> {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Recover stuck approvals at the agent level without requiring a concrete
|
||||
* conversation ID. This is the fallback for default/alias conversations.
|
||||
*/
|
||||
export async function recoverPendingApprovalsForAgent(
|
||||
agentId: string,
|
||||
reason = 'Session was interrupted - retrying request'
|
||||
): Promise<{ recovered: boolean; details: string }> {
|
||||
try {
|
||||
const pending = await getPendingApprovals(agentId);
|
||||
if (pending.length === 0) {
|
||||
// Some servers report approval conflicts while omitting pending_approval
|
||||
// details/tool_call IDs. In that case, cancel approval-blocked runs directly.
|
||||
const approvalRunIds = await listAgentApprovalRunIds(agentId);
|
||||
if (approvalRunIds.length === 0) {
|
||||
return { recovered: false, details: 'No pending approvals found on agent' };
|
||||
}
|
||||
const cancelled = await cancelRuns(agentId, approvalRunIds);
|
||||
if (!cancelled) {
|
||||
return {
|
||||
recovered: false,
|
||||
details: `Found ${approvalRunIds.length} approval-blocked run(s) but failed to cancel`,
|
||||
};
|
||||
}
|
||||
return {
|
||||
recovered: true,
|
||||
details: `Cancelled ${approvalRunIds.length} approval-blocked run(s) without tool-call details`,
|
||||
};
|
||||
}
|
||||
|
||||
let rejectedCount = 0;
|
||||
for (const approval of pending) {
|
||||
const ok = await rejectApproval(agentId, {
|
||||
toolCallId: approval.toolCallId,
|
||||
reason,
|
||||
});
|
||||
if (ok) rejectedCount += 1;
|
||||
}
|
||||
|
||||
const runIds = [...new Set(
|
||||
pending
|
||||
.map(a => a.runId)
|
||||
.filter((id): id is string => !!id && id !== 'unknown')
|
||||
)];
|
||||
if (runIds.length > 0) {
|
||||
await cancelRuns(agentId, runIds);
|
||||
}
|
||||
|
||||
if (rejectedCount === 0) {
|
||||
return { recovered: false, details: 'Failed to reject pending approvals' };
|
||||
}
|
||||
|
||||
return {
|
||||
recovered: true,
|
||||
details: `Rejected ${rejectedCount} pending approval(s)${runIds.length > 0 ? ` and cancelled ${runIds.length} run(s)` : ''}`,
|
||||
};
|
||||
} catch (e) {
|
||||
return {
|
||||
recovered: false,
|
||||
details: `Agent-level approval recovery failed: ${e instanceof Error ? e.message : String(e)}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true when a conversation id refers to a concrete conversation record
|
||||
* that can be queried for messages/runs.
|
||||
*/
|
||||
export function isRecoverableConversationId(conversationId?: string | null): conversationId is string {
|
||||
if (typeof conversationId !== 'string') return false;
|
||||
const value = conversationId.trim();
|
||||
if (!value) return false;
|
||||
// SDK/API aliases are not materialized conversation IDs.
|
||||
if (value === 'default' || value === 'shared') return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
// Re-export types that callers use
|
||||
export type LettaTool = Awaited<ReturnType<Letta['tools']['upsert']>>;
|
||||
|
||||
@@ -274,48 +376,51 @@ export async function getPendingApprovals(
|
||||
if ('pending_approval' in agentState) {
|
||||
const pending = agentState.pending_approval;
|
||||
if (!pending) {
|
||||
log.info('No pending approvals on agent');
|
||||
return [];
|
||||
}
|
||||
log.info(`Found pending approval: ${pending.id}, run_id=${pending.run_id}`);
|
||||
|
||||
// Extract tool calls - handle both Array<ToolCall> and ToolCallDelta formats
|
||||
const rawToolCalls = pending.tool_calls;
|
||||
const toolCallsList: Array<{ tool_call_id: string; name: string }> = [];
|
||||
|
||||
if (Array.isArray(rawToolCalls)) {
|
||||
for (const tc of rawToolCalls) {
|
||||
if (tc && 'tool_call_id' in tc && tc.tool_call_id) {
|
||||
log.info('No pending approvals on agent; falling back to run scan');
|
||||
} else {
|
||||
log.info(`Found pending approval: ${pending.id}, run_id=${pending.run_id}`);
|
||||
|
||||
// Extract tool calls - handle both Array<ToolCall> and ToolCallDelta formats
|
||||
const rawToolCalls = pending.tool_calls;
|
||||
const toolCallsList: Array<{ tool_call_id: string; name: string }> = [];
|
||||
|
||||
if (Array.isArray(rawToolCalls)) {
|
||||
for (const tc of rawToolCalls) {
|
||||
if (tc && 'tool_call_id' in tc && tc.tool_call_id) {
|
||||
toolCallsList.push({ tool_call_id: tc.tool_call_id, name: tc.name || 'unknown' });
|
||||
}
|
||||
}
|
||||
} else if (rawToolCalls && typeof rawToolCalls === 'object' && 'tool_call_id' in rawToolCalls && rawToolCalls.tool_call_id) {
|
||||
// ToolCallDelta case
|
||||
toolCallsList.push({ tool_call_id: rawToolCalls.tool_call_id, name: rawToolCalls.name || 'unknown' });
|
||||
}
|
||||
|
||||
// Fallback to deprecated singular tool_call field
|
||||
if (toolCallsList.length === 0 && pending.tool_call) {
|
||||
const tc = pending.tool_call;
|
||||
if ('tool_call_id' in tc && tc.tool_call_id) {
|
||||
toolCallsList.push({ tool_call_id: tc.tool_call_id, name: tc.name || 'unknown' });
|
||||
}
|
||||
}
|
||||
} else if (rawToolCalls && typeof rawToolCalls === 'object' && 'tool_call_id' in rawToolCalls && rawToolCalls.tool_call_id) {
|
||||
// ToolCallDelta case
|
||||
toolCallsList.push({ tool_call_id: rawToolCalls.tool_call_id, name: rawToolCalls.name || 'unknown' });
|
||||
}
|
||||
|
||||
// Fallback to deprecated singular tool_call field
|
||||
if (toolCallsList.length === 0 && pending.tool_call) {
|
||||
const tc = pending.tool_call;
|
||||
if ('tool_call_id' in tc && tc.tool_call_id) {
|
||||
toolCallsList.push({ tool_call_id: tc.tool_call_id, name: tc.name || 'unknown' });
|
||||
|
||||
const seen = new Set<string>();
|
||||
const approvals: PendingApproval[] = [];
|
||||
for (const tc of toolCallsList) {
|
||||
if (seen.has(tc.tool_call_id)) continue;
|
||||
seen.add(tc.tool_call_id);
|
||||
approvals.push({
|
||||
runId: pending.run_id || 'unknown',
|
||||
toolCallId: tc.tool_call_id,
|
||||
toolName: tc.name || 'unknown',
|
||||
messageId: pending.id,
|
||||
});
|
||||
}
|
||||
if (approvals.length > 0) {
|
||||
log.info(`Extracted ${approvals.length} pending approval(s): ${approvals.map(a => a.toolName).join(', ')}`);
|
||||
return approvals;
|
||||
}
|
||||
log.warn('Agent pending_approval had no tool_call_ids; falling back to run scan');
|
||||
}
|
||||
|
||||
const seen = new Set<string>();
|
||||
const approvals: PendingApproval[] = [];
|
||||
for (const tc of toolCallsList) {
|
||||
if (seen.has(tc.tool_call_id)) continue;
|
||||
seen.add(tc.tool_call_id);
|
||||
approvals.push({
|
||||
runId: pending.run_id || 'unknown',
|
||||
toolCallId: tc.tool_call_id,
|
||||
toolName: tc.name || 'unknown',
|
||||
messageId: pending.id,
|
||||
});
|
||||
}
|
||||
log.info(`Extracted ${approvals.length} pending approval(s): ${approvals.map(a => a.toolName).join(', ')}`);
|
||||
return approvals;
|
||||
}
|
||||
} catch (e) {
|
||||
log.warn('Failed to retrieve agent pending_approval, falling back to run scan:', e);
|
||||
@@ -673,6 +778,13 @@ export async function recoverOrphanedConversationApproval(
|
||||
deepScan = false
|
||||
): Promise<{ recovered: boolean; details: string }> {
|
||||
try {
|
||||
if (!isRecoverableConversationId(conversationId)) {
|
||||
return {
|
||||
recovered: false,
|
||||
details: `Conversation is not recoverable: ${conversationId || '(empty)'}`,
|
||||
};
|
||||
}
|
||||
|
||||
const client = getClient();
|
||||
|
||||
// List recent messages from the conversation to find orphaned approvals.
|
||||
|
||||
Reference in New Issue
Block a user