fix(ws): remote interrupt recovery for listen-client [LET-7796] (#1272)

Co-authored-by: Letta Code <noreply@letta.com>
This commit is contained in:
Charles Packer
2026-03-04 22:35:17 -08:00
committed by GitHub
parent 0092e6ab76
commit 95a628eaa0
3 changed files with 1501 additions and 15 deletions

View File

@@ -0,0 +1,747 @@
/**
* Tests for the interrupt queue mechanism (LET-7796).
*
* Structure:
* 1. Structural tests — field initialization, teardown, epoch guards
* 2. Behavior-path tests — exercises populateInterruptQueue + consumeInterruptQueue
* through the same state sequences as the production cancel/resume flow:
* - Cancel during tool execution (Path A) → next turn consumes queued results
* - Cancel during approval wait (Path B) → next turn consumes synthesized denials
* - Post-cancel next turn → no repeated error loop (queue consumed once)
* - Stale-ID guard: clearing IDs after send prevents stale Path-B denials
*/
import { describe, expect, test } from "bun:test";
import WebSocket from "ws";
import type { ApprovalResult } from "../../agent/approval-execution";
import {
__listenClientTestUtils,
rejectPendingApprovalResolvers,
} from "../../websocket/listen-client";
const {
createRuntime,
stopRuntime,
rememberPendingApprovalBatchIds,
populateInterruptQueue,
consumeInterruptQueue,
extractInterruptToolReturns,
emitInterruptToolReturnMessage,
getInterruptApprovalsForEmission,
} = __listenClientTestUtils;
class MockSocket {
readyState: number;
closeCalls = 0;
removeAllListenersCalls = 0;
sentPayloads: string[] = [];
constructor(readyState: number = WebSocket.OPEN) {
this.readyState = readyState;
}
send(data: string): void {
this.sentPayloads.push(data);
}
close(): void {
this.closeCalls += 1;
}
removeAllListeners(): this {
this.removeAllListenersCalls += 1;
return this;
}
}
// ---------------------------------------------------------------------------
// 1. Structural tests
// ---------------------------------------------------------------------------
describe("ListenerRuntime interrupt queue fields", () => {
test("createRuntime initializes interrupt queue fields to safe defaults", () => {
const runtime = createRuntime();
expect(runtime.pendingInterruptedResults).toBeNull();
expect(runtime.pendingInterruptedContext).toBeNull();
expect(runtime.continuationEpoch).toBe(0);
});
});
describe("stopRuntime teardown", () => {
test("clears pendingInterruptedResults, context, and batch map", () => {
const runtime = createRuntime();
runtime.socket = new MockSocket(WebSocket.OPEN) as unknown as WebSocket;
runtime.pendingInterruptedResults = [
{
type: "approval",
tool_call_id: "call-1",
approve: false,
reason: "interrupted",
},
];
runtime.pendingInterruptedContext = {
agentId: "agent-1",
conversationId: "conv-1",
continuationEpoch: 0,
};
runtime.pendingApprovalBatchByToolCallId.set("call-1", "batch-1");
stopRuntime(runtime, true);
expect(runtime.pendingInterruptedResults).toBeNull();
expect(runtime.pendingInterruptedContext).toBeNull();
expect(runtime.pendingApprovalBatchByToolCallId.size).toBe(0);
});
test("increments continuationEpoch on each stop", () => {
const runtime = createRuntime();
runtime.socket = new MockSocket(WebSocket.OPEN) as unknown as WebSocket;
expect(runtime.continuationEpoch).toBe(0);
stopRuntime(runtime, true);
expect(runtime.continuationEpoch).toBe(1);
runtime.socket = new MockSocket(WebSocket.OPEN) as unknown as WebSocket;
runtime.intentionallyClosed = false;
stopRuntime(runtime, true);
expect(runtime.continuationEpoch).toBe(2);
});
});
describe("pendingApprovalBatchByToolCallId survives rejectPendingApprovalResolvers", () => {
test("batch map preserved after resolver rejection (used for Path B IDs)", () => {
const runtime = createRuntime();
runtime.pendingApprovalBatchByToolCallId.set("call-1", "batch-1");
runtime.pendingApprovalResolvers.set("perm-1", {
resolve: () => {},
reject: () => {},
});
rejectPendingApprovalResolvers(runtime, "cancelled");
expect(runtime.pendingApprovalResolvers.size).toBe(0);
expect(runtime.pendingApprovalBatchByToolCallId.size).toBe(1);
});
});
describe("extractInterruptToolReturns", () => {
test("maps completed tool execution results into tool_return payloads", () => {
const results: ApprovalResult[] = [
{
type: "tool",
tool_call_id: "call-ok",
status: "success",
tool_return: "704",
} as ApprovalResult,
{
type: "tool",
tool_call_id: "call-err",
status: "error",
tool_return: "User interrupted the stream",
stderr: ["interrupted"],
} as ApprovalResult,
];
const mapped = extractInterruptToolReturns(results);
expect(mapped).toEqual([
{
tool_call_id: "call-ok",
status: "success",
tool_return: "704",
},
{
tool_call_id: "call-err",
status: "error",
tool_return: "User interrupted the stream",
stderr: ["interrupted"],
},
]);
});
test("maps synthesized approval denials into terminal error tool returns", () => {
const results: ApprovalResult[] = [
{
type: "approval",
tool_call_id: "call-denied",
approve: false,
reason: "User interrupted the stream",
} as ApprovalResult,
];
const mapped = extractInterruptToolReturns(results);
expect(mapped).toEqual([
{
tool_call_id: "call-denied",
status: "error",
tool_return: "User interrupted the stream",
},
]);
});
test("emitInterruptToolReturnMessage emits deterministic per-tool terminal messages", () => {
const runtime = createRuntime();
const socket = new MockSocket(WebSocket.OPEN) as unknown as WebSocket;
const approvals: ApprovalResult[] = [
{
type: "tool",
tool_call_id: "call-a",
status: "success",
tool_return: "704",
} as ApprovalResult,
{
type: "approval",
tool_call_id: "call-b",
approve: false,
reason: "User interrupted the stream",
} as ApprovalResult,
];
emitInterruptToolReturnMessage(socket, runtime, approvals, "run-1");
const parsed = (socket as unknown as MockSocket).sentPayloads.map((raw) =>
JSON.parse(raw),
);
const toolReturnFrames = parsed.filter(
(payload) => payload.message_type === "tool_return_message",
);
expect(toolReturnFrames).toHaveLength(2);
expect(toolReturnFrames[0]).toMatchObject({
run_id: "run-1",
tool_call_id: "call-a",
status: "success",
tool_returns: [{ tool_call_id: "call-a", status: "success" }],
});
expect(toolReturnFrames[1]).toMatchObject({
run_id: "run-1",
tool_call_id: "call-b",
status: "error",
tool_returns: [{ tool_call_id: "call-b", status: "error" }],
});
});
});
describe("getInterruptApprovalsForEmission", () => {
test("prefers lastExecutionResults when available", () => {
const runtime = createRuntime();
runtime.pendingInterruptedResults = [
{
type: "approval",
tool_call_id: "call-old",
approve: false,
},
];
runtime.pendingInterruptedContext = {
agentId: "agent-1",
conversationId: "conv-1",
continuationEpoch: runtime.continuationEpoch,
};
const result = getInterruptApprovalsForEmission(runtime, {
lastExecutionResults: [
{
type: "approval",
tool_call_id: "call-new",
approve: true,
},
],
agentId: "agent-1",
conversationId: "conv-1",
});
expect(result?.[0]).toMatchObject({ tool_call_id: "call-new" });
});
test("falls back to pendingInterruptedResults only when context matches", () => {
const runtime = createRuntime();
runtime.pendingInterruptedResults = [
{
type: "approval",
tool_call_id: "call-pending",
approve: false,
},
];
runtime.pendingInterruptedContext = {
agentId: "agent-1",
conversationId: "conv-1",
continuationEpoch: runtime.continuationEpoch,
};
const matching = getInterruptApprovalsForEmission(runtime, {
lastExecutionResults: null,
agentId: "agent-1",
conversationId: "conv-1",
});
expect(matching?.[0]).toMatchObject({ tool_call_id: "call-pending" });
const mismatched = getInterruptApprovalsForEmission(runtime, {
lastExecutionResults: null,
agentId: "agent-2",
conversationId: "conv-1",
});
expect(mismatched).toBeNull();
});
});
// ---------------------------------------------------------------------------
// 2. Behavior-path tests using extracted helpers
// ---------------------------------------------------------------------------
describe("Path A: cancel during tool execution → next turn consumes actual results", () => {
test("full sequence: populate with execution results → consume on next turn", () => {
const runtime = createRuntime();
const agentId = "agent-abc";
const conversationId = "conv-xyz";
// Simulate: executeApprovalBatch completed, results captured
const executionResults: ApprovalResult[] = [
{ type: "approval", tool_call_id: "call-1", approve: true },
{
tool_call_id: "call-2",
status: "success",
tool_return: "file written",
} as unknown as ApprovalResult,
];
// Cancel fires: populateInterruptQueue (Path A — has execution results)
const populated = populateInterruptQueue(runtime, {
lastExecutionResults: executionResults,
lastNeedsUserInputToolCallIds: ["call-1", "call-2"],
agentId,
conversationId,
});
expect(populated).toBe(true);
expect(runtime.pendingInterruptedResults).toBe(executionResults);
expect(runtime.pendingInterruptedContext).toMatchObject({
agentId,
conversationId,
continuationEpoch: 0,
});
// Next user message arrives: consumeInterruptQueue
const consumed = consumeInterruptQueue(runtime, agentId, conversationId);
expect(consumed).not.toBeNull();
expect(consumed?.type).toBe("approval");
expect(consumed?.approvals).toBe(executionResults);
expect(consumed?.approvals).toHaveLength(2);
// Queue is atomically cleared after consumption
expect(runtime.pendingInterruptedResults).toBeNull();
expect(runtime.pendingInterruptedContext).toBeNull();
});
test("Path A takes priority over Path B even when both sources available", () => {
const runtime = createRuntime();
// Both execution results AND batch map IDs exist
const executionResults: ApprovalResult[] = [
{ type: "approval", tool_call_id: "call-1", approve: true },
];
runtime.pendingApprovalBatchByToolCallId.set("call-1", "batch-1");
const populated = populateInterruptQueue(runtime, {
lastExecutionResults: executionResults,
lastNeedsUserInputToolCallIds: ["call-1"],
agentId: "agent-1",
conversationId: "conv-1",
});
expect(populated).toBe(true);
// Should contain the execution results (Path A), not synthesized denials (Path B)
expect(runtime.pendingInterruptedResults?.[0]).toMatchObject({
approve: true, // Path A preserves actual approval state
});
});
});
describe("Path B: cancel during approval wait → next turn consumes synthesized denials", () => {
test("full sequence: populate from batch map IDs → consume synthesized denials", () => {
const runtime = createRuntime();
const agentId = "agent-abc";
const conversationId = "conv-xyz";
// Simulate: approvals classified, batch IDs remembered, waiting for user input
rememberPendingApprovalBatchIds(
runtime,
[{ toolCallId: "call-1" }, { toolCallId: "call-2" }],
"batch-42",
);
// Cancel fires during approval wait: no execution results
const populated = populateInterruptQueue(runtime, {
lastExecutionResults: null,
lastNeedsUserInputToolCallIds: [],
agentId,
conversationId,
});
expect(populated).toBe(true);
expect(runtime.pendingInterruptedResults).toHaveLength(2);
// Verify synthesized denials
const pendingResults = runtime.pendingInterruptedResults ?? [];
for (const result of pendingResults) {
expect(result).toMatchObject({
type: "approval",
approve: false,
reason: "User interrupted the stream",
});
}
const ids = runtime.pendingInterruptedResults?.map(
(r) => "tool_call_id" in r && r.tool_call_id,
);
expect(ids).toContain("call-1");
expect(ids).toContain("call-2");
// Next user message: consume
const consumed = consumeInterruptQueue(runtime, agentId, conversationId);
expect(consumed).not.toBeNull();
expect(consumed?.approvals).toHaveLength(2);
// Queue cleared
expect(runtime.pendingInterruptedResults).toBeNull();
});
test("fallback to lastNeedsUserInputToolCallIds when batch map empty", () => {
const runtime = createRuntime();
// No batch map entries, but we have the snapshot IDs
const populated = populateInterruptQueue(runtime, {
lastExecutionResults: null,
lastNeedsUserInputToolCallIds: ["call-a", "call-b"],
agentId: "agent-1",
conversationId: "conv-1",
});
expect(populated).toBe(true);
expect(runtime.pendingInterruptedResults).toHaveLength(2);
const ids = runtime.pendingInterruptedResults?.map(
(r) => "tool_call_id" in r && r.tool_call_id,
);
expect(ids).toEqual(["call-a", "call-b"]);
});
test("returns false when both ID sources empty (no-op edge case)", () => {
const runtime = createRuntime();
const populated = populateInterruptQueue(runtime, {
lastExecutionResults: null,
lastNeedsUserInputToolCallIds: [],
agentId: "agent-1",
conversationId: "conv-1",
});
expect(populated).toBe(false);
expect(runtime.pendingInterruptedResults).toBeNull();
});
});
describe("post-cancel next turn: queue consumed exactly once (no error loop)", () => {
test("second consumeInterruptQueue returns null after first consumption", () => {
const runtime = createRuntime();
const agentId = "agent-1";
const convId = "conv-1";
// Populate
populateInterruptQueue(runtime, {
lastExecutionResults: [
{
type: "approval",
tool_call_id: "call-1",
approve: false,
reason: "cancelled",
},
],
lastNeedsUserInputToolCallIds: [],
agentId,
conversationId: convId,
});
// First consume — gets the results
const first = consumeInterruptQueue(runtime, agentId, convId);
expect(first).not.toBeNull();
// Second consume — queue is empty, returns null
const second = consumeInterruptQueue(runtime, agentId, convId);
expect(second).toBeNull();
});
test("third message also gets null (queue stays drained)", () => {
const runtime = createRuntime();
const agentId = "agent-1";
const convId = "conv-1";
populateInterruptQueue(runtime, {
lastExecutionResults: [
{ type: "approval", tool_call_id: "call-1", approve: true },
],
lastNeedsUserInputToolCallIds: [],
agentId,
conversationId: convId,
});
consumeInterruptQueue(runtime, agentId, convId); // first
consumeInterruptQueue(runtime, agentId, convId); // second
const third = consumeInterruptQueue(runtime, agentId, convId);
expect(third).toBeNull();
});
});
describe("idempotency: first cancel populates, second is no-op", () => {
test("second populateInterruptQueue returns false and preserves first results", () => {
const runtime = createRuntime();
const first = populateInterruptQueue(runtime, {
lastExecutionResults: [
{ type: "approval", tool_call_id: "call-first", approve: true },
],
lastNeedsUserInputToolCallIds: [],
agentId: "agent-1",
conversationId: "conv-1",
});
expect(first).toBe(true);
const second = populateInterruptQueue(runtime, {
lastExecutionResults: [
{
type: "approval",
tool_call_id: "call-second",
approve: false,
reason: "x",
},
],
lastNeedsUserInputToolCallIds: [],
agentId: "agent-1",
conversationId: "conv-1",
});
expect(second).toBe(false);
// First results preserved
expect(runtime.pendingInterruptedResults?.[0]).toMatchObject({
tool_call_id: "call-first",
});
});
test("populate succeeds again after consume clears the queue", () => {
const runtime = createRuntime();
populateInterruptQueue(runtime, {
lastExecutionResults: [
{ type: "approval", tool_call_id: "call-1", approve: true },
],
lastNeedsUserInputToolCallIds: [],
agentId: "agent-1",
conversationId: "conv-1",
});
// Consume clears
consumeInterruptQueue(runtime, "agent-1", "conv-1");
// Re-populate succeeds
const repopulated = populateInterruptQueue(runtime, {
lastExecutionResults: [
{ type: "approval", tool_call_id: "call-2", approve: true },
],
lastNeedsUserInputToolCallIds: [],
agentId: "agent-1",
conversationId: "conv-1",
});
expect(repopulated).toBe(true);
expect(runtime.pendingInterruptedResults?.[0]).toMatchObject({
tool_call_id: "call-2",
});
});
});
describe("epoch guard: stale context discarded on consume", () => {
test("consume returns null for queue populated in earlier epoch", () => {
const runtime = createRuntime();
runtime.socket = new MockSocket(WebSocket.OPEN) as unknown as WebSocket;
// Populate at epoch 0
populateInterruptQueue(runtime, {
lastExecutionResults: [
{ type: "approval", tool_call_id: "call-1", approve: true },
],
lastNeedsUserInputToolCallIds: [],
agentId: "agent-1",
conversationId: "conv-1",
});
// Stop bumps epoch, also clears — but let's test the guard directly:
// Manually bump epoch without clearing (simulating a theoretical race)
runtime.continuationEpoch = 99;
const consumed = consumeInterruptQueue(runtime, "agent-1", "conv-1");
// Context has epoch 0, runtime has epoch 99 → mismatch → no consumption
expect(consumed).toBeNull();
// But queue IS cleared (atomic clear regardless of match)
expect(runtime.pendingInterruptedResults).toBeNull();
});
test("consume returns null for queue with wrong agentId", () => {
const runtime = createRuntime();
populateInterruptQueue(runtime, {
lastExecutionResults: [
{ type: "approval", tool_call_id: "call-1", approve: true },
],
lastNeedsUserInputToolCallIds: [],
agentId: "agent-old",
conversationId: "conv-1",
});
const consumed = consumeInterruptQueue(runtime, "agent-new", "conv-1");
expect(consumed).toBeNull();
// Cleared regardless
expect(runtime.pendingInterruptedResults).toBeNull();
});
test("consume returns null for queue with wrong conversationId", () => {
const runtime = createRuntime();
populateInterruptQueue(runtime, {
lastExecutionResults: [
{ type: "approval", tool_call_id: "call-1", approve: true },
],
lastNeedsUserInputToolCallIds: [],
agentId: "agent-1",
conversationId: "conv-old",
});
const consumed = consumeInterruptQueue(runtime, "agent-1", "conv-new");
expect(consumed).toBeNull();
});
});
describe("stale Path-B IDs: clearing after successful send prevents re-denial", () => {
test("populate with cleared IDs after send produces no Path B denials", () => {
const runtime = createRuntime();
// After successful send: both lastExecutionResults and lastNeedsUserInputToolCallIds cleared
// Also batch map should be cleared by clearPendingApprovalBatchIds
const populated = populateInterruptQueue(runtime, {
lastExecutionResults: null,
lastNeedsUserInputToolCallIds: [], // cleared after send
agentId: "agent-1",
conversationId: "conv-1",
});
expect(populated).toBe(false);
expect(runtime.pendingInterruptedResults).toBeNull();
});
test("batch map as primary Path B source after send still works if not cleared", () => {
const runtime = createRuntime();
// Batch map still has entries (from a NEW approval round that wasn't sent yet)
runtime.pendingApprovalBatchByToolCallId.set("call-new-1", "batch-new");
const populated = populateInterruptQueue(runtime, {
lastExecutionResults: null,
lastNeedsUserInputToolCallIds: [], // cleared from previous send
agentId: "agent-1",
conversationId: "conv-1",
});
expect(populated).toBe(true);
expect(runtime.pendingInterruptedResults).toHaveLength(1);
expect(runtime.pendingInterruptedResults?.[0]).toMatchObject({
tool_call_id: "call-new-1",
approve: false,
});
});
});
describe("cancel-induced stop reason reclassification", () => {
/**
* Mirrors the effectiveStopReason computation from the Case 3 stream path.
* Both the legacy (sendClientMessage) and modern (emitToWS) branches now
* use effectiveStopReason — this test verifies the reclassification logic
* that both branches depend on.
*/
function computeEffectiveStopReason(
cancelRequested: boolean,
rawStopReason: string | null,
): string {
return cancelRequested ? "cancelled" : rawStopReason || "error";
}
test("backend 'error' is reclassified to 'cancelled' when cancel was requested", () => {
const effective = computeEffectiveStopReason(true, "error");
expect(effective).toBe("cancelled");
});
test("backend 'error' is preserved when cancel was NOT requested", () => {
const effective = computeEffectiveStopReason(false, "error");
expect(effective).toBe("error");
});
test("null stop reason defaults to 'error' when cancel was NOT requested", () => {
const effective = computeEffectiveStopReason(false, null);
expect(effective).toBe("error");
});
test("any raw stop reason is overridden to 'cancelled' when cancel was requested", () => {
expect(computeEffectiveStopReason(true, "llm_api_error")).toBe("cancelled");
expect(computeEffectiveStopReason(true, "end_turn")).toBe("cancelled");
expect(computeEffectiveStopReason(true, null)).toBe("cancelled");
});
test("runtime.lastStopReason tracks the effective value after cancel populate", () => {
const runtime = createRuntime();
runtime.cancelRequested = true;
// After cancel, the production code sets:
// runtime.lastStopReason = effectiveStopReason
// where effectiveStopReason = cancelRequested ? "cancelled" : rawStop
const rawFromBackend = "error";
const effective = computeEffectiveStopReason(
runtime.cancelRequested,
rawFromBackend,
);
runtime.lastStopReason = effective;
expect(runtime.lastStopReason).toBe("cancelled");
});
});
describe("consume clears pendingApprovalBatchByToolCallId", () => {
test("batch map is cleared as part of atomic consumption", () => {
const runtime = createRuntime();
runtime.pendingApprovalBatchByToolCallId.set("call-1", "batch-1");
populateInterruptQueue(runtime, {
lastExecutionResults: [
{ type: "approval", tool_call_id: "call-1", approve: true },
],
lastNeedsUserInputToolCallIds: [],
agentId: "agent-1",
conversationId: "conv-1",
});
consumeInterruptQueue(runtime, "agent-1", "conv-1");
expect(runtime.pendingApprovalBatchByToolCallId.size).toBe(0);
});
test("batch map is cleared even when context doesn't match (discard path)", () => {
const runtime = createRuntime();
runtime.pendingApprovalBatchByToolCallId.set("call-1", "batch-1");
populateInterruptQueue(runtime, {
lastExecutionResults: [
{ type: "approval", tool_call_id: "call-1", approve: true },
],
lastNeedsUserInputToolCallIds: [],
agentId: "agent-old",
conversationId: "conv-old",
});
// Different agent → context mismatch → discard, but still clears
consumeInterruptQueue(runtime, "agent-new", "conv-new");
expect(runtime.pendingApprovalBatchByToolCallId.size).toBe(0);
});
});

View File

@@ -0,0 +1,171 @@
/**
* Tests for pending approval recovery semantics (reconnect scenario).
*
* Covers:
* 1. Cold-start recovery: empty batch map → synthetic batch ID generated.
* 2. Warm recovery: existing batch map entries → resolved to single batch ID.
* 3. Ambiguous mapping: conflicting batch IDs → fail-closed (null).
* 4. Idempotency: repeated resolve calls with same state → same behavior.
* 5. isRecoveringApprovals guard prevents concurrent recovery.
*/
import { describe, expect, test } from "bun:test";
import { __listenClientTestUtils } from "../../websocket/listen-client";
const {
createRuntime,
resolveRecoveryBatchId,
resolvePendingApprovalBatchId,
rememberPendingApprovalBatchIds,
} = __listenClientTestUtils;
describe("resolveRecoveryBatchId cold-start", () => {
test("empty batch map returns synthetic recovery-* batch ID", () => {
const runtime = createRuntime();
expect(runtime.pendingApprovalBatchByToolCallId.size).toBe(0);
const batchId = resolveRecoveryBatchId(runtime, [
{ toolCallId: "call-1" },
{ toolCallId: "call-2" },
]);
expect(batchId).not.toBeNull();
expect(batchId?.startsWith("recovery-")).toBe(true);
});
test("each cold-start call generates a unique batch ID", () => {
const runtime = createRuntime();
const id1 = resolveRecoveryBatchId(runtime, [{ toolCallId: "call-1" }]);
const id2 = resolveRecoveryBatchId(runtime, [{ toolCallId: "call-1" }]);
expect(id1).not.toBe(id2);
});
test("cold-start returns synthetic even with empty approval list", () => {
const runtime = createRuntime();
const batchId = resolveRecoveryBatchId(runtime, []);
expect(batchId).not.toBeNull();
expect(batchId?.startsWith("recovery-")).toBe(true);
});
});
describe("resolveRecoveryBatchId warm path", () => {
test("returns existing batch ID when all approvals map to same batch", () => {
const runtime = createRuntime();
rememberPendingApprovalBatchIds(
runtime,
[{ toolCallId: "call-1" }, { toolCallId: "call-2" }],
"batch-1",
);
const batchId = resolveRecoveryBatchId(runtime, [
{ toolCallId: "call-1" },
{ toolCallId: "call-2" },
]);
expect(batchId).toBe("batch-1");
});
test("returns null for ambiguous mapping (multiple batch IDs)", () => {
const runtime = createRuntime();
rememberPendingApprovalBatchIds(
runtime,
[{ toolCallId: "call-1" }],
"batch-1",
);
rememberPendingApprovalBatchIds(
runtime,
[{ toolCallId: "call-2" }],
"batch-2",
);
const batchId = resolveRecoveryBatchId(runtime, [
{ toolCallId: "call-1" },
{ toolCallId: "call-2" },
]);
expect(batchId).toBeNull();
});
test("returns null when approval has no batch mapping", () => {
const runtime = createRuntime();
rememberPendingApprovalBatchIds(
runtime,
[{ toolCallId: "call-1" }],
"batch-1",
);
// call-2 has no mapping
const batchId = resolveRecoveryBatchId(runtime, [
{ toolCallId: "call-1" },
{ toolCallId: "call-2" },
]);
expect(batchId).toBeNull();
});
});
describe("isRecoveringApprovals guard", () => {
test("runtime starts with isRecoveringApprovals = false", () => {
const runtime = createRuntime();
expect(runtime.isRecoveringApprovals).toBe(false);
});
test("guard flag prevents concurrent recovery (production pattern)", () => {
const runtime = createRuntime();
// Simulate first recovery in progress
runtime.isRecoveringApprovals = true;
// Second recovery attempt should observe guard and bail
expect(runtime.isRecoveringApprovals).toBe(true);
// Simulate completion
runtime.isRecoveringApprovals = false;
expect(runtime.isRecoveringApprovals).toBe(false);
});
});
describe("resolvePendingApprovalBatchId original behavior preserved", () => {
test("returns null when map is empty (unchanged behavior)", () => {
const runtime = createRuntime();
const batchId = resolvePendingApprovalBatchId(runtime, [
{ toolCallId: "call-1" },
]);
expect(batchId).toBeNull();
});
test("returns batch ID for single consistent mapping", () => {
const runtime = createRuntime();
rememberPendingApprovalBatchIds(
runtime,
[{ toolCallId: "call-1" }],
"batch-abc",
);
const batchId = resolvePendingApprovalBatchId(runtime, [
{ toolCallId: "call-1" },
]);
expect(batchId).toBe("batch-abc");
});
test("returns null for conflicting mappings (strict fail-closed)", () => {
const runtime = createRuntime();
rememberPendingApprovalBatchIds(
runtime,
[{ toolCallId: "call-1" }],
"batch-a",
);
rememberPendingApprovalBatchIds(
runtime,
[{ toolCallId: "call-2" }],
"batch-b",
);
const batchId = resolvePendingApprovalBatchId(runtime, [
{ toolCallId: "call-1" },
{ toolCallId: "call-2" },
]);
expect(batchId).toBeNull();
});
});

View File

@@ -17,6 +17,7 @@ import {
type ApprovalResult,
executeApprovalBatch,
} from "../agent/approval-execution";
import { fetchRunErrorDetail } from "../agent/approval-recovery";
import { getResumeData } from "../agent/check-approval";
import { getClient } from "../agent/client";
import { getStreamToolContextId, sendMessageStream } from "../agent/message";
@@ -278,6 +279,16 @@ type ListenerRuntime = {
* Used to preserve run attachment continuity across approval recovery.
*/
pendingApprovalBatchByToolCallId: Map<string, string>;
/** Queued interrupted tool-call resolutions from a cancelled turn. Prepended to the next user message. */
pendingInterruptedResults: Array<ApprovalResult> | null;
/** Context for pendingInterruptedResults — prevents replay into wrong conversation. */
pendingInterruptedContext: {
agentId: string;
conversationId: string;
continuationEpoch: number;
} | null;
/** Monotonic epoch for queued continuation validity checks. */
continuationEpoch: number;
};
type ApprovalSlot =
@@ -351,6 +362,9 @@ function createRuntime(): ListenerRuntime {
cancelRequested: false,
isRecoveringApprovals: false,
pendingApprovalBatchByToolCallId: new Map<string, string>(),
pendingInterruptedResults: null,
pendingInterruptedContext: null,
continuationEpoch: 0,
coalescedSkipQueueItemIds: new Set<string>(),
pendingTurns: 0,
// queueRuntime assigned below — needs runtime ref in callbacks
@@ -484,6 +498,22 @@ function resolvePendingApprovalBatchId(
return batchIds.values().next().value ?? null;
}
/**
* Resolve the batch ID for pending approval recovery.
* Cold start (empty map): returns a synthetic batch ID.
* Warm (map has entries): delegates to resolvePendingApprovalBatchId,
* returning null for ambiguous/conflicting mappings (fail-closed).
*/
function resolveRecoveryBatchId(
runtime: ListenerRuntime,
pendingApprovals: Array<{ toolCallId: string }>,
): string | null {
if (runtime.pendingApprovalBatchByToolCallId.size === 0) {
return `recovery-${crypto.randomUUID()}`;
}
return resolvePendingApprovalBatchId(runtime, pendingApprovals);
}
function clearPendingApprovalBatchIds(
runtime: ListenerRuntime,
approvals: Array<{ toolCallId: string }>,
@@ -509,6 +539,11 @@ function stopRuntime(
rejectPendingApprovalResolvers(runtime, "Listener runtime stopped");
runtime.pendingApprovalBatchByToolCallId.clear();
// Clear interrupted queue on true teardown to prevent cross-session leakage.
runtime.pendingInterruptedResults = null;
runtime.pendingInterruptedContext = null;
runtime.continuationEpoch++;
if (!runtime.socket) {
return;
}
@@ -815,6 +850,358 @@ function emitToWS(socket: WebSocket, event: WsProtocolEvent): void {
}
const LLM_API_ERROR_MAX_RETRIES = 3;
const MAX_PRE_STREAM_RECOVERY = 2;
// ---------------------------------------------------------------------------
// Interrupt queue helpers — extracted for testability.
// These are the ONLY places that read/write pendingInterruptedResults.
// ---------------------------------------------------------------------------
interface InterruptPopulateInput {
lastExecutionResults: ApprovalResult[] | null;
lastNeedsUserInputToolCallIds: string[];
agentId: string;
conversationId: string;
}
interface InterruptToolReturn {
tool_call_id: string;
status: "success" | "error";
tool_return: string;
stdout?: string[];
stderr?: string[];
}
function normalizeToolReturnValue(value: unknown): string {
if (typeof value === "string") {
return value;
}
if (value === null || value === undefined) {
return "";
}
try {
return JSON.stringify(value);
} catch {
return String(value);
}
}
function extractInterruptToolReturns(
approvals: ApprovalResult[] | null,
): InterruptToolReturn[] {
if (!approvals || approvals.length === 0) {
return [];
}
return approvals.flatMap((approval): InterruptToolReturn[] => {
if (!approval || typeof approval !== "object") {
return [];
}
if ("type" in approval && approval.type === "tool") {
const toolCallId =
"tool_call_id" in approval && typeof approval.tool_call_id === "string"
? approval.tool_call_id
: null;
if (!toolCallId) {
return [];
}
const status =
"status" in approval && approval.status === "success"
? "success"
: "error";
const stdout =
"stdout" in approval && Array.isArray(approval.stdout)
? approval.stdout.filter(
(entry): entry is string => typeof entry === "string",
)
: undefined;
const stderr =
"stderr" in approval && Array.isArray(approval.stderr)
? approval.stderr.filter(
(entry): entry is string => typeof entry === "string",
)
: undefined;
return [
{
tool_call_id: toolCallId,
status,
tool_return:
"tool_return" in approval
? normalizeToolReturnValue(approval.tool_return)
: "",
...(stdout ? { stdout } : {}),
...(stderr ? { stderr } : {}),
},
];
}
if ("type" in approval && approval.type === "approval") {
const toolCallId =
"tool_call_id" in approval && typeof approval.tool_call_id === "string"
? approval.tool_call_id
: null;
if (!toolCallId) {
return [];
}
const reason =
"reason" in approval && typeof approval.reason === "string"
? approval.reason
: "User interrupted the stream";
return [
{
tool_call_id: toolCallId,
status: "error",
tool_return: reason,
},
];
}
return [];
});
}
function emitInterruptToolReturnMessage(
socket: WebSocket,
runtime: ListenerRuntime,
approvals: ApprovalResult[] | null,
runId?: string | null,
uuidPrefix: string = "interrupt-tool-return",
): void {
const toolReturns = extractInterruptToolReturns(approvals);
if (toolReturns.length === 0) {
return;
}
const resolvedRunId = runId ?? runtime.activeRunId ?? undefined;
for (const toolReturn of toolReturns) {
emitToWS(socket, {
type: "message",
message_type: "tool_return_message",
id: `message-${crypto.randomUUID()}`,
date: new Date().toISOString(),
run_id: resolvedRunId,
tool_call_id: toolReturn.tool_call_id,
tool_return: toolReturn.tool_return,
status: toolReturn.status,
...(toolReturn.stdout ? { stdout: toolReturn.stdout } : {}),
...(toolReturn.stderr ? { stderr: toolReturn.stderr } : {}),
tool_returns: [toolReturn],
session_id: runtime.sessionId,
uuid: `${uuidPrefix}-${crypto.randomUUID()}`,
} as unknown as MessageWire);
}
}
function getInterruptApprovalsForEmission(
runtime: ListenerRuntime,
params: {
lastExecutionResults: ApprovalResult[] | null;
agentId: string;
conversationId: string;
},
): ApprovalResult[] | null {
if (params.lastExecutionResults && params.lastExecutionResults.length > 0) {
return params.lastExecutionResults;
}
const context = runtime.pendingInterruptedContext;
if (
!context ||
context.agentId !== params.agentId ||
context.conversationId !== params.conversationId ||
context.continuationEpoch !== runtime.continuationEpoch
) {
return null;
}
if (
!runtime.pendingInterruptedResults ||
runtime.pendingInterruptedResults.length === 0
) {
return null;
}
return runtime.pendingInterruptedResults;
}
/**
* Populate the interrupt queue on the runtime after a cancel.
* Returns true if the queue was populated, false if skipped (idempotent).
*
* Path A: execution completed before cancel → queue actual results.
* Path B: no execution yet → synthesize denial results from stable ID sources.
*/
function populateInterruptQueue(
runtime: ListenerRuntime,
input: InterruptPopulateInput,
): boolean {
// Idempotency: preserve first cancel's results if already populated.
const shouldPopulate =
!runtime.pendingInterruptedResults ||
runtime.pendingInterruptedResults.length === 0 ||
!runtime.pendingInterruptedContext;
if (!shouldPopulate) return false;
if (input.lastExecutionResults && input.lastExecutionResults.length > 0) {
// Path A: execution happened before cancel — queue actual results
runtime.pendingInterruptedResults = input.lastExecutionResults;
runtime.pendingInterruptedContext = {
agentId: input.agentId,
conversationId: input.conversationId,
continuationEpoch: runtime.continuationEpoch,
};
return true;
}
// Path B: no execution — synthesize denial results from stable ID sources.
const batchToolCallIds = [...runtime.pendingApprovalBatchByToolCallId.keys()];
const pendingIds =
batchToolCallIds.length > 0
? batchToolCallIds
: input.lastNeedsUserInputToolCallIds;
if (pendingIds.length > 0) {
runtime.pendingInterruptedResults = pendingIds.map((toolCallId) => ({
type: "approval" as const,
tool_call_id: toolCallId,
approve: false,
reason: "User interrupted the stream",
}));
runtime.pendingInterruptedContext = {
agentId: input.agentId,
conversationId: input.conversationId,
continuationEpoch: runtime.continuationEpoch,
};
return true;
}
if (process.env.DEBUG) {
console.warn(
"[Listen] Cancel during approval loop but no tool_call_ids available " +
"for interrupted queue — next turn may hit pre-stream conflict. " +
`batchMap=${runtime.pendingApprovalBatchByToolCallId.size}, ` +
`lastNeedsUserInput=${input.lastNeedsUserInputToolCallIds.length}`,
);
}
return false;
}
/**
* Consume queued interrupted results and return an ApprovalCreate to prepend,
* or null if nothing to consume. Always clears the queue atomically.
*
* This is the SOLE consumption point — called at the top of handleIncomingMessage.
*/
function consumeInterruptQueue(
runtime: ListenerRuntime,
agentId: string,
conversationId: string,
): { type: "approval"; approvals: ApprovalResult[] } | null {
if (
!runtime.pendingInterruptedResults ||
runtime.pendingInterruptedResults.length === 0
) {
return null;
}
const ctx = runtime.pendingInterruptedContext;
let result: { type: "approval"; approvals: ApprovalResult[] } | null = null;
if (
ctx &&
ctx.agentId === agentId &&
ctx.conversationId === conversationId &&
ctx.continuationEpoch === runtime.continuationEpoch
) {
result = {
type: "approval",
approvals: runtime.pendingInterruptedResults,
};
}
// Atomic clear — always, regardless of context match.
// Stale results for wrong context are discarded, not retried.
runtime.pendingInterruptedResults = null;
runtime.pendingInterruptedContext = null;
runtime.pendingApprovalBatchByToolCallId.clear();
return result;
}
/**
* Attempt to resolve stale pending approvals by fetching them from the backend
* and auto-denying. This is the Phase 3 bounded recovery mechanism — it does NOT
* touch pendingInterruptedResults (that's exclusively owned by handleIncomingMessage).
*/
async function resolveStaleApprovals(
runtime: ListenerRuntime,
abortSignal: AbortSignal,
): Promise<void> {
if (!runtime.activeAgentId) return;
const client = await getClient();
let agent: Awaited<ReturnType<typeof client.agents.retrieve>>;
try {
agent = await client.agents.retrieve(runtime.activeAgentId);
} catch (err) {
// 404 = agent deleted, 422 = invalid ID — both mean nothing to recover
if (err instanceof APIError && (err.status === 404 || err.status === 422)) {
return;
}
throw err;
}
const requestedConversationId =
runtime.activeConversationId && runtime.activeConversationId !== "default"
? runtime.activeConversationId
: undefined;
let resumeData: Awaited<ReturnType<typeof getResumeData>>;
try {
resumeData = await getResumeData(client, agent, requestedConversationId, {
includeMessageHistory: false,
});
} catch (err) {
// getResumeData rethrows 404/422 for conversations — treat as no approvals
if (err instanceof APIError && (err.status === 404 || err.status === 422)) {
return;
}
throw err;
}
const pendingApprovals = resumeData.pendingApprovals || [];
if (pendingApprovals.length === 0) return;
if (abortSignal.aborted) throw new Error("Cancelled");
const denialResults: ApprovalResult[] = pendingApprovals.map((approval) => ({
type: "approval" as const,
tool_call_id: approval.toolCallId,
approve: false,
reason: "Auto-denied during pre-stream approval recovery",
}));
const recoveryConversationId = runtime.activeConversationId || "default";
const recoveryStream = await sendMessageStream(
recoveryConversationId,
[{ type: "approval", approvals: denialResults }],
{
agentId: runtime.activeAgentId,
streamTokens: true,
background: true,
},
{ maxRetries: 0, signal: abortSignal },
);
const drainResult = await drainStreamWithResume(
recoveryStream as Stream<LettaStreamingResponse>,
createBuffers(runtime.activeAgentId),
() => {},
abortSignal,
);
if (drainResult.stopReason === "error") {
throw new Error("Pre-stream approval recovery drain ended with error");
}
}
/**
* Wrap sendMessageStream with pre-stream error handling (retry/recovery).
@@ -830,6 +1217,7 @@ async function sendMessageStreamWithRetry(
): Promise<Awaited<ReturnType<typeof sendMessageStream>>> {
let transientRetries = 0;
let conversationBusyRetries = 0;
let preStreamRecoveryAttempts = 0;
const MAX_CONVERSATION_BUSY_RETRIES = 1;
// eslint-disable-next-line no-constant-condition
@@ -868,9 +1256,31 @@ async function sendMessageStreamWithRetry(
);
if (action === "resolve_approval_pending") {
// Listener can't auto-resolve pending approvals like headless does.
// Rethrow — the cloud will resend with the approval.
throw preStreamError;
// Abort check first — don't let recovery mask a user cancel
if (abortSignal?.aborted) throw new Error("Cancelled by user");
// Attempt bounded recovery: fetch pending approvals and auto-deny them.
// This does NOT touch pendingInterruptedResults (sole owner: handleIncomingMessage).
if (
abortSignal &&
preStreamRecoveryAttempts < MAX_PRE_STREAM_RECOVERY
) {
preStreamRecoveryAttempts++;
try {
await resolveStaleApprovals(runtime, abortSignal);
continue; // Retry send after resolving
} catch (_recoveryError) {
if (abortSignal.aborted) throw new Error("Cancelled by user");
// Recovery failed — fall through to structured error
}
}
// Unrecoverable — emit structured error instead of blind rethrow
const detail = await fetchRunErrorDetail(runtime.activeRunId);
throw new Error(
detail ||
`Pre-stream approval conflict (resolve_approval_pending) after ${preStreamRecoveryAttempts} recovery attempts`,
);
}
if (action === "retry_transient") {
@@ -1071,6 +1481,17 @@ async function recoverPendingApprovals(
socket: WebSocket,
msg: RecoverPendingApprovalsMessage,
): Promise<void> {
console.debug(
"[listener] recover_pending_approvals received",
JSON.stringify({
agentId: msg.agentId,
conversationId: msg.conversationId ?? null,
isProcessing: runtime.isProcessing,
isRecovering: runtime.isRecoveringApprovals,
batchMapSize: runtime.pendingApprovalBatchByToolCallId.size,
}),
);
if (runtime.isProcessing || runtime.isRecoveringApprovals) {
return;
}
@@ -1108,15 +1529,12 @@ async function recoverPendingApprovals(
return;
}
const recoveryBatchId = resolvePendingApprovalBatchId(
runtime,
pendingApprovals,
);
const recoveryBatchId = resolveRecoveryBatchId(runtime, pendingApprovals);
if (!recoveryBatchId) {
emitToWS(socket, {
type: "error",
message:
"Unable to recover pending approvals without originating batch correlation",
"Unable to recover pending approvals: ambiguous batch correlation",
stop_reason: "error",
session_id: runtime.sessionId,
uuid: `error-${crypto.randomUUID()}`,
@@ -1189,6 +1607,11 @@ async function recoverPendingApprovals(
return;
}
// Reflect approval-wait state in runtime snapshot while control
// requests are pending, so state_response queries see
// requires_approval even during the WS round-trip.
runtime.lastStopReason = "requires_approval";
for (const ac of needsUserInput) {
const requestId = `perm-${ac.approval.toolCallId}`;
const diffs = await computeDiffPreviews(
@@ -1501,6 +1924,25 @@ async function connectWithRetry(
if (hasPendingApprovals) {
rejectPendingApprovalResolvers(runtime, "Cancelled by user");
}
// Backend cancel parity with TUI (App.tsx:5932-5941).
// Fire-and-forget — local cancel + queued results are the primary mechanism.
const cancelConversationId = runtime.activeConversationId;
const cancelAgentId = runtime.activeAgentId;
if (cancelAgentId) {
getClient()
.then((client) => {
const cancelId =
cancelConversationId === "default" || !cancelConversationId
? cancelAgentId
: cancelConversationId;
return client.conversations.cancel(cancelId);
})
.catch(() => {
// Fire-and-forget
});
}
emitCancelAck(socket, runtime, {
requestId,
accepted: true,
@@ -1794,6 +2236,11 @@ async function handleIncomingMessage(
let msgTurnCount = 0;
const msgRunIds: string[] = [];
// Track last approval-loop state for cancel-time queueing (Phase 1.2).
// Hoisted before try so the cancel catch block can access them.
let lastExecutionResults: ApprovalResult[] | null = null;
let lastNeedsUserInputToolCallIds: string[] = [];
runtime.isProcessing = true;
runtime.cancelRequested = false;
runtime.activeAbortController = new AbortController();
@@ -1824,9 +2271,21 @@ async function handleIncomingMessage(
onStatusChange?.("processing", connectionId);
}
let messagesToSend: Array<MessageCreate | ApprovalCreate> = msg.messages;
let messagesToSend: Array<MessageCreate | ApprovalCreate> = [];
let turnToolContextId: string | null = null;
// Prepend queued interrupted results from a prior cancelled turn.
const consumed = consumeInterruptQueue(
runtime,
agentId || "",
conversationId,
);
if (consumed) {
messagesToSend.push(consumed);
}
messagesToSend.push(...msg.messages);
const firstMessage = msg.messages[0];
const isApprovalMessage =
firstMessage &&
@@ -1891,6 +2350,15 @@ async function handleIncomingMessage(
approvals: rebuiltApprovals,
},
];
// Emit terminal tool outcomes immediately so WS consumers can close
// tool-call UI state without waiting for follow-up hydration.
emitInterruptToolReturnMessage(
socket,
runtime,
rebuiltApprovals,
runtime.activeRunId ?? undefined,
"tool-return",
);
}
let stream = await sendMessageStreamWithRetry(
@@ -2034,16 +2502,62 @@ async function handleIncomingMessage(
break;
}
// Case 3: Error
// Case 3: Error (or cancel-induced error)
if (stopReason !== "requires_approval") {
runtime.lastStopReason = stopReason;
// Cancel-induced errors should be treated as cancellation, not error.
// This handles the race where cancel fires during stream drain and the
// backend returns "error" instead of "cancelled".
// We're already inside `stopReason !== "requires_approval"`, so this
// is a true non-approval stop. If cancel was requested, treat as cancelled.
const effectiveStopReason: StopReasonType = runtime.cancelRequested
? "cancelled"
: (stopReason as StopReasonType) || "error";
// If effective stop reason is cancelled, route through cancelled semantics (Case 2).
if (effectiveStopReason === "cancelled") {
runtime.lastStopReason = "cancelled";
runtime.isProcessing = false;
clearActiveRunState(runtime);
if (runtime.controlResponseCapable) {
emitToWS(socket, {
type: "result",
subtype: "interrupted",
agent_id: agentId,
conversation_id: conversationId,
duration_ms: performance.now() - msgStartTime,
duration_api_ms: 0,
num_turns: msgTurnCount,
result: null,
run_ids: msgRunIds,
usage: null,
stop_reason: "cancelled",
session_id: runtime.sessionId,
uuid: `result-${crypto.randomUUID()}`,
});
} else {
sendClientMessage(socket, {
type: "result",
success: false,
stopReason: "cancelled",
});
}
break;
}
runtime.lastStopReason = effectiveStopReason;
runtime.isProcessing = false;
clearActiveRunState(runtime);
// Try to fetch richer error detail from the run metadata
const errorDetail = await fetchRunErrorDetail(runId).catch(() => null);
const errorMessage =
errorDetail || `Unexpected stop reason: ${stopReason}`;
emitToWS(socket, {
type: "error",
message: `Unexpected stop reason: ${stopReason}`,
stop_reason: (stopReason as StopReasonType) || "error",
message: errorMessage,
stop_reason: effectiveStopReason,
run_id: runId,
session_id: runtime.sessionId,
uuid: `error-${crypto.randomUUID()}`,
@@ -2060,7 +2574,7 @@ async function handleIncomingMessage(
result: null,
run_ids: msgRunIds,
usage: null,
stop_reason: (stopReason as StopReasonType) || "error",
stop_reason: effectiveStopReason,
session_id: runtime.sessionId,
uuid: `result-${crypto.randomUUID()}`,
});
@@ -2068,7 +2582,7 @@ async function handleIncomingMessage(
sendClientMessage(socket, {
type: "result",
success: false,
stopReason,
stopReason: effectiveStopReason,
});
}
break;
@@ -2103,6 +2617,13 @@ async function handleIncomingMessage(
requireArgsForAutoApprove: true,
});
// Snapshot all tool_call_ids before entering approval wait so cancel can
// synthesize denial results even after pendingApprovalResolvers is cleared.
lastNeedsUserInputToolCallIds = needsUserInput.map(
(ac) => ac.approval.toolCallId,
);
lastExecutionResults = null;
// Build decisions list (before needsUserInput gate so both paths accumulate here)
type Decision =
| {
@@ -2255,6 +2776,19 @@ async function handleIncomingMessage(
abortSignal: runtime.activeAbortController.signal,
},
);
lastExecutionResults = executionResults;
// WS-first parity: publish tool-return terminal outcomes immediately on
// normal approval execution, before continuation stream send.
emitInterruptToolReturnMessage(
socket,
runtime,
executionResults,
runtime.activeRunId ||
runId ||
msgRunIds[msgRunIds.length - 1] ||
undefined,
"tool-return",
);
clearPendingApprovalBatchIds(
runtime,
decisions.map((decision) => decision.approval),
@@ -2274,12 +2808,40 @@ async function handleIncomingMessage(
runtime,
runtime.activeAbortController.signal,
);
// Results were successfully submitted to the backend — clear both so a
// cancel during the subsequent stream drain won't queue already-sent
// results (Path A) or re-deny already-resolved tool calls (Path B).
lastExecutionResults = null;
lastNeedsUserInputToolCallIds = [];
turnToolContextId = getStreamToolContextId(
stream as Stream<LettaStreamingResponse>,
);
}
} catch (error) {
if (runtime.cancelRequested) {
// Queue interrupted tool-call resolutions for the next message turn.
populateInterruptQueue(runtime, {
lastExecutionResults,
lastNeedsUserInputToolCallIds,
agentId: agentId || "",
conversationId,
});
const approvalsForEmission = getInterruptApprovalsForEmission(runtime, {
lastExecutionResults,
agentId: agentId || "",
conversationId,
});
if (approvalsForEmission) {
emitInterruptToolReturnMessage(
socket,
runtime,
approvalsForEmission,
runtime.activeRunId || msgRunIds[msgRunIds.length - 1] || undefined,
);
}
runtime.lastStopReason = "cancelled";
runtime.isProcessing = false;
clearActiveRunState(runtime);
@@ -2381,5 +2943,11 @@ export const __listenClientTestUtils = {
emitToWS,
rememberPendingApprovalBatchIds,
resolvePendingApprovalBatchId,
resolveRecoveryBatchId,
clearPendingApprovalBatchIds,
populateInterruptQueue,
consumeInterruptQueue,
extractInterruptToolReturns,
emitInterruptToolReturnMessage,
getInterruptApprovalsForEmission,
};