From f38bd1b133898d7765ee3f6e7c0cf050963b8cf6 Mon Sep 17 00:00:00 2001 From: Charles Packer Date: Thu, 18 Dec 2025 20:52:07 -0800 Subject: [PATCH] fix: auto-retry when tools missing on server + graceful error handling (#315) Co-authored-by: Letta --- src/headless.ts | 108 ++++++++++++++++++++++++--------- src/index.ts | 141 +++++++++++++++++++++++++++++++++---------- src/tools/manager.ts | 34 +++++++++++ 3 files changed, 224 insertions(+), 59 deletions(-) diff --git a/src/headless.ts b/src/headless.ts index fed441c..e017daa 100644 --- a/src/headless.ts +++ b/src/headless.ts @@ -22,7 +22,11 @@ import { formatErrorDetails } from "./cli/helpers/errorFormatter"; import { safeJsonParseOr } from "./cli/helpers/safeJsonParse"; import { drainStreamWithResume } from "./cli/helpers/stream"; import { settingsManager } from "./settings-manager"; -import { checkToolPermission } from "./tools/manager"; +import { + checkToolPermission, + forceUpsertTools, + isToolsNotFoundError, +} from "./tools/manager"; // Maximum number of times to retry a turn when the backend // reports an `llm_api_error` stop reason. This helps smooth @@ -100,6 +104,12 @@ export async function handleHeadlessCommand( const client = await getClient(); + // Get base URL for tool upsert operations + const baseURL = + process.env.LETTA_BASE_URL || + settings.env?.LETTA_BASE_URL || + "https://api.letta.com"; + // Resolve agent (same logic as interactive mode) let agent: AgentState | null = null; const specifiedAgentId = values.agent as string | undefined; @@ -190,19 +200,41 @@ export async function handleHeadlessCommand( // Priority 3: Check if --new flag was passed (skip all resume logic) if (!agent && forceNew) { const updateArgs = getModelUpdateArgs(model); - const result = await createAgent( - undefined, - model, - undefined, - updateArgs, - skillsDirectory, - true, // parallelToolCalls always enabled - sleeptimeFlag ?? settings.enableSleeptime, - specifiedSystem, - initBlocks, - baseTools, - ); - agent = result.agent; + try { + const result = await createAgent( + undefined, + model, + undefined, + updateArgs, + skillsDirectory, + true, // parallelToolCalls always enabled + sleeptimeFlag ?? settings.enableSleeptime, + specifiedSystem, + initBlocks, + baseTools, + ); + agent = result.agent; + } catch (err) { + if (isToolsNotFoundError(err)) { + console.warn("Tools missing on server, re-uploading and retrying..."); + await forceUpsertTools(client, baseURL); + const result = await createAgent( + undefined, + model, + undefined, + updateArgs, + skillsDirectory, + true, + sleeptimeFlag ?? settings.enableSleeptime, + specifiedSystem, + initBlocks, + baseTools, + ); + agent = result.agent; + } else { + throw err; + } + } } // Priority 4: Try to resume from project settings (.letta/settings.local.json) @@ -234,19 +266,41 @@ export async function handleHeadlessCommand( // Priority 6: Create a new agent if (!agent) { const updateArgs = getModelUpdateArgs(model); - const result = await createAgent( - undefined, - model, - undefined, - updateArgs, - skillsDirectory, - true, // parallelToolCalls always enabled - sleeptimeFlag ?? settings.enableSleeptime, - specifiedSystem, - undefined, - undefined, - ); - agent = result.agent; + try { + const result = await createAgent( + undefined, + model, + undefined, + updateArgs, + skillsDirectory, + true, // parallelToolCalls always enabled + sleeptimeFlag ?? settings.enableSleeptime, + specifiedSystem, + undefined, + undefined, + ); + agent = result.agent; + } catch (err) { + if (isToolsNotFoundError(err)) { + console.warn("Tools missing on server, re-uploading and retrying..."); + await forceUpsertTools(client, baseURL); + const result = await createAgent( + undefined, + model, + undefined, + updateArgs, + skillsDirectory, + true, + sleeptimeFlag ?? settings.enableSleeptime, + specifiedSystem, + undefined, + undefined, + ); + agent = result.agent; + } else { + throw err; + } + } } // Check if we're resuming an existing agent (not creating a new one) diff --git a/src/index.ts b/src/index.ts index 55b2cd6..a3f0e58 100755 --- a/src/index.ts +++ b/src/index.ts @@ -8,7 +8,12 @@ import type { AgentProvenance } from "./agent/create"; import { LETTA_CLOUD_API_URL } from "./auth/oauth"; import { permissionMode } from "./permissions/mode"; import { settingsManager } from "./settings-manager"; -import { loadTools, upsertToolsIfNeeded } from "./tools/manager"; +import { + forceUpsertTools, + isToolsNotFoundError, + loadTools, + upsertToolsIfNeeded, +} from "./tools/manager"; function printHelp() { // Keep this plaintext (no colors) so output pipes cleanly @@ -204,7 +209,7 @@ function getModelForToolLoading( return specifiedModel; } -async function main() { +async function main(): Promise { // Initialize settings manager (loads settings once into memory) await settingsManager.initialize(); const settings = settingsManager.getSettings(); @@ -437,7 +442,16 @@ async function main() { const { runSetup } = await import("./auth/setup"); await runSetup(); // After setup, restart main flow - return main(); + return main().catch((err: unknown) => { + // Handle top-level errors gracefully without raw stack traces + const message = + err instanceof Error ? err.message : "An unexpected error occurred"; + console.error(`\nError: ${message}`); + if (process.env.DEBUG) { + console.error(err); + } + process.exit(1); + }); } if (!apiKey && baseURL === LETTA_CLOUD_API_URL) { @@ -765,20 +779,47 @@ async function main() { // Priority 3: Check if --new flag was passed - create new agent if (!agent && forceNew) { const updateArgs = getModelUpdateArgs(model); - const result = await createAgent( - undefined, - model, - undefined, - updateArgs, - skillsDirectory, - true, // parallelToolCalls always enabled - sleeptimeFlag ?? settings.enableSleeptime, - system, - initBlocks, - baseTools, - ); - agent = result.agent; - setAgentProvenance(result.provenance); + try { + const result = await createAgent( + undefined, + model, + undefined, + updateArgs, + skillsDirectory, + true, // parallelToolCalls always enabled + sleeptimeFlag ?? settings.enableSleeptime, + system, + initBlocks, + baseTools, + ); + agent = result.agent; + setAgentProvenance(result.provenance); + } catch (err) { + // Check if tools are missing on server (stale hash cache) + if (isToolsNotFoundError(err)) { + console.warn( + "Tools missing on server, re-uploading and retrying...", + ); + await forceUpsertTools(client, baseURL); + // Retry agent creation + const result = await createAgent( + undefined, + model, + undefined, + updateArgs, + skillsDirectory, + true, + sleeptimeFlag ?? settings.enableSleeptime, + system, + initBlocks, + baseTools, + ); + agent = result.agent; + setAgentProvenance(result.provenance); + } else { + throw err; + } + } } // Priority 4: Try to resume from project settings LRU (.letta/settings.local.json) @@ -815,20 +856,47 @@ async function main() { // Priority 7: Create a new agent if (!agent) { const updateArgs = getModelUpdateArgs(model); - const result = await createAgent( - undefined, - model, - undefined, - updateArgs, - skillsDirectory, - true, // parallelToolCalls always enabled - sleeptimeFlag ?? settings.enableSleeptime, - system, - undefined, - undefined, - ); - agent = result.agent; - setAgentProvenance(result.provenance); + try { + const result = await createAgent( + undefined, + model, + undefined, + updateArgs, + skillsDirectory, + true, // parallelToolCalls always enabled + sleeptimeFlag ?? settings.enableSleeptime, + system, + undefined, + undefined, + ); + agent = result.agent; + setAgentProvenance(result.provenance); + } catch (err) { + // Check if tools are missing on server (stale hash cache) + if (isToolsNotFoundError(err)) { + console.warn( + "Tools missing on server, re-uploading and retrying...", + ); + await forceUpsertTools(client, baseURL); + // Retry agent creation + const result = await createAgent( + undefined, + model, + undefined, + updateArgs, + skillsDirectory, + true, + sleeptimeFlag ?? settings.enableSleeptime, + system, + undefined, + undefined, + ); + agent = result.agent; + setAgentProvenance(result.provenance); + } else { + throw err; + } + } } // Ensure local project settings are loaded before updating @@ -952,7 +1020,16 @@ async function main() { setLoadingState("ready"); } - init(); + init().catch((err) => { + // Handle errors gracefully without showing raw stack traces + const message = + err instanceof Error ? err.message : "An unexpected error occurred"; + console.error(`\nError during initialization: ${message}`); + if (process.env.DEBUG) { + console.error(err); + } + process.exit(1); + }); }, [ continueSession, forceNew, diff --git a/src/tools/manager.ts b/src/tools/manager.ts index 296db69..f211fb0 100644 --- a/src/tools/manager.ts +++ b/src/tools/manager.ts @@ -715,6 +715,40 @@ export async function upsertToolsIfNeeded( return true; } +/** + * Force upsert tools by clearing the hash cache for the server. + * Use this when tools are missing on the server despite the hash matching. + * + * @param client - Letta client instance + * @param serverUrl - The server URL (used as cache key) + */ +export async function forceUpsertTools( + client: Letta, + serverUrl: string, +): Promise { + const { settingsManager } = await import("../settings-manager"); + const cachedHashes = settingsManager.getSetting("toolUpsertHashes") || {}; + + // Clear the hash for this server to force re-upsert + delete cachedHashes[serverUrl]; + settingsManager.updateSettings({ toolUpsertHashes: cachedHashes }); + + // Now upsert (will always run since hash was cleared) + await upsertToolsIfNeeded(client, serverUrl); +} + +/** + * Check if an error indicates tools are missing on the server. + * This can happen when the local hash cache is stale (tools were deleted server-side). + */ +export function isToolsNotFoundError(error: unknown): boolean { + if (error && typeof error === "object" && "message" in error) { + const message = String((error as { message: string }).message); + return message.includes("Tools not found by name"); + } + return false; +} + /** * Helper to clip tool return text to a reasonable display size * Used by UI components to truncate long responses for display