fix: auto-retry when tools missing on server + graceful error handling (#315)

Co-authored-by: Letta <noreply@letta.com>
2025-12-18 20:52:07 -08:00
parent d82a29a044
commit f38bd1b133
3 changed files with 224 additions and 59 deletions
--- a/src/headless.ts
+++ b/src/headless.ts
@@ -22,7 +22,11 @@ import { formatErrorDetails } from "./cli/helpers/errorFormatter";
 import { safeJsonParseOr } from "./cli/helpers/safeJsonParse";
 import { drainStreamWithResume } from "./cli/helpers/stream";
 import { settingsManager } from "./settings-manager";
-import { checkToolPermission } from "./tools/manager";
+import {
+  checkToolPermission,
+  forceUpsertTools,
+  isToolsNotFoundError,
+} from "./tools/manager";

 // Maximum number of times to retry a turn when the backend
 // reports an `llm_api_error` stop reason. This helps smooth
@@ -100,6 +104,12 @@ export async function handleHeadlessCommand(

  const client = await getClient();

+  // Get base URL for tool upsert operations
+  const baseURL =
+    process.env.LETTA_BASE_URL ||
+    settings.env?.LETTA_BASE_URL ||
+    "https://api.letta.com";
+
  // Resolve agent (same logic as interactive mode)
  let agent: AgentState | null = null;
  const specifiedAgentId = values.agent as string | undefined;
@@ -190,19 +200,41 @@ export async function handleHeadlessCommand(
  // Priority 3: Check if --new flag was passed (skip all resume logic)
  if (!agent && forceNew) {
    const updateArgs = getModelUpdateArgs(model);
-    const result = await createAgent(
-      undefined,
-      model,
-      undefined,
-      updateArgs,
-      skillsDirectory,
-      true, // parallelToolCalls always enabled
-      sleeptimeFlag ?? settings.enableSleeptime,
-      specifiedSystem,
-      initBlocks,
-      baseTools,
-    );
-    agent = result.agent;
+    try {
+      const result = await createAgent(
+        undefined,
+        model,
+        undefined,
+        updateArgs,
+        skillsDirectory,
+        true, // parallelToolCalls always enabled
+        sleeptimeFlag ?? settings.enableSleeptime,
+        specifiedSystem,
+        initBlocks,
+        baseTools,
+      );
+      agent = result.agent;
+    } catch (err) {
+      if (isToolsNotFoundError(err)) {
+        console.warn("Tools missing on server, re-uploading and retrying...");
+        await forceUpsertTools(client, baseURL);
+        const result = await createAgent(
+          undefined,
+          model,
+          undefined,
+          updateArgs,
+          skillsDirectory,
+          true,
+          sleeptimeFlag ?? settings.enableSleeptime,
+          specifiedSystem,
+          initBlocks,
+          baseTools,
+        );
+        agent = result.agent;
+      } else {
+        throw err;
+      }
+    }
  }

  // Priority 4: Try to resume from project settings (.letta/settings.local.json)
@@ -234,19 +266,41 @@ export async function handleHeadlessCommand(
  // Priority 6: Create a new agent
  if (!agent) {
    const updateArgs = getModelUpdateArgs(model);
-    const result = await createAgent(
-      undefined,
-      model,
-      undefined,
-      updateArgs,
-      skillsDirectory,
-      true, // parallelToolCalls always enabled
-      sleeptimeFlag ?? settings.enableSleeptime,
-      specifiedSystem,
-      undefined,
-      undefined,
-    );
-    agent = result.agent;
+    try {
+      const result = await createAgent(
+        undefined,
+        model,
+        undefined,
+        updateArgs,
+        skillsDirectory,
+        true, // parallelToolCalls always enabled
+        sleeptimeFlag ?? settings.enableSleeptime,
+        specifiedSystem,
+        undefined,
+        undefined,
+      );
+      agent = result.agent;
+    } catch (err) {
+      if (isToolsNotFoundError(err)) {
+        console.warn("Tools missing on server, re-uploading and retrying...");
+        await forceUpsertTools(client, baseURL);
+        const result = await createAgent(
+          undefined,
+          model,
+          undefined,
+          updateArgs,
+          skillsDirectory,
+          true,
+          sleeptimeFlag ?? settings.enableSleeptime,
+          specifiedSystem,
+          undefined,
+          undefined,
+        );
+        agent = result.agent;
+      } else {
+        throw err;
+      }
+    }
  }

  // Check if we're resuming an existing agent (not creating a new one)
--- a/src/index.ts
+++ b/src/index.ts
@@ -8,7 +8,12 @@ import type { AgentProvenance } from "./agent/create";
 import { LETTA_CLOUD_API_URL } from "./auth/oauth";
 import { permissionMode } from "./permissions/mode";
 import { settingsManager } from "./settings-manager";
-import { loadTools, upsertToolsIfNeeded } from "./tools/manager";
+import {
+  forceUpsertTools,
+  isToolsNotFoundError,
+  loadTools,
+  upsertToolsIfNeeded,
+} from "./tools/manager";

 function printHelp() {
  // Keep this plaintext (no colors) so output pipes cleanly
@@ -204,7 +209,7 @@ function getModelForToolLoading(
  return specifiedModel;
 }

-async function main() {
+async function main(): Promise<void> {
  // Initialize settings manager (loads settings once into memory)
  await settingsManager.initialize();
  const settings = settingsManager.getSettings();
@@ -437,7 +442,16 @@ async function main() {
    const { runSetup } = await import("./auth/setup");
    await runSetup();
    // After setup, restart main flow
-    return main();
+    return main().catch((err: unknown) => {
+      // Handle top-level errors gracefully without raw stack traces
+      const message =
+        err instanceof Error ? err.message : "An unexpected error occurred";
+      console.error(`\nError: ${message}`);
+      if (process.env.DEBUG) {
+        console.error(err);
+      }
+      process.exit(1);
+    });
  }

  if (!apiKey && baseURL === LETTA_CLOUD_API_URL) {
@@ -765,20 +779,47 @@ async function main() {
        // Priority 3: Check if --new flag was passed - create new agent
        if (!agent && forceNew) {
          const updateArgs = getModelUpdateArgs(model);
-          const result = await createAgent(
-            undefined,
-            model,
-            undefined,
-            updateArgs,
-            skillsDirectory,
-            true, // parallelToolCalls always enabled
-            sleeptimeFlag ?? settings.enableSleeptime,
-            system,
-            initBlocks,
-            baseTools,
-          );
-          agent = result.agent;
-          setAgentProvenance(result.provenance);
+          try {
+            const result = await createAgent(
+              undefined,
+              model,
+              undefined,
+              updateArgs,
+              skillsDirectory,
+              true, // parallelToolCalls always enabled
+              sleeptimeFlag ?? settings.enableSleeptime,
+              system,
+              initBlocks,
+              baseTools,
+            );
+            agent = result.agent;
+            setAgentProvenance(result.provenance);
+          } catch (err) {
+            // Check if tools are missing on server (stale hash cache)
+            if (isToolsNotFoundError(err)) {
+              console.warn(
+                "Tools missing on server, re-uploading and retrying...",
+              );
+              await forceUpsertTools(client, baseURL);
+              // Retry agent creation
+              const result = await createAgent(
+                undefined,
+                model,
+                undefined,
+                updateArgs,
+                skillsDirectory,
+                true,
+                sleeptimeFlag ?? settings.enableSleeptime,
+                system,
+                initBlocks,
+                baseTools,
+              );
+              agent = result.agent;
+              setAgentProvenance(result.provenance);
+            } else {
+              throw err;
+            }
+          }
        }

        // Priority 4: Try to resume from project settings LRU (.letta/settings.local.json)
@@ -815,20 +856,47 @@ async function main() {
        // Priority 7: Create a new agent
        if (!agent) {
          const updateArgs = getModelUpdateArgs(model);
-          const result = await createAgent(
-            undefined,
-            model,
-            undefined,
-            updateArgs,
-            skillsDirectory,
-            true, // parallelToolCalls always enabled
-            sleeptimeFlag ?? settings.enableSleeptime,
-            system,
-            undefined,
-            undefined,
-          );
-          agent = result.agent;
-          setAgentProvenance(result.provenance);
+          try {
+            const result = await createAgent(
+              undefined,
+              model,
+              undefined,
+              updateArgs,
+              skillsDirectory,
+              true, // parallelToolCalls always enabled
+              sleeptimeFlag ?? settings.enableSleeptime,
+              system,
+              undefined,
+              undefined,
+            );
+            agent = result.agent;
+            setAgentProvenance(result.provenance);
+          } catch (err) {
+            // Check if tools are missing on server (stale hash cache)
+            if (isToolsNotFoundError(err)) {
+              console.warn(
+                "Tools missing on server, re-uploading and retrying...",
+              );
+              await forceUpsertTools(client, baseURL);
+              // Retry agent creation
+              const result = await createAgent(
+                undefined,
+                model,
+                undefined,
+                updateArgs,
+                skillsDirectory,
+                true,
+                sleeptimeFlag ?? settings.enableSleeptime,
+                system,
+                undefined,
+                undefined,
+              );
+              agent = result.agent;
+              setAgentProvenance(result.provenance);
+            } else {
+              throw err;
+            }
+          }
        }

        // Ensure local project settings are loaded before updating
@@ -952,7 +1020,16 @@ async function main() {
        setLoadingState("ready");
      }

-      init();
+      init().catch((err) => {
+        // Handle errors gracefully without showing raw stack traces
+        const message =
+          err instanceof Error ? err.message : "An unexpected error occurred";
+        console.error(`\nError during initialization: ${message}`);
+        if (process.env.DEBUG) {
+          console.error(err);
+        }
+        process.exit(1);
+      });
    }, [
      continueSession,
      forceNew,
--- a/src/tools/manager.ts
+++ b/src/tools/manager.ts
@@ -715,6 +715,40 @@ export async function upsertToolsIfNeeded(
  return true;
 }

+/**
+ * Force upsert tools by clearing the hash cache for the server.
+ * Use this when tools are missing on the server despite the hash matching.
+ *
+ * @param client - Letta client instance
+ * @param serverUrl - The server URL (used as cache key)
+ */
+export async function forceUpsertTools(
+  client: Letta,
+  serverUrl: string,
+): Promise<void> {
+  const { settingsManager } = await import("../settings-manager");
+  const cachedHashes = settingsManager.getSetting("toolUpsertHashes") || {};
+
+  // Clear the hash for this server to force re-upsert
+  delete cachedHashes[serverUrl];
+  settingsManager.updateSettings({ toolUpsertHashes: cachedHashes });
+
+  // Now upsert (will always run since hash was cleared)
+  await upsertToolsIfNeeded(client, serverUrl);
+}
+
+/**
+ * Check if an error indicates tools are missing on the server.
+ * This can happen when the local hash cache is stale (tools were deleted server-side).
+ */
+export function isToolsNotFoundError(error: unknown): boolean {
+  if (error && typeof error === "object" && "message" in error) {
+    const message = String((error as { message: string }).message);
+    return message.includes("Tools not found by name");
+  }
+  return false;
+}
+
 /**
 * Helper to clip tool return text to a reasonable display size
 * Used by UI components to truncate long responses for display