fix: get context windows from server (#787)

Co-authored-by: cpacker <packercharles@gmail.com>
Co-authored-by: Letta <noreply@letta.com>

src/agent/available-models.ts

@@ -109,9 +109,15 @@ export function prefetchAvailableModelHandles(): void {
 }
 
 /**
- * Get the max_context_window for a model handle from the cached API response.
- * Returns undefined if not cached or handle not found.
+ * Get the max_context_window for a model handle from the API.
+ * Ensures the cache is populated before reading.
+ * Returns undefined if handle not found in the API response.
  */
-export function getModelContextWindow(handle: string): number | undefined {
+export async function getModelContextWindow(
+  handle: string,
+): Promise<number | undefined> {
+  if (!cache) {
+    await getAvailableModelHandles();
+  }
   return cache?.contextWindows.get(handle);
 }

src/agent/create.ts

@@ -8,6 +8,7 @@ import type {
   AgentType,
 } from "@letta-ai/letta-client/resources/agents/agents";
 import { DEFAULT_AGENT_NAME } from "../constants";
+import { getModelContextWindow } from "./available-models";
 import { getClient } from "./client";
 import { getDefaultMemoryBlocks } from "./memory";
 import {
@@ -282,10 +283,11 @@ export async function createAgent(
   }
 
   // Get the model's context window from its configuration (if known)
-  // For unknown models (e.g., from self-hosted servers), don't set a context window
-  // and let the server use its default
+  // First try models.json, then fall back to API-cached context window for BYOK models
   const modelUpdateArgs = getModelUpdateArgs(modelHandle);
-  const contextWindow = modelUpdateArgs?.context_window as number | undefined;
+  const contextWindow =
+    (modelUpdateArgs?.context_window as number | undefined) ??
+    (await getModelContextWindow(modelHandle));
 
   // Resolve system prompt content:
   // 1. If systemPromptCustom is provided, use it as-is

src/agent/modify.ts

@@ -9,6 +9,7 @@ import type {
 } from "@letta-ai/letta-client/resources/agents/agents";
 import type { LlmConfig } from "@letta-ai/letta-client/resources/models/models";
 import { OPENAI_CODEX_PROVIDER_NAME } from "../providers/openai-codex-provider";
+import { getModelContextWindow } from "./available-models";
 import { getClient } from "./client";
 
 type ModelSettings =
@@ -164,7 +165,10 @@ export async function updateAgentLLMConfig(
   const client = await getClient();
 
   const modelSettings = buildModelSettings(modelHandle, updateArgs);
-  const contextWindow = updateArgs?.context_window as number | undefined;
+  // First try updateArgs, then fall back to API-cached context window for BYOK models
+  const contextWindow =
+    (updateArgs?.context_window as number | undefined) ??
+    (await getModelContextWindow(modelHandle));
   const hasModelSettings = Object.keys(modelSettings).length > 0;
 
   await client.agents.update(agentId, {

src/cli/App.tsx

@@ -8823,7 +8823,7 @@ ${SYSTEM_REMINDER_CLOSE}
             const { getModelContextWindow } = await import(
               "../agent/available-models"
             );
-            const apiContextWindow = getModelContextWindow(modelId);
+            const apiContextWindow = await getModelContextWindow(modelId);
 
             selectedModel = {
               id: modelId,