diff --git a/src/agent/available-models.ts b/src/agent/available-models.ts index 52fdf51..f456a55 100644 --- a/src/agent/available-models.ts +++ b/src/agent/available-models.ts @@ -109,9 +109,15 @@ export function prefetchAvailableModelHandles(): void { } /** - * Get the max_context_window for a model handle from the cached API response. - * Returns undefined if not cached or handle not found. + * Get the max_context_window for a model handle from the API. + * Ensures the cache is populated before reading. + * Returns undefined if handle not found in the API response. */ -export function getModelContextWindow(handle: string): number | undefined { +export async function getModelContextWindow( + handle: string, +): Promise { + if (!cache) { + await getAvailableModelHandles(); + } return cache?.contextWindows.get(handle); } diff --git a/src/agent/create.ts b/src/agent/create.ts index 0280210..2fbf057 100644 --- a/src/agent/create.ts +++ b/src/agent/create.ts @@ -8,6 +8,7 @@ import type { AgentType, } from "@letta-ai/letta-client/resources/agents/agents"; import { DEFAULT_AGENT_NAME } from "../constants"; +import { getModelContextWindow } from "./available-models"; import { getClient } from "./client"; import { getDefaultMemoryBlocks } from "./memory"; import { @@ -282,10 +283,11 @@ export async function createAgent( } // Get the model's context window from its configuration (if known) - // For unknown models (e.g., from self-hosted servers), don't set a context window - // and let the server use its default + // First try models.json, then fall back to API-cached context window for BYOK models const modelUpdateArgs = getModelUpdateArgs(modelHandle); - const contextWindow = modelUpdateArgs?.context_window as number | undefined; + const contextWindow = + (modelUpdateArgs?.context_window as number | undefined) ?? + (await getModelContextWindow(modelHandle)); // Resolve system prompt content: // 1. If systemPromptCustom is provided, use it as-is diff --git a/src/agent/modify.ts b/src/agent/modify.ts index 45e263f..7eb3b97 100644 --- a/src/agent/modify.ts +++ b/src/agent/modify.ts @@ -9,6 +9,7 @@ import type { } from "@letta-ai/letta-client/resources/agents/agents"; import type { LlmConfig } from "@letta-ai/letta-client/resources/models/models"; import { OPENAI_CODEX_PROVIDER_NAME } from "../providers/openai-codex-provider"; +import { getModelContextWindow } from "./available-models"; import { getClient } from "./client"; type ModelSettings = @@ -164,7 +165,10 @@ export async function updateAgentLLMConfig( const client = await getClient(); const modelSettings = buildModelSettings(modelHandle, updateArgs); - const contextWindow = updateArgs?.context_window as number | undefined; + // First try updateArgs, then fall back to API-cached context window for BYOK models + const contextWindow = + (updateArgs?.context_window as number | undefined) ?? + (await getModelContextWindow(modelHandle)); const hasModelSettings = Object.keys(modelSettings).length > 0; await client.agents.update(agentId, { diff --git a/src/cli/App.tsx b/src/cli/App.tsx index 72e61f5..8b462af 100644 --- a/src/cli/App.tsx +++ b/src/cli/App.tsx @@ -8823,7 +8823,7 @@ ${SYSTEM_REMINDER_CLOSE} const { getModelContextWindow } = await import( "../agent/available-models" ); - const apiContextWindow = getModelContextWindow(modelId); + const apiContextWindow = await getModelContextWindow(modelId); selectedModel = { id: modelId,