From ea175e1dbb05191f6aeb15d9c14dc20effa4043c Mon Sep 17 00:00:00 2001 From: Kevin Lin Date: Tue, 16 Dec 2025 11:31:54 -0800 Subject: [PATCH] feat: Only show available models in selector (#238) --- src/agent/available-models.ts | 101 +++++++++++++++ src/cli/App.tsx | 6 + src/cli/components/ModelSelector.tsx | 179 +++++++++++++++++++++++---- 3 files changed, 260 insertions(+), 26 deletions(-) create mode 100644 src/agent/available-models.ts diff --git a/src/agent/available-models.ts b/src/agent/available-models.ts new file mode 100644 index 0000000..653107b --- /dev/null +++ b/src/agent/available-models.ts @@ -0,0 +1,101 @@ +import { getClient } from "./client"; + +const CACHE_TTL_MS = 5 * 60 * 1000; // 5 minutes + +type CacheEntry = { + handles: Set; + fetchedAt: number; +}; + +let cache: CacheEntry | null = null; +let inflight: Promise | null = null; + +function isFresh(now = Date.now()) { + return cache !== null && now - cache.fetchedAt < CACHE_TTL_MS; +} + +export type AvailableModelHandlesResult = { + handles: Set; + source: "cache" | "network"; + fetchedAt: number; +}; + +export function clearAvailableModelsCache() { + cache = null; +} + +export function getAvailableModelsCacheInfo(): { + hasCache: boolean; + isFresh: boolean; + fetchedAt: number | null; + ageMs: number | null; + ttlMs: number; +} { + const now = Date.now(); + return { + hasCache: cache !== null, + isFresh: isFresh(now), + fetchedAt: cache?.fetchedAt ?? null, + ageMs: cache ? now - cache.fetchedAt : null, + ttlMs: CACHE_TTL_MS, + }; +} + +async function fetchFromNetwork(): Promise { + const client = await getClient(); + const modelsList = await client.models.list(); + const handles = new Set( + modelsList.map((m) => m.handle).filter((h): h is string => !!h), + ); + return { handles, fetchedAt: Date.now() }; +} + +export async function getAvailableModelHandles(options?: { + forceRefresh?: boolean; +}): Promise { + const forceRefresh = options?.forceRefresh === true; + const now = Date.now(); + + if (!forceRefresh && isFresh(now) && cache) { + return { + handles: cache.handles, + source: "cache", + fetchedAt: cache.fetchedAt, + }; + } + + if (!forceRefresh && inflight) { + const entry = await inflight; + return { + handles: entry.handles, + source: "network", + fetchedAt: entry.fetchedAt, + }; + } + + inflight = fetchFromNetwork() + .then((entry) => { + cache = entry; + return entry; + }) + .finally(() => { + inflight = null; + }); + + const entry = await inflight; + return { + handles: entry.handles, + source: "network", + fetchedAt: entry.fetchedAt, + }; +} + +/** + * Best-effort prefetch to warm the cache (no throw). + * This is intentionally fire-and-forget. + */ +export function prefetchAvailableModelHandles(): void { + void getAvailableModelHandles().catch(() => { + // Ignore failures; UI will handle errors on-demand. + }); +} diff --git a/src/cli/App.tsx b/src/cli/App.tsx index 89b9e91..739a01d 100644 --- a/src/cli/App.tsx +++ b/src/cli/App.tsx @@ -14,6 +14,7 @@ import type { LlmConfig } from "@letta-ai/letta-client/resources/models/models"; import { Box, Static, Text } from "ink"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import type { ApprovalResult } from "../agent/approval-execution"; +import { prefetchAvailableModelHandles } from "../agent/available-models"; import { getResumeData } from "../agent/check-approval"; import { getClient } from "../agent/client"; import { setCurrentAgentId } from "../agent/context"; @@ -276,6 +277,11 @@ export default function App({ tokenStreaming?: boolean; agentProvenance?: AgentProvenance | null; }) { + // Warm the model-access cache in the background so /model is fast on first open. + useEffect(() => { + prefetchAvailableModelHandles(); + }, []); + // Track current agent (can change when swapping) const [agentId, setAgentId] = useState(initialAgentId); const [agentState, setAgentState] = useState(initialAgentState); diff --git a/src/cli/components/ModelSelector.tsx b/src/cli/components/ModelSelector.tsx index 467c711..45ea782 100644 --- a/src/cli/components/ModelSelector.tsx +++ b/src/cli/components/ModelSelector.tsx @@ -1,6 +1,11 @@ // Import useInput from vendored Ink for bracketed paste support import { Box, Text, useInput } from "ink"; import { useEffect, useMemo, useRef, useState } from "react"; +import { + clearAvailableModelsCache, + getAvailableModelHandles, + getAvailableModelsCacheInfo, +} from "../../agent/available-models"; import { models } from "../../agent/model"; import { colors } from "./colors"; @@ -30,17 +35,79 @@ export function ModelSelector({ const typedModels = models as UiModel[]; const [showAll, setShowAll] = useState(false); const [selectedIndex, setSelectedIndex] = useState(0); + // undefined: not loaded yet (show spinner) + // Set: loaded and filtered + // null: error fallback (show all models + warning) + const [availableModels, setAvailableModels] = useState< + Set | null | undefined + >(undefined); + const [isLoading, setIsLoading] = useState(true); + const [error, setError] = useState(null); + const [isCached, setIsCached] = useState(false); + const [refreshing, setRefreshing] = useState(false); + + const mountedRef = useRef(true); + useEffect(() => { + mountedRef.current = true; + return () => { + mountedRef.current = false; + }; + }, []); + + // Fetch available models from the API (with caching + inflight dedupe) + const loadModels = useRef(async (forceRefresh = false) => { + try { + if (forceRefresh) { + clearAvailableModelsCache(); + if (mountedRef.current) { + setRefreshing(true); + setError(null); + } + } + + const cacheInfoBefore = getAvailableModelsCacheInfo(); + const result = await getAvailableModelHandles({ forceRefresh }); + + if (!mountedRef.current) return; + + setAvailableModels(result.handles); + setIsCached(!forceRefresh && cacheInfoBefore.isFresh); + setIsLoading(false); + setRefreshing(false); + } catch (err) { + if (!mountedRef.current) return; + setError(err instanceof Error ? err.message : "Failed to load models"); + setIsLoading(false); + setRefreshing(false); + // Fallback: show all models if API fails + setAvailableModels(null); + } + }); + + useEffect(() => { + loadModels.current(false); + }, []); + + // Filter models based on availability + const filteredModels = useMemo(() => { + // Not loaded yet: render nothing (avoid briefly showing unfiltered models) + if (availableModels === undefined) return []; + // Error fallback: show all models with warning + if (availableModels === null) return typedModels; + // Loaded: filter to only show models the user has access to + return typedModels.filter((model) => availableModels.has(model.handle)); + }, [typedModels, availableModels]); const featuredModels = useMemo( - () => typedModels.filter((model) => model.isFeatured), - [typedModels], + () => filteredModels.filter((model) => model.isFeatured), + [filteredModels], ); const visibleModels = useMemo(() => { - if (showAll) return typedModels; + if (showAll) return filteredModels; if (featuredModels.length > 0) return featuredModels; - return typedModels.slice(0, 5); - }, [featuredModels, showAll, typedModels]); + return filteredModels.slice(0, 5); + }, [featuredModels, showAll, filteredModels]); // Set initial selection to current model on mount const initializedRef = useRef(false); @@ -54,36 +121,94 @@ export function ModelSelector({ } }, [visibleModels, currentModel]); - const totalItems = showAll ? visibleModels.length : visibleModels.length + 1; + const hasMoreModels = + !showAll && filteredModels.length > visibleModels.length; + const totalItems = hasMoreModels + ? visibleModels.length + 1 + : visibleModels.length; - useInput((_input, key) => { - if (key.upArrow) { - setSelectedIndex((prev) => Math.max(0, prev - 1)); - } else if (key.downArrow) { - setSelectedIndex((prev) => Math.min(totalItems - 1, prev + 1)); - } else if (key.return) { - if (!showAll && selectedIndex === visibleModels.length) { - setShowAll(true); - setSelectedIndex(0); - } else { - const selectedModel = visibleModels[selectedIndex]; - if (selectedModel) { - onSelect(selectedModel.id); + useInput( + (input, key) => { + // Allow ESC even while loading + if (key.escape) { + onCancel(); + return; + } + + // Allow 'r' to refresh even while loading (but not while already refreshing) + if (input === "r" && !refreshing) { + loadModels.current(true); + return; + } + + // Disable other inputs while loading + if (isLoading || refreshing || visibleModels.length === 0) { + return; + } + + if (key.upArrow) { + setSelectedIndex((prev) => Math.max(0, prev - 1)); + } else if (key.downArrow) { + setSelectedIndex((prev) => Math.min(totalItems - 1, prev + 1)); + } else if (key.return) { + if (hasMoreModels && selectedIndex === visibleModels.length) { + setShowAll(true); + setSelectedIndex(0); + } else { + const selectedModel = visibleModels[selectedIndex]; + if (selectedModel) { + onSelect(selectedModel.id); + } } } - } else if (key.escape) { - onCancel(); - } - }); + }, + // Keep active so ESC and 'r' work while loading. + { isActive: true }, + ); return ( - + Select Model (↑↓ to navigate, Enter to select, ESC to cancel) + {!isLoading && !refreshing && ( + + {isCached + ? "Cached models (press 'r' to refresh)" + : "Press 'r' to refresh"} + + )} + {isLoading && ( + + Loading available models... + + )} + + {refreshing && ( + + Refreshing models... + + )} + + {error && ( + + + Warning: Could not fetch available models. Showing all models. + + + )} + + {!isLoading && visibleModels.length === 0 && ( + + + No models available. Please check your Letta configuration. + + + )} + {visibleModels.map((model, index) => { const isSelected = index === selectedIndex; @@ -132,7 +257,7 @@ export function ModelSelector({ ); })} - {!showAll && ( + {!showAll && filteredModels.length > visibleModels.length && ( {selectedIndex === visibleModels.length ? "›" : " "} - Show all models + + Show all models ({filteredModels.length} available) + )}