diff --git a/package.json b/package.json index 54b70a1..c69b273 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "test": "vitest", "test:run": "vitest run --exclude 'e2e/**'", "lint:console": "bash scripts/no-console.sh", + "repro:context-window-reset": "tsx scripts/repro-context-window-reset.ts", "test:e2e": "vitest run e2e/", "skills": "tsx src/cli.ts skills", "skills:list": "tsx src/cli.ts skills list", diff --git a/scripts/repro-context-window-reset.ts b/scripts/repro-context-window-reset.ts new file mode 100644 index 0000000..7e0cdfb --- /dev/null +++ b/scripts/repro-context-window-reset.ts @@ -0,0 +1,313 @@ +#!/usr/bin/env tsx + +import { createHash } from 'node:crypto'; + +import { Letta } from '@letta-ai/letta-client'; +import { createAgent, resumeSession } from '@letta-ai/letta-code-sdk'; + +type ParsedArgs = { + agentId?: string; + model?: string; + targetWindow: number; + iterations: number; + preSessionIdleMs: number; + keepAgent: boolean; + skipControl: boolean; + baseUrl?: string; + apiKey?: string; + includeDirectSystemPatch: boolean; +}; + +type AgentSnapshot = { + at: string; + contextWindowLimit: number | null; + llmContextWindow: number | null; + effectiveContextWindow: number | null; + systemHash: string; + systemLength: number; + compactionSettings: unknown; +}; + +type StepResult = { + step: string; + snapshot: AgentSnapshot; + changedFromTarget: boolean; +}; + +function printUsage(): void { + console.log(` +Repro: context_window_limit drift after SDK memfs toggle updates. + +Usage: + npm run repro:context-window-reset -- [options] + +Options: + --agent-id Use an existing agent instead of creating one. + --model Model handle for created agent. + --target-window Context window limit to pin before triggers (default: 38000). + --iterations Number of memfs-false init cycles (default: 3). + --pre-idle-ms Idle wait after pin and before SDK steps (default: 2000). + --keep-agent Keep auto-created agent (default: delete it). + --skip-control Skip control init with memfs omitted. + --direct-system-patch Also run direct {system:} patch via API client. + --base-url Override LETTA_BASE_URL. + --api-key Override LETTA_API_KEY. + --help Show this message. + +Required env (unless passed via flags): + LETTA_API_KEY +Optional env: + LETTA_BASE_URL (default: https://api.letta.com) +`); +} + +function parseArgs(argv: string[]): ParsedArgs { + const out: ParsedArgs = { + targetWindow: 38000, + iterations: 3, + preSessionIdleMs: 2000, + keepAgent: false, + skipControl: false, + includeDirectSystemPatch: false, + }; + + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + if (arg === '--help' || arg === '-h') { + printUsage(); + process.exit(0); + } + if (arg === '--agent-id') { + out.agentId = argv[++i]; + continue; + } + if (arg === '--model') { + out.model = argv[++i]; + continue; + } + if (arg === '--target-window') { + out.targetWindow = Number(argv[++i]); + continue; + } + if (arg === '--iterations') { + out.iterations = Number(argv[++i]); + continue; + } + if (arg === '--pre-idle-ms') { + out.preSessionIdleMs = Number(argv[++i]); + continue; + } + if (arg === '--keep-agent') { + out.keepAgent = true; + continue; + } + if (arg === '--skip-control') { + out.skipControl = true; + continue; + } + if (arg === '--direct-system-patch') { + out.includeDirectSystemPatch = true; + continue; + } + if (arg === '--base-url') { + out.baseUrl = argv[++i]; + continue; + } + if (arg === '--api-key') { + out.apiKey = argv[++i]; + continue; + } + throw new Error(`Unknown argument: ${arg}`); + } + + if (!Number.isFinite(out.targetWindow) || out.targetWindow <= 0) { + throw new Error(`--target-window must be a positive number, got: ${out.targetWindow}`); + } + if (!Number.isFinite(out.iterations) || out.iterations <= 0) { + throw new Error(`--iterations must be a positive number, got: ${out.iterations}`); + } + if (!Number.isFinite(out.preSessionIdleMs) || out.preSessionIdleMs < 0) { + throw new Error(`--pre-idle-ms must be >= 0, got: ${out.preSessionIdleMs}`); + } + + return out; +} + +function hashText(value: string): string { + return createHash('sha256').update(value).digest('hex').slice(0, 12); +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function getSnapshot(client: Letta, agentId: string): Promise { + const state = await client.agents.retrieve(agentId); + const unsafe = state as Record; + const contextWindowLimit = typeof unsafe.context_window_limit === 'number' + ? unsafe.context_window_limit + : null; + const llmConfig = unsafe.llm_config as Record | undefined; + const llmContextWindow = llmConfig && typeof llmConfig.context_window === 'number' + ? llmConfig.context_window + : null; + const effectiveContextWindow = contextWindowLimit ?? llmContextWindow; + const system = typeof unsafe.system === 'string' ? unsafe.system : ''; + const compactionSettings = unsafe.compaction_settings ?? null; + + return { + at: new Date().toISOString(), + contextWindowLimit, + llmContextWindow, + effectiveContextWindow, + systemHash: hashText(system), + systemLength: system.length, + compactionSettings, + }; +} + +function isWindowChanged(snapshot: AgentSnapshot, targetWindow: number): boolean { + return snapshot.effectiveContextWindow !== targetWindow; +} + +async function initializeAndClose(agentId: string, memfs: boolean | undefined): Promise { + const opts = memfs === undefined ? {} : { memfs }; + const session = resumeSession(agentId, opts); + try { + await session.initialize(); + } finally { + session.close(); + } +} + +async function main(): Promise { + const args = parseArgs(process.argv.slice(2)); + const baseURL = args.baseUrl || process.env.LETTA_BASE_URL || 'https://api.letta.com'; + const apiKey = args.apiKey || process.env.LETTA_API_KEY; + + if (!apiKey) { + throw new Error('LETTA_API_KEY is required (set env or pass --api-key).'); + } + + process.env.LETTA_BASE_URL = baseURL; + process.env.LETTA_API_KEY = apiKey; + + const client = new Letta({ apiKey, baseURL }); + const startedAt = new Date().toISOString(); + + let agentId = args.agentId; + let createdAgent = false; + + if (!agentId) { + agentId = await createAgent({ + ...(args.model ? { model: args.model } : {}), + // Keep baseline deterministic: avoid cloud auto-memfs behavior on new agents. + memfs: false, + tags: ['origin:context-window-repro'], + }); + createdAgent = true; + } + + console.log(`Using agent: ${agentId}${createdAgent ? ' (created for repro)' : ''}`); + + const initial = await getSnapshot(client, agentId); + + await client.agents.update(agentId, { context_window_limit: args.targetWindow }); + await sleep(750); + const afterPin = await getSnapshot(client, agentId); + + const steps: StepResult[] = []; + + if (args.preSessionIdleMs > 0) { + await sleep(args.preSessionIdleMs); + const idle = await getSnapshot(client, agentId); + steps.push({ + step: `control: idle wait ${args.preSessionIdleMs}ms (no SDK session)`, + snapshot: idle, + changedFromTarget: isWindowChanged(idle, args.targetWindow), + }); + } + + if (!args.skipControl) { + await initializeAndClose(agentId, undefined); + await sleep(750); + const control = await getSnapshot(client, agentId); + steps.push({ + step: 'control: sdk init with memfs omitted', + snapshot: control, + changedFromTarget: isWindowChanged(control, args.targetWindow), + }); + } + + for (let i = 1; i <= args.iterations; i += 1) { + await initializeAndClose(agentId, false); + await sleep(750); + const snap = await getSnapshot(client, agentId); + steps.push({ + step: `trigger ${i}: sdk init with memfs=false (--no-memfs)`, + snapshot: snap, + changedFromTarget: isWindowChanged(snap, args.targetWindow), + }); + } + + if (args.includeDirectSystemPatch) { + const latest = await getSnapshot(client, agentId); + const state = await client.agents.retrieve(agentId); + const system = typeof (state as Record).system === 'string' + ? (state as Record).system as string + : ''; + await client.agents.update(agentId, { system }); + await sleep(750); + const afterDirectPatch = await getSnapshot(client, agentId); + steps.push({ + step: 'trigger: direct client.agents.update({system: sameText})', + snapshot: afterDirectPatch, + changedFromTarget: isWindowChanged(afterDirectPatch, args.targetWindow), + }); + + if (latest.systemHash !== afterDirectPatch.systemHash) { + console.warn('Note: system hash changed across direct system patch step.'); + } + } + + const reproduced = steps.some((s) => s.changedFromTarget); + const finishedAt = new Date().toISOString(); + + const report = { + scenario: 'context-window-limit drift on partial agent updates', + startedAt, + finishedAt, + baseURL, + agentId, + createdAgent, + targetWindow: args.targetWindow, + initial, + afterPin, + steps, + note: 'effectiveContextWindow uses context_window_limit when available, otherwise llm_config.context_window.', + reproduced, + summary: reproduced + ? 'BUG REPRODUCED: context_window_limit changed away from pinned value.' + : 'No drift observed in this run.', + }; + + console.log('\n=== Repro Report (JSON) ==='); + console.log(JSON.stringify(report, null, 2)); + + if (createdAgent && !args.keepAgent) { + await client.agents.delete(agentId); + console.log(`Deleted temporary agent: ${agentId}`); + } else if (createdAgent && args.keepAgent) { + console.log(`Kept temporary agent for inspection: ${agentId}`); + } + + if (reproduced) { + process.exitCode = 2; + } +} + +main().catch((error) => { + const message = error instanceof Error ? error.stack || error.message : String(error); + console.error(message); + process.exit(1); +}); diff --git a/src/config/memfs.test.ts b/src/config/memfs.test.ts new file mode 100644 index 0000000..ea0c9aa --- /dev/null +++ b/src/config/memfs.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from 'vitest'; + +import { resolveSessionMemfs } from './memfs.js'; + +describe('resolveSessionMemfs', () => { + it('uses explicit agent config first', () => { + const result = resolveSessionMemfs({ + configuredMemfs: true, + envMemfs: 'false', + serverMode: 'api', + }); + + expect(result).toEqual({ value: true, source: 'config' }); + }); + + it('uses LETTABOT_MEMFS env override when config is unset', () => { + const result = resolveSessionMemfs({ + envMemfs: 'false', + serverMode: 'api', + }); + + expect(result).toEqual({ value: false, source: 'env' }); + }); + + it('defaults to memfs false in docker/selfhosted mode when unset', () => { + const result = resolveSessionMemfs({ + serverMode: 'selfhosted', + }); + + expect(result).toEqual({ value: false, source: 'default-docker' }); + }); + + it('leaves memfs unchanged in api/cloud mode when unset', () => { + const result = resolveSessionMemfs({ + serverMode: 'cloud', + }); + + expect(result).toEqual({ value: undefined, source: 'unset' }); + }); + + it('ignores invalid LETTABOT_MEMFS values', () => { + const result = resolveSessionMemfs({ + envMemfs: 'yes', + serverMode: 'api', + }); + + expect(result).toEqual({ value: undefined, source: 'unset' }); + }); + + it('treats null configured memfs as unset and applies docker default', () => { + const result = resolveSessionMemfs({ + configuredMemfs: null as unknown as boolean, + serverMode: 'selfhosted', + }); + + expect(result).toEqual({ value: false, source: 'default-docker' }); + }); +}); diff --git a/src/config/memfs.ts b/src/config/memfs.ts new file mode 100644 index 0000000..69ff82d --- /dev/null +++ b/src/config/memfs.ts @@ -0,0 +1,48 @@ +import { isDockerServerMode, type ServerMode } from './types.js'; + +export type ResolvedMemfsSource = 'config' | 'env' | 'default-docker' | 'unset'; + +export interface ResolveSessionMemfsInput { + configuredMemfs?: boolean; + envMemfs?: string; + serverMode?: ServerMode; +} + +export interface ResolveSessionMemfsResult { + value: boolean | undefined; + source: ResolvedMemfsSource; +} + +function parseBooleanEnv(value?: string): boolean | undefined { + if (value === 'true') return true; + if (value === 'false') return false; + return undefined; +} + +/** + * Resolve the memfs value forwarded to SDK session options. + * + * Precedence: + * 1) Per-agent config (`features.memfs`) + * 2) `LETTABOT_MEMFS` env var (`true`/`false`) + * 3) Default `false` in docker/self-hosted mode (safety) + * 4) `undefined` in API mode (leave agent memfs unchanged) + */ +export function resolveSessionMemfs(input: ResolveSessionMemfsInput): ResolveSessionMemfsResult { + // Runtime config parsing can surface non-boolean values (e.g. YAML `memfs:` -> null). + // Only treat explicit booleans as configured; everything else falls through. + if (typeof input.configuredMemfs === 'boolean') { + return { value: input.configuredMemfs, source: 'config' }; + } + + const envMemfs = parseBooleanEnv(input.envMemfs); + if (envMemfs !== undefined) { + return { value: envMemfs, source: 'env' }; + } + + if (isDockerServerMode(input.serverMode)) { + return { value: false, source: 'default-docker' }; + } + + return { value: undefined, source: 'unset' }; +} diff --git a/src/core/session-manager.ts b/src/core/session-manager.ts index 4c73883..70a3792 100644 --- a/src/core/session-manager.ts +++ b/src/core/session-manager.ts @@ -20,6 +20,23 @@ import { createLogger } from '../logger.js'; const log = createLogger('Session'); +function formatMemfsStartupOption(memfs: boolean | undefined): string { + if (memfs === true) return 'enabled (--memfs)'; + if (memfs === false) return 'disabled (--no-memfs)'; + return 'unchanged (omitted)'; +} + +function formatSleeptimeStartupOption( + sleeptime: BotConfig['sleeptime'], +): string { + if (!sleeptime) return 'none'; + const parts: string[] = []; + if (sleeptime.trigger) parts.push(`trigger=${sleeptime.trigger}`); + if (sleeptime.behavior) parts.push(`behavior=${sleeptime.behavior}`); + if (sleeptime.stepCount !== undefined) parts.push(`stepCount=${sleeptime.stepCount}`); + return parts.length > 0 ? parts.join(', ') : 'configured'; +} + function toConcreteConversationId(value: string | null | undefined): string | null { if (!value) return null; const trimmed = value.trim(); @@ -313,6 +330,11 @@ export class SessionManager { } // Initialize eagerly so the subprocess is ready before the first send() + log.info( + `Session startup options (key=${key}): ` + + `memfs=${formatMemfsStartupOption(this.config.memfs)}, ` + + `sleeptime=${formatSleeptimeStartupOption(this.config.sleeptime)}`, + ); log.info(`Initializing session subprocess (key=${key})...`); try { await this.withSessionTimeout(session.initialize(), `Session initialize (key=${key})`); diff --git a/src/main.ts b/src/main.ts index a897ac3..398ec4a 100644 --- a/src/main.ts +++ b/src/main.ts @@ -24,6 +24,7 @@ import { serverModeLabel, wasLoadedFromFleetConfig, } from './config/index.js'; +import { resolveSessionMemfs } from './config/memfs.js'; import { getCronDataDir, getDataDir, getWorkingDir, hasRailwayVolume, resolveWorkingDirPath } from './utils/paths.js'; import { parseCsvList, parseNonNegativeNumber } from './utils/parse.js'; import { createLogger, setLogLevel } from './logger.js'; @@ -299,21 +300,23 @@ async function main() { for (const agentConfig of agents) { log.info(`Configuring agent: ${agentConfig.name}`); - // Resolve memfs: YAML config takes precedence, then env var, then default false. - // Default false prevents the SDK from auto-enabling memfs, which crashes on - // self-hosted Letta servers that don't have the git endpoint. - const resolvedMemfs = agentConfig.features?.memfs ?? (process.env.LETTABOT_MEMFS === 'true' ? true : false); + const resolvedMemfsResult = resolveSessionMemfs({ + configuredMemfs: agentConfig.features?.memfs, + envMemfs: process.env.LETTABOT_MEMFS, + serverMode: yamlConfig.server.mode, + }); + const resolvedMemfs = resolvedMemfsResult.value; const configuredSleeptime = agentConfig.features?.sleeptime; // Treat missing trigger as active (conservative): only `trigger: 'off'` explicitly disables. const sleeptimeRequiresMemfs = !!configuredSleeptime && configuredSleeptime.trigger !== 'off'; - const effectiveSleeptime = !resolvedMemfs && sleeptimeRequiresMemfs + const effectiveSleeptime = resolvedMemfs === false && sleeptimeRequiresMemfs ? undefined : configuredSleeptime; - if (!resolvedMemfs && sleeptimeRequiresMemfs) { + if (resolvedMemfs === false && sleeptimeRequiresMemfs) { log.warn( `Agent ${agentConfig.name}: sleeptime is configured but memfs is disabled; ` + - `sleeptime will be ignored. Enable features.memfs (or LETTABOT_MEMFS=true) to use sleeptime.` + `sleeptime will be ignored. Enable features.memfs (or set LETTABOT_MEMFS=true) to use sleeptime.` ); } @@ -359,8 +362,14 @@ async function main() { // Log memfs config (from either YAML or env var) if (resolvedMemfs !== undefined) { - const source = agentConfig.features?.memfs !== undefined ? '' : ' (from LETTABOT_MEMFS env)'; + const source = resolvedMemfsResult.source === 'config' + ? '' + : resolvedMemfsResult.source === 'env' + ? ' (from LETTABOT_MEMFS env)' + : ' (default for docker/selfhosted mode)'; log.info(`Agent ${agentConfig.name}: memfs ${resolvedMemfs ? 'enabled' : 'disabled'}${source}`); + } else { + log.info(`Agent ${agentConfig.name}: memfs unchanged (not explicitly configured)`); } // Apply explicit agent ID from config (before store verification)