From 457ccd898893e3e0cff4bf0a658fd8f1087fb74c Mon Sep 17 00:00:00 2001 From: Sarah Wooders Date: Mon, 23 Mar 2026 14:54:24 -0700 Subject: [PATCH] feat(cli): add /doctor command for memory context health check (#1488) Co-authored-by: Letta Code --- src/cli/App.tsx | 46 +++++ src/cli/commands/registry.ts | 9 + src/cli/helpers/initCommand.ts | 24 +++ src/skills/builtin/context_doctor/SKILL.md | 132 +++++++++++++ .../scripts/estimate_system_tokens.ts | 181 ++++++++++++++++++ src/websocket/terminalHandler.ts | 29 ++- 6 files changed, 419 insertions(+), 2 deletions(-) create mode 100644 src/skills/builtin/context_doctor/SKILL.md create mode 100755 src/skills/builtin/context_doctor/scripts/estimate_system_tokens.ts diff --git a/src/cli/App.tsx b/src/cli/App.tsx index d864424..014163b 100644 --- a/src/cli/App.tsx +++ b/src/cli/App.tsx @@ -235,6 +235,7 @@ import { import { formatCompact } from "./helpers/format"; import { parsePatchOperations } from "./helpers/formatArgsDisplay"; import { + buildDoctorMessage, buildInitMessage, fireAutoInit, gatherInitGitContext, @@ -9687,6 +9688,51 @@ export default function App({ return { submitted: true }; } + // Special handling for /doctor command + if (trimmed === "/doctor") { + const cmd = commandRunner.start(msg, "Gathering project context..."); + + const approvalCheck = await checkPendingApprovalsForSlashCommand(); + if (approvalCheck.blocked) { + cmd.fail( + "Pending approval(s). Resolve approvals before running /doctor.", + ); + return { submitted: false }; + } + + setCommandRunning(true); + try { + cmd.finish( + "Running memory doctor... I'll ask a few questions to refine memory structure.", + true, + ); + + const { context: gitContext } = gatherInitGitContext(); + const memoryDir = settingsManager.isMemfsEnabled(agentId) + ? getMemoryFilesystemRoot(agentId) + : undefined; + + const doctorMessage = buildDoctorMessage({ + gitContext, + memoryDir, + }); + + await processConversation([ + { + type: "message", + role: "user", + content: buildTextParts(doctorMessage), + }, + ]); + } catch (error) { + const errorDetails = formatErrorDetails(error, agentId); + cmd.fail(`Failed: ${errorDetails}`); + } finally { + setCommandRunning(false); + } + return { submitted: true }; + } + if (trimmed.startsWith("/feedback")) { const maybeMsg = msg.slice("/feedback".length).trim(); setFeedbackPrefill(maybeMsg); diff --git a/src/cli/commands/registry.ts b/src/cli/commands/registry.ts index 7382c62..94cd4ce 100644 --- a/src/cli/commands/registry.ts +++ b/src/cli/commands/registry.ts @@ -40,6 +40,15 @@ export const commands: Record = { return "Initializing memory..."; }, }, + "/doctor": { + desc: "Audit and refine your memory structure", + order: 12.1, + noArgs: true, + handler: () => { + // Handled specially in App.tsx to send doctor prompt + return "Running memory doctor..."; + }, + }, "/remember": { desc: "Remember something from the conversation (/remember [instructions])", order: 13, diff --git a/src/cli/helpers/initCommand.ts b/src/cli/helpers/initCommand.ts index 3f1a416..2cdc650 100644 --- a/src/cli/helpers/initCommand.ts +++ b/src/cli/helpers/initCommand.ts @@ -291,3 +291,27 @@ Once invoked, follow the instructions from the \`initializing-memory\` skill to ${args.gitContext} ${SYSTEM_REMINDER_CLOSE}`; } + +/** Message for the primary agent via processConversation when user runs /doctor. */ +export function buildDoctorMessage(args: { + gitContext: string; + memoryDir?: string; +}): string { + const memfsSection = args.memoryDir + ? `\n## Memory filesystem\n\nMemory filesystem is enabled. Memory directory: \`${args.memoryDir}\`\n` + : ""; + + return `${SYSTEM_REMINDER_OPEN} +The user has requested a memory structure check via /doctor. +${memfsSection} +## 1. Invoke the context_doctor skill + +Use the \`Skill\` tool with \`skill: "context_doctor"\` to load guidance for memory structure refinement. + +## 2. Follow the skill instructions + +Once invoked, follow the instructions from the \`context_doctor\` skill. + +${args.gitContext} +${SYSTEM_REMINDER_CLOSE}`; +} diff --git a/src/skills/builtin/context_doctor/SKILL.md b/src/skills/builtin/context_doctor/SKILL.md new file mode 100644 index 0000000..11b014d --- /dev/null +++ b/src/skills/builtin/context_doctor/SKILL.md @@ -0,0 +1,132 @@ +--- +name: Context Doctor +id: context_doctor +description: Identify and repair degradation in system prompt, external memory, and skills preventing you from following instructions or remembering information as well as you should. +--- + +# Context Doctor +Your context is managed by yourself, along with additional memory subagents. Your context includes: +- Your system prompt and instructions (contained in `system/`) +- Your external memory +- Your skills (procedural memory) + +Over time, context can degrade — bloat and poor prompt quality erode your ability to remember the right things and follow instructions properly. This skill helps you identify issues with your context and repair them collaboratively with the user. + +## Operating Procedure + +### Step 1: Identifying and resolving context issues +Explore your memory files to identify issues. Consider what is confusing about your own prompts and context, and resolve the issues. + +Below are additional common issues with context and how they can be resolved: + +### Context quality +Your system prompt and memory filesystem should be well structured and clear. + +**Questions to ask**: +- Is my system prompt clear and well formatted? +- Are there wasteful or unnecessary tokens in my prompts? +- Do I know when to load which files in my memory filesystem? + +#### System prompt bloat +Prompts that are compiled as part of the system prompt (contained in `system/`) should only take up about 10% of the total context size, though this is a recommendation, not a hard requirement. Usually this means about 15-20k tokens. + +Use the following script to evaluate the token usage of the system prompt: +```bash +bun scripts/estimate_system_tokens.ts --memory-dir "$MEMORY_DIR" +``` + +**Questions to ask**: +- Do all these tokens need to be passed to the LLM on every turn, or can they be retrieved when needed through being part of external memory of my conversation history? +- Do any of these prompts confuse or distract me? +- Am I able to effectively follow critical instructions (e.g. persona information, user preferences) given the current prompt structure and contents? + +**Solution**: Reduce the size of the system prompt if needed: +- Move files outside of `system/` so they are no longer part of the system prompt +- Compact information to be more information dense or eliminate redundancy +- Leverage progressive disclosure: move some context outside of `system/` and reference it to pull in dynamically + +**Scope**: You may refine, tighten, and restructure prompts to improve clarity and adherence — but do not change the intended semantics. The goal is better signal, not different behavior. +- Do not alter persona-defining content (who you are, how you communicate) +- Do not remove or change user identity or preferences (e.g. the human's name, their stated goals) +- Do not rewrite instructions in ways that shift their meaning — only reduce noise and improve structure + +#### Context redundancy and unclear organization +The context in the memory filesystem should have a clear structure, with a well-defined purpose for each file. Memory file descriptions should be precise and non-overlapping. Their contents should be consistent with the description, and have non-overlapping content to other files. + +**Questions to ask**: +- Do the descriptions make clear what file is for what? +- Do the contents of the file match the descriptions? (you can ask subagents to check) + +**Solution**: Read all memory files (use subagents for efficiency), then: +- Consolidate redundant files +- Reorganize files and rewrite descriptions to have clear separation of concerns +- Avoid duplication by referencing common files from multiple places (e.g. `[[reference/api]]`) +- Rewrite unclear or low-quality content + +#### Invalid context format +Files in the memory filesystem must follow certain structural requirements: +- Must have a `system/persona.md` +- Must NOT have overlapping file and folder names (e.g. `system/human.md` and `system/human/identity.md`) +- Must follow specification for skills (e.g. `skills/{skill_name}/`) with the format: +``` +skill-name/ +├── SKILL.md # Required: metadata + instructions +├── scripts/ # Optional: executable code +├── references/ # Optional: documentation +├── assets/ # Optional: templates, resources +└── ... # Any additional files or directories +``` + +**Solution**: Reorganize files to follow the required structure + +### Poor use of progressive disclosure +Only critical information should be in the system prompt, since it's passed on every turn. Use progressive disclosure so that context only *sometimes* needed can be dynamically retrieved. + +Files that are outside of `system/` are not part of the system prompt, and must be dynamically loaded. You must index your files to ensure your future self can discover them: for example, make sure that files have informative names and descriptions, or are referenced from parts of your system prompt. Otherwise, you will never discover the external context or make use of it. + +**Solution**: +- Reference external skills from the relevant parts of in-context memory: +``` +When running a migration, always use the skill [[skills/db-migrations]] +``` +or external memory files: +``` +Sarah's active projects are: Letta Code [[projects/letta_code.md]] and Letta Cloud [[projects/letta_cloud]] +``` +- Ensure that contents of files match the file name and descriptions +- Make sure your future self will be able to find and load external files when needed. + +### Step 2: Implement context fixes +Create a plan for what fixes you want to make, then implement them. + +Before moving on, verify: +- [ ] System prompt token budget reviewed (target ~10% of context, usually 15-20k tokens) +- [ ] No overlapping or redundant files remain +- [ ] All file descriptions are unique, accurate, and match their contents +- [ ] Moved-out knowledge has references from in-context memory so it can be discovered +- [ ] No semantic changes to persona, user identity, or behavioral instructions + +### Step 3: Commit and push +Review changes, then commit with a descriptive message: + +```bash +cd $MEMORY_DIR +git status # Review what changed before staging +git add # Stage targeted paths — avoid blind `git add -A` +git commit --author=" <@letta.com>" -m "fix(doctor): 🏥 + +" + +git push +``` + +### Step 4: Final checklist and message +Tell the user what issues you identitied, the fixes you made, the commit you made, and also recommend that they run `/recompile` to apply these changes to the current system prompt. + +Before finishing make sure you: +- [ ] Resolved all the identified context issues +- [ ] Pushed your changes successfully +- [ ] Told the user to run `/recompile` to refresh the system prompt and apply changes + +## Critical information +- **Ask the user about their goals for you, not the implementation**: You understand your own context best, and should follow the guidelines in this document. Do NOT ask the user about their structural preferences - the context is for YOU, not them. Ask them how they want YOU to behave or know instead. diff --git a/src/skills/builtin/context_doctor/scripts/estimate_system_tokens.ts b/src/skills/builtin/context_doctor/scripts/estimate_system_tokens.ts new file mode 100755 index 0000000..05aca74 --- /dev/null +++ b/src/skills/builtin/context_doctor/scripts/estimate_system_tokens.ts @@ -0,0 +1,181 @@ +#!/usr/bin/env bun + +import { existsSync, readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { getClient } from "../../../../agent/client"; +import { settingsManager } from "../../../../settings-manager"; + +const BYTES_PER_TOKEN = 4; + +type FileEstimate = { + path: string; + tokens: number; +}; + +type ParsedArgs = { + memoryDir?: string; + agentId?: string; + top: number; +}; + +function parseArgs(argv: string[]): ParsedArgs { + const parsed: ParsedArgs = { top: 20 }; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--memory-dir") { + parsed.memoryDir = argv[i + 1]; + i++; + continue; + } + if (arg === "--agent-id") { + parsed.agentId = argv[i + 1]; + i++; + continue; + } + if (arg === "--top") { + const raw = argv[i + 1]; + const value = Number.parseInt(raw ?? "", 10); + if (!Number.isNaN(value) && value >= 0) { + parsed.top = value; + } + i++; + } + } + + return parsed; +} + +function estimateTokens(text: string): number { + return Math.ceil(Buffer.byteLength(text, "utf8") / BYTES_PER_TOKEN); +} + +function normalizePath(value: string): string { + return value.replaceAll("\\", "/"); +} + +function walkMarkdownFiles(dir: string): string[] { + if (!existsSync(dir)) { + return []; + } + + const out: string[] = []; + const entries = readdirSync(dir, { withFileTypes: true }); + + for (const entry of entries) { + if (entry.name.startsWith(".")) { + continue; + } + const full = join(dir, entry.name); + if (entry.isDirectory()) { + if (entry.name === ".git") { + continue; + } + out.push(...walkMarkdownFiles(full)); + continue; + } + if (entry.isFile() && entry.name.endsWith(".md")) { + out.push(full); + } + } + + return out; +} + +function inferAgentIdFromMemoryDir(memoryDir: string): string | null { + const parts = normalizePath(memoryDir).split("/"); + for (let i = 0; i < parts.length - 1; i++) { + if (parts[i] === "agents" && parts[i + 1]?.startsWith("agent-")) { + return parts[i + 1]; + } + } + + const maybe = parts.at(-2); + return maybe?.startsWith("agent-") ? maybe : null; +} + +async function resolveAgentId( + memoryDir: string, + cliAgentId?: string, +): Promise { + if (cliAgentId) { + return cliAgentId; + } + + if (process.env.AGENT_ID) { + return process.env.AGENT_ID; + } + + const inferred = inferAgentIdFromMemoryDir(memoryDir); + if (inferred) { + return inferred; + } + + const fromSession = settingsManager.getEffectiveLastAgentId(process.cwd()); + if (fromSession) { + return fromSession; + } + + throw new Error( + "Unable to resolve agent ID. Pass --agent-id or set AGENT_ID.", + ); +} + +function formatNumber(value: number): string { + return value.toLocaleString("en-US"); +} + +async function main(): Promise { + await settingsManager.initialize(); + + const args = parseArgs(process.argv.slice(2)); + const memoryDir = args.memoryDir || process.env.MEMORY_DIR; + + if (!memoryDir) { + throw new Error("Missing memory dir. Pass --memory-dir or set MEMORY_DIR."); + } + + const systemDir = join(memoryDir, "system"); + if (!existsSync(systemDir)) { + throw new Error(`Missing system directory: ${systemDir}`); + } + + const agentId = await resolveAgentId(memoryDir, args.agentId); + + // Use the SDK auth path used by letta-code (OAuth + API key handling via getClient). + const client = await getClient(); + await client.agents.retrieve(agentId); + + const files = walkMarkdownFiles(systemDir).sort(); + const rows: FileEstimate[] = []; + + for (const filePath of files) { + const text = readFileSync(filePath, "utf8"); + const rel = normalizePath(filePath.slice(memoryDir.length + 1)); + rows.push({ path: rel, tokens: estimateTokens(text) }); + } + + const estimatedTotalTokens = rows.reduce((sum, row) => sum + row.tokens, 0); + + console.log("Estimated total tokens"); + console.log(` ${formatNumber(estimatedTotalTokens)}`); + + console.log("\nPer-file token estimates"); + console.log(` ${"tokens".padStart(8)} path`); + + const sortedRows = [...rows].sort((a, b) => b.tokens - a.tokens); + for (const row of sortedRows.slice(0, Math.max(0, args.top))) { + console.log(` ${formatNumber(row.tokens).padStart(8)} ${row.path}`); + } + + return 0; +} + +main() + .then((code) => { + process.exit(code); + }) + .catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + }); diff --git a/src/websocket/terminalHandler.ts b/src/websocket/terminalHandler.ts index 044dd47..74ab02d 100644 --- a/src/websocket/terminalHandler.ts +++ b/src/websocket/terminalHandler.ts @@ -27,6 +27,31 @@ interface TerminalSession { spawnedAt: number; } +type NodePtyExitEvent = { exitCode?: number; signal?: number }; + +type NodePtyProcess = { + pid: number; + write: (data: string) => void; + resize: (cols: number, rows: number) => void; + kill: () => void; + onData: (listener: (data: string) => void) => void; + onExit: (listener: (event: NodePtyExitEvent) => void) => void; +}; + +type NodePtyModule = { + spawn: ( + file: string, + args: string[], + options: { + name: string; + cols: number; + rows: number; + cwd: string; + env: Record; + }, + ) => NodePtyProcess; +}; + const terminals = new Map(); function getDefaultShell(): string { @@ -160,7 +185,7 @@ function spawnNodePty( socket: WebSocket, ): TerminalSession { // eslint-disable-next-line @typescript-eslint/no-require-imports - const pty = require("node-pty") as typeof import("node-pty"); + const pty = require("node-pty") as NodePtyModule; const handleData = makeOutputBatcher((data) => sendTerminalMessage(socket, { type: "terminal_output", terminal_id, data }), @@ -180,7 +205,7 @@ function spawnNodePty( ptyProcess.onData(handleData); - ptyProcess.onExit(({ exitCode }) => { + ptyProcess.onExit(({ exitCode }: NodePtyExitEvent) => { const current = terminals.get(terminal_id); if (current && current.pid === ptyProcess.pid) { terminals.delete(terminal_id);