From d1a6eeb40a12583d4e82b78f9769ec9bfcdeb54d Mon Sep 17 00:00:00 2001 From: Charles Packer Date: Tue, 10 Feb 2026 18:06:05 -0800 Subject: [PATCH] feat: memory filesystem sync (#905) Co-authored-by: Letta --- src/agent/memoryFilesystem.ts | 1264 +---------------- src/agent/memoryGit.ts | 435 ++++++ src/agent/prompts/system_prompt_memfs.txt | 87 +- src/cli/App.tsx | 470 ++---- src/cli/components/MemfsTreeViewer.tsx | 9 +- src/cli/subcommands/memfs.ts | 615 +------- src/headless.ts | 32 +- src/index.ts | 11 + .../syncing-memory-filesystem/SKILL.md | 160 ++- .../memoryFilesystem.sync.integration.test.ts | 658 +-------- src/tests/agent/memoryFilesystem.test.ts | 108 +- src/tests/agent/memoryGit.precommit.test.ts | 305 ++++ src/tools/impl/shellEnv.ts | 10 +- 13 files changed, 1085 insertions(+), 3079 deletions(-) create mode 100644 src/agent/memoryGit.ts create mode 100644 src/tests/agent/memoryGit.precommit.test.ts diff --git a/src/agent/memoryFilesystem.ts b/src/agent/memoryFilesystem.ts index 58577de..de0a92a 100644 --- a/src/agent/memoryFilesystem.ts +++ b/src/agent/memoryFilesystem.ts @@ -1,87 +1,20 @@ -import { createHash } from "node:crypto"; -import { existsSync, mkdirSync, readFileSync } from "node:fs"; -import { readdir, readFile, unlink, writeFile } from "node:fs/promises"; +/** + * Memory filesystem helpers. + * + * With git-backed memory, most sync/hash logic is removed. + * This module retains: directory helpers and tree rendering. + */ + +import { existsSync, mkdirSync } from "node:fs"; import { homedir } from "node:os"; -import { dirname, join, relative } from "node:path"; +import { join } from "node:path"; -import type { Block } from "@letta-ai/letta-client/resources/agents/blocks"; -import { getClient } from "./client"; -import { parseMdxFrontmatter, READ_ONLY_BLOCK_LABELS } from "./memory"; - -export const MEMORY_FILESYSTEM_BLOCK_LABEL = "memory_filesystem"; export const MEMORY_FS_ROOT = ".letta"; export const MEMORY_FS_AGENTS_DIR = "agents"; export const MEMORY_FS_MEMORY_DIR = "memory"; export const MEMORY_SYSTEM_DIR = "system"; -/** @deprecated Detached blocks now go at root level, not in /user/ */ -export const MEMORY_USER_DIR = "user"; -export const MEMORY_FS_STATE_FILE = ".sync-state.json"; -/** - * Block labels that are managed by the memfs system itself and have special - * update logic (updateMemoryFilesystemBlock). These are skipped in the main - * sync loop but written separately. - */ -const MEMFS_MANAGED_LABELS = new Set([MEMORY_FILESYSTEM_BLOCK_LABEL]); - -// Unified sync state - no system/detached split -// The attached/detached distinction is derived at runtime from API and FS -type SyncState = { - blockHashes: Record; // label → content hash - fileHashes: Record; // label → content hash - blockIds: Record; // label → block ID - lastSync: string | null; -}; - -// Legacy format for migration -type LegacySyncState = { - systemBlocks?: Record; - systemFiles?: Record; - detachedBlocks?: Record; - detachedFiles?: Record; - detachedBlockIds?: Record; - blocks?: Record; - files?: Record; - lastSync?: string | null; -}; - -export type MemorySyncConflict = { - label: string; - blockValue: string | null; - fileValue: string | null; -}; - -export type MemfsSyncStatus = { - /** Blocks where both file and block changed since last sync */ - conflicts: MemorySyncConflict[]; - /** Labels where only the file changed (would auto-resolve to block) */ - pendingFromFile: string[]; - /** Labels where only the block changed (would auto-resolve to file) */ - pendingFromBlock: string[]; - /** Labels where a file exists but no block */ - newFiles: string[]; - /** Labels where a block exists but no file */ - newBlocks: string[]; - /** Labels where file location doesn't match block attachment (would auto-sync) */ - locationMismatches: string[]; - /** True when there are no conflicts or pending changes */ - isClean: boolean; -}; - -export type MemorySyncResult = { - updatedBlocks: string[]; - createdBlocks: string[]; - deletedBlocks: string[]; - updatedFiles: string[]; - createdFiles: string[]; - deletedFiles: string[]; - conflicts: MemorySyncConflict[]; -}; - -export type MemorySyncResolution = { - label: string; - resolution: "file" | "block"; -}; +// ----- Directory helpers ----- export function getMemoryFilesystemRoot( agentId: string, @@ -103,25 +36,6 @@ export function getMemorySystemDir( return join(getMemoryFilesystemRoot(agentId, homeDir), MEMORY_SYSTEM_DIR); } -/** - * Get the directory for detached (non-attached) blocks. - * In the flat structure, detached blocks go directly in the memory root. - */ -export function getMemoryDetachedDir( - agentId: string, - homeDir: string = homedir(), -): string { - // Detached blocks go at root level (flat structure) - return getMemoryFilesystemRoot(agentId, homeDir); -} - -function getMemoryStatePath( - agentId: string, - homeDir: string = homedir(), -): string { - return join(getMemoryFilesystemRoot(agentId, homeDir), MEMORY_FS_STATE_FILE); -} - export function ensureMemoryFilesystemDirs( agentId: string, homeDir: string = homedir(), @@ -135,345 +49,21 @@ export function ensureMemoryFilesystemDirs( if (!existsSync(systemDir)) { mkdirSync(systemDir, { recursive: true }); } - // Note: detached blocks go directly in root, no separate directory needed } -function hashContent(content: string): string { - return createHash("sha256").update(content).digest("hex"); -} - -/** - * Hash just the body content of a file (excluding frontmatter). - * Used for "content matches" checks where we compare file body to block value. - */ -function hashFileBody(content: string): string { - const { body } = parseMdxFrontmatter(content); - return hashContent(body); -} - -function loadSyncState( - agentId: string, - homeDir: string = homedir(), -): SyncState { - const statePath = getMemoryStatePath(agentId, homeDir); - const emptyState: SyncState = { - blockHashes: {}, - fileHashes: {}, - blockIds: {}, - lastSync: null, - }; - - if (!existsSync(statePath)) { - return emptyState; - } - - try { - const raw = readFileSync(statePath, "utf-8"); - const parsed = JSON.parse(raw) as LegacySyncState & Partial; - - // New format - return directly - if (parsed.blockHashes !== undefined) { - return { - blockHashes: parsed.blockHashes || {}, - fileHashes: parsed.fileHashes || {}, - blockIds: parsed.blockIds || {}, - lastSync: parsed.lastSync || null, - }; - } - - // Migrate from legacy format: merge system + detached into unified maps - const blockHashes: Record = { - ...(parsed.systemBlocks || parsed.blocks || {}), - ...(parsed.detachedBlocks || {}), - }; - const fileHashes: Record = { - ...(parsed.systemFiles || parsed.files || {}), - ...(parsed.detachedFiles || {}), - }; - const blockIds: Record = { - ...(parsed.detachedBlockIds || {}), - }; - - return { - blockHashes, - fileHashes, - blockIds, - lastSync: parsed.lastSync || null, - }; - } catch { - return emptyState; - } -} - -async function saveSyncState( - state: SyncState, - agentId: string, - homeDir: string = homedir(), -) { - const statePath = getMemoryStatePath(agentId, homeDir); - await writeFile(statePath, JSON.stringify(state, null, 2)); -} - -async function scanMdFiles( - dir: string, - baseDir = dir, - excludeDirs: string[] = [], -): Promise { - const entries = await readdir(dir, { withFileTypes: true }); - const results: string[] = []; - - for (const entry of entries) { - const fullPath = join(dir, entry.name); - if (entry.isDirectory()) { - // Skip excluded directories (e.g., "system" when scanning for detached files) - if (excludeDirs.includes(entry.name)) { - continue; - } - results.push(...(await scanMdFiles(fullPath, baseDir, excludeDirs))); - } else if (entry.isFile() && entry.name.endsWith(".md")) { - results.push(relative(baseDir, fullPath)); - } - } - - return results; -} +// ----- Path helpers ----- export function labelFromRelativePath(relativePath: string): string { const normalized = relativePath.replace(/\\/g, "/"); return normalized.replace(/\.md$/, ""); } -/** - * Parse file content and extract block creation data. - * Handles YAML frontmatter for label, description, limit, and read_only. - */ -export function parseBlockFromFileContent( - fileContent: string, - defaultLabel: string, -): { - label: string; - value: string; - description: string; - limit: number; - read_only?: boolean; -} { - const { frontmatter, body } = parseMdxFrontmatter(fileContent); - - // Use frontmatter label if provided, otherwise use default (from file path) - const label = frontmatter.label || defaultLabel; - - // Use frontmatter description if provided, otherwise generate from label - const description = frontmatter.description || `Memory block: ${label}`; - - // Use frontmatter limit if provided and valid, otherwise default to 20000 - let limit = 20000; - if (frontmatter.limit) { - const parsed = Number.parseInt(frontmatter.limit, 10); - if (!Number.isNaN(parsed) && parsed > 0) { - limit = parsed; - } - } - - // Check if block should be read-only (from frontmatter or known read-only labels) - const isReadOnly = - frontmatter.read_only === "true" || - (READ_ONLY_BLOCK_LABELS as readonly string[]).includes(label); - - return { - label, - value: body, - description, - limit, - ...(isReadOnly && { read_only: true }), - }; -} - -async function readMemoryFiles( - dir: string, - excludeDirs: string[] = [], -): Promise> { - const files = await scanMdFiles(dir, dir, excludeDirs); - const entries = new Map(); - - for (const relativePath of files) { - const label = labelFromRelativePath(relativePath); - const fullPath = join(dir, relativePath); - const content = await readFile(fullPath, "utf-8"); - entries.set(label, { content, path: fullPath }); - } - - return entries; -} - -async function ensureFilePath(filePath: string) { - const parent = dirname(filePath); - if (!existsSync(parent)) { - mkdirSync(parent, { recursive: true }); - } -} - -async function writeMemoryFile(dir: string, label: string, content: string) { - const filePath = join(dir, `${label}.md`); - await ensureFilePath(filePath); - await writeFile(filePath, content, "utf-8"); -} +// ----- Tree rendering ----- /** - * Serialize a block to file content with YAML frontmatter. - * Includes description, limit, and read_only (if true) for round-trip preservation. - * Label is omitted since it's implied by the file path. + * Render a tree visualization of the memory filesystem. + * Takes system labels (under system/) and detached labels (at root). */ -export function renderBlockToFileContent(block: { - value?: string | null; - description?: string | null; - limit?: number | null; - read_only?: boolean | null; -}): string { - const lines: string[] = ["---"]; - - // Always include description (so it round-trips and doesn't silently reset) - if (block.description) { - // Escape description for YAML if it contains special chars - const desc = - block.description.includes(":") || block.description.includes("\n") - ? `"${block.description.replace(/"/g, '\\"')}"` - : block.description; - lines.push(`description: ${desc}`); - } - - // Always include limit - if (block.limit) { - lines.push(`limit: ${block.limit}`); - } - - // Only include read_only if true (avoid cluttering frontmatter) - if (block.read_only === true) { - lines.push("read_only: true"); - } - - lines.push("---"); - lines.push(""); // blank line after frontmatter - lines.push(block.value || ""); - - return lines.join("\n"); -} - -/** - * Parse file content for UPDATING an existing block. - * Only includes metadata fields that are explicitly present in frontmatter. - * This prevents overwriting existing API metadata when frontmatter is absent. - */ -export function parseBlockUpdateFromFileContent( - fileContent: string, - defaultLabel: string, -): { - label: string; - value: string; - description?: string; - limit?: number; - read_only?: boolean; - // Flags indicating which fields were explicitly present - hasDescription: boolean; - hasLimit: boolean; - hasReadOnly: boolean; -} { - const { frontmatter, body } = parseMdxFrontmatter(fileContent); - - const label = frontmatter.label || defaultLabel; - - // Check explicit presence using hasOwnProperty - const hasDescription = Object.hasOwn(frontmatter, "description"); - const hasLimit = Object.hasOwn(frontmatter, "limit"); - const hasReadOnly = Object.hasOwn(frontmatter, "read_only"); - - let limit: number | undefined; - if (hasLimit && frontmatter.limit) { - const parsed = Number.parseInt(frontmatter.limit, 10); - if (!Number.isNaN(parsed) && parsed > 0) { - limit = parsed; - } - } - - return { - label, - value: body, - ...(hasDescription && { description: frontmatter.description }), - ...(hasLimit && limit !== undefined && { limit }), - ...(hasReadOnly && { read_only: frontmatter.read_only === "true" }), - hasDescription, - hasLimit, - hasReadOnly, - }; -} - -async function deleteMemoryFile(dir: string, label: string) { - const filePath = join(dir, `${label}.md`); - if (existsSync(filePath)) { - await unlink(filePath); - } -} - -async function fetchAgentBlocks(agentId: string): Promise { - const client = await getClient(); - // Use high limit - SDK's async iterator has a bug that causes infinite loops - const page = await client.agents.blocks.list(agentId, { limit: 1000 }); - - // Handle both array response and paginated response - if (Array.isArray(page)) { - return page; - } - - // Extract items from paginated response - const items = - (page as { items?: Block[] }).items || - (page as { blocks?: Block[] }).blocks || - []; - return items; -} - -/** - * Fetch all blocks owned by this agent (via owner tag). - * This includes both attached and detached blocks. - */ -async function fetchOwnedBlocks(agentId: string): Promise { - const client = await getClient(); - const ownerTag = `owner:${agentId}`; - const page = await client.blocks.list({ tags: [ownerTag], limit: 1000 }); - - // Handle both array response and paginated response - if (Array.isArray(page)) { - return page; - } - - const items = - (page as { items?: Block[] }).items || - (page as { blocks?: Block[] }).blocks || - []; - return items; -} - -/** - * Backfill owner tags on blocks that don't have them. - * This ensures backwards compatibility with blocks created before tagging. - */ -async function backfillOwnerTags( - agentId: string, - blocks: Block[], -): Promise { - const client = await getClient(); - const ownerTag = `owner:${agentId}`; - - for (const block of blocks) { - if (!block.id) continue; - const tags = block.tags || []; - if (!tags.includes(ownerTag)) { - await client.blocks.update(block.id, { - tags: [...tags, ownerTag], - }); - } - } -} - export function renderMemoryFilesystemTree( systemLabels: string[], detachedLabels: string[], @@ -484,7 +74,6 @@ export function renderMemoryFilesystemTree( const root = makeNode(); const insertPath = (base: string | null, label: string) => { - // If base is null, insert at root level const parts = base ? [base, ...label.split("/")] : label.split("/"); let current = root; for (const [i, partName] of parts.entries()) { @@ -499,11 +88,9 @@ export function renderMemoryFilesystemTree( } }; - // System blocks go in /system/ for (const label of systemLabels) { insertPath(MEMORY_SYSTEM_DIR, label); } - // Detached blocks go at root level (flat structure) for (const label of detachedLabels) { insertPath(null, label); } @@ -542,826 +129,3 @@ export function renderMemoryFilesystemTree( return lines.join("\n"); } - -function buildStateHashes( - allBlocks: Map, - allFiles: Map, -): SyncState { - const blockHashes: Record = {}; - const fileHashes: Record = {}; - const blockIds: Record = {}; - - allBlocks.forEach((block, label) => { - blockHashes[label] = hashContent(block.value || ""); - if (block.id) { - blockIds[label] = block.id; - } - }); - - allFiles.forEach((file, label) => { - fileHashes[label] = hashContent(file.content || ""); - }); - - return { - blockHashes, - fileHashes, - blockIds, - lastSync: new Date().toISOString(), - }; -} - -export async function syncMemoryFilesystem( - agentId: string, - options: { homeDir?: string; resolutions?: MemorySyncResolution[] } = {}, -): Promise { - const homeDir = options.homeDir ?? homedir(); - ensureMemoryFilesystemDirs(agentId, homeDir); - - const systemDir = getMemorySystemDir(agentId, homeDir); - const detachedDir = getMemoryDetachedDir(agentId, homeDir); - const systemFiles = await readMemoryFiles(systemDir); - const detachedFiles = await readMemoryFiles(detachedDir, [MEMORY_SYSTEM_DIR]); - systemFiles.delete(MEMORY_FILESYSTEM_BLOCK_LABEL); - - const attachedBlocks = await fetchAgentBlocks(agentId); - const systemBlockMap = new Map( - attachedBlocks - .filter((block) => block.label) - .map((block) => [block.label as string, block]), - ); - systemBlockMap.delete(MEMORY_FILESYSTEM_BLOCK_LABEL); - - const lastState = loadSyncState(agentId, homeDir); - const conflicts: MemorySyncConflict[] = []; - - const updatedBlocks: string[] = []; - const createdBlocks: string[] = []; - const deletedBlocks: string[] = []; - const updatedFiles: string[] = []; - const createdFiles: string[] = []; - const deletedFiles: string[] = []; - - const resolutions = new Map( - (options.resolutions ?? []).map((resolution) => [ - resolution.label, - resolution, - ]), - ); - - const client = await getClient(); - - // Backfill owner tags on attached blocks (for backwards compat) - await backfillOwnerTags(agentId, attachedBlocks); - - // Discover detached blocks via owner tag - const allOwnedBlocks = await fetchOwnedBlocks(agentId); - const attachedIds = new Set(attachedBlocks.map((b) => b.id)); - const detachedBlocks = allOwnedBlocks.filter((b) => !attachedIds.has(b.id)); - - // Build detached block map - const detachedBlockMap = new Map(); - for (const block of detachedBlocks) { - if (block.label && block.id) { - // Skip memfs-managed blocks (memory_filesystem has special handling) - if (MEMFS_MANAGED_LABELS.has(block.label)) { - continue; - } - // Skip blocks whose label matches a system block (prevents duplicates) - // This can happen when a system block is detached but keeps its owner tag - if (systemBlockMap.has(block.label)) { - continue; - } - detachedBlockMap.set(block.label, block); - } - } - - // Unified sync loop - collect all labels and process once - // The attached/detached distinction is determined at runtime - const allLabels = new Set([ - ...Array.from(systemFiles.keys()), - ...Array.from(detachedFiles.keys()), - ...Array.from(systemBlockMap.keys()), - ...Array.from(detachedBlockMap.keys()), - ...Object.keys(lastState.blockHashes), - ...Object.keys(lastState.fileHashes), - ]); - - // Track all blocks for state saving - const allBlocksMap = new Map< - string, - { value?: string | null; id?: string } - >(); - const allFilesMap = new Map(); - - for (const label of Array.from(allLabels).sort()) { - // Skip memfs-managed blocks (memory_filesystem has special handling) - if (MEMFS_MANAGED_LABELS.has(label)) { - continue; - } - - // Determine current state at runtime - const systemFile = systemFiles.get(label); - const detachedFile = detachedFiles.get(label); - const attachedBlock = systemBlockMap.get(label); - const detachedBlock = detachedBlockMap.get(label); - - // Derive file and block entries - const fileEntry = systemFile || detachedFile; - const fileInSystem = !!systemFile; - const blockEntry = attachedBlock || detachedBlock; - const isAttached = !!attachedBlock; - const isReadOnlyLabel = ( - READ_ONLY_BLOCK_LABELS as readonly string[] - ).includes(label); - const effectiveReadOnly = !!blockEntry?.read_only || isReadOnlyLabel; - - // Get directory for file operations - const fileDir = fileInSystem ? systemDir : detachedDir; - - const fileHash = fileEntry ? hashContent(fileEntry.content) : null; - // Body hash excludes frontmatter - used for "content matches" checks - const fileBodyHash = fileEntry ? hashFileBody(fileEntry.content) : null; - const blockHash = blockEntry ? hashContent(blockEntry.value || "") : null; - - // Use unified hash lookup - const lastFileHash = lastState.fileHashes[label] || null; - const lastBlockHash = lastState.blockHashes[label] || null; - - const fileChanged = fileHash !== lastFileHash; - const blockChanged = blockHash !== lastBlockHash; - - const resolution = resolutions.get(label); - - // Track for state saving - if (blockEntry) { - allBlocksMap.set(label, { value: blockEntry.value, id: blockEntry.id }); - } - if (fileEntry) { - allFilesMap.set(label, { content: fileEntry.content }); - } - - // Case 1: File exists, no block - if (fileEntry && !blockEntry) { - if (lastBlockHash && !fileChanged) { - // Block was deleted elsewhere; delete file - await deleteMemoryFile(fileDir, label); - deletedFiles.push(label); - allFilesMap.delete(label); - continue; - } - - // Read-only labels are API-authoritative. If file exists but block doesn't, delete the file. - if ((READ_ONLY_BLOCK_LABELS as readonly string[]).includes(label)) { - await deleteMemoryFile(fileDir, label); - deletedFiles.push(label); - allFilesMap.delete(label); - continue; - } - - // Create block from file - const blockData = parseBlockFromFileContent(fileEntry.content, label); - const createdBlock = await client.blocks.create({ - ...blockData, - tags: [`owner:${agentId}`], - }); - if (createdBlock.id) { - // Policy: attach if file is in system/, don't attach if at root - if (fileInSystem) { - await client.agents.blocks.attach(createdBlock.id, { - agent_id: agentId, - }); - } - allBlocksMap.set(label, { - value: createdBlock.value, - id: createdBlock.id, - }); - } - createdBlocks.push(blockData.label); - continue; - } - - // Case 2: Block exists, no file - if (!fileEntry && blockEntry) { - // Read-only blocks: never delete/un-tag. Always recreate file instead. - if (effectiveReadOnly) { - const targetDir = isAttached ? systemDir : detachedDir; - const fileContent = renderBlockToFileContent(blockEntry); - await writeMemoryFile(targetDir, label, fileContent); - createdFiles.push(label); - allFilesMap.set(label, { content: fileContent }); - continue; - } - - if (lastFileHash && !blockChanged) { - // File deleted, block unchanged → remove owner tag so file doesn't resurrect - if (blockEntry.id) { - try { - if (isAttached) { - // Detach the attached block first - await client.agents.blocks.detach(blockEntry.id, { - agent_id: agentId, - }); - } - // Remove owner tag from block - const currentTags = blockEntry.tags || []; - const newTags = currentTags.filter( - (tag) => !tag.startsWith(`owner:${agentId}`), - ); - await client.blocks.update(blockEntry.id, { tags: newTags }); - allBlocksMap.delete(label); - deletedBlocks.push(label); - } catch (err) { - if (!(err instanceof Error && err.message.includes("Not Found"))) { - throw err; - } - } - } - continue; - } - - // Create file from block - use block's attached status to determine location - const targetDir = isAttached ? systemDir : detachedDir; - const fileContent = renderBlockToFileContent(blockEntry); - await writeMemoryFile(targetDir, label, fileContent); - createdFiles.push(label); - allFilesMap.set(label, { content: fileContent }); - continue; - } - - // Case 3: Neither exists (was in lastState but now gone) - if (!fileEntry || !blockEntry) { - continue; - } - - // Case 4: Both exist - check for sync/conflict/location mismatch - - // Check for location mismatch: file location doesn't match block attachment - const locationMismatch = - (fileInSystem && !isAttached) || (!fileInSystem && isAttached); - - // If content matches but location mismatches, sync attachment to match file location - // Use body hash (excludes frontmatter) for "content matches" check - if (fileBodyHash === blockHash) { - if (locationMismatch && blockEntry.id) { - if (fileInSystem && !isAttached) { - // File in system/, block detached → attach block - await client.agents.blocks.attach(blockEntry.id, { - agent_id: agentId, - }); - } else if (!fileInSystem && isAttached) { - // File at root, block attached → detach block - await client.agents.blocks.detach(blockEntry.id, { - agent_id: agentId, - }); - } - } - - // Frontmatter-only change: update metadata even when body matches - if (fileChanged) { - // Read-only blocks: ignore local changes, overwrite file with API content - if (effectiveReadOnly) { - const fileContent = renderBlockToFileContent(blockEntry); - await writeMemoryFile(fileDir, label, fileContent); - updatedFiles.push(label); - allFilesMap.set(label, { content: fileContent }); - continue; - } - - if (blockEntry.id) { - const parsed = parseBlockUpdateFromFileContent( - fileEntry.content, - label, - ); - const updatePayload: Record = {}; - if (parsed.hasDescription) - updatePayload.description = parsed.description; - if (parsed.hasLimit) updatePayload.limit = parsed.limit; - if (parsed.hasReadOnly) updatePayload.read_only = parsed.read_only; - // For detached blocks, keep label in sync - if (!isAttached) updatePayload.label = label; - - if (Object.keys(updatePayload).length > 0) { - await client.blocks.update(blockEntry.id, updatePayload); - updatedBlocks.push(label); - allBlocksMap.set(label, { - value: parsed.value, - id: blockEntry.id, - }); - } - } - } - continue; - } - - // "FS wins all" policy: if file changed, file wins (even if block also changed) - // Only conflict if explicit resolution provided but doesn't match - if ( - fileChanged && - blockChanged && - resolution && - resolution.resolution === "block" - ) { - // User explicitly requested block wins via resolution for CONTENT - // But FS still wins for LOCATION (attachment status) - const fileContent = renderBlockToFileContent(blockEntry); - await writeMemoryFile(fileDir, label, fileContent); - updatedFiles.push(label); - allFilesMap.set(label, { content: fileContent }); - - // Sync attachment status to match file location (FS wins for location) - if (locationMismatch && blockEntry.id) { - if (fileInSystem && !isAttached) { - await client.agents.blocks.attach(blockEntry.id, { - agent_id: agentId, - }); - } else if (!fileInSystem && isAttached) { - await client.agents.blocks.detach(blockEntry.id, { - agent_id: agentId, - }); - } - } - continue; - } - - // Handle explicit resolution override - if (resolution?.resolution === "block") { - // Block wins for CONTENT, but FS wins for LOCATION - const fileContent = renderBlockToFileContent(blockEntry); - await writeMemoryFile(fileDir, label, fileContent); - updatedFiles.push(label); - allFilesMap.set(label, { content: fileContent }); - - // Sync attachment status to match file location (FS wins for location) - if (locationMismatch && blockEntry.id) { - if (fileInSystem && !isAttached) { - await client.agents.blocks.attach(blockEntry.id, { - agent_id: agentId, - }); - } else if (!fileInSystem && isAttached) { - await client.agents.blocks.detach(blockEntry.id, { - agent_id: agentId, - }); - } - } - continue; - } - - // "FS wins all": if file changed at all, file wins (update block from file) - // EXCEPT for read_only blocks - those are API → file only (ignore local changes) - // Also sync attachment status to match file location - if (fileChanged) { - // Read-only blocks: ignore local changes, overwrite file with API content - if (effectiveReadOnly) { - const fileContent = renderBlockToFileContent(blockEntry); - await writeMemoryFile(fileDir, label, fileContent); - updatedFiles.push(label); - allFilesMap.set(label, { content: fileContent }); - continue; - } - - if (blockEntry.id) { - try { - // Use update-mode parsing to preserve metadata not in frontmatter - const parsed = parseBlockUpdateFromFileContent( - fileEntry.content, - label, - ); - const updatePayload: Record = { - value: parsed.value, - }; - // Only include metadata if explicitly present in frontmatter - if (parsed.hasDescription) - updatePayload.description = parsed.description; - if (parsed.hasLimit) updatePayload.limit = parsed.limit; - if (parsed.hasReadOnly) updatePayload.read_only = parsed.read_only; - // For detached blocks, also update label if changed - if (!isAttached) updatePayload.label = label; - await client.blocks.update(blockEntry.id, updatePayload); - updatedBlocks.push(label); - allBlocksMap.set(label, { - value: parsed.value, - id: blockEntry.id, - }); - - // Sync attachment status to match file location (FS wins for location too) - if (locationMismatch) { - if (fileInSystem && !isAttached) { - await client.agents.blocks.attach(blockEntry.id, { - agent_id: agentId, - }); - } else if (!fileInSystem && isAttached) { - await client.agents.blocks.detach(blockEntry.id, { - agent_id: agentId, - }); - } - } - } catch (err) { - if (err instanceof Error && err.message.includes("Not Found")) { - // Block was deleted - create a new one - const blockData = parseBlockFromFileContent( - fileEntry.content, - label, - ); - const createdBlock = await client.blocks.create({ - ...blockData, - tags: [`owner:${agentId}`], - }); - if (createdBlock.id) { - if (fileInSystem) { - await client.agents.blocks.attach(createdBlock.id, { - agent_id: agentId, - }); - } - allBlocksMap.set(label, { - value: createdBlock.value, - id: createdBlock.id, - }); - } - createdBlocks.push(blockData.label); - } else { - throw err; - } - } - } - continue; - } - - // Only block changed (file unchanged) → update file from block - // Also sync attachment status to match file location - if (blockChanged) { - const fileContent = renderBlockToFileContent(blockEntry); - await writeMemoryFile(fileDir, label, fileContent); - updatedFiles.push(label); - allFilesMap.set(label, { content: fileContent }); - - // Sync attachment status to match file location (FS wins for location) - if (locationMismatch && blockEntry.id) { - if (fileInSystem && !isAttached) { - await client.agents.blocks.attach(blockEntry.id, { - agent_id: agentId, - }); - } else if (!fileInSystem && isAttached) { - await client.agents.blocks.detach(blockEntry.id, { - agent_id: agentId, - }); - } - } - } - } - - // Save state if no conflicts - if (conflicts.length === 0) { - const nextState = buildStateHashes(allBlocksMap, allFilesMap); - await saveSyncState(nextState, agentId, homeDir); - } - - return { - updatedBlocks, - createdBlocks, - deletedBlocks, - updatedFiles, - createdFiles, - deletedFiles, - conflicts, - }; -} - -export async function updateMemoryFilesystemBlock( - agentId: string, - homeDir: string = homedir(), -) { - const systemDir = getMemorySystemDir(agentId, homeDir); - const detachedDir = getMemoryDetachedDir(agentId, homeDir); - - const systemFiles = await readMemoryFiles(systemDir); - const detachedFiles = await readMemoryFiles(detachedDir, [MEMORY_SYSTEM_DIR]); - - const tree = renderMemoryFilesystemTree( - Array.from(systemFiles.keys()).filter( - (label) => label !== MEMORY_FILESYSTEM_BLOCK_LABEL, - ), - Array.from(detachedFiles.keys()), - ); - - // Prepend memory directory path (tilde format for readability) - const memoryPath = `~/.letta/agents/${agentId}/memory`; - const content = `Memory Directory: ${memoryPath}\n\n${tree}`; - - const client = await getClient(); - const blocks = await fetchAgentBlocks(agentId); - const memfsBlock = blocks.find( - (block) => block.label === MEMORY_FILESYSTEM_BLOCK_LABEL, - ); - - if (memfsBlock?.id) { - await client.blocks.update(memfsBlock.id, { value: content }); - - // Write file with frontmatter (consistent with other blocks) - const fileContent = renderBlockToFileContent({ - value: content, - description: memfsBlock.description, - limit: memfsBlock.limit, - read_only: memfsBlock.read_only, - }); - await writeMemoryFile( - systemDir, - MEMORY_FILESYSTEM_BLOCK_LABEL, - fileContent, - ); - } -} - -export async function ensureMemoryFilesystemBlock(agentId: string) { - const client = await getClient(); - const blocks = await fetchAgentBlocks(agentId); - const exists = blocks.some( - (block) => block.label === MEMORY_FILESYSTEM_BLOCK_LABEL, - ); - - if (exists) { - return; - } - - const createdBlock = await client.blocks.create({ - label: MEMORY_FILESYSTEM_BLOCK_LABEL, - value: "/memory/", - description: "Filesystem view of memory blocks", - limit: 20000, - read_only: true, - tags: [`owner:${agentId}`], - }); - - if (createdBlock.id) { - await client.agents.blocks.attach(createdBlock.id, { agent_id: agentId }); - } -} - -export async function refreshMemoryFilesystemTree( - agentId: string, - homeDir: string = homedir(), -) { - ensureMemoryFilesystemDirs(agentId, homeDir); - await updateMemoryFilesystemBlock(agentId, homeDir); -} - -export async function collectMemorySyncConflicts( - agentId: string, - homeDir: string = homedir(), -): Promise { - const result = await syncMemoryFilesystem(agentId, { homeDir }); - return result.conflicts; -} - -export function formatMemorySyncSummary(result: MemorySyncResult): string { - const lines: string[] = []; - const pushCount = (label: string, count: number) => { - if (count > 0) { - lines.push(`⎿ ${label}: ${count}`); - } - }; - - pushCount("Blocks updated", result.updatedBlocks.length); - pushCount("Blocks created", result.createdBlocks.length); - pushCount("Blocks deleted", result.deletedBlocks.length); - pushCount("Files updated", result.updatedFiles.length); - pushCount("Files created", result.createdFiles.length); - pushCount("Files deleted", result.deletedFiles.length); - - if (result.conflicts.length > 0) { - lines.push(`⎿ Conflicts: ${result.conflicts.length}`); - } - - if (lines.length === 0) { - return "Memory filesystem sync complete (no changes needed)"; - } - - return `Memory filesystem sync complete:\n${lines.join("\n")}`; -} - -/** - * Read-only check of the current memFS sync status. - * Does NOT modify any blocks, files, or sync state. - * Safe to call frequently (e.g., after every turn). - */ -export async function checkMemoryFilesystemStatus( - agentId: string, - options?: { homeDir?: string }, -): Promise { - const homeDir = options?.homeDir ?? homedir(); - ensureMemoryFilesystemDirs(agentId, homeDir); - - const systemDir = getMemorySystemDir(agentId, homeDir); - const detachedDir = getMemoryDetachedDir(agentId, homeDir); - const systemFiles = await readMemoryFiles(systemDir); - const detachedFiles = await readMemoryFiles(detachedDir, [MEMORY_SYSTEM_DIR]); - systemFiles.delete(MEMORY_FILESYSTEM_BLOCK_LABEL); - - const attachedBlocks = await fetchAgentBlocks(agentId); - const systemBlockMap = new Map( - attachedBlocks - .filter((block) => block.label) - .map((block) => [block.label as string, block]), - ); - systemBlockMap.delete(MEMORY_FILESYSTEM_BLOCK_LABEL); - - const lastState = loadSyncState(agentId, homeDir); - - const conflicts: MemorySyncConflict[] = []; - const pendingFromFile: string[] = []; - const pendingFromBlock: string[] = []; - const newFiles: string[] = []; - const newBlocks: string[] = []; - const locationMismatches: string[] = []; - - // Discover detached blocks via owner tag - const allOwnedBlocks = await fetchOwnedBlocks(agentId); - const attachedIds = new Set(attachedBlocks.map((b) => b.id)); - const detachedBlocks = allOwnedBlocks.filter((b) => !attachedIds.has(b.id)); - - const detachedBlockMap = new Map(); - for (const block of detachedBlocks) { - if (block.label) { - // Skip memfs-managed blocks - if (MEMFS_MANAGED_LABELS.has(block.label)) { - continue; - } - // Skip blocks whose label matches a system block (prevents duplicates) - if (systemBlockMap.has(block.label)) { - continue; - } - detachedBlockMap.set(block.label, block); - } - } - - // Unified label check - collect all labels and classify once - const allLabels = new Set([ - ...Array.from(systemFiles.keys()), - ...Array.from(detachedFiles.keys()), - ...Array.from(systemBlockMap.keys()), - ...Array.from(detachedBlockMap.keys()), - ...Object.keys(lastState.blockHashes), - ...Object.keys(lastState.fileHashes), - ]); - - for (const label of Array.from(allLabels).sort()) { - // Skip memfs-managed blocks (memory_filesystem has special handling) - if (MEMFS_MANAGED_LABELS.has(label)) continue; - - // Determine current state at runtime - const systemFile = systemFiles.get(label); - const detachedFile = detachedFiles.get(label); - const attachedBlock = systemBlockMap.get(label); - const detachedBlock = detachedBlockMap.get(label); - - const fileContent = systemFile?.content ?? detachedFile?.content ?? null; - const blockValue = attachedBlock?.value ?? detachedBlock?.value ?? null; - const blockReadOnly = - (attachedBlock?.read_only ?? detachedBlock?.read_only ?? false) || - (READ_ONLY_BLOCK_LABELS as readonly string[]).includes(label); - - const fileInSystem = !!systemFile; - const isAttached = !!attachedBlock; - - // Check for location mismatch (both file and block exist but location doesn't match) - if (fileContent !== null && blockValue !== null) { - const locationMismatch = - (fileInSystem && !isAttached) || (!fileInSystem && isAttached); - if (locationMismatch) { - locationMismatches.push(label); - } - } - - classifyLabel( - label, - fileContent, - blockValue, - lastState.fileHashes[label] ?? null, - lastState.blockHashes[label] ?? null, - conflicts, - pendingFromFile, - pendingFromBlock, - newFiles, - newBlocks, - blockReadOnly, - ); - } - - const isClean = - conflicts.length === 0 && - pendingFromFile.length === 0 && - pendingFromBlock.length === 0 && - newFiles.length === 0 && - newBlocks.length === 0 && - locationMismatches.length === 0; - - return { - conflicts, - pendingFromFile, - pendingFromBlock, - newFiles, - newBlocks, - locationMismatches, - isClean, - }; -} - -/** - * Classify a single label's sync status (read-only). - * Pushes into the appropriate output array based on file/block state comparison. - */ -function classifyLabel( - label: string, - fileContent: string | null, - blockValue: string | null, - lastFileHash: string | null, - lastBlockHash: string | null, - _conflicts: MemorySyncConflict[], // Unused with "FS wins all" policy (kept for API compatibility) - pendingFromFile: string[], - pendingFromBlock: string[], - newFiles: string[], - newBlocks: string[], - blockReadOnly: boolean, -): void { - const fileHash = fileContent !== null ? hashContent(fileContent) : null; - const fileBodyHash = fileContent !== null ? hashFileBody(fileContent) : null; - const blockHash = blockValue !== null ? hashContent(blockValue) : null; - - const fileChanged = fileHash !== lastFileHash; - const blockChanged = blockHash !== lastBlockHash; - - if (fileContent !== null && blockValue === null) { - if (lastBlockHash && !fileChanged) { - // Block was deleted, file unchanged — would delete file - return; - } - // Ignore file-only read_only labels (API is authoritative, file will be deleted on sync) - if ((READ_ONLY_BLOCK_LABELS as readonly string[]).includes(label)) { - return; - } - newFiles.push(label); - return; - } - - if (fileContent === null && blockValue !== null) { - if (blockReadOnly) { - // Read-only blocks: missing file should be recreated - pendingFromFile.push(label); - return; - } - if (lastFileHash && !blockChanged) { - // File was deleted, block unchanged — would delete block - return; - } - newBlocks.push(label); - return; - } - - if (fileContent === null || blockValue === null) { - return; - } - - // Both exist — check for differences - if (blockReadOnly) { - if (blockChanged) { - pendingFromBlock.push(label); - } - return; - } - - if (fileBodyHash === blockHash) { - if (fileChanged) { - pendingFromFile.push(label); // frontmatter-only change - } - return; // In sync - } - - // "FS wins all" policy: if file changed at all, file wins - // So both-changed is treated as pendingFromFile, not a conflict - if (fileChanged) { - pendingFromFile.push(label); - return; - } - - // Only block changed - if (blockChanged) { - pendingFromBlock.push(label); - } -} - -/** - * Detach the memory_filesystem block from an agent. - * Used when disabling memfs. - */ -export async function detachMemoryFilesystemBlock( - agentId: string, -): Promise { - const client = await getClient(); - const blocks = await fetchAgentBlocks(agentId); - const memfsBlock = blocks.find( - (block) => block.label === MEMORY_FILESYSTEM_BLOCK_LABEL, - ); - - if (memfsBlock?.id) { - await client.agents.blocks.detach(memfsBlock.id, { agent_id: agentId }); - } -} diff --git a/src/agent/memoryGit.ts b/src/agent/memoryGit.ts new file mode 100644 index 0000000..1d79028 --- /dev/null +++ b/src/agent/memoryGit.ts @@ -0,0 +1,435 @@ +/** + * Git operations for git-backed agent memory. + * + * When memFS is enabled, the agent's memory is stored in a git repo + * on the server at $LETTA_BASE_URL/v1/git/$AGENT_ID/state.git. + * This module provides the CLI harness helpers: clone on first run, + * pull on startup, and status check for system reminders. + * + * The agent itself handles commit/push via Bash tool calls. + */ + +import { execFile as execFileCb } from "node:child_process"; +import { + chmodSync, + existsSync, + mkdirSync, + renameSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { debugLog, debugWarn } from "../utils/debug"; +import { getClient, getServerUrl } from "./client"; + +const execFile = promisify(execFileCb); + +const GIT_MEMORY_ENABLED_TAG = "git-memory-enabled"; + +/** Get the agent root directory (~/.letta/agents/{id}/) */ +export function getAgentRootDir(agentId: string): string { + return join(homedir(), ".letta", "agents", agentId); +} + +/** Get the git repo directory for memory (now ~/.letta/agents/{id}/memory/) */ +export function getMemoryRepoDir(agentId: string): string { + return join(getAgentRootDir(agentId), "memory"); +} + +/** Git remote URL for the agent's state repo */ +function getGitRemoteUrl(agentId: string): string { + const baseUrl = getServerUrl(); + return `${baseUrl}/v1/git/${agentId}/state.git`; +} + +/** + * Get a fresh auth token for git operations. + * Reuses the same token resolution flow as getClient() + * (env var → settings → OAuth refresh). + */ +async function getAuthToken(): Promise { + const client = await getClient(); + // The client constructor resolves the token; extract it + // biome-ignore lint/suspicious/noExplicitAny: accessing internal client options + return (client as any)._options?.apiKey ?? ""; +} + +/** + * Run a git command in the given directory. + * If a token is provided, passes it as an auth header. + */ +async function runGit( + cwd: string, + args: string[], + token?: string, +): Promise<{ stdout: string; stderr: string }> { + const authArgs = token + ? [ + "-c", + `http.extraHeader=Authorization: Basic ${Buffer.from(`letta:${token}`).toString("base64")}`, + ] + : []; + const allArgs = [...authArgs, ...args]; + + debugLog("memfs-git", `git ${args.join(" ")} (in ${cwd})`); + + const result = await execFile("git", allArgs, { + cwd, + maxBuffer: 10 * 1024 * 1024, // 10MB + timeout: 60_000, // 60s + }); + + return { + stdout: result.stdout?.toString() ?? "", + stderr: result.stderr?.toString() ?? "", + }; +} + +/** + * Configure a local credential helper in the repo's .git/config + * so plain `git push` / `git pull` work without auth prefixes. + */ +async function configureLocalCredentialHelper( + dir: string, + token: string, +): Promise { + const baseUrl = getServerUrl(); + const helper = `!f() { echo "username=letta"; echo "password=${token}"; }; f`; + await runGit(dir, ["config", `credential.${baseUrl}.helper`, helper]); + debugLog("memfs-git", "Configured local credential helper"); +} + +/** + * Bash pre-commit hook that validates frontmatter in memory .md files. + * + * Rules: + * - Frontmatter is REQUIRED (must start with ---) + * - Must be properly closed with --- + * - Required fields: description (non-empty string), limit (positive integer) + * - read_only is a PROTECTED field: agent cannot add, remove, or change it. + * Files where HEAD has read_only: true cannot be modified at all. + * - Only allowed agent-editable keys: description, limit + * - read_only may exist (from server) but agent must not change it + */ +export const PRE_COMMIT_HOOK_SCRIPT = `#!/usr/bin/env bash +# Validate frontmatter in staged memory .md files +# Installed by Letta Code CLI + +AGENT_EDITABLE_KEYS="description limit" +PROTECTED_KEYS="read_only" +ALL_KNOWN_KEYS="description limit read_only" +errors="" + +# Helper: extract a frontmatter value from content +get_fm_value() { + local content="$1" key="$2" + local closing_line + closing_line=$(echo "$content" | tail -n +2 | grep -n '^---$' | head -1 | cut -d: -f1) + [ -z "$closing_line" ] && return + echo "$content" | tail -n +2 | head -n $((closing_line - 1)) | grep "^$key:" | cut -d: -f2- | sed 's/^ *//;s/ *$//' +} + +for file in $(git diff --cached --name-only --diff-filter=ACM | grep '^memory/.*\\.md$'); do + staged=$(git show ":$file") + + # Frontmatter is required + first_line=$(echo "$staged" | head -1) + if [ "$first_line" != "---" ]; then + errors="$errors\\n $file: missing frontmatter (must start with ---)" + continue + fi + + # Check frontmatter is properly closed + closing_line=$(echo "$staged" | tail -n +2 | grep -n '^---$' | head -1 | cut -d: -f1) + if [ -z "$closing_line" ]; then + errors="$errors\\n $file: frontmatter opened but never closed (missing closing ---)" + continue + fi + + # Check read_only protection against HEAD version + head_content=$(git show "HEAD:$file" 2>/dev/null || true) + if [ -n "$head_content" ]; then + head_ro=$(get_fm_value "$head_content" "read_only") + if [ "$head_ro" = "true" ]; then + errors="$errors\\n $file: file is read_only and cannot be modified" + continue + fi + fi + + # Extract frontmatter lines + frontmatter=$(echo "$staged" | tail -n +2 | head -n $((closing_line - 1))) + + # Track required fields + has_description=false + has_limit=false + + # Validate each line + while IFS= read -r line; do + [ -z "$line" ] && continue + + key=$(echo "$line" | cut -d: -f1 | tr -d ' ') + value=$(echo "$line" | cut -d: -f2- | sed 's/^ *//;s/ *$//') + + # Check key is known + known=false + for k in $ALL_KNOWN_KEYS; do + if [ "$key" = "$k" ]; then + known=true + break + fi + done + if [ "$known" = "false" ]; then + errors="$errors\\n $file: unknown frontmatter key '$key' (allowed: $ALL_KNOWN_KEYS)" + continue + fi + + # Check if agent is trying to modify a protected key + for k in $PROTECTED_KEYS; do + if [ "$key" = "$k" ]; then + # Compare against HEAD — if value changed (or key was added), reject + if [ -n "$head_content" ]; then + head_val=$(get_fm_value "$head_content" "$key") + if [ "$value" != "$head_val" ]; then + errors="$errors\\n $file: '$key' is a protected field and cannot be changed by the agent" + fi + else + # New file with read_only — agent shouldn't set this + errors="$errors\\n $file: '$key' is a protected field and cannot be set by the agent" + fi + fi + done + + # Validate value types + case "$key" in + limit) + has_limit=true + if ! echo "$value" | grep -qE '^[0-9]+$' || [ "$value" = "0" ]; then + errors="$errors\\n $file: 'limit' must be a positive integer, got '$value'" + fi + ;; + description) + has_description=true + if [ -z "$value" ]; then + errors="$errors\\n $file: 'description' must not be empty" + fi + ;; + esac + done <<< "$frontmatter" + + # Check required fields + if [ "$has_description" = "false" ]; then + errors="$errors\\n $file: missing required field 'description'" + fi + if [ "$has_limit" = "false" ]; then + errors="$errors\\n $file: missing required field 'limit'" + fi + + # Check if protected keys were removed (existed in HEAD but not in staged) + if [ -n "$head_content" ]; then + for k in $PROTECTED_KEYS; do + head_val=$(get_fm_value "$head_content" "$k") + if [ -n "$head_val" ]; then + staged_val=$(get_fm_value "$staged" "$k") + if [ -z "$staged_val" ]; then + errors="$errors\\n $file: '$k' is a protected field and cannot be removed by the agent" + fi + fi + done + fi +done + +if [ -n "$errors" ]; then + echo "Frontmatter validation failed:" + echo -e "$errors" + exit 1 +fi +`; + +/** + * Install the pre-commit hook for frontmatter validation. + */ +function installPreCommitHook(dir: string): void { + const hooksDir = join(dir, ".git", "hooks"); + const hookPath = join(hooksDir, "pre-commit"); + + if (!existsSync(hooksDir)) { + mkdirSync(hooksDir, { recursive: true }); + } + + writeFileSync(hookPath, PRE_COMMIT_HOOK_SCRIPT, "utf-8"); + chmodSync(hookPath, 0o755); + debugLog("memfs-git", "Installed pre-commit hook"); +} + +/** Check if the memory directory is a git repo */ +export function isGitRepo(agentId: string): boolean { + return existsSync(join(getMemoryRepoDir(agentId), ".git")); +} + +/** + * Clone the agent's state repo into the memory directory. + * + * Git root is ~/.letta/agents/{id}/memory/ (not the agent root). + */ +export async function cloneMemoryRepo(agentId: string): Promise { + const token = await getAuthToken(); + const url = getGitRemoteUrl(agentId); + const dir = getMemoryRepoDir(agentId); + + debugLog("memfs-git", `Cloning ${url} → ${dir}`); + + if (!existsSync(dir)) { + // Fresh clone into new memory directory + mkdirSync(dir, { recursive: true }); + await runGit(dir, ["clone", url, "."], token); + } else if (!existsSync(join(dir, ".git"))) { + // Directory exists but isn't a git repo (legacy local layout) + // Clone to temp, move .git/ into existing dir, then checkout files. + const tmpDir = `${dir}-git-clone-tmp`; + try { + if (existsSync(tmpDir)) { + rmSync(tmpDir, { recursive: true, force: true }); + } + mkdirSync(tmpDir, { recursive: true }); + await runGit(tmpDir, ["clone", url, "."], token); + + // Move .git into the existing memory directory + renameSync(join(tmpDir, ".git"), join(dir, ".git")); + + // Reset to match remote state + await runGit(dir, ["checkout", "--", "."], token); + + debugLog("memfs-git", "Migrated existing memory directory to git repo"); + } finally { + if (existsSync(tmpDir)) { + rmSync(tmpDir, { recursive: true, force: true }); + } + } + } + + // Configure local credential helper so the agent can do plain + // `git push` / `git pull` without auth prefixes. + await configureLocalCredentialHelper(dir, token); + + // Install pre-commit hook to validate frontmatter + installPreCommitHook(dir); +} + +/** + * Pull latest changes from the server. + * Called on startup to ensure local state is current. + */ +export async function pullMemory( + agentId: string, +): Promise<{ updated: boolean; summary: string }> { + const token = await getAuthToken(); + const dir = getMemoryRepoDir(agentId); + + // Self-healing: ensure credential helper and pre-commit hook are configured + await configureLocalCredentialHelper(dir, token); + installPreCommitHook(dir); + + try { + const { stdout, stderr } = await runGit(dir, ["pull", "--ff-only"]); + const output = stdout + stderr; + const updated = !output.includes("Already up to date"); + return { + updated, + summary: updated ? output.trim() : "Already up to date", + }; + } catch { + // If ff-only fails (diverged), try rebase + debugWarn("memfs-git", "Fast-forward pull failed, trying rebase"); + try { + const { stdout, stderr } = await runGit(dir, ["pull", "--rebase"]); + return { updated: true, summary: (stdout + stderr).trim() }; + } catch (rebaseErr) { + const msg = + rebaseErr instanceof Error ? rebaseErr.message : String(rebaseErr); + debugWarn("memfs-git", `Pull failed: ${msg}`); + return { updated: false, summary: `Pull failed: ${msg}` }; + } + } +} + +export interface MemoryGitStatus { + /** Uncommitted changes in working tree */ + dirty: boolean; + /** Local commits not pushed to remote */ + aheadOfRemote: boolean; + /** Human-readable summary for system reminder */ + summary: string; +} + +/** + * Check git status of the memory directory. + * Used to decide whether to inject a sync reminder. + */ +export async function getMemoryGitStatus( + agentId: string, +): Promise { + const dir = getMemoryRepoDir(agentId); + + // Check for uncommitted changes + const { stdout: statusOut } = await runGit(dir, ["status", "--porcelain"]); + const dirty = statusOut.trim().length > 0; + + // Check if local is ahead of remote + let aheadOfRemote = false; + try { + const { stdout: revListOut } = await runGit(dir, [ + "rev-list", + "--count", + "@{u}..HEAD", + ]); + const aheadCount = parseInt(revListOut.trim(), 10); + aheadOfRemote = aheadCount > 0; + } catch { + // No upstream configured or other error - ignore + } + + // Build summary + const parts: string[] = []; + if (dirty) { + const changedFiles = statusOut + .trim() + .split("\n") + .filter((l) => l.trim()) + .map((l) => l.trim()); + parts.push(`${changedFiles.length} uncommitted change(s)`); + } + if (aheadOfRemote) { + parts.push("local commits not pushed to remote"); + } + + return { + dirty, + aheadOfRemote, + summary: parts.length > 0 ? parts.join(", ") : "clean", + }; +} + +/** + * Add the git-memory-enabled tag to an agent. + * This triggers the backend to create the git repo. + */ +export async function addGitMemoryTag(agentId: string): Promise { + const client = await getClient(); + try { + const agent = await client.agents.retrieve(agentId); + const tags = agent.tags || []; + if (!tags.includes(GIT_MEMORY_ENABLED_TAG)) { + await client.agents.update(agentId, { + tags: [...tags, GIT_MEMORY_ENABLED_TAG], + }); + debugLog("memfs-git", `Added ${GIT_MEMORY_ENABLED_TAG} tag`); + } + } catch (err) { + debugWarn( + "memfs-git", + `Failed to add git-memory tag: ${err instanceof Error ? err.message : String(err)}`, + ); + } +} diff --git a/src/agent/prompts/system_prompt_memfs.txt b/src/agent/prompts/system_prompt_memfs.txt index 5fc3f19..6c7e225 100644 --- a/src/agent/prompts/system_prompt_memfs.txt +++ b/src/agent/prompts/system_prompt_memfs.txt @@ -1,65 +1,44 @@ -## Memory Filesystem (memFS) - -Your memory blocks are mirrored as Markdown files on disk at: -`~/.letta/agents//memory/` - -This provides: -- **Persistent storage**: memory edits survive restarts and can be version-controlled -- **Two-way sync**: edits in files sync to memory blocks, and edits in blocks sync to files -- **Visibility**: a `memory_filesystem` block shows the current on-disk tree +## Memory Filesystem +Your memory is stored in a git repository at `~/.letta/agents//memory/`. This provides full version control, sync with the server, and branching for parallel edits. ### Structure - ``` -~/.letta/agents//memory/ -├── system/ # Attached system blocks (in system prompt) -│ ├── persona/ # Namespaced blocks (e.g. persona/git_safety.md) -│ ├── human.md -│ └── ... -├──