diff --git a/src/agent/memoryFilesystem.ts b/src/agent/memoryFilesystem.ts index e87de10..19abde6 100644 --- a/src/agent/memoryFilesystem.ts +++ b/src/agent/memoryFilesystem.ts @@ -814,6 +814,41 @@ export async function syncMemoryFilesystem( }); } } + + // Frontmatter-only change: update metadata even when body matches + if (fileChanged) { + // Read-only blocks: ignore local changes, overwrite file with API content + if (blockEntry.read_only) { + const fileContent = renderBlockToFileContent(blockEntry); + await writeMemoryFile(fileDir, label, fileContent); + updatedFiles.push(label); + allFilesMap.set(label, { content: fileContent }); + continue; + } + + if (blockEntry.id) { + const parsed = parseBlockUpdateFromFileContent( + fileEntry.content, + label, + ); + const updatePayload: Record = {}; + if (parsed.hasDescription) + updatePayload.description = parsed.description; + if (parsed.hasLimit) updatePayload.limit = parsed.limit; + if (parsed.hasReadOnly) updatePayload.read_only = parsed.read_only; + // For detached blocks, keep label in sync + if (!isAttached) updatePayload.label = label; + + if (Object.keys(updatePayload).length > 0) { + await client.blocks.update(blockEntry.id, updatePayload); + updatedBlocks.push(label); + allBlocksMap.set(label, { + value: parsed.value, + id: blockEntry.id, + }); + } + } + } continue; } @@ -1179,6 +1214,8 @@ export async function checkMemoryFilesystemStatus( const fileContent = systemFile?.content ?? detachedFile?.content ?? null; const blockValue = attachedBlock?.value ?? detachedBlock?.value ?? null; + const blockReadOnly = + attachedBlock?.read_only ?? detachedBlock?.read_only ?? false; const fileInSystem = !!systemFile; const isAttached = !!attachedBlock; @@ -1203,6 +1240,7 @@ export async function checkMemoryFilesystemStatus( pendingFromBlock, newFiles, newBlocks, + blockReadOnly, ); } @@ -1240,8 +1278,10 @@ function classifyLabel( pendingFromBlock: string[], newFiles: string[], newBlocks: string[], + blockReadOnly: boolean, ): void { const fileHash = fileContent !== null ? hashContent(fileContent) : null; + const fileBodyHash = fileContent !== null ? hashFileBody(fileContent) : null; const blockHash = blockValue !== null ? hashContent(blockValue) : null; const fileChanged = fileHash !== lastFileHash; @@ -1274,7 +1314,17 @@ function classifyLabel( } // Both exist — check for differences - if (fileHash === blockHash) { + if (blockReadOnly) { + if (blockChanged) { + pendingFromBlock.push(label); + } + return; + } + + if (fileBodyHash === blockHash) { + if (fileChanged) { + pendingFromFile.push(label); // frontmatter-only change + } return; // In sync } diff --git a/src/skills/builtin/syncing-memory-filesystem/scripts/memfs-diff.ts b/src/skills/builtin/syncing-memory-filesystem/scripts/memfs-diff.ts index 9dbe58b..1759fbb 100644 --- a/src/skills/builtin/syncing-memory-filesystem/scripts/memfs-diff.ts +++ b/src/skills/builtin/syncing-memory-filesystem/scripts/memfs-diff.ts @@ -45,12 +45,11 @@ function getApiKey(): string { const MEMORY_FS_STATE_FILE = ".sync-state.json"; +// Unified sync state format (matches main memoryFilesystem.ts) type SyncState = { - systemBlocks: Record; - systemFiles: Record; - detachedBlocks: Record; - detachedFiles: Record; - detachedBlockIds: Record; + blockHashes: Record; + fileHashes: Record; + blockIds: Record; lastSync: string | null; }; @@ -58,6 +57,50 @@ function hashContent(content: string): string { return createHash("sha256").update(content).digest("hex"); } +/** + * Parse frontmatter from file content. + */ +function parseFrontmatter(content: string): { + frontmatter: Record; + body: string; +} { + const frontmatterRegex = /^---\n([\s\S]*?)\n---\n([\s\S]*)$/; + const match = content.match(frontmatterRegex); + + if (!match || !match[1] || !match[2]) { + return { frontmatter: {}, body: content }; + } + + const frontmatterText = match[1]; + const body = match[2]; + const frontmatter: Record = {}; + + for (const line of frontmatterText.split("\n")) { + const colonIdx = line.indexOf(":"); + if (colonIdx > 0) { + const key = line.slice(0, colonIdx).trim(); + let value = line.slice(colonIdx + 1).trim(); + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + value = value.slice(1, -1); + } + frontmatter[key] = value; + } + } + + return { frontmatter, body }; +} + +/** + * Hash just the body of file content (excluding frontmatter). + */ +function hashFileBody(content: string): string { + const { body } = parseFrontmatter(content); + return hashContent(body); +} + function getMemoryRoot(agentId: string): string { return join(homedir(), ".letta", "agents", agentId, "memory"); } @@ -66,49 +109,45 @@ function loadSyncState(agentId: string): SyncState { const statePath = join(getMemoryRoot(agentId), MEMORY_FS_STATE_FILE); if (!existsSync(statePath)) { return { - systemBlocks: {}, - systemFiles: {}, - detachedBlocks: {}, - detachedFiles: {}, - detachedBlockIds: {}, + blockHashes: {}, + fileHashes: {}, + blockIds: {}, lastSync: null, }; } try { const raw = readFileSync(statePath, "utf-8"); - const parsed = JSON.parse(raw) as Partial & { - blocks?: Record; - files?: Record; - }; + const parsed = JSON.parse(raw); return { - systemBlocks: parsed.systemBlocks || parsed.blocks || {}, - systemFiles: parsed.systemFiles || parsed.files || {}, - detachedBlocks: parsed.detachedBlocks || {}, - detachedFiles: parsed.detachedFiles || {}, - detachedBlockIds: parsed.detachedBlockIds || {}, + blockHashes: parsed.blockHashes || {}, + fileHashes: parsed.fileHashes || {}, + blockIds: parsed.blockIds || {}, lastSync: parsed.lastSync || null, }; } catch { return { - systemBlocks: {}, - systemFiles: {}, - detachedBlocks: {}, - detachedFiles: {}, - detachedBlockIds: {}, + blockHashes: {}, + fileHashes: {}, + blockIds: {}, lastSync: null, }; } } -async function scanMdFiles(dir: string, baseDir = dir): Promise { +async function scanMdFiles( + dir: string, + baseDir = dir, + excludeDirs: string[] = [], +): Promise { if (!existsSync(dir)) return []; const entries = await readdir(dir, { withFileTypes: true }); const results: string[] = []; for (const entry of entries) { const fullPath = join(dir, entry.name); if (entry.isDirectory()) { - results.push(...(await scanMdFiles(fullPath, baseDir))); + if (excludeDirs.includes(entry.name)) continue; + results.push(...(await scanMdFiles(fullPath, baseDir, excludeDirs))); } else if (entry.isFile() && entry.name.endsWith(".md")) { results.push(relative(baseDir, fullPath)); } @@ -122,8 +161,9 @@ function labelFromPath(relativePath: string): string { async function readMemoryFiles( dir: string, + excludeDirs: string[] = [], ): Promise> { - const files = await scanMdFiles(dir); + const files = await scanMdFiles(dir, dir, excludeDirs); const entries = new Map(); for (const rel of files) { const label = labelFromPath(rel); @@ -133,11 +173,8 @@ async function readMemoryFiles( return entries; } -const MANAGED_LABELS = new Set([ - "memory_filesystem", - "skills", - "loaded_skills", -]); +// Only memory_filesystem is managed by memfs itself +const MEMFS_MANAGED_LABELS = new Set(["memory_filesystem"]); interface Conflict { label: string; @@ -145,6 +182,12 @@ interface Conflict { blockContent: string; } +interface MetadataChange { + label: string; + fileContent: string; + blockContent: string; +} + /** * Get the overflow directory following the same pattern as tool output overflow. * Pattern: ~/.letta/projects//agent-tools/ @@ -160,122 +203,161 @@ function getOverflowDirectory(): string { return join(homedir(), ".letta", "projects", sanitizedPath, "agent-tools"); } -async function findConflicts(agentId: string): Promise { +async function findConflicts(agentId: string): Promise<{ + conflicts: Conflict[]; + metadataOnly: MetadataChange[]; +}> { const baseUrl = process.env.LETTA_BASE_URL || "https://api.letta.com"; const client = new Letta({ apiKey: getApiKey(), baseUrl }); const root = getMemoryRoot(agentId); const systemDir = join(root, "system"); - // Detached files go at root level (flat structure) const detachedDir = root; for (const dir of [root, systemDir]) { if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); } + // Read files from both locations const systemFiles = await readMemoryFiles(systemDir); - const detachedFiles = await readMemoryFiles(detachedDir); - systemFiles.delete("memory_filesystem"); + const detachedFiles = await readMemoryFiles(detachedDir, ["system", "user"]); + // Fetch attached blocks const blocksResponse = await client.agents.blocks.list(agentId, { limit: 1000, }); - const blocks = Array.isArray(blocksResponse) + const attachedBlocks = Array.isArray(blocksResponse) ? blocksResponse : ((blocksResponse as { items?: unknown[] }).items as Array<{ + id?: string; label?: string; value?: string; + read_only?: boolean; }>) || []; - const systemBlockMap = new Map( - blocks - .filter((b: { label?: string }) => b.label) - .map((b: { label?: string; value?: string }) => [ - b.label as string, - b.value || "", - ]), - ); - systemBlockMap.delete("memory_filesystem"); + const systemBlockMap = new Map< + string, + { value: string; id: string; read_only?: boolean } + >(); + for (const block of attachedBlocks) { + if (block.label && block.id) { + systemBlockMap.set(block.label, { + value: block.value || "", + id: block.id, + read_only: block.read_only, + }); + } + } + + // Fetch detached blocks via owner tag + const ownedBlocksResponse = await client.blocks.list({ + tags: [`owner:${agentId}`], + limit: 1000, + }); + const ownedBlocks = Array.isArray(ownedBlocksResponse) + ? ownedBlocksResponse + : ((ownedBlocksResponse as { items?: unknown[] }).items as Array<{ + id?: string; + label?: string; + value?: string; + read_only?: boolean; + }>) || []; + + const attachedIds = new Set(attachedBlocks.map((b) => b.id)); + const detachedBlockMap = new Map< + string, + { value: string; id: string; read_only?: boolean } + >(); + for (const block of ownedBlocks) { + if (block.label && block.id && !attachedIds.has(block.id)) { + if (!systemBlockMap.has(block.label)) { + detachedBlockMap.set(block.label, { + value: block.value || "", + id: block.id, + read_only: block.read_only, + }); + } + } + } const lastState = loadSyncState(agentId); - - const detachedBlockMap = new Map(); - for (const [label, blockId] of Object.entries(lastState.detachedBlockIds)) { - try { - const block = await client.blocks.retrieve(blockId); - detachedBlockMap.set(label, block.value || ""); - } catch { - // Block no longer exists - } - } - const conflicts: Conflict[] = []; + const metadataOnly: MetadataChange[] = []; - function checkConflict( - label: string, - fileContent: string | null, - blockValue: string | null, - lastFileHash: string | null, - lastBlockHash: string | null, - ) { - if (fileContent === null || blockValue === null) return; - const fileHash = hashContent(fileContent); - const blockHash = hashContent(blockValue); - if (fileHash === blockHash) return; + // Collect all labels + const allLabels = new Set([ + ...systemFiles.keys(), + ...detachedFiles.keys(), + ...systemBlockMap.keys(), + ...detachedBlockMap.keys(), + ...Object.keys(lastState.blockHashes), + ...Object.keys(lastState.fileHashes), + ]); + + for (const label of [...allLabels].sort()) { + if (MEMFS_MANAGED_LABELS.has(label)) continue; + + const systemFile = systemFiles.get(label); + const detachedFile = detachedFiles.get(label); + const attachedBlock = systemBlockMap.get(label); + const detachedBlock = detachedBlockMap.get(label); + + const fileEntry = systemFile || detachedFile; + const blockEntry = attachedBlock || detachedBlock; + + if (!fileEntry || !blockEntry) continue; + + // read_only blocks are API-authoritative; no conflicts possible + if (blockEntry.read_only) continue; + + // Full file hash for "file changed" check + const fileHash = hashContent(fileEntry.content); + // Body hash for "content matches" check + const fileBodyHash = hashFileBody(fileEntry.content); + const blockHash = hashContent(blockEntry.value); + + const lastFileHash = lastState.fileHashes[label] ?? null; + const lastBlockHash = lastState.blockHashes[label] ?? null; const fileChanged = fileHash !== lastFileHash; const blockChanged = blockHash !== lastBlockHash; + + // Content matches - check for frontmatter-only changes + if (fileBodyHash === blockHash) { + if (fileChanged) { + metadataOnly.push({ + label, + fileContent: fileEntry.content, + blockContent: blockEntry.value, + }); + } + continue; + } + + // Conflict only if both changed if (fileChanged && blockChanged) { - conflicts.push({ label, fileContent, blockContent: blockValue }); + conflicts.push({ + label, + fileContent: fileEntry.content, + blockContent: blockEntry.value, + }); } } - // Check system labels - const systemLabels = new Set([ - ...systemFiles.keys(), - ...systemBlockMap.keys(), - ...Object.keys(lastState.systemBlocks), - ...Object.keys(lastState.systemFiles), - ]); - - for (const label of [...systemLabels].sort()) { - if (MANAGED_LABELS.has(label)) continue; - checkConflict( - label, - systemFiles.get(label)?.content ?? null, - systemBlockMap.get(label) ?? null, - lastState.systemFiles[label] ?? null, - lastState.systemBlocks[label] ?? null, - ); - } - - // Check user labels - const userLabels = new Set([ - ...detachedFiles.keys(), - ...detachedBlockMap.keys(), - ...Object.keys(lastState.detachedBlocks), - ...Object.keys(lastState.detachedFiles), - ]); - - for (const label of [...userLabels].sort()) { - checkConflict( - label, - detachedFiles.get(label)?.content ?? null, - detachedBlockMap.get(label) ?? null, - lastState.detachedFiles[label] ?? null, - lastState.detachedBlocks[label] ?? null, - ); - } - - return conflicts; + return { conflicts, metadataOnly }; } -function formatDiffFile(conflicts: Conflict[], agentId: string): string { +function formatDiffFile( + conflicts: Conflict[], + metadataOnly: MetadataChange[], + agentId: string, +): string { const lines: string[] = [ `# Memory Filesystem Diff`, ``, `Agent: ${agentId}`, `Generated: ${new Date().toISOString()}`, `Conflicts: ${conflicts.length}`, + `Metadata-only changes: ${metadataOnly.length}`, ``, `---`, ``, @@ -298,6 +380,32 @@ function formatDiffFile(conflicts: Conflict[], agentId: string): string { lines.push(``); } + if (metadataOnly.length > 0) { + lines.push(`## Metadata-only Changes`); + lines.push(``); + lines.push( + `Frontmatter changed while body content stayed the same (file wins).`, + ); + lines.push(``); + + for (const change of metadataOnly) { + lines.push(`### ${change.label}`); + lines.push(``); + lines.push(`#### File Version (with frontmatter)`); + lines.push(`\`\`\``); + lines.push(change.fileContent); + lines.push(`\`\`\``); + lines.push(``); + lines.push(`#### Block Version (body only)`); + lines.push(`\`\`\``); + lines.push(change.blockContent); + lines.push(`\`\`\``); + lines.push(``); + lines.push(`---`); + lines.push(``); + } + } + return lines.join("\n"); } @@ -329,13 +437,13 @@ Output: Path to the diff file, or a message if no conflicts exist. } findConflicts(agentId) - .then((conflicts) => { - if (conflicts.length === 0) { + .then(({ conflicts, metadataOnly }) => { + if (conflicts.length === 0 && metadataOnly.length === 0) { console.log("No conflicts found. Memory filesystem is clean."); return; } - const diffContent = formatDiffFile(conflicts, agentId); + const diffContent = formatDiffFile(conflicts, metadataOnly, agentId); // Write to overflow directory (same pattern as tool output overflow) const overflowDir = getOverflowDirectory(); @@ -348,7 +456,7 @@ Output: Path to the diff file, or a message if no conflicts exist. writeFileSync(diffPath, diffContent, "utf-8"); console.log( - `Diff (${conflicts.length} conflict${conflicts.length === 1 ? "" : "s"}) written to: ${diffPath}`, + `Diff (${conflicts.length} conflict${conflicts.length === 1 ? "" : "s"}, ${metadataOnly.length} metadata-only change${metadataOnly.length === 1 ? "" : "s"}) written to: ${diffPath}`, ); }) .catch((error) => { diff --git a/src/skills/builtin/syncing-memory-filesystem/scripts/memfs-resolve.ts b/src/skills/builtin/syncing-memory-filesystem/scripts/memfs-resolve.ts index af0d14d..a2c9e37 100644 --- a/src/skills/builtin/syncing-memory-filesystem/scripts/memfs-resolve.ts +++ b/src/skills/builtin/syncing-memory-filesystem/scripts/memfs-resolve.ts @@ -55,12 +55,11 @@ interface Resolution { const MEMORY_FS_STATE_FILE = ".sync-state.json"; +// Unified sync state format (matches main memoryFilesystem.ts) type SyncState = { - systemBlocks: Record; - systemFiles: Record; - detachedBlocks: Record; - detachedFiles: Record; - detachedBlockIds: Record; + blockHashes: Record; + fileHashes: Record; + blockIds: Record; lastSync: string | null; }; @@ -68,47 +67,111 @@ function hashContent(content: string): string { return createHash("sha256").update(content).digest("hex"); } -function getMemoryRoot(agentId: string): string { - return join(homedir(), ".letta", "agents", agentId, "memory"); +/** + * Parse frontmatter from file content. + */ +function parseFrontmatter(content: string): { + frontmatter: Record; + body: string; +} { + const frontmatterRegex = /^---\n([\s\S]*?)\n---\n([\s\S]*)$/; + const match = content.match(frontmatterRegex); + + if (!match || !match[1] || !match[2]) { + return { frontmatter: {}, body: content }; + } + + const frontmatterText = match[1]; + const body = match[2]; + const frontmatter: Record = {}; + + for (const line of frontmatterText.split("\n")) { + const colonIdx = line.indexOf(":"); + if (colonIdx > 0) { + const key = line.slice(0, colonIdx).trim(); + let value = line.slice(colonIdx + 1).trim(); + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + value = value.slice(1, -1); + } + frontmatter[key] = value; + } + } + + return { frontmatter, body }; } -function loadSyncState(agentId: string): SyncState { - const statePath = join(getMemoryRoot(agentId), MEMORY_FS_STATE_FILE); - if (!existsSync(statePath)) { - return { - systemBlocks: {}, - systemFiles: {}, - detachedBlocks: {}, - detachedFiles: {}, - detachedBlockIds: {}, - lastSync: null, - }; +/** + * Parse block update from file content (update-mode: only update metadata if present in frontmatter). + */ +function parseBlockUpdateFromFileContent( + fileContent: string, + defaultLabel: string, +): { + label: string; + value: string; + description?: string; + limit?: number; + read_only?: boolean; + hasDescription: boolean; + hasLimit: boolean; + hasReadOnly: boolean; +} { + const { frontmatter, body } = parseFrontmatter(fileContent); + const label = frontmatter.label || defaultLabel; + const hasDescription = Object.hasOwn(frontmatter, "description"); + const hasLimit = Object.hasOwn(frontmatter, "limit"); + const hasReadOnly = Object.hasOwn(frontmatter, "read_only"); + + let limit: number | undefined; + if (hasLimit && frontmatter.limit) { + const parsed = parseInt(frontmatter.limit, 10); + if (!Number.isNaN(parsed) && parsed > 0) { + limit = parsed; + } } - try { - const raw = readFileSync(statePath, "utf-8"); - const parsed = JSON.parse(raw) as Partial & { - blocks?: Record; - files?: Record; - }; - return { - systemBlocks: parsed.systemBlocks || parsed.blocks || {}, - systemFiles: parsed.systemFiles || parsed.files || {}, - detachedBlocks: parsed.detachedBlocks || {}, - detachedFiles: parsed.detachedFiles || {}, - detachedBlockIds: parsed.detachedBlockIds || {}, - lastSync: parsed.lastSync || null, - }; - } catch { - return { - systemBlocks: {}, - systemFiles: {}, - detachedBlocks: {}, - detachedFiles: {}, - detachedBlockIds: {}, - lastSync: null, - }; + return { + label, + value: body, + ...(hasDescription && { description: frontmatter.description }), + ...(hasLimit && limit !== undefined && { limit }), + ...(hasReadOnly && { read_only: frontmatter.read_only === "true" }), + hasDescription, + hasLimit, + hasReadOnly, + }; +} + +/** + * Render block to file content with frontmatter. + */ +function renderBlockToFileContent(block: { + value?: string | null; + description?: string | null; + limit?: number | null; + read_only?: boolean | null; +}): string { + const lines: string[] = ["---"]; + if (block.description) { + // Escape quotes in description + const escaped = block.description.replace(/"/g, '\\"'); + lines.push(`description: "${escaped}"`); } + if (block.limit) { + lines.push(`limit: ${block.limit}`); + } + if (block.read_only === true) { + lines.push("read_only: true"); + } + lines.push("---", "", block.value || ""); + return lines.join("\n"); +} + +function getMemoryRoot(agentId: string): string { + return join(homedir(), ".letta", "agents", agentId, "memory"); } function saveSyncState(state: SyncState, agentId: string): void { @@ -116,14 +179,19 @@ function saveSyncState(state: SyncState, agentId: string): void { writeFileSync(statePath, JSON.stringify(state, null, 2), "utf-8"); } -async function scanMdFiles(dir: string, baseDir = dir): Promise { +async function scanMdFiles( + dir: string, + baseDir = dir, + excludeDirs: string[] = [], +): Promise { if (!existsSync(dir)) return []; const entries = await readdir(dir, { withFileTypes: true }); const results: string[] = []; for (const entry of entries) { const fullPath = join(dir, entry.name); if (entry.isDirectory()) { - results.push(...(await scanMdFiles(fullPath, baseDir))); + if (excludeDirs.includes(entry.name)) continue; + results.push(...(await scanMdFiles(fullPath, baseDir, excludeDirs))); } else if (entry.isFile() && entry.name.endsWith(".md")) { results.push(relative(baseDir, fullPath)); } @@ -137,8 +205,9 @@ function labelFromPath(relativePath: string): string { async function readMemoryFiles( dir: string, + excludeDirs: string[] = [], ): Promise> { - const files = await scanMdFiles(dir); + const files = await scanMdFiles(dir, dir, excludeDirs); const entries = new Map(); for (const rel of files) { const label = labelFromPath(rel); @@ -171,7 +240,6 @@ async function resolveConflicts( const root = getMemoryRoot(agentId); const systemDir = join(root, "system"); - // Detached files go at root level (flat structure) const detachedDir = root; for (const dir of [root, systemDir]) { @@ -180,40 +248,83 @@ async function resolveConflicts( // Read current state const systemFiles = await readMemoryFiles(systemDir); - const detachedFiles = await readMemoryFiles(detachedDir); - systemFiles.delete("memory_filesystem"); + const detachedFiles = await readMemoryFiles(detachedDir, ["system", "user"]); + // Fetch attached blocks const blocksResponse = await client.agents.blocks.list(agentId, { limit: 1000, }); - const blocks = Array.isArray(blocksResponse) + const attachedBlocks = Array.isArray(blocksResponse) ? blocksResponse : ((blocksResponse as { items?: unknown[] }).items as Array<{ id?: string; label?: string; value?: string; + description?: string | null; + limit?: number | null; + read_only?: boolean; }>) || []; - const systemBlockMap = new Map( - blocks - .filter((b: { label?: string }) => b.label) - .map((b: { id?: string; label?: string; value?: string }) => [ - b.label as string, - { id: b.id || "", value: b.value || "" }, - ]), - ); - - const lastState = loadSyncState(agentId); - const detachedBlockMap = new Map(); - for (const [label, blockId] of Object.entries(lastState.detachedBlockIds)) { - try { - const block = await client.blocks.retrieve(blockId); - detachedBlockMap.set(label, { - id: block.id || "", + const systemBlockMap = new Map< + string, + { + id: string; + value: string; + description?: string | null; + limit?: number | null; + read_only?: boolean; + } + >(); + for (const block of attachedBlocks) { + if (block.label && block.id) { + systemBlockMap.set(block.label, { + id: block.id, value: block.value || "", + description: block.description, + limit: block.limit, + read_only: block.read_only, }); - } catch { - // Block no longer exists + } + } + + // Fetch detached blocks via owner tag + const ownedBlocksResponse = await client.blocks.list({ + tags: [`owner:${agentId}`], + limit: 1000, + }); + const ownedBlocks = Array.isArray(ownedBlocksResponse) + ? ownedBlocksResponse + : ((ownedBlocksResponse as { items?: unknown[] }).items as Array<{ + id?: string; + label?: string; + value?: string; + description?: string | null; + limit?: number | null; + read_only?: boolean; + }>) || []; + + const attachedIds = new Set(attachedBlocks.map((b) => b.id)); + const detachedBlockMap = new Map< + string, + { + id: string; + value: string; + description?: string | null; + limit?: number | null; + read_only?: boolean; + } + >(); + for (const block of ownedBlocks) { + if (block.label && block.id && !attachedIds.has(block.id)) { + if (!systemBlockMap.has(block.label)) { + detachedBlockMap.set(block.label, { + id: block.id, + value: block.value || "", + description: block.description, + limit: block.limit, + read_only: block.read_only, + }); + } } } @@ -221,7 +332,7 @@ async function resolveConflicts( for (const { label, resolution } of resolutions) { try { - // Check system blocks/files first, then user blocks/files + // Check system blocks/files first, then detached blocks/files const systemBlock = systemBlockMap.get(label); const systemFile = systemFiles.get(label); const detachedBlock = detachedBlockMap.get(label); @@ -245,20 +356,42 @@ async function resolveConflicts( } if (resolution === "file") { - // Overwrite block with file content - await client.blocks.update(block.id, { value: file.content }); + // read_only blocks: ignore local edits, overwrite file from API + if (block.read_only) { + const fileContent = renderBlockToFileContent(block); + writeMemoryFile(dir, label, fileContent); + result.resolved.push({ + label, + resolution: "block", + action: "read_only: kept API version (file overwritten)", + }); + continue; + } + + // Use update-mode parsing (only update metadata if present in frontmatter) + const parsed = parseBlockUpdateFromFileContent(file.content, label); + const updatePayload: Record = { value: parsed.value }; + if (parsed.hasDescription) + updatePayload.description = parsed.description; + if (parsed.hasLimit) updatePayload.limit = parsed.limit; + if (parsed.hasReadOnly) updatePayload.read_only = parsed.read_only; + // For detached blocks, also update label if changed + if (!systemBlock) updatePayload.label = label; + + await client.blocks.update(block.id, updatePayload); result.resolved.push({ label, resolution: "file", - action: "Updated block with file content", + action: "Updated block from file", }); } else if (resolution === "block") { - // Overwrite file with block content - writeMemoryFile(dir, label, block.value); + // Overwrite file with block content (including frontmatter) + const fileContent = renderBlockToFileContent(block); + writeMemoryFile(dir, label, fileContent); result.resolved.push({ label, resolution: "block", - action: "Updated file with block content", + action: "Updated file from block", }); } } catch (error) { @@ -269,59 +402,48 @@ async function resolveConflicts( } } - // Update sync state after resolving all conflicts - // Re-read everything to capture the new state + // Rebuild sync state in unified format const updatedSystemFiles = await readMemoryFiles(systemDir); - const updatedDetachedFiles = await readMemoryFiles(detachedDir); - updatedSystemFiles.delete("memory_filesystem"); + const updatedDetachedFiles = await readMemoryFiles(detachedDir, [ + "system", + "user", + ]); - const updatedBlocksResponse = await client.agents.blocks.list(agentId, { + // Re-fetch all owned blocks + const updatedOwnedResp = await client.blocks.list({ + tags: [`owner:${agentId}`], limit: 1000, }); - const updatedBlocks = Array.isArray(updatedBlocksResponse) - ? updatedBlocksResponse - : ((updatedBlocksResponse as { items?: unknown[] }).items as Array<{ + const updatedOwnedBlocks = Array.isArray(updatedOwnedResp) + ? updatedOwnedResp + : ((updatedOwnedResp as { items?: unknown[] }).items as Array<{ + id?: string; label?: string; value?: string; }>) || []; - const systemBlockHashes: Record = {}; - const systemFileHashes: Record = {}; - const detachedBlockHashes: Record = {}; - const detachedFileHashes: Record = {}; - - for (const block of updatedBlocks.filter( - (b: { label?: string }) => b.label && b.label !== "memory_filesystem", - )) { - systemBlockHashes[block.label as string] = hashContent( - (block as { value?: string }).value || "", - ); - } - - for (const [label, file] of updatedSystemFiles) { - systemFileHashes[label] = hashContent(file.content); - } - - for (const [label, blockId] of Object.entries(lastState.detachedBlockIds)) { - try { - const block = await client.blocks.retrieve(blockId); - detachedBlockHashes[label] = hashContent(block.value || ""); - } catch { - // Block gone + const blockHashes: Record = {}; + const blockIds: Record = {}; + for (const b of updatedOwnedBlocks) { + if (b.label && b.id) { + blockHashes[b.label] = hashContent(b.value || ""); + blockIds[b.label] = b.id; } } - for (const [label, file] of updatedDetachedFiles) { - detachedFileHashes[label] = hashContent(file.content); + const fileHashes: Record = {}; + for (const [lbl, f] of updatedSystemFiles) { + fileHashes[lbl] = hashContent(f.content); + } + for (const [lbl, f] of updatedDetachedFiles) { + fileHashes[lbl] = hashContent(f.content); } saveSyncState( { - systemBlocks: systemBlockHashes, - systemFiles: systemFileHashes, - detachedBlocks: detachedBlockHashes, - detachedFiles: detachedFileHashes, - detachedBlockIds: lastState.detachedBlockIds, + blockHashes, + fileHashes, + blockIds, lastSync: new Date().toISOString(), }, agentId, @@ -353,6 +475,8 @@ Resolution options: "file" — Overwrite the memory block with the file contents "block" — Overwrite the file with the memory block contents +Note: read_only blocks always resolve to "block" (API is authoritative). + Example: npx tsx memfs-resolve.ts $LETTA_AGENT_ID --resolutions '[{"label":"persona/soul","resolution":"block"}]' `); @@ -399,8 +523,8 @@ Example: } resolveConflicts(agentId, resolutions) - .then((result) => { - console.log(JSON.stringify(result, null, 2)); + .then((res) => { + console.log(JSON.stringify(res, null, 2)); }) .catch((error) => { console.error( diff --git a/src/skills/builtin/syncing-memory-filesystem/scripts/memfs-status.ts b/src/skills/builtin/syncing-memory-filesystem/scripts/memfs-status.ts index d0771bb..0583f7f 100644 --- a/src/skills/builtin/syncing-memory-filesystem/scripts/memfs-status.ts +++ b/src/skills/builtin/syncing-memory-filesystem/scripts/memfs-status.ts @@ -12,10 +12,12 @@ * Output: JSON object with sync status */ -import { readFileSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, readFileSync } from "node:fs"; +import { readdir, readFile } from "node:fs/promises"; import { createRequire } from "node:module"; import { homedir } from "node:os"; -import { join } from "node:path"; +import { join, relative } from "node:path"; const require = createRequire(import.meta.url); const Letta = require("@letta-ai/letta-client") @@ -41,24 +43,13 @@ function getApiKey(): string { ); } -// We can't import checkMemoryFilesystemStatus directly since it relies on -// getClient() which uses the CLI's auth chain. Instead, we reimplement the -// status check logic using the standalone client pattern. -// This keeps the script fully standalone and runnable outside the CLI process. - -import { createHash } from "node:crypto"; -import { existsSync, mkdirSync } from "node:fs"; -import { readdir, readFile } from "node:fs/promises"; -import { relative } from "node:path"; - const MEMORY_FS_STATE_FILE = ".sync-state.json"; +// Unified sync state format (matches main memoryFilesystem.ts) type SyncState = { - systemBlocks: Record; - systemFiles: Record; - detachedBlocks: Record; - detachedFiles: Record; - detachedBlockIds: Record; + blockHashes: Record; + fileHashes: Record; + blockIds: Record; lastSync: string | null; }; @@ -66,6 +57,50 @@ function hashContent(content: string): string { return createHash("sha256").update(content).digest("hex"); } +/** + * Parse frontmatter from file content. + */ +function parseFrontmatter(content: string): { + frontmatter: Record; + body: string; +} { + const frontmatterRegex = /^---\n([\s\S]*?)\n---\n([\s\S]*)$/; + const match = content.match(frontmatterRegex); + + if (!match || !match[1] || !match[2]) { + return { frontmatter: {}, body: content }; + } + + const frontmatterText = match[1]; + const body = match[2]; + const frontmatter: Record = {}; + + for (const line of frontmatterText.split("\n")) { + const colonIdx = line.indexOf(":"); + if (colonIdx > 0) { + const key = line.slice(0, colonIdx).trim(); + let value = line.slice(colonIdx + 1).trim(); + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + value = value.slice(1, -1); + } + frontmatter[key] = value; + } + } + + return { frontmatter, body }; +} + +/** + * Hash just the body of file content (excluding frontmatter). + */ +function hashFileBody(content: string): string { + const { body } = parseFrontmatter(content); + return hashContent(body); +} + function getMemoryRoot(agentId: string): string { return join(homedir(), ".letta", "agents", agentId, "memory"); } @@ -74,49 +109,45 @@ function loadSyncState(agentId: string): SyncState { const statePath = join(getMemoryRoot(agentId), MEMORY_FS_STATE_FILE); if (!existsSync(statePath)) { return { - systemBlocks: {}, - systemFiles: {}, - detachedBlocks: {}, - detachedFiles: {}, - detachedBlockIds: {}, + blockHashes: {}, + fileHashes: {}, + blockIds: {}, lastSync: null, }; } try { const raw = readFileSync(statePath, "utf-8"); - const parsed = JSON.parse(raw) as Partial & { - blocks?: Record; - files?: Record; - }; + const parsed = JSON.parse(raw); return { - systemBlocks: parsed.systemBlocks || parsed.blocks || {}, - systemFiles: parsed.systemFiles || parsed.files || {}, - detachedBlocks: parsed.detachedBlocks || {}, - detachedFiles: parsed.detachedFiles || {}, - detachedBlockIds: parsed.detachedBlockIds || {}, + blockHashes: parsed.blockHashes || {}, + fileHashes: parsed.fileHashes || {}, + blockIds: parsed.blockIds || {}, lastSync: parsed.lastSync || null, }; } catch { return { - systemBlocks: {}, - systemFiles: {}, - detachedBlocks: {}, - detachedFiles: {}, - detachedBlockIds: {}, + blockHashes: {}, + fileHashes: {}, + blockIds: {}, lastSync: null, }; } } -async function scanMdFiles(dir: string, baseDir = dir): Promise { +async function scanMdFiles( + dir: string, + baseDir = dir, + excludeDirs: string[] = [], +): Promise { if (!existsSync(dir)) return []; const entries = await readdir(dir, { withFileTypes: true }); const results: string[] = []; for (const entry of entries) { const fullPath = join(dir, entry.name); if (entry.isDirectory()) { - results.push(...(await scanMdFiles(fullPath, baseDir))); + if (excludeDirs.includes(entry.name)) continue; + results.push(...(await scanMdFiles(fullPath, baseDir, excludeDirs))); } else if (entry.isFile() && entry.name.endsWith(".md")) { results.push(relative(baseDir, fullPath)); } @@ -130,8 +161,9 @@ function labelFromPath(relativePath: string): string { async function readMemoryFiles( dir: string, + excludeDirs: string[] = [], ): Promise> { - const files = await scanMdFiles(dir); + const files = await scanMdFiles(dir, dir, excludeDirs); const entries = new Map(); for (const rel of files) { const label = labelFromPath(rel); @@ -141,10 +173,13 @@ async function readMemoryFiles( return entries; } -const MANAGED_LABELS = new Set([ - "memory_filesystem", +// Only memory_filesystem is managed by memfs itself +const MEMFS_MANAGED_LABELS = new Set(["memory_filesystem"]); +// Read-only labels are API-authoritative (file-only copies should be ignored) +const READ_ONLY_LABELS = new Set([ "skills", "loaded_skills", + "memory_filesystem", ]); interface StatusResult { @@ -153,6 +188,7 @@ interface StatusResult { pendingFromBlock: string[]; newFiles: string[]; newBlocks: string[]; + locationMismatches: string[]; isClean: boolean; lastSync: string | null; } @@ -163,7 +199,6 @@ async function checkStatus(agentId: string): Promise { const root = getMemoryRoot(agentId); const systemDir = join(root, "system"); - // Detached files go at root level (flat structure) const detachedDir = root; // Ensure directories exist @@ -171,30 +206,67 @@ async function checkStatus(agentId: string): Promise { if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); } + // Read files from both locations const systemFiles = await readMemoryFiles(systemDir); - const detachedFiles = await readMemoryFiles(detachedDir); - systemFiles.delete("memory_filesystem"); + const detachedFiles = await readMemoryFiles(detachedDir, ["system", "user"]); // Fetch attached blocks const blocksResponse = await client.agents.blocks.list(agentId, { limit: 1000, }); - const blocks = Array.isArray(blocksResponse) + const attachedBlocks = Array.isArray(blocksResponse) ? blocksResponse : ((blocksResponse as { items?: unknown[] }).items as Array<{ + id?: string; label?: string; value?: string; + read_only?: boolean; }>) || []; - const systemBlockMap = new Map( - blocks - .filter((b: { label?: string }) => b.label) - .map((b: { label?: string; value?: string }) => [ - b.label as string, - b.value || "", - ]), - ); - systemBlockMap.delete("memory_filesystem"); + const systemBlockMap = new Map< + string, + { value: string; id: string; read_only?: boolean } + >(); + for (const block of attachedBlocks) { + if (block.label && block.id) { + systemBlockMap.set(block.label, { + value: block.value || "", + id: block.id, + read_only: block.read_only, + }); + } + } + + // Fetch detached blocks via owner tag + const ownedBlocksResponse = await client.blocks.list({ + tags: [`owner:${agentId}`], + limit: 1000, + }); + const ownedBlocks = Array.isArray(ownedBlocksResponse) + ? ownedBlocksResponse + : ((ownedBlocksResponse as { items?: unknown[] }).items as Array<{ + id?: string; + label?: string; + value?: string; + read_only?: boolean; + }>) || []; + + const attachedIds = new Set(attachedBlocks.map((b) => b.id)); + const detachedBlockMap = new Map< + string, + { value: string; id: string; read_only?: boolean } + >(); + for (const block of ownedBlocks) { + if (block.label && block.id && !attachedIds.has(block.id)) { + if (!systemBlockMap.has(block.label)) { + detachedBlockMap.set(block.label, { + value: block.value || "", + id: block.id, + read_only: block.read_only, + }); + } + } + } const lastState = loadSyncState(agentId); @@ -203,98 +275,102 @@ async function checkStatus(agentId: string): Promise { const pendingFromBlock: string[] = []; const newFiles: string[] = []; const newBlocks: string[] = []; + const locationMismatches: string[] = []; - // Fetch user blocks - const detachedBlockMap = new Map(); - for (const [label, blockId] of Object.entries(lastState.detachedBlockIds)) { - try { - const block = await client.blocks.retrieve(blockId); - detachedBlockMap.set(label, block.value || ""); - } catch { - // Block no longer exists + // Collect all labels + const allLabels = new Set([ + ...systemFiles.keys(), + ...detachedFiles.keys(), + ...systemBlockMap.keys(), + ...detachedBlockMap.keys(), + ...Object.keys(lastState.blockHashes), + ...Object.keys(lastState.fileHashes), + ]); + + for (const label of [...allLabels].sort()) { + if (MEMFS_MANAGED_LABELS.has(label)) continue; + + const systemFile = systemFiles.get(label); + const detachedFile = detachedFiles.get(label); + const attachedBlock = systemBlockMap.get(label); + const detachedBlock = detachedBlockMap.get(label); + + const fileEntry = systemFile || detachedFile; + const fileInSystem = !!systemFile; + const blockEntry = attachedBlock || detachedBlock; + const isAttached = !!attachedBlock; + + // Check for location mismatch + if (fileEntry && blockEntry) { + const locationMismatch = + (fileInSystem && !isAttached) || (!fileInSystem && isAttached); + if (locationMismatch) { + locationMismatches.push(label); + } } - } - function classify( - label: string, - fileContent: string | null, - blockValue: string | null, - lastFileHash: string | null, - lastBlockHash: string | null, - ) { - const fileHash = fileContent !== null ? hashContent(fileContent) : null; - const blockHash = blockValue !== null ? hashContent(blockValue) : null; + // Compute hashes + // Full file hash for "file changed" check (matches what's stored in fileHashes) + const fileHash = fileEntry ? hashContent(fileEntry.content) : null; + // Body hash for "content matches" check (compares to block value) + const fileBodyHash = fileEntry ? hashFileBody(fileEntry.content) : null; + const blockHash = blockEntry ? hashContent(blockEntry.value) : null; + + const lastFileHash = lastState.fileHashes[label] ?? null; + const lastBlockHash = lastState.blockHashes[label] ?? null; + const fileChanged = fileHash !== lastFileHash; const blockChanged = blockHash !== lastBlockHash; - if (fileContent !== null && blockValue === null) { - if (lastBlockHash && !fileChanged) return; + // Classify + if (fileEntry && !blockEntry) { + if (READ_ONLY_LABELS.has(label)) continue; // API authoritative, file-only will be deleted on sync + if (lastBlockHash && !fileChanged) continue; // Block deleted, file unchanged newFiles.push(label); - return; + continue; } - if (fileContent === null && blockValue !== null) { - if (lastFileHash && !blockChanged) return; + + if (!fileEntry && blockEntry) { + if (lastFileHash && !blockChanged) continue; // File deleted, block unchanged newBlocks.push(label); - return; + continue; } - if (fileContent === null || blockValue === null) return; - if (fileHash === blockHash) return; - if (fileChanged && blockChanged) { - conflicts.push({ label }); - return; + + if (!fileEntry || !blockEntry) continue; + + // Both exist - read_only blocks are API-authoritative + if (blockEntry.read_only) { + if (blockChanged) pendingFromBlock.push(label); + continue; } - if (fileChanged && !blockChanged) { + + // Both exist - check if content matches (body vs block value) + if (fileBodyHash === blockHash) { + if (fileChanged) { + // Frontmatter-only change; content matches + pendingFromFile.push(label); + } + continue; + } + + // "FS wins all" policy: if file changed, treat as pendingFromFile + if (fileChanged) { pendingFromFile.push(label); - return; + continue; } - if (!fileChanged && blockChanged) { + + if (blockChanged) { pendingFromBlock.push(label); } } - // Check system labels - const systemLabels = new Set([ - ...systemFiles.keys(), - ...systemBlockMap.keys(), - ...Object.keys(lastState.systemBlocks), - ...Object.keys(lastState.systemFiles), - ]); - - for (const label of [...systemLabels].sort()) { - if (MANAGED_LABELS.has(label)) continue; - classify( - label, - systemFiles.get(label)?.content ?? null, - systemBlockMap.get(label) ?? null, - lastState.systemFiles[label] ?? null, - lastState.systemBlocks[label] ?? null, - ); - } - - // Check user labels - const userLabels = new Set([ - ...detachedFiles.keys(), - ...detachedBlockMap.keys(), - ...Object.keys(lastState.detachedBlocks), - ...Object.keys(lastState.detachedFiles), - ]); - - for (const label of [...userLabels].sort()) { - classify( - label, - detachedFiles.get(label)?.content ?? null, - detachedBlockMap.get(label) ?? null, - lastState.detachedFiles[label] ?? null, - lastState.detachedBlocks[label] ?? null, - ); - } - const isClean = conflicts.length === 0 && pendingFromFile.length === 0 && pendingFromBlock.length === 0 && newFiles.length === 0 && - newBlocks.length === 0; + newBlocks.length === 0 && + locationMismatches.length === 0; return { conflicts, @@ -302,6 +378,7 @@ async function checkStatus(agentId: string): Promise { pendingFromBlock, newFiles, newBlocks, + locationMismatches, isClean, lastSync: lastState.lastSync, }; @@ -328,6 +405,7 @@ Output: JSON object with: - pendingFromBlock: block changed, file didn't - newFiles: file exists without a block - newBlocks: block exists without a file + - locationMismatches: file/block location doesn't match attachment - isClean: true if everything is in sync - lastSync: timestamp of last sync `);