feat: change lettaignore logic (#1375)

Co-authored-by: Letta Code <noreply@letta.com>

src/cli/helpers/fileSearchConfig.ts

@@ -4,68 +4,73 @@ import {
   readLettaIgnorePatterns,
 } from "./ignoredDirectories";
 
-/**
+interface CwdConfig {
- * Hardcoded defaults — always excluded from both the file index and disk scans.
+  nameMatchers: picomatch.Matcher[];
- * These cover the most common build/dependency directories across ecosystems.
+  pathMatchers: picomatch.Matcher[];
- * Matched case-insensitively against the entry name.
+}
- */
-const DEFAULT_EXCLUDED = new Set([
-  // JavaScript / Node
-  "node_modules",
-  "bower_components",
-  // Build outputs
-  "dist",
-  "build",
-  "out",
-  "coverage",
-  // Frameworks
-  ".next",
-  ".nuxt",
-  // Python
-  "venv",
-  ".venv",
-  "__pycache__",
-  ".tox",
-  // Rust / Maven / Java
-  "target",
-  // Version control & tooling
-  ".git",
-  ".cache",
-]);
 
 /**
- * Pre-compiled matchers from .lettaignore, split by whether the pattern
+ * Cache of compiled matchers keyed by absolute cwd path.
- * is name-based (no slash → match against entry name) or path-based
+ * Compiled once per unique cwd for performance, re-built when cwd changes.
- * (contains slash → match against the full relative path).
- * Compiled once at module load for performance.
  */
-const { nameMatchers, pathMatchers } = (() => {
+const cwdConfigCache = new Map<string, CwdConfig>();
-  // Create .lettaignore with defaults if the project doesn't have one yet.
+
-  // Must run before readLettaIgnorePatterns() so the file exists when we read it.
+function buildConfig(cwd: string): CwdConfig {
-  ensureLettaIgnoreFile();
+  const patterns = readLettaIgnorePatterns(cwd);
-  const patterns = readLettaIgnorePatterns();
   const nameMatchers: picomatch.Matcher[] = [];
   const pathMatchers: picomatch.Matcher[] = [];
 
   for (const raw of patterns) {
     const normalized = raw.replace(/\/$/, ""); // strip trailing slash
     if (normalized.includes("/")) {
+      // Path-based patterns: match against the full relative path
       pathMatchers.push(picomatch(normalized, { dot: true }));
     } else {
-      nameMatchers.push(picomatch(normalized, { dot: true }));
+      // Name-based patterns: match against the entry basename, case-insensitively
+      // so that e.g. "node_modules" also matches "Node_Modules" on case-sensitive FSes.
+      nameMatchers.push(picomatch(normalized, { dot: true, nocase: true }));
     }
   }
 
   return { nameMatchers, pathMatchers };
+}
+
+/**
+ * Returns the compiled matchers for the current working directory.
+ * Builds and caches on first access per cwd; returns cached result thereafter.
+ */
+function getConfig(): CwdConfig {
+  const cwd = process.cwd();
+  const cached = cwdConfigCache.get(cwd);
+  if (cached) return cached;
+
+  const config = buildConfig(cwd);
+  cwdConfigCache.set(cwd, config);
+  return config;
+}
+
+// On module load: ensure .lettaignore exists for the initial cwd and prime the cache.
+(() => {
+  const cwd = process.cwd();
+  ensureLettaIgnoreFile(cwd);
+  cwdConfigCache.set(cwd, buildConfig(cwd));
 })();
 
+/**
+ * Invalidate the cached config for a given directory so it is re-read on the
+ * next call to shouldExcludeEntry / shouldHardExcludeEntry. Call this after
+ * writing or deleting .letta/.lettaignore in that directory.
+ */
+export function invalidateFileSearchConfig(cwd: string = process.cwd()): void {
+  cwdConfigCache.delete(cwd);
+}
+
 /**
  * Returns true if the given entry should be excluded from the file index.
- * Applies both the hardcoded defaults and any .lettaignore patterns.
+ * Applies patterns from .letta/.lettaignore for the current working directory.
  *
- * Use this when building the index — .lettaignore controls what gets cached,
+ * Use this when building the index. For disk scan fallback paths, use
- * not what the user can ever find. For disk scan fallback paths, use
+ * shouldHardExcludeEntry() which matches against entry names only.
- * shouldHardExcludeEntry() so .lettaignore-matched files remain discoverable.
  *
  * @param name         - The entry's basename (e.g. "node_modules", ".env")
  * @param relativePath - Optional path relative to cwd (e.g. "src/generated/foo.ts").
@@ -75,8 +80,7 @@ export function shouldExcludeEntry(
   name: string,
   relativePath?: string,
 ): boolean {
-  // Fast path: hardcoded defaults (O(1) Set lookup)
+  const { nameMatchers, pathMatchers } = getConfig();
-  if (DEFAULT_EXCLUDED.has(name.toLowerCase())) return true;
 
   // Name-based .lettaignore patterns (e.g. *.log, vendor)
   if (nameMatchers.length > 0 && nameMatchers.some((m) => m(name))) return true;
@@ -94,11 +98,12 @@ export function shouldExcludeEntry(
 
 /**
  * Returns true if the given entry should be excluded from disk scan fallbacks.
- * Only applies the hardcoded defaults — .lettaignore patterns are intentionally
+ * Applies name-based .lettaignore patterns only (no path patterns, since only
- * skipped here so users can still find those files with an explicit @ search.
+ * the entry name is available during a shallow disk scan).
  *
  * @param name - The entry's basename (e.g. "node_modules", "dist")
  */
 export function shouldHardExcludeEntry(name: string): boolean {
-  return DEFAULT_EXCLUDED.has(name.toLowerCase());
+  const { nameMatchers } = getConfig();
+  return nameMatchers.length > 0 && nameMatchers.some((m) => m(name));
 }

src/cli/helpers/ignoredDirectories.ts

@@ -1,54 +1,77 @@
-import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 
 const DEFAULT_LETTAIGNORE = `\
 # .lettaignore — Letta Code file index exclusions
 #
 # Files and directories matching these patterns are excluded from the @ file
-# search index (cache). They won't appear in autocomplete results by default,
+# search index and disk scan fallback. Comment out or remove a line to bring
-# but can still be found if you type their path explicitly.
+# it back into search results. Add new patterns to exclude more.
 #
 # Syntax: one pattern per line, supports globs (e.g. *.log, src/generated/**)
 # Lines starting with # are comments.
 #
-# The following are always excluded (even from explicit search) and do not need
+# --- Dependency directories ---
-# to be listed here:
+node_modules
-#   node_modules  dist  build  out  coverage  target  bower_components
+bower_components
-#   .git  .cache  .next  .nuxt  venv  .venv  __pycache__  .tox
+vendor
 
-# Lock files
+# --- Build outputs ---
+dist
+build
+out
+coverage
+target
+.next
+.nuxt
+
+# --- Python ---
+venv
+.venv
+__pycache__
+.tox
+
+# --- Version control & tooling ---
+.git
+.cache
+.letta
+
+# --- Lock files ---
 package-lock.json
 yarn.lock
 pnpm-lock.yaml
 poetry.lock
 Cargo.lock
 
-# Logs
+# --- Logs ---
 *.log
 
-# OS artifacts
+# --- OS artifacts ---
 .DS_Store
 Thumbs.db
 `;
 
 /**
- * Create a .lettaignore file in the project root with sensible defaults
+ * Create a .lettaignore file in the project's .letta directory with a
- * if one does not already exist. Safe to call multiple times.
+ * commented-out template if one does not already exist.
+ * All patterns in the generated file are commented out — nothing is excluded
+ * by default. Users uncomment the patterns they want.
  */
 export function ensureLettaIgnoreFile(cwd: string = process.cwd()): void {
-  const filePath = join(cwd, ".lettaignore");
+  const lettaDir = join(cwd, ".letta");
+  const filePath = join(lettaDir, ".lettaignore");
   if (existsSync(filePath)) return;
 
   try {
+    mkdirSync(lettaDir, { recursive: true });
     writeFileSync(filePath, DEFAULT_LETTAIGNORE, "utf-8");
   } catch {
-    // If we can't write (e.g. read-only fs), silently skip — the
+    // If we can't write (e.g. read-only fs), silently skip.
-    // hardcoded defaults in fileSearchConfig.ts still apply.
   }
 }
 
 /**
- * Read glob patterns from a .lettaignore file in the given directory.
+ * Read glob patterns from the project's .letta/.lettaignore file.
  * Returns an empty array if the file is missing or unreadable.
  *
  * Syntax:
@@ -58,7 +81,7 @@ export function ensureLettaIgnoreFile(cwd: string = process.cwd()): void {
  *   - A trailing / is treated as a directory hint and stripped before matching
  */
 export function readLettaIgnorePatterns(cwd: string = process.cwd()): string[] {
-  const filePath = join(cwd, ".lettaignore");
+  const filePath = join(cwd, ".letta", ".lettaignore");
   if (!existsSync(filePath)) return [];
 
   try {

src/tests/cli/fileIndex.test.ts

@@ -37,6 +37,15 @@ beforeEach(() => {
   writeFileSync(join(TEST_DIR, "src/components/Input.tsx"), "export Input");
   writeFileSync(join(TEST_DIR, "tests/app.test.ts"), "test()");
 
+  // Provide a .lettaignore so the file index respects exclusions.
+  // .letta itself is listed so this directory doesn't affect entry counts.
+  mkdirSync(join(TEST_DIR, ".letta"), { recursive: true });
+  writeFileSync(
+    join(TEST_DIR, ".letta", ".lettaignore"),
+    "node_modules\n.git\nvenv\n.venv\n__pycache__\ndist\nbuild\n.letta\n",
+    "utf-8",
+  );
+
   process.chdir(TEST_DIR);
 });
 

src/tests/cli/fileSearchConfig.test.ts

@@ -1,16 +1,43 @@
-import { describe, expect, test } from "bun:test";
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+import { mkdirSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
 import {
   shouldExcludeEntry,
   shouldHardExcludeEntry,
 } from "../../cli/helpers/fileSearchConfig";
 
+// These tests rely on there being NO .letta/.lettaignore in the working
+// directory — they verify that nothing is excluded unless the user explicitly
+// opts in via .letta/.lettaignore.  Each test therefore runs from a fresh
+// temporary directory that contains no ignore file.
+
+let testDir: string;
+let originalCwd: string;
+
+beforeEach(() => {
+  originalCwd = process.cwd();
+  testDir = join(
+    tmpdir(),
+    `letta-fsc-test-${Date.now()}-${Math.random().toString(36).slice(2)}`,
+  );
+  mkdirSync(testDir, { recursive: true });
+  process.chdir(testDir);
+});
+
+afterEach(() => {
+  process.chdir(originalCwd);
+  rmSync(testDir, { recursive: true, force: true });
+});
+
 // ---------------------------------------------------------------------------
-// shouldExcludeEntry — hardcoded defaults
+// shouldExcludeEntry — driven by .lettaignore only (no hardcoded defaults)
 // ---------------------------------------------------------------------------
 
 describe("shouldExcludeEntry", () => {
-  describe("hardcoded defaults", () => {
+  describe("no hardcoded defaults", () => {
-    const hardcoded = [
+    // Without a .lettaignore, none of these entries are excluded.
+    const formerlyHardcoded = [
       "node_modules",
       "bower_components",
       "dist",
@@ -28,18 +55,11 @@ describe("shouldExcludeEntry", () => {
       ".cache",
     ];
 
-    for (const name of hardcoded) {
+    for (const name of formerlyHardcoded) {
-      test(`excludes "${name}"`, () => {
+      test(`does not exclude "${name}" without a .lettaignore entry`, () => {
-        expect(shouldExcludeEntry(name)).toBe(true);
+        expect(shouldExcludeEntry(name)).toBe(false);
       });
     }
-
-    test("exclusion is case-insensitive", () => {
-      expect(shouldExcludeEntry("Node_Modules")).toBe(true);
-      expect(shouldExcludeEntry("DIST")).toBe(true);
-      expect(shouldExcludeEntry("BUILD")).toBe(true);
-      expect(shouldExcludeEntry(".GIT")).toBe(true);
-    });
   });
 
   describe("non-excluded entries", () => {
@@ -59,19 +79,14 @@ describe("shouldExcludeEntry", () => {
 });
 
 // ---------------------------------------------------------------------------
-// shouldHardExcludeEntry — hardcoded only, no .lettaignore
+// shouldHardExcludeEntry — driven by .lettaignore name patterns only
 // ---------------------------------------------------------------------------
 
 describe("shouldHardExcludeEntry", () => {
-  test("excludes hardcoded defaults", () => {
+  test("does not exclude previously hardcoded entries without a .lettaignore entry", () => {
-    expect(shouldHardExcludeEntry("node_modules")).toBe(true);
+    expect(shouldHardExcludeEntry("node_modules")).toBe(false);
-    expect(shouldHardExcludeEntry(".git")).toBe(true);
+    expect(shouldHardExcludeEntry(".git")).toBe(false);
-    expect(shouldHardExcludeEntry("dist")).toBe(true);
+    expect(shouldHardExcludeEntry("dist")).toBe(false);
-  });
-
-  test("exclusion is case-insensitive", () => {
-    expect(shouldHardExcludeEntry("Node_Modules")).toBe(true);
-    expect(shouldHardExcludeEntry("DIST")).toBe(true);
   });
 
   test("does not exclude normal entries", () => {

src/tests/cli/ignoredDirectories.test.ts

@@ -1,5 +1,5 @@
 import { afterEach, describe, expect, test } from "bun:test";
-import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import {
   ensureLettaIgnoreFile,
@@ -16,22 +16,27 @@ describe("ensureLettaIgnoreFile", () => {
 
   test("creates .lettaignore when missing", () => {
     testDir = new TestDirectory();
-    const filePath = join(testDir.path, ".lettaignore");
+    const filePath = join(testDir.path, ".letta", ".lettaignore");
 
     expect(existsSync(filePath)).toBe(false);
     ensureLettaIgnoreFile(testDir.path);
     expect(existsSync(filePath)).toBe(true);
 
-    const content = readFileSync(filePath, "utf-8");
+    // Common patterns are active by default
-    expect(content).toContain("package-lock.json");
+    const activePatterns = readLettaIgnorePatterns(testDir.path);
-    expect(content).toContain("*.log");
+    expect(activePatterns).toContain("node_modules");
-    expect(content).toContain(".DS_Store");
+    expect(activePatterns).toContain("dist");
+    expect(activePatterns).toContain(".git");
+    expect(activePatterns).toContain("*.log");
+    expect(activePatterns).toContain("package-lock.json");
   });
 
   test("does not overwrite existing .lettaignore", () => {
     testDir = new TestDirectory();
-    const filePath = join(testDir.path, ".lettaignore");
+    const lettaDir = join(testDir.path, ".letta");
+    const filePath = join(lettaDir, ".lettaignore");
 
+    mkdirSync(lettaDir, { recursive: true });
     writeFileSync(filePath, "custom-pattern\n", "utf-8");
     ensureLettaIgnoreFile(testDir.path);
 
@@ -55,8 +60,10 @@ describe("readLettaIgnorePatterns", () => {
 
   test("parses patterns from file", () => {
     testDir = new TestDirectory();
+    const lettaDir = join(testDir.path, ".letta");
+    mkdirSync(lettaDir, { recursive: true });
     writeFileSync(
-      join(testDir.path, ".lettaignore"),
+      join(lettaDir, ".lettaignore"),
       "*.log\nvendor\nsrc/generated/**\n",
       "utf-8",
     );
@@ -67,8 +74,10 @@ describe("readLettaIgnorePatterns", () => {
 
   test("skips comments and blank lines", () => {
     testDir = new TestDirectory();
+    const lettaDir = join(testDir.path, ".letta");
+    mkdirSync(lettaDir, { recursive: true });
     writeFileSync(
-      join(testDir.path, ".lettaignore"),
+      join(lettaDir, ".lettaignore"),
       "# This is a comment\n\n  \npattern1\n# Another comment\npattern2\n",
       "utf-8",
     );
@@ -79,8 +88,10 @@ describe("readLettaIgnorePatterns", () => {
 
   test("skips negation patterns", () => {
     testDir = new TestDirectory();
+    const lettaDir = join(testDir.path, ".letta");
+    mkdirSync(lettaDir, { recursive: true });
     writeFileSync(
-      join(testDir.path, ".lettaignore"),
+      join(lettaDir, ".lettaignore"),
       "*.log\n!important.log\nvendor\n",
       "utf-8",
     );
@@ -91,8 +102,10 @@ describe("readLettaIgnorePatterns", () => {
 
   test("trims whitespace from patterns", () => {
     testDir = new TestDirectory();
+    const lettaDir = join(testDir.path, ".letta");
+    mkdirSync(lettaDir, { recursive: true });
     writeFileSync(
-      join(testDir.path, ".lettaignore"),
+      join(lettaDir, ".lettaignore"),
       "  *.log  \n  vendor  \n",
       "utf-8",
     );

src/tests/fileSearch.test.ts

@@ -20,6 +20,14 @@ beforeEach(() => {
   writeFileSync(join(TEST_DIR, "src/App.tsx"), "export default App");
   writeFileSync(join(TEST_DIR, "src/components/Button.tsx"), "export Button");
   writeFileSync(join(TEST_DIR, "tests/app.test.ts"), "test()");
+
+  // Provide a .lettaignore so exclusions work when the cwd is changed to TEST_DIR.
+  mkdirSync(join(TEST_DIR, ".letta"), { recursive: true });
+  writeFileSync(
+    join(TEST_DIR, ".letta", ".lettaignore"),
+    "node_modules\nvenv\n.venv\n__pycache__\n.letta\n",
+    "utf-8",
+  );
 });
 
 afterEach(() => {