import { describe, expect, test } from "bun:test"; import { spawn } from "node:child_process"; /** * Integration tests for CLI startup flows. * * These tests verify the boot flow decision tree: * - Flag conflict detection * - --conversation: derives agent from conversation * - --agent: uses specified agent * - --new-agent: creates new agent * - Error messages for invalid inputs * * Note: Tests that depend on settings files (.letta/) are harder to isolate * because the CLI uses process.cwd(). For now, we focus on flag-based tests. */ const projectRoot = process.cwd(); // Helper to run CLI and capture output async function runCli( args: string[], options: { timeoutMs?: number; expectExit?: number; } = {}, ): Promise<{ stdout: string; stderr: string; exitCode: number | null }> { const { timeoutMs = 30000, expectExit } = options; return new Promise((resolve, reject) => { const proc = spawn("bun", ["run", "dev", ...args], { cwd: projectRoot, env: { ...process.env }, }); let stdout = ""; let stderr = ""; proc.stdout?.on("data", (data) => { stdout += data.toString(); }); proc.stderr?.on("data", (data) => { stderr += data.toString(); }); const timeout = setTimeout(() => { proc.kill(); reject( new Error( `Timeout after ${timeoutMs}ms. stdout: ${stdout}, stderr: ${stderr}`, ), ); }, timeoutMs); proc.on("close", (code) => { clearTimeout(timeout); if (expectExit !== undefined && code !== expectExit) { reject( new Error( `Expected exit code ${expectExit}, got ${code}. stdout: ${stdout}, stderr: ${stderr}`, ), ); } else { resolve({ stdout, stderr, exitCode: code }); } }); proc.on("error", (err) => { clearTimeout(timeout); reject(err); }); }); } // ============================================================================ // Flag Conflict Tests (fast, no API calls needed) // ============================================================================ describe("Startup Flow - Flag Conflicts", () => { test("--conversation conflicts with --agent", async () => { const result = await runCli( ["--conversation", "conv-123", "--agent", "agent-123"], { expectExit: 1 }, ); expect(result.stderr).toContain( "--conversation cannot be used with --agent", ); }); test("--conversation conflicts with --new-agent", async () => { const result = await runCli(["--conversation", "conv-123", "--new-agent"], { expectExit: 1, }); expect(result.stderr).toContain( "--conversation cannot be used with --new-agent", ); }); test("--conversation conflicts with --resume", async () => { const result = await runCli(["--conversation", "conv-123", "--resume"], { expectExit: 1, }); expect(result.stderr).toContain( "--conversation cannot be used with --resume", ); }); test("--conversation conflicts with --continue", async () => { const result = await runCli(["--conversation", "conv-123", "--continue"], { expectExit: 1, }); expect(result.stderr).toContain( "--conversation cannot be used with --continue", ); }); test("--conversation conflicts with --from-af", async () => { const result = await runCli( ["--conversation", "conv-123", "--from-af", "test.af"], { expectExit: 1 }, ); expect(result.stderr).toContain( "--conversation cannot be used with --from-af", ); }); test("--conversation conflicts with --name", async () => { const result = await runCli( ["--conversation", "conv-123", "--name", "MyAgent"], { expectExit: 1 }, ); expect(result.stderr).toContain( "--conversation cannot be used with --name", ); }); }); // ============================================================================ // Invalid Input Tests (require API calls but fail fast) // ============================================================================ describe("Startup Flow - Invalid Inputs", () => { test( "--agent with nonexistent ID shows error", async () => { const result = await runCli( ["--agent", "agent-definitely-does-not-exist-12345", "-p", "test"], { expectExit: 1, timeoutMs: 60000 }, ); expect(result.stderr).toContain("not found"); }, { timeout: 70000 }, ); test( "--conversation with nonexistent ID shows error", async () => { const result = await runCli( [ "--conversation", "conversation-definitely-does-not-exist-12345", "-p", "test", ], { expectExit: 1, timeoutMs: 60000 }, ); expect(result.stderr).toContain("not found"); }, { timeout: 70000 }, ); test("--from-af with nonexistent file shows error", async () => { const result = await runCli( ["--from-af", "/nonexistent/path/agent.af", "-p", "test"], { expectExit: 1 }, ); expect(result.stderr).toContain("not found"); }); }); // ============================================================================ // Integration Tests (require API access, create real agents) // ============================================================================ describe("Startup Flow - Integration", () => { // Store created agent/conversation IDs for cleanup and reuse let testAgentId: string | null = null; let testConversationId: string | null = null; test( "--new-agent creates agent and responds", async () => { const result = await runCli( [ "--new-agent", "-m", "haiku", "-p", "Say OK and nothing else", "--output-format", "json", ], { timeoutMs: 120000 }, ); expect(result.exitCode).toBe(0); // stdout includes the bun invocation line, extract just the JSON const jsonStart = result.stdout.indexOf("{"); const output = JSON.parse(result.stdout.slice(jsonStart)); expect(output.agent_id).toBeDefined(); expect(output.result).toBeDefined(); // Save for later tests testAgentId = output.agent_id; testConversationId = output.conversation_id; }, { timeout: 130000 }, ); test( "--agent with valid ID uses that agent", async () => { // Skip if previous test didn't create an agent if (!testAgentId) { console.log("Skipping: no test agent available"); return; } const result = await runCli( [ "--agent", testAgentId, "-m", "haiku", "-p", "Say OK", "--output-format", "json", ], { timeoutMs: 120000 }, ); expect(result.exitCode).toBe(0); const jsonStart = result.stdout.indexOf("{"); const output = JSON.parse(result.stdout.slice(jsonStart)); expect(output.agent_id).toBe(testAgentId); }, { timeout: 130000 }, ); test( "--conversation with valid ID derives agent and uses conversation", async () => { // Skip if previous test didn't create an agent/conversation if (!testAgentId || !testConversationId) { console.log("Skipping: no test conversation available"); return; } const result = await runCli( [ "--conversation", testConversationId, "-m", "haiku", "-p", "Say OK", "--output-format", "json", ], { timeoutMs: 120000 }, ); expect(result.exitCode).toBe(0); const jsonStart = result.stdout.indexOf("{"); const output = JSON.parse(result.stdout.slice(jsonStart)); // Should use the same agent that owns the conversation expect(output.agent_id).toBe(testAgentId); // Should use the specified conversation expect(output.conversation_id).toBe(testConversationId); }, { timeout: 130000 }, ); test( "--new-agent with --init-blocks none creates minimal agent", async () => { const result = await runCli( [ "--new-agent", "--init-blocks", "none", "-m", "haiku", "-p", "Say OK", "--output-format", "json", ], { timeoutMs: 120000 }, ); expect(result.exitCode).toBe(0); // stdout includes the bun invocation line, extract just the JSON const jsonStart = result.stdout.indexOf("{"); const output = JSON.parse(result.stdout.slice(jsonStart)); expect(output.agent_id).toBeDefined(); }, { timeout: 130000 }, ); }); // ============================================================================ // --continue Tests (depend on LRU state, harder to isolate) // ============================================================================ describe("Startup Flow - Continue Flag", () => { test( "--continue with no LRU shows error", async () => { // This test relies on running in a directory with no .letta/ settings // In practice, this might use the project's .letta/ which has an LRU // So we check for either success (if LRU exists) or error (if not) const result = await runCli( ["--continue", "-p", "Say OK", "--output-format", "json"], { timeoutMs: 60000 }, ); // Either succeeds (LRU exists) or fails with specific error if (result.exitCode !== 0) { expect(result.stderr).toContain("No recent session found"); } // If it succeeds, that's also valid (test env has LRU) }, { timeout: 70000 }, ); });