chore: add CLI CI test (#1858)

Co-authored-by: Matt Zhou <mattzhou@Matts-MacBook-Pro.local>

tests/test_cli.py

@@ -1,44 +1,73 @@
-import subprocess
+import os
+import shutil
 import sys
 
-subprocess.check_call([sys.executable, "-m", "pip", "install", "pexpect"])
-from prettytable.colortable import ColorTable
+import pexpect
+import pytest
 
-from letta.cli.cli_config import ListChoice, add, delete
-from letta.cli.cli_config import list as list_command
+from letta.local_llm.constants import (
+    ASSISTANT_MESSAGE_CLI_SYMBOL,
+    INNER_THOUGHTS_CLI_SYMBOL,
+)
 
-# def test_configure_letta():
-#    configure_letta()
-
-options = [ListChoice.agents, ListChoice.sources, ListChoice.humans, ListChoice.personas]
+original_letta_path = os.path.expanduser("~/.letta")
+backup_letta_path = os.path.expanduser("~/.letta_backup")
 
 
-def test_cli_list():
-    for option in options:
-        output = list_command(arg=option)
-        # check if is a list
-        assert isinstance(output, ColorTable)
+@pytest.fixture
+def swap_letta_config():
+    if os.path.exists(backup_letta_path):
+        print("\nDelete the backup ~/.letta directory\n")
+        shutil.rmtree(backup_letta_path)
+
+    if os.path.exists(original_letta_path):
+        print("\nBackup the original ~/.letta directory\n")
+        shutil.move(original_letta_path, backup_letta_path)
+
+    try:
+        # Run the test
+        yield
+    finally:
+        # Ensure this runs no matter what
+        print("\nClean up ~/.letta and restore the original directory\n")
+        if os.path.exists(original_letta_path):
+            shutil.rmtree(original_letta_path)
+
+        if os.path.exists(backup_letta_path):
+            shutil.move(backup_letta_path, original_letta_path)
 
 
-def test_cli_config():
+def test_letta_run_create_new_agent(swap_letta_config):
+    child = pexpect.spawn("poetry run letta run", encoding="utf-8")
+    # Start the letta run command
+    child.logfile = sys.stdout
+    child.expect("Creating new agent", timeout=10)
+    # Optional: LLM model selection
+    try:
+        child.expect("Select LLM model:", timeout=10)
+        child.sendline("\033[B\033[B\033[B\033[B\033[B")
+    except (pexpect.TIMEOUT, pexpect.EOF):
+        print("[WARNING] LLM model selection step was skipped.")
 
-    # test add
-    for option in ["human", "persona"]:
+    # Optional: Embedding model selection
+    try:
+        child.expect("Select embedding model:", timeout=10)
+        child.sendline("text-embedding-ada-002")
+    except (pexpect.TIMEOUT, pexpect.EOF):
+        print("[WARNING] Embedding model selection step was skipped.")
 
-        # create initial
-        add(option=option, name="test", text="test data")
+    child.expect("Created new agent", timeout=10)
+    child.sendline("")
 
-        ## update
-        # filename = "test.txt"
-        # open(filename, "w").write("test data new")
-        # child = pexpect.spawn(f"poetry run letta add --{str(option)} {filename} --name test --strip-ui")
-        # child.expect("Human test already exists. Overwrite?", timeout=TIMEOUT)
-        # child.sendline()
-        # child.expect(pexpect.EOF, timeout=TIMEOUT)  # Wait for child to exit
-        # child.close()
-
-        for row in list_command(arg=ListChoice.humans if option == "human" else ListChoice.personas):
-            if row[0] == "test":
-                assert "test data" in row
-        # delete
-        delete(option=option, name="test")
+    # Get initial response
+    child.expect("Enter your message:", timeout=60)
+    # Capture the output up to this point
+    full_output = child.before
+    # Count occurrences of inner thoughts
+    cloud_emoji_count = full_output.count(INNER_THOUGHTS_CLI_SYMBOL)
+    assert cloud_emoji_count == 1, f"It appears that there are multiple instances of inner thought outputted."
+    # Count occurrences of assistant messages
+    robot = full_output.count(ASSISTANT_MESSAGE_CLI_SYMBOL)
+    assert robot == 1, f"It appears that there are multiple instances of assistant messages outputted."
+    # Make sure the user name was repeated back at least once
+    assert full_output.count("Chad") > 0, f"Chad was not mentioned...please manually inspect the outputs."