From 8247b7a29a90f3da9e09b903cc9b506c169f269d Mon Sep 17 00:00:00 2001
From: Sarah Wooders <sarahwooders@gmail.com>
Date: Mon, 20 Jan 2025 10:40:14 -0800
Subject: [PATCH] chore: remove old tests from core (#696)

---
 .github/workflows/check_for_new_prints.yml |  62 ------------
 .github/workflows/integration_tests.yml    |  80 ---------------
 .github/workflows/test_anthropic.yml       | 102 -------------------
 .github/workflows/test_azure.yml           | 111 ---------------------
 .github/workflows/test_cli.yml             |  67 -------------
 .github/workflows/test_examples.yml        |  69 -------------
 .github/workflows/test_memgpt_hosted.yml   |  31 ------
 .github/workflows/test_ollama.yml          |  87 ----------------
 .github/workflows/test_openai.yml          |  82 ---------------
 .github/workflows/test_together.yml        | 105 -------------------
 10 files changed, 796 deletions(-)
 delete mode 100644 .github/workflows/check_for_new_prints.yml
 delete mode 100644 .github/workflows/integration_tests.yml
 delete mode 100644 .github/workflows/test_anthropic.yml
 delete mode 100644 .github/workflows/test_azure.yml
 delete mode 100644 .github/workflows/test_cli.yml
 delete mode 100644 .github/workflows/test_examples.yml
 delete mode 100644 .github/workflows/test_memgpt_hosted.yml
 delete mode 100644 .github/workflows/test_ollama.yml
 delete mode 100644 .github/workflows/test_openai.yml
 delete mode 100644 .github/workflows/test_together.yml

diff --git a/.github/workflows/check_for_new_prints.yml b/.github/workflows/check_for_new_prints.yml
deleted file mode 100644
index c7bba7a0..00000000
--- a/.github/workflows/check_for_new_prints.yml
+++ /dev/null
@@ -1,62 +0,0 @@
-name: Check for Print Statements
-on:
-  pull_request:
-    paths:
-      - '**.py'
-
-jobs:
-  check-print-statements:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.x'
-
-      - name: Check for new print statements
-        run: |
-          # Get the files changed in this PR
-          git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} > changed_files.txt
-
-          # Filter for only Python files, excluding tests directory
-          grep "\.py$" changed_files.txt | grep -v "^tests/" > python_files.txt || true
-
-          # Initialize error flag
-          ERROR=0
-
-          # Check each changed Python file
-          while IFS= read -r file; do
-            if [ "$file" == "letta/main.py" ]; then
-              echo "Skipping $file for print statement checks."
-              continue
-            fi
-
-            if [ -f "$file" ]; then
-              echo "Checking $file for new print statements..."
-
-              # Get diff and look for added lines containing print statements
-              NEW_PRINTS=$(git diff ${{ github.event.pull_request.base.sha }} ${{ github.sha }} "$file" | \
-                          grep "^+" | \
-                          grep -v "^+++" | \
-                          grep -E "(^|\s)print\(" || true)
-
-              if [ ! -z "$NEW_PRINTS" ]; then
-                echo "❌ Found new print statements in $file:"
-                echo "$NEW_PRINTS"
-                ERROR=1
-              fi
-            fi
-          done < python_files.txt
-
-          # Exit with error if print statements were found
-          if [ $ERROR -eq 1 ]; then
-            echo "::error::New print statements were found in the changes"
-            exit 1
-          fi
-
-          echo "✅ No new print statements found"
diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
deleted file mode 100644
index a094f267..00000000
--- a/.github/workflows/integration_tests.yml
+++ /dev/null
@@ -1,80 +0,0 @@
-name: Integration Tests
-
-env:
-  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-  COMPOSIO_API_KEY: ${{ secrets.COMPOSIO_API_KEY }}
-  ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-  GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
-  GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
-  AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
-  AZURE_BASE_URL: ${{ secrets.AZURE_BASE_URL }}
-  E2B_API_KEY: ${{ secrets.E2B_API_KEY }}
-  E2B_SANDBOX_TEMPLATE_ID: ${{ secrets.E2B_SANDBOX_TEMPLATE_ID }}
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  integ-run:
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-    strategy:
-      fail-fast: false
-      matrix:
-        integration_test_suite:
-          - "integration_test_summarizer.py"
-          - "integration_test_tool_execution_sandbox.py"
-          - "integration_test_offline_memory_agent.py"
-          - "integration_test_agent_tool_graph.py"
-    services:
-      qdrant:
-        image: qdrant/qdrant
-        ports:
-          - 6333:6333
-      postgres:
-        image: pgvector/pgvector:pg17
-        ports:
-          - 5432:5432
-        env:
-          POSTGRES_HOST_AUTH_METHOD: trust
-          POSTGRES_DB: postgres
-          POSTGRES_USER: postgres
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Setup Python, Poetry, and Dependencies
-        uses: packetcoders/action-setup-cache-python-poetry@main
-        with:
-          python-version: "3.12"
-          poetry-version: "1.8.2"
-          install-args: "-E dev -E postgres -E external-tools -E tests -E cloud-tool-sandbox"
-      - name: Migrate database
-        env:
-          LETTA_PG_PORT: 5432
-          LETTA_PG_USER: postgres
-          LETTA_PG_PASSWORD: postgres
-          LETTA_PG_DB: postgres
-          LETTA_PG_HOST: localhost
-        run: |
-          psql -h localhost -U postgres -d postgres -c 'CREATE EXTENSION vector'
-          poetry run alembic upgrade head
-      - name: Run core unit tests
-        env:
-          LETTA_PG_PORT: 5432
-          LETTA_PG_USER: postgres
-          LETTA_PG_PASSWORD: postgres
-          LETTA_PG_DB: postgres
-          LETTA_PG_HOST: localhost
-          LETTA_SERVER_PASS: test_server_token
-        run: |
-          poetry run pytest -s -vv tests/${{ matrix.integration_test_suite }}
diff --git a/.github/workflows/test_anthropic.yml b/.github/workflows/test_anthropic.yml
deleted file mode 100644
index b3993301..00000000
--- a/.github/workflows/test_anthropic.yml
+++ /dev/null
@@ -1,102 +0,0 @@
-name: Anthropic Claude Opus 3 Capabilities Test
-
-env:
-  ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-  COMPOSIO_API_KEY: ${{ secrets.COMPOSIO_API_KEY }}
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v4
-
-    - name: "Setup Python, Poetry and Dependencies"
-      uses: packetcoders/action-setup-cache-python-poetry@main
-      with:
-        python-version: "3.12"
-        poetry-version: "1.8.2"
-        install-args: "-E dev -E external-tools"
-
-    - name: Test first message contains expected function call and inner monologue
-      id: test_first_message
-      env:
-        ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-      run: |
-        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_claude_opus_3_returns_valid_first_message
-        echo "TEST_FIRST_MESSAGE_EXIT_CODE=$?" >> $GITHUB_ENV
-      continue-on-error: true
-
-    - name: Test model sends message with keyword
-      id: test_keyword_message
-      env:
-        ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-      run: |
-        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_claude_opus_3_returns_keyword
-        echo "TEST_KEYWORD_MESSAGE_EXIT_CODE=$?" >> $GITHUB_ENV
-      continue-on-error: true
-
-    - name: Test model uses external tool correctly
-      id: test_external_tool
-      env:
-        ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-      run: |
-        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_claude_opus_3_uses_external_tool
-        echo "TEST_EXTERNAL_TOOL_EXIT_CODE=$?" >> $GITHUB_ENV
-      continue-on-error: true
-
-    - name: Test model recalls chat memory
-      id: test_chat_memory
-      env:
-        ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-      run: |
-        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_claude_opus_3_recall_chat_memory
-        echo "TEST_CHAT_MEMORY_EXIT_CODE=$?" >> $GITHUB_ENV
-      continue-on-error: true
-
-    - name: Test model uses 'archival_memory_search' to find secret
-      id: test_archival_memory
-      env:
-        ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-      run: |
-        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_claude_opus_3_archival_memory_retrieval
-        echo "TEST_ARCHIVAL_MEMORY_EXIT_CODE=$?" >> $GITHUB_ENV
-      continue-on-error: true
-
-    - name: Test model can edit core memories
-      id: test_core_memory
-      env:
-        ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-      run: |
-        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_claude_opus_3_edit_core_memory
-        echo "TEST_CORE_MEMORY_EXIT_CODE=$?" >> $GITHUB_ENV
-      continue-on-error: true
-
-    - name: Summarize test results
-      if: always()
-      run: |
-        echo "Test Results Summary:"
-        echo "Test first message: $([[ $TEST_FIRST_MESSAGE_EXIT_CODE -eq 0 ]] && echo ✅ || echo ❌)"
-        echo "Test model sends message with keyword: $([[ $TEST_KEYWORD_MESSAGE_EXIT_CODE -eq 0 ]] && echo ✅ || echo ❌)"
-        echo "Test model uses external tool: $([[ $TEST_EXTERNAL_TOOL_EXIT_CODE -eq 0 ]] && echo ✅ || echo ❌)"
-        echo "Test model recalls chat memory: $([[ $TEST_CHAT_MEMORY_EXIT_CODE -eq 0 ]] && echo ✅ || echo ❌)"
-        echo "Test model uses 'archival_memory_search' to find secret: $([[ $TEST_ARCHIVAL_MEMORY_EXIT_CODE -eq 0 ]] && echo ✅ || echo ❌)"
-        echo "Test model can edit core memories: $([[ $TEST_CORE_MEMORY_EXIT_CODE -eq 0 ]] && echo ✅ || echo ❌)"
-
-        # Check if any test failed
-        if [[ $TEST_FIRST_MESSAGE_EXIT_CODE -ne 0 || \
-              $TEST_KEYWORD_MESSAGE_EXIT_CODE -ne 0 || \
-              $TEST_EXTERNAL_TOOL_EXIT_CODE -ne 0 || \
-              $TEST_CHAT_MEMORY_EXIT_CODE -ne 0 || \
-              $TEST_ARCHIVAL_MEMORY_EXIT_CODE -ne 0 || \
-              $TEST_CORE_MEMORY_EXIT_CODE -ne 0 ]]; then
-          echo "Some tests failed."
-          exit 78
-        fi
diff --git a/.github/workflows/test_azure.yml b/.github/workflows/test_azure.yml
deleted file mode 100644
index 7ea6982c..00000000
--- a/.github/workflows/test_azure.yml
+++ /dev/null
@@ -1,111 +0,0 @@
-name: Azure OpenAI GPT-4o Mini Capabilities Test
-
-env:
-  AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
-  AZURE_BASE_URL: ${{ secrets.AZURE_BASE_URL }}
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v4
-
-    - name: "Setup Python, Poetry and Dependencies"
-      uses: packetcoders/action-setup-cache-python-poetry@main
-      with:
-        python-version: "3.12"
-        poetry-version: "1.8.2"
-        install-args: "-E dev -E external-tools"
-
-    - name: Test first message contains expected function call and inner monologue
-      id: test_first_message
-      env:
-        AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
-        AZURE_BASE_URL: ${{ secrets.AZURE_BASE_URL }}
-      run: |
-        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_azure_gpt_4o_mini_returns_valid_first_message
-        echo "TEST_FIRST_MESSAGE_EXIT_CODE=$?" >> $GITHUB_ENV
-      continue-on-error: true
-
-    - name: Test model sends message with keyword
-      id: test_keyword_message
-      env:
-        AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
-        AZURE_BASE_URL: ${{ secrets.AZURE_BASE_URL }}
-      run: |
-        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_azure_gpt_4o_mini_returns_keyword
-        echo "TEST_KEYWORD_MESSAGE_EXIT_CODE=$?" >> $GITHUB_ENV
-      continue-on-error: true
-
-    - name: Test model uses external tool correctly
-      id: test_external_tool
-      env:
-        AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
-        AZURE_BASE_URL: ${{ secrets.AZURE_BASE_URL }}
-      run: |
-        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_azure_gpt_4o_mini_uses_external_tool
-        echo "TEST_EXTERNAL_TOOL_EXIT_CODE=$?" >> $GITHUB_ENV
-      continue-on-error: true
-
-    - name: Test model recalls chat memory
-      id: test_chat_memory
-      env:
-        AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
-        AZURE_BASE_URL: ${{ secrets.AZURE_BASE_URL }}
-      run: |
-        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_azure_gpt_4o_mini_recall_chat_memory
-        echo "TEST_CHAT_MEMORY_EXIT_CODE=$?" >> $GITHUB_ENV
-      continue-on-error: true
-
-    - name: Test model uses 'archival_memory_search' to find secret
-      id: test_archival_memory
-      env:
-        AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
-        AZURE_BASE_URL: ${{ secrets.AZURE_BASE_URL }}
-      run: |
-        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_azure_gpt_4o_mini_archival_memory_retrieval
-        echo "TEST_ARCHIVAL_MEMORY_EXIT_CODE=$?" >> $GITHUB_ENV
-      continue-on-error: true
-
-    - name: Test model can edit core memories
-      id: test_core_memory
-      env:
-        AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
-        AZURE_BASE_URL: ${{ secrets.AZURE_BASE_URL }}
-      run: |
-        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_azure_gpt_4o_mini_edit_core_memory
-        echo "TEST_CORE_MEMORY_EXIT_CODE=$?" >> $GITHUB_ENV
-      continue-on-error: true
-
-    - name: Summarize test results
-      if: always()
-      run: |
-        echo "Test Results Summary:"
-
-        # If the exit code is empty, treat it as a failure (❌)
-        echo "Test first message: $([[ -z $TEST_FIRST_MESSAGE_EXIT_CODE || $TEST_FIRST_MESSAGE_EXIT_CODE -ne 0 ]] && echo ❌ || echo ✅)"
-        echo "Test model sends message with keyword: $([[ -z $TEST_KEYWORD_MESSAGE_EXIT_CODE || $TEST_KEYWORD_MESSAGE_EXIT_CODE -ne 0 ]] && echo ❌ || echo ✅)"
-        echo "Test model uses external tool: $([[ -z $TEST_EXTERNAL_TOOL_EXIT_CODE || $TEST_EXTERNAL_TOOL_EXIT_CODE -ne 0 ]] && echo ❌ || echo ✅)"
-        echo "Test model recalls chat memory: $([[ -z $TEST_CHAT_MEMORY_EXIT_CODE || $TEST_CHAT_MEMORY_EXIT_CODE -ne 0 ]] && echo ❌ || echo ✅)"
-        echo "Test model uses 'archival_memory_search' to find secret: $([[ -z $TEST_ARCHIVAL_MEMORY_EXIT_CODE || $TEST_ARCHIVAL_MEMORY_EXIT_CODE -ne 0 ]] && echo ❌ || echo ✅)"
-        echo "Test model can edit core memories: $([[ -z $TEST_CORE_MEMORY_EXIT_CODE || $TEST_CORE_MEMORY_EXIT_CODE -ne 0 ]] && echo ❌ || echo ✅)"
-
-        # Check if any test failed (either non-zero or unset exit code)
-        if [[ -z $TEST_FIRST_MESSAGE_EXIT_CODE || $TEST_FIRST_MESSAGE_EXIT_CODE -ne 0 || \
-              -z $TEST_KEYWORD_MESSAGE_EXIT_CODE || $TEST_KEYWORD_MESSAGE_EXIT_CODE -ne 0 || \
-              -z $TEST_EXTERNAL_TOOL_EXIT_CODE || $TEST_EXTERNAL_TOOL_EXIT_CODE -ne 0 || \
-              -z $TEST_CHAT_MEMORY_EXIT_CODE || $TEST_CHAT_MEMORY_EXIT_CODE -ne 0 || \
-              -z $TEST_ARCHIVAL_MEMORY_EXIT_CODE || $TEST_ARCHIVAL_MEMORY_EXIT_CODE -ne 0 || \
-              -z $TEST_CORE_MEMORY_EXIT_CODE || $TEST_CORE_MEMORY_EXIT_CODE -ne 0 ]]; then
-          echo "Some tests failed."
-          exit 78
-        fi
-      continue-on-error: true
diff --git a/.github/workflows/test_cli.yml b/.github/workflows/test_cli.yml
deleted file mode 100644
index c7cd5240..00000000
--- a/.github/workflows/test_cli.yml
+++ /dev/null
@@ -1,67 +0,0 @@
-name: Test CLI
-
-env:
-  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  test-cli:
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-
-    services:
-      qdrant:
-        image: qdrant/qdrant
-        ports:
-          - 6333:6333
-      postgres:
-        image: pgvector/pgvector:pg17
-        ports:
-          - 5432:5432
-        env:
-          POSTGRES_HOST_AUTH_METHOD: trust
-          POSTGRES_DB: postgres
-          POSTGRES_USER: postgres
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v4
-
-    - name: "Setup Python, Poetry and Dependencies"
-      uses: packetcoders/action-setup-cache-python-poetry@main
-      with:
-        python-version: "3.12"
-        poetry-version: "1.8.2"
-        install-args: "-E dev -E postgres -E tests"
-
-    - name: Migrate database
-      env:
-        LETTA_PG_PORT: 5432
-        LETTA_PG_USER: postgres
-        LETTA_PG_PASSWORD: postgres
-        LETTA_PG_DB: postgres
-        LETTA_PG_HOST: localhost
-      run: |
-        psql -h localhost -U postgres -d postgres -c 'CREATE EXTENSION vector'
-        poetry run alembic upgrade head
-
-    - name: Test `letta run` up until first message
-      env:
-        LETTA_PG_PORT: 5432
-        LETTA_PG_USER: postgres
-        LETTA_PG_PASSWORD: postgres
-        LETTA_PG_DB: postgres
-        LETTA_PG_HOST: localhost
-        LETTA_SERVER_PASS: test_server_token
-      run: |
-        poetry run pytest -s -vv tests/test_cli.py::test_letta_run_create_new_agent
diff --git a/.github/workflows/test_examples.yml b/.github/workflows/test_examples.yml
deleted file mode 100644
index dada60dd..00000000
--- a/.github/workflows/test_examples.yml
+++ /dev/null
@@ -1,69 +0,0 @@
-name: Examples (documentation)
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v4
-    - name: Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.12'
-
-    - name: Set permissions for log directory
-      run: |
-        mkdir -p /home/runner/.letta/logs
-        sudo chown -R $USER:$USER /home/runner/.letta/logs
-        chmod -R 755 /home/runner/.letta/logs
-
-    - name: Build and run docker dev server
-      env:
-        LETTA_PG_DB:   letta
-        LETTA_PG_USER: letta
-        LETTA_PG_PASSWORD: letta
-        LETTA_PG_PORT: 8888
-        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-
-      run: docker compose -f dev-compose.yaml up --build -d
-    #- name: "Setup Python, Poetry and Dependencies"
-    #  uses: packetcoders/action-setup-cache-python-poetry@v1.2.0
-    #  with:
-    #    python-version: "3.12"
-    #    poetry-version: "1.8.2"
-    #    install-args: "--all-extras"
-
-    - name: Wait for service
-      run: bash scripts/wait_for_service.sh http://localhost:8283 -- echo "Service is ready"
-
-    - name: Run tests with pytest
-      env:
-        LETTA_PG_DB:   letta
-        LETTA_PG_USER: letta
-        LETTA_PG_PASSWORD: letta
-        LETTA_PG_PORT: 8888
-        LETTA_SERVER_PASS: test_server_token
-        LETTA_SERVER_URL: http://localhost:8283
-        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-        PYTHONPATH: ${{ github.workspace }}:${{ env.PYTHONPATH }}
-      run: |
-        pipx install poetry==1.8.2
-        poetry install -E dev -E postgres -E external-tools
-        poetry run python examples/docs/agent_advanced.py
-        poetry run python examples/docs/agent_basic.py
-        poetry run python examples/docs/memory.py
-        poetry run python examples/docs/rest_client.py
-        poetry run python examples/docs/tools.py
-
-    - name: Print docker logs if tests fail
-      if: failure()
-      run: |
-        echo "Printing Docker Logs..."
-        docker compose -f dev-compose.yaml logs
diff --git a/.github/workflows/test_memgpt_hosted.yml b/.github/workflows/test_memgpt_hosted.yml
deleted file mode 100644
index 191ace57..00000000
--- a/.github/workflows/test_memgpt_hosted.yml
+++ /dev/null
@@ -1,31 +0,0 @@
-name: Endpoint (Letta)
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v4
-
-    - name: "Setup Python, Poetry and Dependencies"
-      uses: packetcoders/action-setup-cache-python-poetry@main
-      with:
-        python-version: "3.12"
-        poetry-version: "1.8.2"
-        install-args: "-E dev"
-
-    - name: Test LLM endpoint
-      run: |
-        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_llm_endpoint_letta_hosted
-      continue-on-error: true
-
-    - name: Test embedding endpoint
-      run: |
-        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_embedding_endpoint_letta_hosted
diff --git a/.github/workflows/test_ollama.yml b/.github/workflows/test_ollama.yml
deleted file mode 100644
index e76dc5dc..00000000
--- a/.github/workflows/test_ollama.yml
+++ /dev/null
@@ -1,87 +0,0 @@
-name: Endpoint (Ollama)
-
-env:
-  OLLAMA_BASE_URL: "http://localhost:11434"
-  COMPOSIO_API_KEY: ${{ secrets.COMPOSIO_API_KEY }}
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@v4
-
-      - name: Install Ollama
-        run: |
-          set -e
-          set -x
-          curl -vfsSL https://ollama.com/install.sh -o install.sh
-          chmod +x install.sh
-          bash -x install.sh
-          if ! command -v ollama; then
-            echo "Ollama binary not found in PATH after installation."
-            exit 1
-          fi
-          echo "Ollama installed successfully."
-
-      - name: Start Ollama Server
-        run: |
-          set -e
-          set -x
-          ollama serve >ollama_server.log 2>&1 &
-          sleep 15
-          if ! curl -v http://localhost:11434; then
-            echo "Server logs (if available):"
-            [ -f ollama_server.log ] && cat ollama_server.log || echo "No logs found."
-            exit 1
-          fi
-          echo "Ollama server started successfully."
-
-      - name: Pull Models
-        run: |
-          set -e
-          set -x
-          for attempt in {1..3}; do
-            ollama pull thewindmom/hermes-3-llama-3.1-8b && break || sleep 5
-          done
-          for attempt in {1..3}; do
-            ollama pull mxbai-embed-large && break || sleep 5
-          done
-
-      - name: Debug Logs on Failure
-        if: failure()
-        run: |
-          echo "Debugging logs on failure:"
-          [ -f ollama_server.log ] && cat ollama_server.log || echo "No server logs available."
-
-      - name: Setup Python, Poetry, and Dependencies
-        uses: packetcoders/action-setup-cache-python-poetry@main
-        with:
-          python-version: "3.12"
-          poetry-version: "1.8.2"
-          install-args: "-E dev"
-
-      - name: Test LLM Endpoint
-        run: |
-          set -e
-          set -x
-          poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_llm_endpoint_ollama
-
-      - name: Test Embedding Endpoint
-        run: |
-          set -e
-          set -x
-          poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_embedding_endpoint_ollama
-
-      - name: Test Provider
-        run: |
-          set -e
-          set -x
-          poetry run pytest -s -vv tests/test_providers.py::test_ollama
diff --git a/.github/workflows/test_openai.yml b/.github/workflows/test_openai.yml
deleted file mode 100644
index bd42fa7d..00000000
--- a/.github/workflows/test_openai.yml
+++ /dev/null
@@ -1,82 +0,0 @@
-name: OpenAI GPT-4 Capabilities Test
-
-env:
-  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-  COMPOSIO_API_KEY: ${{ secrets.COMPOSIO_API_KEY }}
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v4
-
-    - name: "Setup Python, Poetry and Dependencies"
-      uses: packetcoders/action-setup-cache-python-poetry@main
-      with:
-        python-version: "3.12"
-        poetry-version: "1.8.2"
-        install-args: "-E dev -E external-tools"
-
-    - name: Test first message contains expected function call and inner monologue
-      id: test_first_message
-      env:
-        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-      run: |
-        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_openai_gpt_4o_returns_valid_first_message
-
-    - name: Test model sends message with keyword
-      id: test_keyword_message
-      env:
-        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-      run: |
-        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_openai_gpt_4o_returns_keyword
-
-    - name: Test model uses external tool correctly
-      id: test_external_tool
-      env:
-        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-      run: |
-        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_openai_gpt_4o_uses_external_tool
-
-    - name: Test model recalls chat memory
-      id: test_chat_memory
-      env:
-        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-      run: |
-        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_openai_gpt_4o_recall_chat_memory
-
-    - name: Test model uses 'archival_memory_search' to find secret
-      id: test_archival_memory_search
-      env:
-        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-      run: |
-        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_openai_gpt_4o_archival_memory_retrieval
-
-    - name: Test model uses 'archival_memory_insert' to insert archival memories
-      id: test_archival_memory_insert
-      env:
-        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-      run: |
-        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_openai_gpt_4o_archival_memory_insert
-
-    - name: Test model can edit core memories
-      id: test_core_memory
-      env:
-        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-      run: |
-        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_openai_gpt_4o_edit_core_memory
-
-    - name: Test embedding endpoint
-      id: test_embedding_endpoint
-      env:
-        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-      run: |
-        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_embedding_endpoint_openai
diff --git a/.github/workflows/test_together.yml b/.github/workflows/test_together.yml
deleted file mode 100644
index 222ce40c..00000000
--- a/.github/workflows/test_together.yml
+++ /dev/null
@@ -1,105 +0,0 @@
-name: Together Llama 3.1 70b Capabilities Test
-
-env:
-  TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
-  COMPOSIO_API_KEY: ${{ secrets.COMPOSIO_API_KEY }}
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v4
-
-    - name: "Setup Python, Poetry and Dependencies"
-      uses: packetcoders/action-setup-cache-python-poetry@main
-      with:
-        python-version: "3.12"
-        poetry-version: "1.8.2"
-        install-args: "-E dev -E external-tools"
-
-    - name: Test first message contains expected function call and inner monologue
-      id: test_first_message
-      env:
-        TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
-      run: |
-        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_together_llama_3_70b_returns_valid_first_message
-        echo "TEST_FIRST_MESSAGE_EXIT_CODE=$?" >> $GITHUB_ENV
-      continue-on-error: true
-
-    - name: Test model sends message with keyword
-      id: test_keyword_message
-      env:
-        TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
-      run: |
-        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_together_llama_3_70b_returns_keyword
-        echo "TEST_KEYWORD_MESSAGE_EXIT_CODE=$?" >> $GITHUB_ENV
-      continue-on-error: true
-
-    - name: Test model uses external tool correctly
-      id: test_external_tool
-      env:
-        TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
-      run: |
-        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_together_llama_3_70b_uses_external_tool
-        echo "TEST_EXTERNAL_TOOL_EXIT_CODE=$?" >> $GITHUB_ENV
-      continue-on-error: true
-
-    - name: Test model recalls chat memory
-      id: test_chat_memory
-      env:
-        TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
-      run: |
-        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_together_llama_3_70b_recall_chat_memory
-        echo "TEST_CHAT_MEMORY_EXIT_CODE=$?" >> $GITHUB_ENV
-      continue-on-error: true
-
-    - name: Test model uses 'archival_memory_search' to find secret
-      id: test_archival_memory
-      env:
-        TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
-      run: |
-        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_together_llama_3_70b_archival_memory_retrieval
-        echo "TEST_ARCHIVAL_MEMORY_EXIT_CODE=$?" >> $GITHUB_ENV
-      continue-on-error: true
-
-    - name: Test model can edit core memories
-      id: test_core_memory
-      env:
-        TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
-      run: |
-        poetry run pytest -s -vv tests/test_model_letta_perfomance.py::test_together_llama_3_70b_edit_core_memory
-        echo "TEST_CORE_MEMORY_EXIT_CODE=$?" >> $GITHUB_ENV
-      continue-on-error: true
-
-    - name: Summarize test results
-      if: always()
-      run: |
-        echo "Test Results Summary:"
-
-        # If the exit code is empty, treat it as a failure (❌)
-        echo "Test first message: $([[ -z $TEST_FIRST_MESSAGE_EXIT_CODE || $TEST_FIRST_MESSAGE_EXIT_CODE -ne 0 ]] && echo ❌ || echo ✅)"
-        echo "Test model sends message with keyword: $([[ -z $TEST_KEYWORD_MESSAGE_EXIT_CODE || $TEST_KEYWORD_MESSAGE_EXIT_CODE -ne 0 ]] && echo ❌ || echo ✅)"
-        echo "Test model uses external tool: $([[ -z $TEST_EXTERNAL_TOOL_EXIT_CODE || $TEST_EXTERNAL_TOOL_EXIT_CODE -ne 0 ]] && echo ❌ || echo ✅)"
-        echo "Test model recalls chat memory: $([[ -z $TEST_CHAT_MEMORY_EXIT_CODE || $TEST_CHAT_MEMORY_EXIT_CODE -ne 0 ]] && echo ❌ || echo ✅)"
-        echo "Test model uses 'archival_memory_search' to find secret: $([[ -z $TEST_ARCHIVAL_MEMORY_EXIT_CODE || $TEST_ARCHIVAL_MEMORY_EXIT_CODE -ne 0 ]] && echo ❌ || echo ✅)"
-        echo "Test model can edit core memories: $([[ -z $TEST_CORE_MEMORY_EXIT_CODE || $TEST_CORE_MEMORY_EXIT_CODE -ne 0 ]] && echo ❌ || echo ✅)"
-
-        # Check if any test failed (either non-zero or unset exit code)
-        if [[ -z $TEST_FIRST_MESSAGE_EXIT_CODE || $TEST_FIRST_MESSAGE_EXIT_CODE -ne 0 || \
-              -z $TEST_KEYWORD_MESSAGE_EXIT_CODE || $TEST_KEYWORD_MESSAGE_EXIT_CODE -ne 0 || \
-              -z $TEST_EXTERNAL_TOOL_EXIT_CODE || $TEST_EXTERNAL_TOOL_EXIT_CODE -ne 0 || \
-              -z $TEST_CHAT_MEMORY_EXIT_CODE || $TEST_CHAT_MEMORY_EXIT_CODE -ne 0 || \
-              -z $TEST_ARCHIVAL_MEMORY_EXIT_CODE || $TEST_ARCHIVAL_MEMORY_EXIT_CODE -ne 0 || \
-              -z $TEST_CORE_MEMORY_EXIT_CODE || $TEST_CORE_MEMORY_EXIT_CODE -ne 0 ]]; then
-          echo "Some tests failed."
-          exit 78
-        fi
-      continue-on-error: true