chore: merge oss (#3712)

.github/workflows/model-sweep.yaml

@@ -0,0 +1,144 @@
+name: Model Sweep
+on:
+  workflow_dispatch:
+    inputs:
+      branch-name:
+        required: true
+        type: string
+
+jobs:
+  model-sweep:
+    runs-on: [self-hosted, medium]
+    services:
+      qdrant:
+        image: qdrant/qdrant
+        ports:
+          - 6333:6333
+      postgres:
+        image: pgvector/pgvector:pg17
+        ports:
+          - 5432:5432
+        env:
+          POSTGRES_HOST_AUTH_METHOD: trust
+          POSTGRES_DB: postgres
+          POSTGRES_USER: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - name: Check if gh is installed
+        run: |
+          if ! command -v gh >/dev/null 2>&1
+          then
+            echo "gh could not be found, installing now..."
+            # install gh cli
+            (type -p wget >/dev/null || (sudo apt update && sudo apt-get install wget -y)) \
+              && sudo mkdir -p -m 755 /etc/apt/keyrings \
+              && out=$(mktemp) && wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg \
+              && cat $out | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \
+              && sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \
+              && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
+              && sudo apt update \
+              && sudo apt install gh -y
+          fi
+
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Inject env vars into environment
+        run: |
+          # Get secrets and mask them before adding to environment
+          while IFS= read -r line || [[ -n "$line" ]]; do
+            if [[ -n "$line" ]]; then
+              value=$(echo "$line" | cut -d= -f2-)
+              echo "::add-mask::$value"
+              echo "$line" >> $GITHUB_ENV
+            fi
+          done < <(letta_secrets_helper --env dev --service ci)
+
+      - name: Install dependencies
+        shell: bash
+        run: poetry install --no-interaction --no-root ${{ inputs.install-args || '-E dev -E postgres -E external-tools -E tests -E cloud-tool-sandbox -E google' }}
+      - name: Migrate database
+        env:
+          LETTA_PG_PORT: 5432
+          LETTA_PG_USER: postgres
+          LETTA_PG_PASSWORD: postgres
+          LETTA_PG_DB: postgres
+          LETTA_PG_HOST: localhost
+        run: |
+          psql -h localhost -U postgres -d postgres -c 'CREATE EXTENSION vector'
+          poetry run alembic upgrade head
+
+      - name: Run integration tests
+        # if any of the 1000+ test cases fail, pytest reports exit code 1 and won't procces/upload the results
+        continue-on-error: true
+        env:
+          LETTA_PG_PORT: 5432
+          LETTA_PG_USER: postgres
+          LETTA_PG_PASSWORD: postgres
+          LETTA_PG_DB: postgres
+          LETTA_PG_HOST: localhost
+          LETTA_SERVER_PASS: test_server_token
+          OPENAI_API_KEY: ${{ env.OPENAI_API_KEY }}
+          ANTHROPIC_API_KEY: ${{ env.ANTHROPIC_API_KEY }}
+          AZURE_API_KEY: ${{ env.AZURE_API_KEY }}
+          AZURE_BASE_URL: ${{ secrets.AZURE_BASE_URL }}
+          GEMINI_API_KEY: ${{ env.GEMINI_API_KEY }}
+          COMPOSIO_API_KEY: ${{ env.COMPOSIO_API_KEY }}
+          GOOGLE_CLOUD_PROJECT: ${{ secrets.GOOGLE_CLOUD_PROJECT}}
+          GOOGLE_CLOUD_LOCATION: ${{ secrets.GOOGLE_CLOUD_LOCATION}}
+          DEEPSEEK_API_KEY: ${{ env.DEEPSEEK_API_KEY}}
+          LETTA_USE_EXPERIMENTAL: 1
+        run: |
+          poetry run pytest \
+            -s -vv \
+            .github/scripts/model-sweep/model_sweep.py \
+            --json-report --json-report-file=.github/scripts/model-sweep/model_sweep_report.json --json-report-indent=4
+
+      - name: Convert report to markdown
+        continue-on-error: true
+        # file path args to generate_model_sweep_markdown.py are relative to the script
+        run: |
+          poetry run python \
+            .github/scripts/model-sweep/generate_model_sweep_markdown.py \
+            .github/scripts/model-sweep/model_sweep_report.json \
+            .github/scripts/model-sweep/supported-models.mdx
+          echo "Model sweep report saved to .github/scripts/model-sweep/supported-models.mdx"
+
+      - id: date
+        run: echo "date=$(date +%Y-%m-%d)" >> $GITHUB_OUTPUT
+
+      - name: commit and open pull request
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          BRANCH_NAME=model-sweep/${{ inputs.branch-name || format('{0}', steps.date.outputs.date) }}
+          gh auth setup-git
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+          git checkout -b $BRANCH_NAME
+          git add .github/scripts/model-sweep/supported-models.mdx
+          git commit -m "Update model sweep report"
+          # only push if changes were made
+          if git diff main --quiet; then
+            echo "No changes detected, skipping push"
+            exit 0
+          else
+            git push origin $BRANCH_NAME
+            gh pr create \
+              --base main \
+              --head $BRANCH_NAME \
+              --title "chore: update model sweep report" \
+              --body "Automated PR to update model sweep report"
+          fi
+
+      - name: Upload model sweep report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: model-sweep-report
+          path: .github/scripts/model-sweep/model_sweep_report.json

.github/workflows/send-message-integration-tests.yaml

@@ -0,0 +1,155 @@
+name: Send Message SDK Tests
+on:
+  pull_request_target:
+    # branches: [main] # TODO: uncomment before merge
+    types: [labeled]
+    paths:
+      - 'letta/**'
+
+jobs:
+  send-messages:
+    # Only run when the "safe to test" label is applied
+    if: contains(github.event.pull_request.labels.*.name, 'safe to test')
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix:
+        config_file:
+          - "openai-gpt-4o-mini.json"
+          - "azure-gpt-4o-mini.json"
+          - "claude-3-5-sonnet.json"
+          - "claude-3-7-sonnet.json"
+          - "claude-3-7-sonnet-extended.json"
+          - "gemini-pro.json"
+          - "gemini-vertex.json"
+    services:
+      qdrant:
+        image: qdrant/qdrant
+        ports:
+          - 6333:6333
+      postgres:
+        image: pgvector/pgvector:pg17
+        ports:
+          - 5432:5432
+        env:
+          POSTGRES_HOST_AUTH_METHOD: trust
+          POSTGRES_DB: postgres
+          POSTGRES_USER: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      # Ensure secrets don't leak
+      - name: Configure git to hide secrets
+        run: |
+          git config --global core.logAllRefUpdates false
+          git config --global log.hideCredentials true
+      - name: Set up secret masking
+        run: |
+          # Automatically mask any environment variable ending with _KEY
+          for var in $(env | grep '_KEY=' | cut -d= -f1); do
+            value="${!var}"
+            if [[ -n "$value" ]]; then
+              # Mask the full value
+              echo "::add-mask::$value"
+
+              # Also mask partial values (first and last several characters)
+              # This helps when only parts of keys appear in logs
+              if [[ ${#value} -gt 8 ]]; then
+                echo "::add-mask::${value:0:8}"
+                echo "::add-mask::${value:(-8)}"
+              fi
+
+              # Also mask with common formatting changes
+              # Some logs might add quotes or other characters
+              echo "::add-mask::\"$value\""
+              echo "::add-mask::$value\""
+              echo "::add-mask::\"$value"
+
+              echo "Masked secret: $var (length: ${#value})"
+            fi
+          done
+
+      # Check out base repository code, not the PR's code (for security)
+      - name: Checkout base repository
+        uses: actions/checkout@v4 # No ref specified means it uses base branch
+
+      # Only extract relevant files from the PR (for security, specifically prevent modification of workflow files)
+      - name: Extract PR schema files
+        run: |
+          # Fetch PR without checking it out
+          git fetch origin pull/${{ github.event.pull_request.number }}/head:pr-${{ github.event.pull_request.number }}
+
+          # Extract ONLY the schema files
+          git checkout pr-${{ github.event.pull_request.number }} -- letta/
+      - name: Set up python 3.12
+        id: setup-python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+      - name: Load cached Poetry Binary
+        id: cached-poetry-binary
+        uses: actions/cache@v4
+        with:
+          path: ~/.local
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-1.8.3
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: 1.8.3
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v4
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}${{ inputs.install-args || '-E dev -E postgres -E external-tools -E tests -E cloud-tool-sandbox' }}
+          # Restore cache with this prefix if not exact match with key
+          # Note cache-hit returns false in this case, so the below step will run
+          restore-keys: |
+            venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-
+      - name: Install dependencies
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        shell: bash
+        run: poetry install --no-interaction --no-root ${{ inputs.install-args || '-E dev -E postgres -E external-tools -E tests -E cloud-tool-sandbox -E google' }}
+      - name: Install letta packages via Poetry
+        run: |
+          poetry run pip install --upgrade letta-client letta
+      - name: Migrate database
+        env:
+          LETTA_PG_PORT: 5432
+          LETTA_PG_USER: postgres
+          LETTA_PG_PASSWORD: postgres
+          LETTA_PG_DB: postgres
+          LETTA_PG_HOST: localhost
+        run: |
+          psql -h localhost -U postgres -d postgres -c 'CREATE EXTENSION vector'
+          poetry run alembic upgrade head
+      - name: Run integration tests for ${{ matrix.config_file }}
+        env:
+          LLM_CONFIG_FILE: ${{ matrix.config_file }}
+          LETTA_PG_PORT: 5432
+          LETTA_PG_USER: postgres
+          LETTA_PG_PASSWORD: postgres
+          LETTA_PG_DB: postgres
+          LETTA_PG_HOST: localhost
+          LETTA_SERVER_PASS: test_server_token
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
+          AZURE_BASE_URL: ${{ secrets.AZURE_BASE_URL }}
+          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
+          COMPOSIO_API_KEY: ${{ secrets.COMPOSIO_API_KEY }}
+          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
+          GOOGLE_CLOUD_PROJECT: ${{ secrets.GOOGLE_CLOUD_PROJECT }}
+          GOOGLE_CLOUD_LOCATION: ${{ secrets.GOOGLE_CLOUD_LOCATION }}
+        run: |
+          poetry run pytest \
+            -s -vv \
+            tests/integration_test_send_message.py \
+            --maxfail=1 --durations=10