From e4456acec13003d611b7f38c59398cdeb61de81b Mon Sep 17 00:00:00 2001
From: cthomas <caren@letta.com>
Date: Mon, 17 Mar 2025 17:58:38 -0700
Subject: [PATCH] feat: add grafana support for local dev (#1318)

---
 letta/tracing.py                          | 10 ++++++
 otel-collector-config-clickhouse-dev.yaml | 43 +++++++++++++++++++++++
 otel-collector-config-file-dev.yaml       | 30 ++++++++++++++++
 project.json                              |  3 +-
 start-otel-collector.sh                   | 26 ++++++++++++++
 5 files changed, 111 insertions(+), 1 deletion(-)
 create mode 100644 otel-collector-config-clickhouse-dev.yaml
 create mode 100644 otel-collector-config-file-dev.yaml
 create mode 100755 start-otel-collector.sh

diff --git a/letta/tracing.py b/letta/tracing.py
index e0dda3d5..0144f1f5 100644
--- a/letta/tracing.py
+++ b/letta/tracing.py
@@ -112,6 +112,16 @@ def setup_tracing(
     global _is_tracing_initialized
 
     provider = TracerProvider(resource=Resource.create({"service.name": service_name}))
+    import uuid
+
+    provider = TracerProvider(
+        resource=Resource.create(
+            {
+                "service.name": service_name,
+                "device.id": uuid.getnode(),  # MAC address as unique device identifier
+            }
+        )
+    )
     if endpoint:
         provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint)))
         _is_tracing_initialized = True
diff --git a/otel-collector-config-clickhouse-dev.yaml b/otel-collector-config-clickhouse-dev.yaml
new file mode 100644
index 00000000..1c9c0cca
--- /dev/null
+++ b/otel-collector-config-clickhouse-dev.yaml
@@ -0,0 +1,43 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+
+processors:
+  batch:
+    timeout: 1s
+    send_batch_size: 1024
+
+exporters:
+  file:
+    path: ${HOME}/.letta/logs/traces.json
+    rotation:
+      max_megabytes: 100
+      max_days: 7
+      max_backups: 5
+  clickhouse:
+    endpoint: ${CLICKHOUSE_ENDPOINT}
+    database: ${CLICKHOUSE_DATABASE}
+    username: ${CLICKHOUSE_USERNAME}
+    password: ${CLICKHOUSE_PASSWORD}
+    timeout: 5s
+    sending_queue:
+      queue_size: 100
+    retry_on_failure:
+      enabled: true
+      initial_interval: 5s
+      max_interval: 30s
+      max_elapsed_time: 300s
+
+service:
+  telemetry:
+    logs:
+      level: error
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [file, clickhouse]
diff --git a/otel-collector-config-file-dev.yaml b/otel-collector-config-file-dev.yaml
new file mode 100644
index 00000000..dbb21454
--- /dev/null
+++ b/otel-collector-config-file-dev.yaml
@@ -0,0 +1,30 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: localhost:4317
+      http:
+        endpoint: localhost:4318
+
+processors:
+  batch:
+    timeout: 1s
+    send_batch_size: 1024
+
+exporters:
+  file:
+    path: ${HOME}/.letta/logs/traces.json
+    rotation:
+      max_megabytes: 100
+      max_days: 7
+      max_backups: 5
+
+service:
+  telemetry:
+    logs:
+      level: error
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [file]
diff --git a/project.json b/project.json
index 18b70617..88637f0f 100644
--- a/project.json
+++ b/project.json
@@ -26,7 +26,8 @@
     "dev": {
       "executor": "@nxlv/python:run-commands",
       "options": {
-        "command": "poetry run letta server",
+        "commands": ["./start-otel-collector.sh", "poetry run letta server"],
+        "parallel": true,
         "cwd": "apps/core"
       }
     },
diff --git a/start-otel-collector.sh b/start-otel-collector.sh
new file mode 100755
index 00000000..58d5225d
--- /dev/null
+++ b/start-otel-collector.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+set -e  # Exit on any error
+
+# Create bin directory if it doesn't exist
+mkdir -p bin
+
+# Download and extract collector if not already present
+if [ ! -f "bin/otelcol-contrib" ]; then
+    echo "Downloading OpenTelemetry Collector..."
+    curl -L https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.96.0/otelcol-contrib_0.96.0_darwin_amd64.tar.gz -o otelcol.tar.gz
+    tar xzf otelcol.tar.gz -C bin/
+    rm otelcol.tar.gz
+    chmod +x bin/otelcol-contrib
+fi
+
+# Start OpenTelemetry Collector
+if [ -n "$CLICKHOUSE_ENDPOINT" ] && [ -n "$CLICKHOUSE_PASSWORD" ]; then
+    echo "Starting OpenTelemetry Collector with Clickhouse export..."
+    CONFIG_FILE="otel-collector-config-clickhouse-dev.yaml"
+else
+    echo "Starting OpenTelemetry Collector with file export only..."
+    CONFIG_FILE="otel-collector-config-file-dev.yaml"
+fi
+
+# Run collector
+exec ./bin/otelcol-contrib --config "$CONFIG_FILE"