feat: add grafana support for local dev (#1318)

letta/tracing.py

@@ -112,6 +112,16 @@ def setup_tracing(
     global _is_tracing_initialized
 
     provider = TracerProvider(resource=Resource.create({"service.name": service_name}))
+    import uuid
+
+    provider = TracerProvider(
+        resource=Resource.create(
+            {
+                "service.name": service_name,
+                "device.id": uuid.getnode(),  # MAC address as unique device identifier
+            }
+        )
+    )
     if endpoint:
         provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint)))
         _is_tracing_initialized = True

otel-collector-config-clickhouse-dev.yaml

@@ -0,0 +1,43 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+
+processors:
+  batch:
+    timeout: 1s
+    send_batch_size: 1024
+
+exporters:
+  file:
+    path: ${HOME}/.letta/logs/traces.json
+    rotation:
+      max_megabytes: 100
+      max_days: 7
+      max_backups: 5
+  clickhouse:
+    endpoint: ${CLICKHOUSE_ENDPOINT}
+    database: ${CLICKHOUSE_DATABASE}
+    username: ${CLICKHOUSE_USERNAME}
+    password: ${CLICKHOUSE_PASSWORD}
+    timeout: 5s
+    sending_queue:
+      queue_size: 100
+    retry_on_failure:
+      enabled: true
+      initial_interval: 5s
+      max_interval: 30s
+      max_elapsed_time: 300s
+
+service:
+  telemetry:
+    logs:
+      level: error
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [file, clickhouse]

otel-collector-config-file-dev.yaml

@@ -0,0 +1,30 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: localhost:4317
+      http:
+        endpoint: localhost:4318
+
+processors:
+  batch:
+    timeout: 1s
+    send_batch_size: 1024
+
+exporters:
+  file:
+    path: ${HOME}/.letta/logs/traces.json
+    rotation:
+      max_megabytes: 100
+      max_days: 7
+      max_backups: 5
+
+service:
+  telemetry:
+    logs:
+      level: error
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [file]

project.json

@@ -26,7 +26,8 @@
     "dev": {
       "executor": "@nxlv/python:run-commands",
       "options": {
-        "command": "poetry run letta server",
+        "commands": ["./start-otel-collector.sh", "poetry run letta server"],
+        "parallel": true,
         "cwd": "apps/core"
       }
     },

start-otel-collector.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -e  # Exit on any error
+
+# Create bin directory if it doesn't exist
+mkdir -p bin
+
+# Download and extract collector if not already present
+if [ ! -f "bin/otelcol-contrib" ]; then
+    echo "Downloading OpenTelemetry Collector..."
+    curl -L https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.96.0/otelcol-contrib_0.96.0_darwin_amd64.tar.gz -o otelcol.tar.gz
+    tar xzf otelcol.tar.gz -C bin/
+    rm otelcol.tar.gz
+    chmod +x bin/otelcol-contrib
+fi
+
+# Start OpenTelemetry Collector
+if [ -n "$CLICKHOUSE_ENDPOINT" ] && [ -n "$CLICKHOUSE_PASSWORD" ]; then
+    echo "Starting OpenTelemetry Collector with Clickhouse export..."
+    CONFIG_FILE="otel-collector-config-clickhouse-dev.yaml"
+else
+    echo "Starting OpenTelemetry Collector with file export only..."
+    CONFIG_FILE="otel-collector-config-file-dev.yaml"
+fi
+
+# Run collector
+exec ./bin/otelcol-contrib --config "$CONFIG_FILE"