feat: Ship traces to datadog and add trace correlation (#6311)

* mvp works locally

* move to vars

* remove local dev datadog shipper

letta/log.py

@@ -71,6 +71,20 @@ class JSONFormatter(logging.Formatter):
         if hasattr(record, "dd.version"):
             log_data["dd.version"] = getattr(record, "dd.version")
 
+        # Add OpenTelemetry trace correlation (for OTEL → Datadog integration)
+        try:
+            from opentelemetry import trace
+
+            span = trace.get_current_span()
+            if span and span.get_span_context().is_valid:
+                ctx = span.get_span_context()
+                # Format trace_id and span_id as Datadog expects (decimal strings)
+                log_data["dd.trace_id"] = str(ctx.trace_id)
+                log_data["dd.span_id"] = str(ctx.span_id)
+        except Exception:
+            # Fail silently if OTEL is not available
+            pass
+
         # Add exception info if present
         if record.exc_info:
             log_data["exception"] = {

otel/otel-collector-config-clickhouse-prod.yaml

@@ -53,6 +53,16 @@ exporters:
       max_interval: 30s
       max_elapsed_time: 300s
 
+  datadog:
+    api:
+      site: ${env:DD_SITE}
+      key: ${env:DD_API_KEY}
+    traces:
+      span_name_as_resource_name: true
+      span_name_remappings:
+        http.method: http.request.method
+        http.status_code: http.response.status_code
+
 extensions:
   health_check:
   pprof:
@@ -69,11 +79,11 @@ service:
     traces:
       receivers: [otlp]
       processors: [memory_limiter, batch]
-      exporters: [clickhouse]
+      exporters: [clickhouse, datadog]
     logs:
       receivers: [filelog]
       processors: [resource, memory_limiter, batch]
-      exporters: [clickhouse]
+      exporters: [clickhouse, datadog]
     metrics:
       receivers: [otlp]
       processors: [memory_limiter, batch]