From abfe742665196b399a96d0c21f8e8914c2eb7e3d Mon Sep 17 00:00:00 2001
From: cthomas <caren@letta.com>
Date: Thu, 31 Jul 2025 17:26:20 -0700
Subject: [PATCH] feat: remove file export for otel traces (#3691)

---
 otel/otel-collector-config-clickhouse-prod.yaml | 7 +++++++
 otel/otel-collector-config-clickhouse.yaml      | 8 +-------
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/otel/otel-collector-config-clickhouse-prod.yaml b/otel/otel-collector-config-clickhouse-prod.yaml
index b9ca8282..53129e25 100644
--- a/otel/otel-collector-config-clickhouse-prod.yaml
+++ b/otel/otel-collector-config-clickhouse-prod.yaml
@@ -10,6 +10,13 @@ receivers:
       - /root/.letta/logs/Letta.log
     multiline:
       line_start_pattern: ^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}
+    operators:
+      # Extract timestamp and other fields
+      - type: regex_parser
+        regex: '^(?P<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})\s+.*'
+      - type: time_parser
+        parse_from: attributes.timestamp
+        layout: '%Y-%m-%d %H:%M:%S,%L'
 
 processors:
   batch:
diff --git a/otel/otel-collector-config-clickhouse.yaml b/otel/otel-collector-config-clickhouse.yaml
index aa00ce0d..332ef340 100644
--- a/otel/otel-collector-config-clickhouse.yaml
+++ b/otel/otel-collector-config-clickhouse.yaml
@@ -12,12 +12,6 @@ processors:
     send_batch_size: 1024
 
 exporters:
-  file:
-    path: /root/.letta/logs/traces.json
-    rotation:
-      max_megabytes: 100
-      max_days: 7
-      max_backups: 5
   clickhouse:
     endpoint: ${CLICKHOUSE_ENDPOINT}
     database: ${CLICKHOUSE_DATABASE}
@@ -40,4 +34,4 @@ service:
     traces:
       receivers: [otlp]
       processors: [batch]
-      exporters: [file, clickhouse]
+      exporters: [clickhouse]