fix: prevent huge otel spans causing pods to be OOMKilled (#5871)

* otel fix

* add unit test

* log the resource id

* iterables support and fix unittest

* fix some edge cases

tests/test_otel_tracing.py

@@ -0,0 +1,301 @@
+"""
+Unit tests for OTEL tracing span attribute handling.
+
+Tests that the @trace_method decorator properly excludes or truncates
+large parameters to prevent memory bloat and RESOURCE_EXHAUSTED errors.
+"""
+
+import pytest
+from opentelemetry import trace
+from opentelemetry.sdk.trace import TracerProvider
+
+from letta.otel.tracing import trace_method
+from letta.schemas.letta_message_content import TextContent
+from letta.schemas.message import Message, MessageRole
+
+
+class CaptureSpanProcessor:
+    """Span processor that captures spans for testing."""
+
+    def __init__(self):
+        self.spans = []
+
+    def on_start(self, span, parent_context):
+        pass
+
+    def on_end(self, span):
+        self.spans.append(span)
+
+    def shutdown(self):
+        pass
+
+    def force_flush(self, timeout_millis=None):
+        pass
+
+
+class MockAgentState:
+    """Mock agent state with configurable size."""
+
+    def __init__(self, size_mb=5):
+        self.id = "agent-test-123"
+        self.name = "Test Agent"
+        self.memory = {"large_data": "X" * int(size_mb * 1024 * 1024)}
+        self.message_ids = [f"msg-{i}" for i in range(1000)]
+
+    def __str__(self):
+        return f"AgentState(id={self.id}, memory_size={len(str(self.memory))})"
+
+
+@pytest.fixture(scope="module")
+def span_processor():
+    """Setup OTEL tracing with span capture."""
+    provider = TracerProvider()
+    processor = CaptureSpanProcessor()
+    provider.add_span_processor(processor)
+    trace.set_tracer_provider(provider)
+
+    # Initialize letta tracing
+    import letta.otel.tracing as tracing_module
+
+    tracing_module._is_tracing_initialized = True
+
+    yield processor
+
+    # Reset
+    tracing_module._is_tracing_initialized = False
+
+
+@pytest.fixture(scope="function", autouse=True)
+def clear_spans(span_processor):
+    """Clear captured spans before each test."""
+    span_processor.spans.clear()
+    yield
+
+
+def create_large_messages(size_mb=5):
+    """Create large message list simulating production data."""
+    large_text = "CRYPTO SECURITY REPORT:\n" + ("X" * int(size_mb * 1024 * 1024))
+
+    return [
+        Message(
+            id="message-12345678",
+            role=MessageRole.user,
+            content=[TextContent(text=large_text)],
+            agent_id="agent-12345678-1234-1234-1234-123456789012",
+        ),
+        Message(
+            id="message-87654321",
+            role=MessageRole.assistant,
+            content=[TextContent(text="I'll analyze that report...")],
+            agent_id="agent-12345678-1234-1234-1234-123456789012",
+        ),
+    ]
+
+
+def get_span_attributes(span):
+    """Extract parameter attributes from a span."""
+    attrs = span.attributes or {}
+    return {key.replace("parameter.", ""): value for key, value in attrs.items() if key.startswith("parameter.")}
+
+
+def test_messages_parameter_excluded(span_processor):
+    """Test that large messages parameter is excluded from span attributes."""
+
+    @trace_method
+    def test_func(messages, agent_state):
+        return "success"
+
+    large_messages = create_large_messages(size_mb=5)
+    agent_state = MockAgentState(size_mb=5)
+
+    result = test_func(messages=large_messages, agent_state=agent_state)
+
+    assert result == "success"
+    assert len(span_processor.spans) == 1
+
+    attrs = get_span_attributes(span_processor.spans[0])
+
+    # Check messages parameter is excluded
+    assert "messages" in attrs
+    messages_value = str(attrs["messages"])
+    assert "excluded" in messages_value
+    assert len(messages_value) < 200  # Should be small (may include IDs)
+
+    # Check agent_state parameter is excluded
+    assert "agent_state" in attrs
+    agent_state_value = str(attrs["agent_state"])
+    assert "excluded" in agent_state_value
+    assert len(agent_state_value) < 200
+
+
+def test_content_parameter_excluded(span_processor):
+    """Test that large content parameter is excluded from span attributes."""
+
+    @trace_method
+    def test_func(content, agent_state):
+        return "success"
+
+    large_content = "C" * (5 * 1024 * 1024)  # 5MB
+    agent_state = MockAgentState(size_mb=3)
+
+    result = test_func(content=large_content, agent_state=agent_state)
+
+    assert result == "success"
+    assert len(span_processor.spans) == 1
+
+    attrs = get_span_attributes(span_processor.spans[0])
+
+    # Check content parameter is excluded
+    assert "content" in attrs
+    content_value = str(attrs["content"])
+    assert "excluded" in content_value
+    assert len(content_value) < 200
+
+
+def test_source_code_parameter_excluded(span_processor):
+    """Test that large source_code parameter is excluded from span attributes."""
+
+    @trace_method
+    def test_func(source_code, tool_name):
+        return "success"
+
+    large_source_code = (
+        '''
+def large_tool(query: str):
+    """Tool with large source code."""
+    padding = """'''
+        + ("X" * (5 * 1024 * 1024))
+        + '''"""
+    return "test"
+'''
+    )
+
+    result = test_func(source_code=large_source_code, tool_name="test_tool")
+
+    assert result == "success"
+    assert len(span_processor.spans) == 1
+
+    attrs = get_span_attributes(span_processor.spans[0])
+
+    # Check source_code parameter is excluded
+    assert "source_code" in attrs
+    source_code_value = str(attrs["source_code"])
+    assert "excluded" in source_code_value
+    assert len(source_code_value) < 200
+
+    # Check small parameter is kept
+    assert "tool_name" in attrs
+    assert attrs["tool_name"] == "test_tool"
+
+
+def test_large_parameter_truncated(span_processor):
+    """Test that non-excluded large parameters are truncated."""
+
+    @trace_method
+    def test_func(data, small_param):
+        return "success"
+
+    # Use a parameter name not in the exclusion list
+    large_data = "D" * (2 * 1024 * 1024)  # 2MB
+    small_param = "test"
+
+    result = test_func(data=large_data, small_param=small_param)
+
+    assert result == "success"
+    assert len(span_processor.spans) == 1
+
+    attrs = get_span_attributes(span_processor.spans[0])
+
+    # Check large data is truncated
+    assert "data" in attrs
+    data_value = str(attrs["data"])
+    assert len(data_value) < 2048  # Should be truncated to ~1KB + message
+    assert "truncated" in data_value
+
+    # Check small param is kept
+    assert "small_param" in attrs
+    assert attrs["small_param"] == "test"
+
+
+def test_small_parameters_kept(span_processor):
+    """Test that small parameters are kept in full."""
+
+    @trace_method
+    def test_func(param1, param2, param3):
+        return "success"
+
+    result = test_func(param1="test", param2=123, param3={"key": "value"})
+
+    assert result == "success"
+    assert len(span_processor.spans) == 1
+
+    attrs = get_span_attributes(span_processor.spans[0])
+
+    # All small params should be present
+    assert "param1" in attrs
+    assert "param2" in attrs
+    assert "param3" in attrs
+
+    # Values should match
+    assert attrs["param1"] == "test"
+    assert "123" in str(attrs["param2"])
+
+
+def test_total_span_size_reasonable(span_processor):
+    """Test that total span attribute size remains reasonable with multiple large params."""
+
+    @trace_method
+    def test_func(messages, agent_state, content, source_code):
+        return "success"
+
+    large_messages = create_large_messages(size_mb=5)
+    agent_state = MockAgentState(size_mb=5)
+    large_content = "C" * (5 * 1024 * 1024)
+    large_source = "def tool(): padding = '" + ("X" * (5 * 1024 * 1024)) + "'"
+
+    result = test_func(messages=large_messages, agent_state=agent_state, content=large_content, source_code=large_source)
+
+    assert result == "success"
+    assert len(span_processor.spans) == 1
+
+    # Calculate total size of all attributes
+    attrs = get_span_attributes(span_processor.spans[0])
+    total_size = sum(len(str(v)) for v in attrs.values())
+
+    # Total should be < 10KB even though input was ~20MB
+    assert total_size < 10 * 1024, f"Total span size {total_size} bytes is too large"
+
+    # Calculate reduction factor
+    input_size = 20 * 1024 * 1024  # 20MB
+    reduction = input_size / total_size
+    assert reduction > 1000, f"Reduction factor {reduction}x is too low"
+
+
+def test_serialization_failure_handled(span_processor):
+    """Test that serialization failures are handled gracefully."""
+
+    class UnserializableObject:
+        def __str__(self):
+            raise Exception("Cannot serialize")
+
+    @trace_method
+    def test_func(obj, normal_param):
+        return "success"
+
+    obj = UnserializableObject()
+
+    # Should not raise exception
+    result = test_func(obj=obj, normal_param="test")
+
+    assert result == "success"
+    assert len(span_processor.spans) == 1
+
+    attrs = get_span_attributes(span_processor.spans[0])
+
+    # Should have a fallback value
+    assert "obj" in attrs
+    assert "serialization failed" in str(attrs["obj"]) or "excluded" in str(attrs["obj"])
+
+    # Normal param should work
+    assert "normal_param" in attrs
+    assert attrs["normal_param"] == "test"