Fimeg/letta-server
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat: add provider trace backend abstraction for multi-backend telemetry (#8814)

Browse Source 
* feat: add provider trace backend abstraction for multi-backend telemetry

Introduces a pluggable backend system for provider traces:
- Base class with async/sync create and read interfaces
- PostgreSQL backend (existing behavior)
- ClickHouse backend (via OTEL instrumentation)
- Socket backend (writes to Unix socket for crouton sidecar)
- Factory for instantiating backends from config

Refactors TelemetryManager to use backends with support for:
- Multi-backend writes (concurrent via asyncio.gather)
- Primary backend for reads (first in config list)
- Graceful error handling per backend

Config: LETTA_TELEMETRY_PROVIDER_TRACE_BACKEND (comma-separated)
Example: "postgres,socket" for dual-write to Postgres and crouton

🐙 Generated with [Letta Code](https://letta.com)

Co-Authored-By: Letta <noreply@letta.com>

* feat: add protocol version to socket backend records

Adds PROTOCOL_VERSION constant to socket backend:
- Included in every telemetry record sent to crouton
- Must match ProtocolVersion in apps/crouton/main.go
- Enables crouton to detect and reject incompatible messages

🐙 Generated with [Letta Code](https://letta.com)

Co-Authored-By: Letta <noreply@letta.com>

* fix: remove organization_id from ProviderTraceCreate calls

The organization_id is now handled via the actor parameter in the
telemetry manager, not through ProviderTraceCreate schema. This fixes
validation errors after changing ProviderTraceCreate to inherit from
BaseProviderTrace which forbids extra fields.

🐙 Generated with [Letta Code](https://letta.com)

Co-Authored-By: Letta <noreply@letta.com>

* consolidate provider trace

* add clickhouse-connect to fix bug on main lmao

* auto generated sdk changes, and deployment details, and clikchouse prefix bug and added fields to runs trace return api

* auto generated sdk changes, and deployment details, and clikchouse prefix bug and added fields to runs trace return api

* consolidate provider trace

* consolidate provider trace bug fix

---------

Co-authored-by: Letta <noreply@letta.com>
This commit is contained in:
Kian Jones
2026-01-16 21:00:52 -08:00
committed by Sarah Wooders
parent 8792e88e8b
commit 9418ab9815
22 changed files with 966 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
alembic/versions/539afa667cff_add_telemetry_context_fields_to_.py Normal file
View File

@@ -0,0 +1,33 @@
"""add telemetry context fields to provider_traces
Revision ID: 539afa667cff
Revises: a1b2c3d4e5f7
Create Date: 2026-01-16 18:29:29.811385
"""
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "539afa667cff"
down_revision: Union[str, None] = "a1b2c3d4e5f7"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column("provider_traces", sa.Column("agent_id", sa.String(), nullable=True))
op.add_column("provider_traces", sa.Column("agent_tags", sa.JSON(), nullable=True))
op.add_column("provider_traces", sa.Column("call_type", sa.String(), nullable=True))
op.add_column("provider_traces", sa.Column("run_id", sa.String(), nullable=True))
def downgrade() -> None:
op.drop_column("provider_traces", "run_id")
op.drop_column("provider_traces", "call_type")
op.drop_column("provider_traces", "agent_tags")
op.drop_column("provider_traces", "agent_id")

4
letta/adapters/letta_llm_request_adapter.py
View File

@@ -5,7 +5,7 @@ from letta.helpers.datetime_helpers import get_utc_timestamp_ns
from letta.otel.tracing import log_attributes, log_event, safe_json_dumps, trace_method
from letta.schemas.letta_message import LettaMessage
from letta.schemas.letta_message_content import OmittedReasoningContent, ReasoningContent, TextContent
from letta.schemas.provider_trace import ProviderTraceCreate
from letta.schemas.provider_trace import ProviderTrace
from letta.schemas.usage import normalize_cache_tokens, normalize_reasoning_tokens
from letta.schemas.user import User
from letta.settings import settings
@@ -120,7 +120,7 @@ class LettaLLMRequestAdapter(LettaLLMAdapter):
safe_create_task(
self.telemetry_manager.create_provider_trace_async(
actor=actor,
provider_trace_create=ProviderTraceCreate(
provider_trace=ProviderTrace(
request_json=self.request_data,
response_json=self.response_data,
step_id=step_id, # Use original step_id for telemetry

4
letta/adapters/letta_llm_stream_adapter.py
View File

@@ -9,7 +9,7 @@ from letta.otel.tracing import log_attributes, safe_json_dumps, trace_method
from letta.schemas.enums import ProviderType
from letta.schemas.letta_message import LettaMessage
from letta.schemas.llm_config import LLMConfig
from letta.schemas.provider_trace import ProviderTraceCreate
from letta.schemas.provider_trace import ProviderTrace
from letta.schemas.usage import LettaUsageStatistics
from letta.schemas.user import User
from letta.settings import settings
@@ -223,7 +223,7 @@ class LettaLLMStreamAdapter(LettaLLMAdapter):
safe_create_task(
self.telemetry_manager.create_provider_trace_async(
actor=actor,
provider_trace_create=ProviderTraceCreate(
provider_trace=ProviderTrace(
request_json=self.request_data,
response_json=response_json,
step_id=step_id, # Use original step_id for telemetry

7
letta/adapters/simple_llm_stream_adapter.py
View File

@@ -13,7 +13,7 @@ from letta.otel.tracing import log_attributes, safe_json_dumps, trace_method
from letta.schemas.enums import ProviderType
from letta.schemas.letta_message import LettaMessage
from letta.schemas.letta_message_content import LettaMessageContentUnion
from letta.schemas.provider_trace import ProviderTraceCreate
from letta.schemas.provider_trace import ProviderTrace
from letta.schemas.usage import LettaUsageStatistics
from letta.schemas.user import User
from letta.settings import settings
@@ -276,11 +276,10 @@ class SimpleLLMStreamAdapter(LettaLLMStreamAdapter):
safe_create_task(
self.telemetry_manager.create_provider_trace_async(
actor=actor,
provider_trace_create=ProviderTraceCreate(
provider_trace=ProviderTrace(
request_json=self.request_data,
response_json=response_json,
step_id=step_id, # Use original step_id for telemetry
organization_id=actor.organization_id,
step_id=step_id,
),
),
label="create_provider_trace",

8
letta/agents/letta_agent.py
View File

@@ -49,7 +49,7 @@ from letta.schemas.openai.chat_completion_response import (
UsageStatisticsCompletionTokenDetails,
UsageStatisticsPromptTokenDetails,
)
from letta.schemas.provider_trace import ProviderTraceCreate
from letta.schemas.provider_trace import ProviderTrace
from letta.schemas.step import StepProgression
from letta.schemas.step_metrics import StepMetrics
from letta.schemas.tool_execution_result import ToolExecutionResult
@@ -411,7 +411,7 @@ class LettaAgent(BaseAgent):
if settings.track_provider_trace:
await self.telemetry_manager.create_provider_trace_async(
actor=self.actor,
provider_trace_create=ProviderTraceCreate(
provider_trace=ProviderTrace(
request_json=request_data,
response_json=response_data,
step_id=step_id, # Use original step_id for telemetry
@@ -756,7 +756,7 @@ class LettaAgent(BaseAgent):
if settings.track_provider_trace:
await self.telemetry_manager.create_provider_trace_async(
actor=self.actor,
provider_trace_create=ProviderTraceCreate(
provider_trace=ProviderTrace(
request_json=request_data,
response_json=response_data,
step_id=step_id, # Use original step_id for telemetry
@@ -1190,7 +1190,7 @@ class LettaAgent(BaseAgent):
if settings.track_provider_trace:
await self.telemetry_manager.create_provider_trace_async(
actor=self.actor,
provider_trace_create=ProviderTraceCreate(
provider_trace=ProviderTrace(
request_json=request_data,
response_json={
"content": {

4
letta/llm_api/llm_api_tools.py
View File

@@ -27,7 +27,7 @@ from letta.schemas.enums import ProviderCategory
from letta.schemas.llm_config import LLMConfig
from letta.schemas.message import Message
from letta.schemas.openai.chat_completion_response import ChatCompletionResponse
from letta.schemas.provider_trace import ProviderTraceCreate
from letta.schemas.provider_trace import ProviderTrace
from letta.services.telemetry_manager import TelemetryManager
from letta.settings import ModelSettings
from letta.streaming_interface import AgentChunkStreamingInterface, AgentRefreshStreamingInterface
@@ -241,7 +241,7 @@ def create(
telemetry_manager.create_provider_trace(
actor=actor,
provider_trace_create=ProviderTraceCreate(
provider_trace=ProviderTrace(
request_json=prepare_openai_payload(data),
response_json=response.model_json_schema(),
step_id=step_id,

6
letta/llm_api/llm_client_base.py
View File

@@ -14,7 +14,7 @@ from letta.schemas.enums import AgentType, ProviderCategory
from letta.schemas.llm_config import LLMConfig
from letta.schemas.message import Message
from letta.schemas.openai.chat_completion_response import ChatCompletionResponse
from letta.schemas.provider_trace import ProviderTraceCreate
from letta.schemas.provider_trace import ProviderTrace
from letta.services.telemetry_manager import TelemetryManager
from letta.settings import settings
@@ -71,7 +71,7 @@ class LLMClientBase:
if step_id and telemetry_manager:
telemetry_manager.create_provider_trace(
actor=self.actor,
provider_trace_create=ProviderTraceCreate(
provider_trace=ProviderTrace(
request_json=request_data,
response_json=response_data,
step_id=step_id,
@@ -104,7 +104,7 @@ class LLMClientBase:
if settings.track_provider_trace and telemetry_manager:
await telemetry_manager.create_provider_trace_async(
actor=self.actor,
provider_trace_create=ProviderTraceCreate(
provider_trace=ProviderTrace(
request_json=request_data,
response_json=response_data,
step_id=step_id,

9
letta/orm/provider_trace.py
View File

@@ -1,4 +1,5 @@
import uuid
from typing import Optional
from sqlalchemy import JSON, Index, String
from sqlalchemy.orm import Mapped, mapped_column, relationship
@@ -20,7 +21,13 @@ class ProviderTrace(SqlalchemyBase, OrganizationMixin):
)
request_json: Mapped[dict] = mapped_column(JSON, doc="JSON content of the provider request")
response_json: Mapped[dict] = mapped_column(JSON, doc="JSON content of the provider response")
step_id: Mapped[str] = mapped_column(String, nullable=True, doc="ID of the step that this trace is associated with")
step_id: Mapped[Optional[str]] = mapped_column(String, nullable=True, doc="ID of the step that this trace is associated with")
# Telemetry context fields
agent_id: Mapped[Optional[str]] = mapped_column(String, nullable=True, doc="ID of the agent that generated this trace")
agent_tags: Mapped[Optional[list]] = mapped_column(JSON, nullable=True, doc="Tags associated with the agent for filtering")
call_type: Mapped[Optional[str]] = mapped_column(String, nullable=True, doc="Type of call (agent_step, summarization, etc.)")
run_id: Mapped[Optional[str]] = mapped_column(String, nullable=True, doc="ID of the run this trace is associated with")
# Relationships
organization: Mapped["Organization"] = relationship("Organization", lazy="selectin")

21
letta/schemas/provider_trace.py
View File

@@ -3,7 +3,7 @@ from __future__ import annotations
from datetime import datetime
from typing import Any, Dict, Optional
from pydantic import BaseModel, Field
from pydantic import Field
from letta.helpers.datetime_helpers import get_utc_time
from letta.schemas.enums import PrimitiveType
@@ -14,14 +14,6 @@ class BaseProviderTrace(OrmMetadataBase):
__id_prefix__ = PrimitiveType.PROVIDER_TRACE.value
class ProviderTraceCreate(BaseModel):
"""Request to create a provider trace"""
request_json: dict[str, Any] = Field(..., description="JSON content of the provider request")
response_json: dict[str, Any] = Field(..., description="JSON content of the provider response")
step_id: str = Field(None, description="ID of the step that this trace is associated with")
class ProviderTrace(BaseProviderTrace):
"""
Letta's internal representation of a provider trace.
@@ -31,6 +23,10 @@ class ProviderTrace(BaseProviderTrace):
request_json (Dict[str, Any]): JSON content of the provider request.
response_json (Dict[str, Any]): JSON content of the provider response.
step_id (str): ID of the step that this trace is associated with.
agent_id (str): ID of the agent that generated this trace.
agent_tags (list[str]): Tags associated with the agent for filtering.
call_type (str): Type of call (agent_step, summarization, etc.).
run_id (str): ID of the run this trace is associated with.
organization_id (str): The unique identifier of the organization.
created_at (datetime): The timestamp when the object was created.
"""
@@ -39,4 +35,11 @@ class ProviderTrace(BaseProviderTrace):
request_json: Dict[str, Any] = Field(..., description="JSON content of the provider request")
response_json: Dict[str, Any] = Field(..., description="JSON content of the provider response")
step_id: Optional[str] = Field(None, description="ID of the step that this trace is associated with")
# Telemetry context fields
agent_id: Optional[str] = Field(None, description="ID of the agent that generated this trace")
agent_tags: Optional[list[str]] = Field(None, description="Tags associated with the agent for filtering")
call_type: Optional[str] = Field(None, description="Type of call (agent_step, summarization, etc.)")
run_id: Optional[str] = Field(None, description="ID of the run this trace is associated with")
created_at: datetime = Field(default_factory=get_utc_time, description="The timestamp when the object was created.")

4
letta/server/rest_api/routers/v1/runs.py
View File

@@ -288,10 +288,10 @@ async def retrieve_trace_for_run(
Requires ClickHouse to be configured for trace storage.
"""
# OTEL traces are only available when ClickHouse is configured
if not settings.use_clickhouse_for_provider_traces:
if not settings.clickhouse_endpoint:
raise HTTPException(
status_code=501,
detail="OTEL traces require ClickHouse. Set use_clickhouse_for_provider_traces=true and configure ClickHouse connection.",
detail="OTEL traces require ClickHouse. Set LETTA_CLICKHOUSE_ENDPOINT and configure ClickHouse connection.",
)
actor = await server.user_manager.get_actor_or_default_async(actor_id=headers.actor_id)

8
letta/services/clickhouse_otel_traces.py
View File

@@ -26,12 +26,9 @@ def _parse_clickhouse_endpoint(endpoint: str) -> tuple[str, int, bool]:
@singleton
class ClickhouseOtelTracesReader:
def __init__(self):
self._client = None
pass
def _get_client(self):
if self._client is not None:
return self._client
import clickhouse_connect
if not settings.clickhouse_endpoint:
@@ -47,7 +44,7 @@ class ClickhouseOtelTracesReader:
if not password:
raise ValueError("CLICKHOUSE_PASSWORD is required")
self._client = clickhouse_connect.get_client(
return clickhouse_connect.get_client(
host=host,
port=port,
username=username,
@@ -56,7 +53,6 @@ class ClickhouseOtelTracesReader:
secure=secure,
verify=True,
)
return self._client
def _get_traces_by_trace_id_sync(self, trace_id: str, limit: int, filter_ui_spans: bool = False) -> list[dict[str, Any]]:
client = self._get_client()

20
letta/services/provider_trace_backends/__init__.py Normal file
View File

@@ -0,0 +1,20 @@
"""
Provider trace backend abstraction.
Supports multiple storage backends for LLM telemetry:
- postgres: Store in PostgreSQL (default)
- clickhouse: Store in ClickHouse via OTEL instrumentation
- socket: Send via Unix socket to external sidecar/service
Multiple backends can be enabled simultaneously for dual-write scenarios.
"""
from letta.services.provider_trace_backends.base import ProviderTraceBackend, ProviderTraceBackendClient
from letta.services.provider_trace_backends.factory import get_provider_trace_backend, get_provider_trace_backends
__all__ = [
"ProviderTraceBackend",
"ProviderTraceBackendClient",
"get_provider_trace_backend",
"get_provider_trace_backends",
]

69
letta/services/provider_trace_backends/base.py Normal file
View File

@@ -0,0 +1,69 @@
"""Base class for provider trace backends."""
from abc import ABC, abstractmethod
from enum import Enum
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from letta.schemas.provider_trace import ProviderTrace
from letta.schemas.user import User
class ProviderTraceBackend(str, Enum):
"""Supported provider trace storage backends."""
POSTGRES = "postgres"
CLICKHOUSE = "clickhouse"
SOCKET = "socket"
class ProviderTraceBackendClient(ABC):
"""Abstract base class for provider trace storage backends."""
@abstractmethod
async def create_async(
self,
actor: "User",
provider_trace: "ProviderTrace",
) -> "ProviderTrace | None":
"""
Store a provider trace record.
Args:
actor: The user/actor creating the trace
provider_trace: The trace data to store
Returns:
The created ProviderTrace, or None if the backend doesn't return it
"""
raise NotImplementedError
@abstractmethod
async def get_by_step_id_async(
self,
step_id: str,
actor: "User",
) -> "ProviderTrace | None":
"""
Retrieve a provider trace by step ID.
Args:
step_id: The step ID to look up
actor: The user/actor requesting the trace
Returns:
The ProviderTrace if found, None otherwise
"""
raise NotImplementedError
def create_sync(
self,
actor: "User",
provider_trace: "ProviderTrace",
) -> "ProviderTrace | None":
"""
Synchronous version of create_async.
Default implementation does nothing. Override if sync support is needed.
"""
return None

42
letta/services/provider_trace_backends/clickhouse.py Normal file
View File

@@ -0,0 +1,42 @@
"""ClickHouse provider trace backend."""
from letta.schemas.provider_trace import ProviderTrace
from letta.schemas.user import User
from letta.services.clickhouse_provider_traces import ClickhouseProviderTraceReader
from letta.services.provider_trace_backends.base import ProviderTraceBackendClient
class ClickhouseProviderTraceBackend(ProviderTraceBackendClient):
"""
Store provider traces in ClickHouse.
Writes flow through OTEL instrumentation, so create_async is a no-op.
Only reads are performed directly against ClickHouse.
"""
def __init__(self):
self._reader = ClickhouseProviderTraceReader()
async def create_async(
self,
actor: User,
provider_trace: ProviderTrace,
) -> ProviderTrace:
# ClickHouse writes flow through OTEL instrumentation, not direct writes.
# Return a ProviderTrace with the same ID for consistency across backends.
return ProviderTrace(
id=provider_trace.id,
step_id=provider_trace.step_id,
request_json=provider_trace.request_json or {},
response_json=provider_trace.response_json or {},
)
async def get_by_step_id_async(
self,
step_id: str,
actor: User,
) -> ProviderTrace | None:
return await self._reader.get_provider_trace_by_step_id_async(
step_id=step_id,
organization_id=actor.organization_id,
)

52
letta/services/provider_trace_backends/factory.py Normal file
View File

@@ -0,0 +1,52 @@
"""Factory for creating provider trace backends."""
from functools import lru_cache
from letta.services.provider_trace_backends.base import ProviderTraceBackend, ProviderTraceBackendClient
def _create_backend(backend: ProviderTraceBackend | str) -> ProviderTraceBackendClient:
"""Create a single backend instance."""
from letta.settings import telemetry_settings
backend_str = backend.value if isinstance(backend, ProviderTraceBackend) else backend
match backend_str:
case "clickhouse":
from letta.services.provider_trace_backends.clickhouse import ClickhouseProviderTraceBackend
return ClickhouseProviderTraceBackend()
case "socket":
from letta.services.provider_trace_backends.socket import SocketProviderTraceBackend
return SocketProviderTraceBackend(socket_path=telemetry_settings.socket_path)
case "postgres" | _:
from letta.services.provider_trace_backends.postgres import PostgresProviderTraceBackend
return PostgresProviderTraceBackend()
@lru_cache(maxsize=1)
def get_provider_trace_backends() -> list[ProviderTraceBackendClient]:
"""
Get all configured provider trace backends.
Returns cached singleton instances for each configured backend.
Supports multiple backends for dual-write scenarios (e.g., migration).
"""
from letta.settings import telemetry_settings
backends = telemetry_settings.provider_trace_backends
return [_create_backend(b) for b in backends]
def get_provider_trace_backend() -> ProviderTraceBackendClient:
"""
Get the primary (first) configured provider trace backend.
For backwards compatibility and read operations.
"""
backends = get_provider_trace_backends()
return backends[0] if backends else _create_backend("postgres")

45
letta/services/provider_trace_backends/postgres.py Normal file
View File

@@ -0,0 +1,45 @@
"""PostgreSQL provider trace backend."""
from letta.helpers.json_helpers import json_dumps, json_loads
from letta.orm.provider_trace import ProviderTrace as ProviderTraceModel
from letta.schemas.provider_trace import ProviderTrace
from letta.schemas.user import User
from letta.server.db import db_registry
from letta.services.provider_trace_backends.base import ProviderTraceBackendClient
class PostgresProviderTraceBackend(ProviderTraceBackendClient):
"""Store provider traces in PostgreSQL."""
async def create_async(
self,
actor: User,
provider_trace: ProviderTrace,
) -> ProviderTrace:
async with db_registry.async_session() as session:
provider_trace_model = ProviderTraceModel(**provider_trace.model_dump())
provider_trace_model.organization_id = actor.organization_id
if provider_trace.request_json:
request_json_str = json_dumps(provider_trace.request_json)
provider_trace_model.request_json = json_loads(request_json_str)
if provider_trace.response_json:
response_json_str = json_dumps(provider_trace.response_json)
provider_trace_model.response_json = json_loads(response_json_str)
await provider_trace_model.create_async(session, actor=actor, no_commit=True, no_refresh=True)
return provider_trace_model.to_pydantic()
async def get_by_step_id_async(
self,
step_id: str,
actor: User,
) -> ProviderTrace | None:
async with db_registry.async_session() as session:
provider_trace_model = await ProviderTraceModel.read_async(
db_session=session,
step_id=step_id,
actor=actor,
)
return provider_trace_model.to_pydantic() if provider_trace_model else None

107
letta/services/provider_trace_backends/socket.py Normal file
View File

@@ -0,0 +1,107 @@
"""Unix socket provider trace backend."""
import json
import os
import socket as socket_module
import threading
from datetime import datetime, timezone
from typing import Any
from letta.log import get_logger
from letta.schemas.provider_trace import ProviderTrace
from letta.schemas.user import User
from letta.services.provider_trace_backends.base import ProviderTraceBackendClient
logger = get_logger(__name__)
# Protocol version for crouton communication.
# Bump this when making breaking changes to the record schema.
# Must match ProtocolVersion in apps/crouton/main.go.
PROTOCOL_VERSION = 1
class SocketProviderTraceBackend(ProviderTraceBackendClient):
"""
Store provider traces via Unix socket.
Sends NDJSON telemetry records to a Unix socket. The receiving service
(sidecar) is responsible for storage (e.g., GCS, S3, local filesystem).
This is a write-only backend - reads are not supported.
"""
def __init__(self, socket_path: str = "/var/run/telemetry/telemetry.sock"):
self.socket_path = socket_path
async def create_async(
self,
actor: User,
provider_trace: ProviderTrace,
) -> ProviderTrace | None:
self._send_to_crouton(provider_trace)
# Return a ProviderTrace with the same ID for consistency across backends
return ProviderTrace(
id=provider_trace.id,
step_id=provider_trace.step_id,
request_json=provider_trace.request_json or {},
response_json=provider_trace.response_json or {},
)
def create_sync(
self,
actor: User,
provider_trace: ProviderTrace,
) -> ProviderTrace | None:
self._send_to_crouton(provider_trace)
return None
async def get_by_step_id_async(
self,
step_id: str,
actor: User,
) -> ProviderTrace | None:
# Socket backend is write-only - reads should go through the storage backend directly.
logger.warning("Socket backend does not support reads")
return None
def _send_to_crouton(self, provider_trace: ProviderTrace) -> None:
"""Build telemetry record and send to Crouton sidecar (fire-and-forget)."""
response = provider_trace.response_json or {}
request = provider_trace.request_json or {}
# Extract error if present
error = response.get("error", {}).get("message") if isinstance(response.get("error"), dict) else None
record = {
"protocol_version": PROTOCOL_VERSION,
"provider_trace_id": provider_trace.id,
"agent_id": provider_trace.agent_id,
"run_id": provider_trace.run_id,
"step_id": provider_trace.step_id,
"tags": provider_trace.agent_tags or [],
"type": provider_trace.call_type or "agent_step",
"request": request,
"response": response if not error else None,
"error": error,
"timestamp": datetime.now(timezone.utc).isoformat(),
}
# Fire-and-forget in background thread
thread = threading.Thread(target=self._send_async, args=(record,), daemon=True)
thread.start()
def _send_async(self, record: dict[str, Any]) -> None:
"""Send record to Unix socket (runs in background thread)."""
try:
if not os.path.exists(self.socket_path):
logger.warning(f"Crouton socket not found at {self.socket_path}")
return
with socket_module.socket(socket_module.AF_UNIX, socket_module.SOCK_STREAM) as sock:
sock.settimeout(5.0)
sock.connect(self.socket_path)
payload = json.dumps(record, default=str) + "\n"
sock.sendall(payload.encode())
except Exception as e:
logger.warning(f"Failed to send telemetry to Crouton: {e}")

141
letta/services/telemetry_manager.py
View File

@@ -1,74 +1,117 @@
from letta.helpers.json_helpers import json_dumps, json_loads
import asyncio
from letta.helpers.singleton import singleton
from letta.orm.provider_trace import ProviderTrace as ProviderTraceModel
from letta.log import get_logger
from letta.otel.tracing import trace_method
from letta.schemas.provider_trace import ProviderTrace as PydanticProviderTrace, ProviderTraceCreate
from letta.schemas.step import Step as PydanticStep
from letta.schemas.provider_trace import ProviderTrace
from letta.schemas.user import User as PydanticUser
from letta.server.db import db_registry
from letta.services.clickhouse_provider_traces import ClickhouseProviderTraceReader
from letta.settings import settings
from letta.services.provider_trace_backends import get_provider_trace_backend, get_provider_trace_backends
from letta.utils import enforce_types
logger = get_logger(__name__)
class TelemetryManager:
"""
Manages provider trace telemetry using configurable backends.
Supports multiple backends for dual-write scenarios (e.g., migration).
Configure via LETTA_TELEMETRY_PROVIDER_TRACE_BACKEND (comma-separated):
- postgres: Store in PostgreSQL (default)
- clickhouse: Store in ClickHouse via OTEL instrumentation
- socket: Store via Unix socket to Crouton sidecar (which writes to GCS)
Example: LETTA_TELEMETRY_PROVIDER_TRACE_BACKEND=postgres,socket
Multi-backend behavior:
- Writes: Sent to ALL configured backends concurrently via asyncio.gather.
Errors in one backend don't affect others (logged but not raised).
- Reads: Only from PRIMARY backend (first in the comma-separated list).
Secondary backends are write-only for this manager.
"""
def __init__(self):
self._backends = get_provider_trace_backends()
self._primary_backend = self._backends[0] if self._backends else get_provider_trace_backend()
@enforce_types
@trace_method
async def get_provider_trace_by_step_id_async(
self,
step_id: str,
actor: PydanticUser,
) -> PydanticProviderTrace | None:
# When ClickHouse is enabled, read only from ClickHouse (no Postgres fallback)
if settings.use_clickhouse_for_provider_traces:
return await ClickhouseProviderTraceReader().get_provider_trace_by_step_id_async(
step_id=step_id,
organization_id=actor.organization_id,
)
# Postgres storage backend
async with db_registry.async_session() as session:
provider_trace = await ProviderTraceModel.read_async(db_session=session, step_id=step_id, actor=actor)
return provider_trace.to_pydantic()
) -> ProviderTrace | None:
# Read from primary backend only
return await self._primary_backend.get_by_step_id_async(step_id=step_id, actor=actor)
@enforce_types
@trace_method
async def create_provider_trace_async(self, actor: PydanticUser, provider_trace_create: ProviderTraceCreate) -> PydanticProviderTrace:
# When ClickHouse is enabled, skip Postgres writes - data flows via OTEL instrumentation
if settings.use_clickhouse_for_provider_traces:
return PydanticProviderTrace(
id=f"provider_trace-{provider_trace_create.step_id}",
step_id=provider_trace_create.step_id,
request_json=provider_trace_create.request_json or {},
response_json=provider_trace_create.response_json or {},
)
async def create_provider_trace_async(
self,
actor: PydanticUser,
provider_trace: ProviderTrace,
) -> ProviderTrace:
# Write to all backends concurrently
tasks = [self._safe_create_async(backend, actor, provider_trace) for backend in self._backends]
results = await asyncio.gather(*tasks)
async with db_registry.async_session() as session:
provider_trace = ProviderTraceModel(**provider_trace_create.model_dump())
provider_trace.organization_id = actor.organization_id
if provider_trace_create.request_json:
request_json_str = json_dumps(provider_trace_create.request_json)
provider_trace.request_json = json_loads(request_json_str)
# Return first non-None result (from primary backend)
return next((r for r in results if r is not None), None)
if provider_trace_create.response_json:
response_json_str = json_dumps(provider_trace_create.response_json)
provider_trace.response_json = json_loads(response_json_str)
await provider_trace.create_async(session, actor=actor, no_commit=True, no_refresh=True)
pydantic_provider_trace = provider_trace.to_pydantic()
return pydantic_provider_trace
async def _safe_create_async(
self,
backend,
actor: PydanticUser,
provider_trace: ProviderTrace,
) -> ProviderTrace | None:
"""Create trace in a backend, catching and logging errors."""
try:
return await backend.create_async(actor=actor, provider_trace=provider_trace)
except Exception as e:
logger.warning(f"Failed to write to {backend.__class__.__name__}: {e}")
return None
def create_provider_trace(
self,
actor: PydanticUser,
provider_trace: ProviderTrace,
) -> ProviderTrace | None:
"""Synchronous version - writes to all backends."""
result = None
for backend in self._backends:
try:
r = backend.create_sync(actor=actor, provider_trace=provider_trace)
if result is None:
result = r
except Exception as e:
logger.warning(f"Failed to write to {backend.__class__.__name__}: {e}")
return result
@singleton
class NoopTelemetryManager(TelemetryManager):
"""
Noop implementation of TelemetryManager.
"""
"""Noop implementation of TelemetryManager."""
async def create_provider_trace_async(self, actor: PydanticUser, provider_trace_create: ProviderTraceCreate) -> PydanticProviderTrace:
return
def __init__(self):
pass # Don't initialize backend
async def get_provider_trace_by_step_id_async(self, step_id: str, actor: PydanticUser) -> PydanticStep:
return
async def create_provider_trace_async(
self,
actor: PydanticUser,
provider_trace: ProviderTrace,
) -> ProviderTrace:
return None
def create_provider_trace(self, actor: PydanticUser, provider_trace_create: ProviderTraceCreate) -> PydanticProviderTrace:
return
async def get_provider_trace_by_step_id_async(
self,
step_id: str,
actor: PydanticUser,
) -> ProviderTrace | None:
return None
def create_provider_trace(
self,
actor: PydanticUser,
provider_trace: ProviderTrace,
) -> ProviderTrace:
return None

53
letta/settings.py
View File

@@ -282,6 +282,28 @@ class Settings(BaseSettings):
# telemetry logging
otel_exporter_otlp_endpoint: str | None = None # otel default: "http://localhost:4317"
# clickhouse (for OTEL traces reader)
clickhouse_endpoint: str | None = Field(
default=None,
validation_alias=AliasChoices("CLICKHOUSE_ENDPOINT", "letta_clickhouse_endpoint"),
description="ClickHouse endpoint URL",
)
clickhouse_database: str | None = Field(
default="otel",
validation_alias=AliasChoices("CLICKHOUSE_DATABASE", "letta_clickhouse_database"),
description="ClickHouse database name",
)
clickhouse_username: str | None = Field(
default="default",
validation_alias=AliasChoices("CLICKHOUSE_USERNAME", "letta_clickhouse_username"),
description="ClickHouse username",
)
clickhouse_password: str | None = Field(
default=None,
validation_alias=AliasChoices("CLICKHOUSE_PASSWORD", "letta_clickhouse_password"),
description="ClickHouse password",
)
otel_preferred_temporality: int | None = Field(
default=1, ge=0, le=2, description="Exported metric temporality. {0: UNSPECIFIED, 1: DELTA, 2: CUMULATIVE}"
)
@@ -292,7 +314,6 @@ class Settings(BaseSettings):
track_stop_reason: bool = Field(default=True, description="Enable tracking stop reason on steps.")
track_agent_run: bool = Field(default=True, description="Enable tracking agent run with cancellation support")
track_provider_trace: bool = Field(default=True, description="Enable tracking raw llm request and response at each step")
use_clickhouse_for_provider_traces: bool = Field(default=False, description="Use ClickHouse backend for provider traces instead of Postgres")
# FastAPI Application Settings
uvicorn_workers: int = 1
@@ -402,6 +423,15 @@ class Settings(BaseSettings):
plugins[name] = {"target": target}
return plugins
@property
def use_clickhouse_for_provider_traces(self) -> bool:
"""Check if ClickHouse backend is configured for provider traces."""
# Access global telemetry_settings (defined at module level after this class)
import sys
module = sys.modules[__name__]
return "clickhouse" in getattr(module, "telemetry_settings").provider_trace_backends
class TestSettings(Settings):
model_config = SettingsConfigDict(env_prefix="letta_test_", extra="ignore")
@@ -459,6 +489,27 @@ class TelemetrySettings(BaseSettings):
description="Primary Python package name for source code linking. Datadog uses this setting to determine which code is 'yours' vs. third-party dependencies.",
)
# Provider trace backend selection (comma-separated for multi-backend support)
provider_trace_backend: str = Field(
default="postgres",
description="Provider trace storage backends (comma-separated): 'postgres', 'clickhouse', 'socket'. Example: 'postgres,socket' for dual-write.",
)
socket_path: str = Field(
default="/var/run/telemetry/telemetry.sock",
validation_alias=AliasChoices("TELEMETRY_SOCKET", "socket_path"),
description="Unix socket path for socket backend.",
)
@property
def provider_trace_backends(self) -> list[str]:
"""Parse comma-separated backend list."""
return [b.strip() for b in self.provider_trace_backend.split(",") if b.strip()]
@property
def socket_backend_enabled(self) -> bool:
"""Check if socket backend is enabled."""
return "socket" in self.provider_trace_backends
# singleton
settings = Settings(_env_parse_none_str="None")

1
pyproject.toml
View File

@@ -74,6 +74,7 @@ dependencies = [
"psutil>=5.9.0",
"fastmcp>=2.12.5",
"ddtrace>=4.2.1",
"clickhouse-connect>=0.10.0",
]
[project.scripts]

332
tests/test_provider_trace_backends.py Normal file
View File

@@ -0,0 +1,332 @@
"""Unit tests for provider trace backends."""
import asyncio
import json
import os
import socket
import tempfile
import threading
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from letta.schemas.provider_trace import ProviderTrace
from letta.schemas.user import User
from letta.services.provider_trace_backends.base import ProviderTraceBackend, ProviderTraceBackendClient
from letta.services.provider_trace_backends.socket import SocketProviderTraceBackend
@pytest.fixture
def mock_actor():
"""Create a mock user/actor."""
return User(
id="user-00000000-0000-4000-8000-000000000000",
organization_id="org-00000000-0000-4000-8000-000000000000",
name="test_user",
)
@pytest.fixture
def sample_provider_trace():
"""Create a sample ProviderTrace."""
return ProviderTrace(
request_json={
"model": "gpt-4o-mini",
"messages": [{"role": "user", "content": "Hello"}],
},
response_json={
"id": "chatcmpl-xyz",
"model": "gpt-4o-mini",
"choices": [{"message": {"content": "Hi!"}}],
"usage": {"prompt_tokens": 10, "completion_tokens": 5},
},
step_id="step-test-789",
run_id="run-test-abc",
)
class TestProviderTraceBackendEnum:
"""Tests for ProviderTraceBackend enum."""
def test_enum_values(self):
assert ProviderTraceBackend.POSTGRES.value == "postgres"
assert ProviderTraceBackend.CLICKHOUSE.value == "clickhouse"
assert ProviderTraceBackend.SOCKET.value == "socket"
def test_enum_string_comparison(self):
assert ProviderTraceBackend.POSTGRES == "postgres"
assert ProviderTraceBackend.SOCKET == "socket"
class TestProviderTrace:
"""Tests for ProviderTrace schema."""
def test_id_generation(self):
"""Test that ID is auto-generated with correct prefix."""
trace = ProviderTrace(
request_json={"model": "test"},
response_json={"id": "test"},
step_id="step-123",
)
assert trace.id.startswith("provider_trace-")
def test_id_uniqueness(self):
"""Test that each instance gets a unique ID."""
trace1 = ProviderTrace(request_json={}, response_json={}, step_id="step-1")
trace2 = ProviderTrace(request_json={}, response_json={}, step_id="step-2")
assert trace1.id != trace2.id
def test_optional_fields(self):
"""Test optional telemetry fields."""
trace = ProviderTrace(
request_json={},
response_json={},
step_id="step-123",
agent_id="agent-456",
agent_tags=["env:dev", "team:ml"],
call_type="summarization",
run_id="run-789",
)
assert trace.agent_id == "agent-456"
assert trace.agent_tags == ["env:dev", "team:ml"]
assert trace.call_type == "summarization"
assert trace.run_id == "run-789"
class TestSocketProviderTraceBackend:
"""Tests for SocketProviderTraceBackend."""
def test_init_default_path(self):
"""Test default socket path."""
backend = SocketProviderTraceBackend()
assert backend.socket_path == "/var/run/telemetry/telemetry.sock"
def test_init_custom_path(self):
"""Test custom socket path."""
backend = SocketProviderTraceBackend(socket_path="/tmp/custom.sock")
assert backend.socket_path == "/tmp/custom.sock"
@pytest.mark.asyncio
async def test_create_async_returns_provider_trace(self, mock_actor, sample_provider_trace):
"""Test that create_async returns a ProviderTrace."""
backend = SocketProviderTraceBackend(socket_path="/nonexistent/path.sock")
result = await backend.create_async(
actor=mock_actor,
provider_trace=sample_provider_trace,
)
assert isinstance(result, ProviderTrace)
assert result.id == sample_provider_trace.id
assert result.step_id == sample_provider_trace.step_id
@pytest.mark.asyncio
async def test_get_by_step_id_returns_none(self, mock_actor):
"""Test that read operations return None (write-only backend)."""
backend = SocketProviderTraceBackend()
result = await backend.get_by_step_id_async(
step_id="step-123",
actor=mock_actor,
)
assert result is None
def test_send_to_socket_with_real_socket(self, sample_provider_trace):
"""Test sending data to a real Unix socket."""
received_data = []
with tempfile.TemporaryDirectory() as tmpdir:
socket_path = os.path.join(tmpdir, "test.sock")
# Create a simple socket server
server_sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
server_sock.bind(socket_path)
server_sock.listen(1)
server_sock.settimeout(5.0)
def accept_connection():
try:
conn, _ = server_sock.accept()
data = conn.recv(65536)
received_data.append(data.decode())
conn.close()
except socket.timeout:
pass # Expected - test socket has short timeout, data may not arrive
finally:
server_sock.close()
# Start server in background
server_thread = threading.Thread(target=accept_connection)
server_thread.start()
# Send data via backend
backend = SocketProviderTraceBackend(socket_path=socket_path)
backend._send_to_crouton(sample_provider_trace)
# Wait for send to complete
server_thread.join(timeout=5.0)
# Verify data was received
assert len(received_data) == 1
record = json.loads(received_data[0].strip())
assert record["provider_trace_id"] == sample_provider_trace.id
assert record["model"] == "gpt-4o-mini"
assert record["provider"] == "openai"
assert record["input_tokens"] == 10
assert record["output_tokens"] == 5
assert record["context"]["step_id"] == "step-test-789"
assert record["context"]["run_id"] == "run-test-abc"
def test_send_to_nonexistent_socket_does_not_raise(self, sample_provider_trace):
"""Test that sending to nonexistent socket fails silently."""
backend = SocketProviderTraceBackend(socket_path="/nonexistent/path.sock")
# Should not raise
backend._send_to_crouton(sample_provider_trace)
def test_record_extracts_usage_from_openai_response(self):
"""Test usage extraction from OpenAI-style response."""
trace = ProviderTrace(
request_json={"model": "gpt-4"},
response_json={
"usage": {
"prompt_tokens": 100,
"completion_tokens": 50,
}
},
step_id="step-123",
)
backend = SocketProviderTraceBackend(socket_path="/fake/path")
# Access internal method to build record
with patch.object(backend, "_send_async"):
backend._send_to_crouton(trace)
def test_record_extracts_usage_from_anthropic_response(self):
"""Test usage extraction from Anthropic-style response."""
trace = ProviderTrace(
request_json={"model": "claude-3"},
response_json={
"usage": {
"input_tokens": 100,
"output_tokens": 50,
}
},
step_id="step-123",
)
backend = SocketProviderTraceBackend(socket_path="/fake/path")
with patch.object(backend, "_send_async"):
backend._send_to_crouton(trace)
def test_record_extracts_error_from_response(self):
"""Test error extraction from response."""
trace = ProviderTrace(
request_json={"model": "gpt-4"},
response_json={
"error": {"message": "Rate limit exceeded"},
},
step_id="step-123",
)
backend = SocketProviderTraceBackend(socket_path="/fake/path")
# Capture the record sent to _send_async
captured_records = []
def capture_record(record):
captured_records.append(record)
with patch.object(backend, "_send_async", side_effect=capture_record):
backend._send_to_crouton(trace)
assert len(captured_records) == 1
assert captured_records[0]["error"] == "Rate limit exceeded"
assert captured_records[0]["response"] is None
class TestBackendFactory:
"""Tests for backend factory."""
def test_get_postgres_backend(self):
"""Test getting postgres backend."""
from letta.services.provider_trace_backends.factory import _create_backend
backend = _create_backend("postgres")
assert backend.__class__.__name__ == "PostgresProviderTraceBackend"
def test_get_socket_backend(self):
"""Test getting socket backend."""
with patch("letta.settings.telemetry_settings") as mock_settings:
mock_settings.socket_path = "/tmp/test.sock"
from letta.services.provider_trace_backends.factory import _create_backend
backend = _create_backend("socket")
assert backend.__class__.__name__ == "SocketProviderTraceBackend"
def test_get_multiple_backends(self):
"""Test getting multiple backends via environment."""
import os
from letta.services.provider_trace_backends.factory import (
get_provider_trace_backends,
)
# Clear cache first
get_provider_trace_backends.cache_clear()
# This test just verifies the factory works - actual backend list
# depends on env var LETTA_TELEMETRY_PROVIDER_TRACE_BACKEND
backends = get_provider_trace_backends()
assert len(backends) >= 1
assert all(hasattr(b, "create_async") and hasattr(b, "get_by_step_id_async") for b in backends)
def test_unknown_backend_defaults_to_postgres(self):
"""Test that unknown backend type defaults to postgres."""
from letta.services.provider_trace_backends.factory import _create_backend
backend = _create_backend("unknown_backend")
assert backend.__class__.__name__ == "PostgresProviderTraceBackend"
class TestTelemetrySettings:
"""Tests for telemetry settings."""
def test_provider_trace_backends_parsing(self):
"""Test parsing comma-separated backend list."""
from letta.settings import TelemetrySettings
# Create a fresh settings object and set the value directly
settings = TelemetrySettings(provider_trace_backend="postgres,socket")
backends = settings.provider_trace_backends
assert backends == ["postgres", "socket"]
def test_provider_trace_backends_single(self):
"""Test single backend."""
from letta.settings import TelemetrySettings
settings = TelemetrySettings(provider_trace_backend="postgres")
backends = settings.provider_trace_backends
assert backends == ["postgres"]
def test_provider_trace_backends_with_whitespace(self):
"""Test backend list with whitespace."""
from letta.settings import TelemetrySettings
settings = TelemetrySettings(provider_trace_backend="postgres , socket , clickhouse")
backends = settings.provider_trace_backends
assert backends == ["postgres", "socket", "clickhouse"]
def test_socket_backend_enabled(self):
"""Test socket_backend_enabled property."""
from letta.settings import TelemetrySettings
settings1 = TelemetrySettings(provider_trace_backend="postgres")
assert settings1.socket_backend_enabled is False
settings2 = TelemetrySettings(provider_trace_backend="postgres,socket")
assert settings2.socket_backend_enabled is True

81
uv.lock generated
View File

@@ -679,6 +679,45 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/85/32/10bb5764d90a8eee674e9dc6f4db6a0ab47c8c4d0d83c27f7c39ac415a4d/click-8.2.1-py3-none-any.whl", hash = "sha256:61a3265b914e850b85317d0b3109c7f8cd35a670f963866005d6ef1d5175a12b", size = 102215, upload-time = "2025-05-20T23:19:47.796Z" },
]
[[package]]
name = "clickhouse-connect"
version = "0.10.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "certifi" },
{ name = "lz4" },
{ name = "pytz" },
{ name = "urllib3" },
{ name = "zstandard" },
]
sdist = { url = "https://files.pythonhosted.org/packages/7b/fd/f8bea1157d40f117248dcaa9abdbf68c729513fcf2098ab5cb4aa58768b8/clickhouse_connect-0.10.0.tar.gz", hash = "sha256:a0256328802c6e5580513e197cef7f9ba49a99fc98e9ba410922873427569564", size = 104753, upload-time = "2025-11-14T20:31:00.947Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/bf/4e/f90caf963d14865c7a3f0e5d80b77e67e0fe0bf39b3de84110707746fa6b/clickhouse_connect-0.10.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:195f1824405501b747b572e1365c6265bb1629eeb712ce91eda91da3c5794879", size = 272911, upload-time = "2025-11-14T20:29:57.129Z" },
{ url = "https://files.pythonhosted.org/packages/50/c7/e01bd2dd80ea4fbda8968e5022c60091a872fd9de0a123239e23851da231/clickhouse_connect-0.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7907624635fe7f28e1b85c7c8b125a72679a63ecdb0b9f4250b704106ef438f8", size = 265938, upload-time = "2025-11-14T20:29:58.443Z" },
{ url = "https://files.pythonhosted.org/packages/f4/07/8b567b949abca296e118331d13380bbdefa4225d7d1d32233c59d4b4b2e1/clickhouse_connect-0.10.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:60772faa54d56f0fa34650460910752a583f5948f44dddeabfafaecbca21fc54", size = 1113548, upload-time = "2025-11-14T20:29:59.781Z" },
{ url = "https://files.pythonhosted.org/packages/9c/13/11f2d37fc95e74d7e2d80702cde87666ce372486858599a61f5209e35fc5/clickhouse_connect-0.10.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fe2a6cd98517330c66afe703fb242c0d3aa2c91f2f7dc9fb97c122c5c60c34b", size = 1135061, upload-time = "2025-11-14T20:30:01.244Z" },
{ url = "https://files.pythonhosted.org/packages/a0/d0/517181ea80060f84d84cff4d42d330c80c77bb352b728fb1f9681fbad291/clickhouse_connect-0.10.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a2427d312bc3526520a0be8c648479af3f6353da7a33a62db2368d6203b08efd", size = 1105105, upload-time = "2025-11-14T20:30:02.679Z" },
{ url = "https://files.pythonhosted.org/packages/7c/b2/4ad93e898562725b58c537cad83ab2694c9b1c1ef37fa6c3f674bdad366a/clickhouse_connect-0.10.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:63bbb5721bfece698e155c01b8fa95ce4377c584f4d04b43f383824e8a8fa129", size = 1150791, upload-time = "2025-11-14T20:30:03.824Z" },
{ url = "https://files.pythonhosted.org/packages/45/a4/fdfbfacc1fa67b8b1ce980adcf42f9e3202325586822840f04f068aff395/clickhouse_connect-0.10.0-cp311-cp311-win32.whl", hash = "sha256:48554e836c6b56fe0854d9a9f565569010583d4960094d60b68a53f9f83042f0", size = 244014, upload-time = "2025-11-14T20:30:05.157Z" },
{ url = "https://files.pythonhosted.org/packages/08/50/cf53f33f4546a9ce2ab1b9930db4850aa1ae53bff1e4e4fa97c566cdfa19/clickhouse_connect-0.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:9eb8df083e5fda78ac7249938691c2c369e8578b5df34c709467147e8289f1d9", size = 262356, upload-time = "2025-11-14T20:30:06.478Z" },
{ url = "https://files.pythonhosted.org/packages/9e/59/fadbbf64f4c6496cd003a0a3c9223772409a86d0eea9d4ff45d2aa88aabf/clickhouse_connect-0.10.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b090c7d8e602dd084b2795265cd30610461752284763d9ad93a5d619a0e0ff21", size = 276401, upload-time = "2025-11-14T20:30:07.469Z" },
{ url = "https://files.pythonhosted.org/packages/1c/e3/781f9970f2ef202410f0d64681e42b2aecd0010097481a91e4df186a36c7/clickhouse_connect-0.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b8a708d38b81dcc8c13bb85549c904817e304d2b7f461246fed2945524b7a31b", size = 268193, upload-time = "2025-11-14T20:30:08.503Z" },
{ url = "https://files.pythonhosted.org/packages/f0/e0/64ab66b38fce762b77b5203a4fcecc603595f2a2361ce1605fc7bb79c835/clickhouse_connect-0.10.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3646fc9184a5469b95cf4a0846e6954e6e9e85666f030a5d2acae58fa8afb37e", size = 1123810, upload-time = "2025-11-14T20:30:09.62Z" },
{ url = "https://files.pythonhosted.org/packages/f5/03/19121aecf11a30feaf19049be96988131798c54ac6ba646a38e5faecaa0a/clickhouse_connect-0.10.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fe7e6be0f40a8a77a90482944f5cc2aa39084c1570899e8d2d1191f62460365b", size = 1153409, upload-time = "2025-11-14T20:30:10.855Z" },
{ url = "https://files.pythonhosted.org/packages/ce/ee/63870fd8b666c6030393950ad4ee76b7b69430f5a49a5d3fa32a70b11942/clickhouse_connect-0.10.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:88b4890f13163e163bf6fa61f3a013bb974c95676853b7a4e63061faf33911ac", size = 1104696, upload-time = "2025-11-14T20:30:12.187Z" },
{ url = "https://files.pythonhosted.org/packages/e9/bc/fcd8da1c4d007ebce088783979c495e3d7360867cfa8c91327ed235778f5/clickhouse_connect-0.10.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6286832cc79affc6fddfbf5563075effa65f80e7cd1481cf2b771ce317c67d08", size = 1156389, upload-time = "2025-11-14T20:30:13.385Z" },
{ url = "https://files.pythonhosted.org/packages/4e/33/7cb99cc3fc503c23fd3a365ec862eb79cd81c8dc3037242782d709280fa9/clickhouse_connect-0.10.0-cp312-cp312-win32.whl", hash = "sha256:92b8b6691a92d2613ee35f5759317bd4be7ba66d39bf81c4deed620feb388ca6", size = 243682, upload-time = "2025-11-14T20:30:14.52Z" },
{ url = "https://files.pythonhosted.org/packages/48/5c/12eee6a1f5ecda2dfc421781fde653c6d6ca6f3080f24547c0af40485a5a/clickhouse_connect-0.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:1159ee2c33e7eca40b53dda917a8b6a2ed889cb4c54f3d83b303b31ddb4f351d", size = 262790, upload-time = "2025-11-14T20:30:15.555Z" },
{ url = "https://files.pythonhosted.org/packages/c3/4a/197abf4d74c914cd11aff644c954f754562546a35015de466592a778f388/clickhouse_connect-0.10.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f927722c5e054cf833a4112cf82d633e37d3b329f01e232754cc2678be268020", size = 273670, upload-time = "2025-11-14T20:30:16.883Z" },
{ url = "https://files.pythonhosted.org/packages/33/ec/5503e235b9588d178ad7777f0a6c61ede9cdfa1dfae1be08beaf18fe336b/clickhouse_connect-0.10.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ef58f431e2ef3c2a91a6d5535484186f2f57f50eff791410548b17017563784b", size = 265346, upload-time = "2025-11-14T20:30:18.154Z" },
{ url = "https://files.pythonhosted.org/packages/a9/83/5bc991215540b9c12c95452783b6b59e074d71391324d2a934c2e25a8afe/clickhouse_connect-0.10.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40b7cf86d016ae6c6c3af6a7b5786f41c18632bfbc9e58d0c4a21a4c5d50c674", size = 1104945, upload-time = "2025-11-14T20:30:19.35Z" },
{ url = "https://files.pythonhosted.org/packages/67/74/93442c805f4ed7ad831ef412c7b94c3df492494fa49f30fa77ed8fd8673a/clickhouse_connect-0.10.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:51193dc39f4169b0dd6da13003bbea60527dea92eb2408aecae7f1fb4ad2c5a4", size = 1137413, upload-time = "2025-11-14T20:30:20.932Z" },
{ url = "https://files.pythonhosted.org/packages/bc/9c/c885d6a726e35e489d874da8a592f0b6b0ac4d5f132e1126466c3143f263/clickhouse_connect-0.10.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b3e393dd95bcce02307f558f6aee53bf2a1bfc83f13030c9b4e47b2045de293f", size = 1086490, upload-time = "2025-11-14T20:30:22.195Z" },
{ url = "https://files.pythonhosted.org/packages/d1/a5/0980c9cb8ad13174840abcf31a085f3e5bb4026efb9271e4c757c40110ab/clickhouse_connect-0.10.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:bd6e1870df82dd57a47bc2a2a6f39c57da8aee43cc291a44d04babfdec5986dc", size = 1140722, upload-time = "2025-11-14T20:30:23.446Z" },
{ url = "https://files.pythonhosted.org/packages/12/6f/716e12a0e9a174dc3e422846a4e2f0ee73bcfc1f47b93bb02cf3d155bf0b/clickhouse_connect-0.10.0-cp313-cp313-win32.whl", hash = "sha256:d69b3f55a3a2f5414db7bed45afcca940e78ce1867cf5cc0c202f7be21cf48e9", size = 242997, upload-time = "2025-11-14T20:30:24.626Z" },
{ url = "https://files.pythonhosted.org/packages/1a/70/d0148812bec753767be39d528cffa48e7510718fb75fd2cf2b4dfdb24f37/clickhouse_connect-0.10.0-cp313-cp313-win_amd64.whl", hash = "sha256:5fa4f3763d46b90dc28b1f38eba8de83fbf6c9928f071dd66074e7d6de80e21b", size = 261840, upload-time = "2025-11-14T20:30:25.686Z" },
]
[[package]]
name = "cobble"
version = "0.1.4"
@@ -2481,6 +2520,7 @@ dependencies = [
{ name = "black", extra = ["jupyter"] },
{ name = "brotli" },
{ name = "certifi" },
{ name = "clickhouse-connect" },
{ name = "colorama" },
{ name = "datadog" },
{ name = "datamodel-code-generator", extra = ["http"] },
@@ -2633,6 +2673,7 @@ requires-dist = [
{ name = "boto3", marker = "extra == 'bedrock'", specifier = ">=1.36.24" },
{ name = "brotli", specifier = ">=1.1.0" },
{ name = "certifi", specifier = ">=2025.6.15" },
{ name = "clickhouse-connect", specifier = ">=0.10.0" },
{ name = "colorama", specifier = ">=0.4.6" },
{ name = "datadog", specifier = ">=0.49.1" },
{ name = "datamodel-code-generator", extras = ["http"], specifier = ">=0.25.0" },
@@ -3084,6 +3125,46 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/4e/0b/942cb7278d6caad79343ad2ddd636ed204a47909b969d19114a3097f5aa3/lxml_html_clean-0.4.2-py3-none-any.whl", hash = "sha256:74ccfba277adcfea87a1e9294f47dd86b05d65b4da7c5b07966e3d5f3be8a505", size = 14184, upload-time = "2025-04-09T11:33:57.988Z" },
]
[[package]]
name = "lz4"
version = "4.4.5"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/57/51/f1b86d93029f418033dddf9b9f79c8d2641e7454080478ee2aab5123173e/lz4-4.4.5.tar.gz", hash = "sha256:5f0b9e53c1e82e88c10d7c180069363980136b9d7a8306c4dca4f760d60c39f0", size = 172886, upload-time = "2025-11-03T13:02:36.061Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/93/5b/6edcd23319d9e28b1bedf32768c3d1fd56eed8223960a2c47dacd2cec2af/lz4-4.4.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d6da84a26b3aa5da13a62e4b89ab36a396e9327de8cd48b436a3467077f8ccd4", size = 207391, upload-time = "2025-11-03T13:01:36.644Z" },
{ url = "https://files.pythonhosted.org/packages/34/36/5f9b772e85b3d5769367a79973b8030afad0d6b724444083bad09becd66f/lz4-4.4.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:61d0ee03e6c616f4a8b69987d03d514e8896c8b1b7cc7598ad029e5c6aedfd43", size = 207146, upload-time = "2025-11-03T13:01:37.928Z" },
{ url = "https://files.pythonhosted.org/packages/04/f4/f66da5647c0d72592081a37c8775feacc3d14d2625bbdaabd6307c274565/lz4-4.4.5-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:33dd86cea8375d8e5dd001e41f321d0a4b1eb7985f39be1b6a4f466cd480b8a7", size = 1292623, upload-time = "2025-11-03T13:01:39.341Z" },
{ url = "https://files.pythonhosted.org/packages/85/fc/5df0f17467cdda0cad464a9197a447027879197761b55faad7ca29c29a04/lz4-4.4.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:609a69c68e7cfcfa9d894dc06be13f2e00761485b62df4e2472f1b66f7b405fb", size = 1279982, upload-time = "2025-11-03T13:01:40.816Z" },
{ url = "https://files.pythonhosted.org/packages/25/3b/b55cb577aa148ed4e383e9700c36f70b651cd434e1c07568f0a86c9d5fbb/lz4-4.4.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:75419bb1a559af00250b8f1360d508444e80ed4b26d9d40ec5b09fe7875cb989", size = 1368674, upload-time = "2025-11-03T13:01:42.118Z" },
{ url = "https://files.pythonhosted.org/packages/fb/31/e97e8c74c59ea479598e5c55cbe0b1334f03ee74ca97726e872944ed42df/lz4-4.4.5-cp311-cp311-win32.whl", hash = "sha256:12233624f1bc2cebc414f9efb3113a03e89acce3ab6f72035577bc61b270d24d", size = 88168, upload-time = "2025-11-03T13:01:43.282Z" },
{ url = "https://files.pythonhosted.org/packages/18/47/715865a6c7071f417bef9b57c8644f29cb7a55b77742bd5d93a609274e7e/lz4-4.4.5-cp311-cp311-win_amd64.whl", hash = "sha256:8a842ead8ca7c0ee2f396ca5d878c4c40439a527ebad2b996b0444f0074ed004", size = 99491, upload-time = "2025-11-03T13:01:44.167Z" },
{ url = "https://files.pythonhosted.org/packages/14/e7/ac120c2ca8caec5c945e6356ada2aa5cfabd83a01e3170f264a5c42c8231/lz4-4.4.5-cp311-cp311-win_arm64.whl", hash = "sha256:83bc23ef65b6ae44f3287c38cbf82c269e2e96a26e560aa551735883388dcc4b", size = 91271, upload-time = "2025-11-03T13:01:45.016Z" },
{ url = "https://files.pythonhosted.org/packages/1b/ac/016e4f6de37d806f7cc8f13add0a46c9a7cfc41a5ddc2bc831d7954cf1ce/lz4-4.4.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:df5aa4cead2044bab83e0ebae56e0944cc7fcc1505c7787e9e1057d6d549897e", size = 207163, upload-time = "2025-11-03T13:01:45.895Z" },
{ url = "https://files.pythonhosted.org/packages/8d/df/0fadac6e5bd31b6f34a1a8dbd4db6a7606e70715387c27368586455b7fc9/lz4-4.4.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6d0bf51e7745484d2092b3a51ae6eb58c3bd3ce0300cf2b2c14f76c536d5697a", size = 207150, upload-time = "2025-11-03T13:01:47.205Z" },
{ url = "https://files.pythonhosted.org/packages/b7/17/34e36cc49bb16ca73fb57fbd4c5eaa61760c6b64bce91fcb4e0f4a97f852/lz4-4.4.5-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7b62f94b523c251cf32aa4ab555f14d39bd1a9df385b72443fd76d7c7fb051f5", size = 1292045, upload-time = "2025-11-03T13:01:48.667Z" },
{ url = "https://files.pythonhosted.org/packages/90/1c/b1d8e3741e9fc89ed3b5f7ef5f22586c07ed6bb04e8343c2e98f0fa7ff04/lz4-4.4.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c3ea562c3af274264444819ae9b14dbbf1ab070aff214a05e97db6896c7597e", size = 1279546, upload-time = "2025-11-03T13:01:50.159Z" },
{ url = "https://files.pythonhosted.org/packages/55/d9/e3867222474f6c1b76e89f3bd914595af69f55bf2c1866e984c548afdc15/lz4-4.4.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:24092635f47538b392c4eaeff14c7270d2c8e806bf4be2a6446a378591c5e69e", size = 1368249, upload-time = "2025-11-03T13:01:51.273Z" },
{ url = "https://files.pythonhosted.org/packages/b2/e7/d667d337367686311c38b580d1ca3d5a23a6617e129f26becd4f5dc458df/lz4-4.4.5-cp312-cp312-win32.whl", hash = "sha256:214e37cfe270948ea7eb777229e211c601a3e0875541c1035ab408fbceaddf50", size = 88189, upload-time = "2025-11-03T13:01:52.605Z" },
{ url = "https://files.pythonhosted.org/packages/a5/0b/a54cd7406995ab097fceb907c7eb13a6ddd49e0b231e448f1a81a50af65c/lz4-4.4.5-cp312-cp312-win_amd64.whl", hash = "sha256:713a777de88a73425cf08eb11f742cd2c98628e79a8673d6a52e3c5f0c116f33", size = 99497, upload-time = "2025-11-03T13:01:53.477Z" },
{ url = "https://files.pythonhosted.org/packages/6a/7e/dc28a952e4bfa32ca16fa2eb026e7a6ce5d1411fcd5986cd08c74ec187b9/lz4-4.4.5-cp312-cp312-win_arm64.whl", hash = "sha256:a88cbb729cc333334ccfb52f070463c21560fca63afcf636a9f160a55fac3301", size = 91279, upload-time = "2025-11-03T13:01:54.419Z" },
{ url = "https://files.pythonhosted.org/packages/2f/46/08fd8ef19b782f301d56a9ccfd7dafec5fd4fc1a9f017cf22a1accb585d7/lz4-4.4.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6bb05416444fafea170b07181bc70640975ecc2a8c92b3b658c554119519716c", size = 207171, upload-time = "2025-11-03T13:01:56.595Z" },
{ url = "https://files.pythonhosted.org/packages/8f/3f/ea3334e59de30871d773963997ecdba96c4584c5f8007fd83cfc8f1ee935/lz4-4.4.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b424df1076e40d4e884cfcc4c77d815368b7fb9ebcd7e634f937725cd9a8a72a", size = 207163, upload-time = "2025-11-03T13:01:57.721Z" },
{ url = "https://files.pythonhosted.org/packages/41/7b/7b3a2a0feb998969f4793c650bb16eff5b06e80d1f7bff867feb332f2af2/lz4-4.4.5-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:216ca0c6c90719731c64f41cfbd6f27a736d7e50a10b70fad2a9c9b262ec923d", size = 1292136, upload-time = "2025-11-03T13:02:00.375Z" },
{ url = "https://files.pythonhosted.org/packages/89/d1/f1d259352227bb1c185288dd694121ea303e43404aa77560b879c90e7073/lz4-4.4.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:533298d208b58b651662dd972f52d807d48915176e5b032fb4f8c3b6f5fe535c", size = 1279639, upload-time = "2025-11-03T13:02:01.649Z" },
{ url = "https://files.pythonhosted.org/packages/d2/fb/ba9256c48266a09012ed1d9b0253b9aa4fe9cdff094f8febf5b26a4aa2a2/lz4-4.4.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:451039b609b9a88a934800b5fc6ee401c89ad9c175abf2f4d9f8b2e4ef1afc64", size = 1368257, upload-time = "2025-11-03T13:02:03.35Z" },
{ url = "https://files.pythonhosted.org/packages/a5/6d/dee32a9430c8b0e01bbb4537573cabd00555827f1a0a42d4e24ca803935c/lz4-4.4.5-cp313-cp313-win32.whl", hash = "sha256:a5f197ffa6fc0e93207b0af71b302e0a2f6f29982e5de0fbda61606dd3a55832", size = 88191, upload-time = "2025-11-03T13:02:04.406Z" },
{ url = "https://files.pythonhosted.org/packages/18/e0/f06028aea741bbecb2a7e9648f4643235279a770c7ffaf70bd4860c73661/lz4-4.4.5-cp313-cp313-win_amd64.whl", hash = "sha256:da68497f78953017deb20edff0dba95641cc86e7423dfadf7c0264e1ac60dc22", size = 99502, upload-time = "2025-11-03T13:02:05.886Z" },
{ url = "https://files.pythonhosted.org/packages/61/72/5bef44afb303e56078676b9f2486f13173a3c1e7f17eaac1793538174817/lz4-4.4.5-cp313-cp313-win_arm64.whl", hash = "sha256:c1cfa663468a189dab510ab231aad030970593f997746d7a324d40104db0d0a9", size = 91285, upload-time = "2025-11-03T13:02:06.77Z" },
{ url = "https://files.pythonhosted.org/packages/49/55/6a5c2952971af73f15ed4ebfdd69774b454bd0dc905b289082ca8664fba1/lz4-4.4.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:67531da3b62f49c939e09d56492baf397175ff39926d0bd5bd2d191ac2bff95f", size = 207348, upload-time = "2025-11-03T13:02:08.117Z" },
{ url = "https://files.pythonhosted.org/packages/4e/d7/fd62cbdbdccc35341e83aabdb3f6d5c19be2687d0a4eaf6457ddf53bba64/lz4-4.4.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a1acbbba9edbcbb982bc2cac5e7108f0f553aebac1040fbec67a011a45afa1ba", size = 207340, upload-time = "2025-11-03T13:02:09.152Z" },
{ url = "https://files.pythonhosted.org/packages/77/69/225ffadaacb4b0e0eb5fd263541edd938f16cd21fe1eae3cd6d5b6a259dc/lz4-4.4.5-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a482eecc0b7829c89b498fda883dbd50e98153a116de612ee7c111c8bcf82d1d", size = 1293398, upload-time = "2025-11-03T13:02:10.272Z" },
{ url = "https://files.pythonhosted.org/packages/c6/9e/2ce59ba4a21ea5dc43460cba6f34584e187328019abc0e66698f2b66c881/lz4-4.4.5-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e099ddfaa88f59dd8d36c8a3c66bd982b4984edf127eb18e30bb49bdba68ce67", size = 1281209, upload-time = "2025-11-03T13:02:12.091Z" },
{ url = "https://files.pythonhosted.org/packages/80/4f/4d946bd1624ec229b386a3bc8e7a85fa9a963d67d0a62043f0af0978d3da/lz4-4.4.5-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2af2897333b421360fdcce895c6f6281dc3fab018d19d341cf64d043fc8d90d", size = 1369406, upload-time = "2025-11-03T13:02:13.683Z" },
{ url = "https://files.pythonhosted.org/packages/02/a2/d429ba4720a9064722698b4b754fb93e42e625f1318b8fe834086c7c783b/lz4-4.4.5-cp313-cp313t-win32.whl", hash = "sha256:66c5de72bf4988e1b284ebdd6524c4bead2c507a2d7f172201572bac6f593901", size = 88325, upload-time = "2025-11-03T13:02:14.743Z" },
{ url = "https://files.pythonhosted.org/packages/4b/85/7ba10c9b97c06af6c8f7032ec942ff127558863df52d866019ce9d2425cf/lz4-4.4.5-cp313-cp313t-win_amd64.whl", hash = "sha256:cdd4bdcbaf35056086d910d219106f6a04e1ab0daa40ec0eeef1626c27d0fddb", size = 99643, upload-time = "2025-11-03T13:02:15.978Z" },
{ url = "https://files.pythonhosted.org/packages/77/4d/a175459fb29f909e13e57c8f475181ad8085d8d7869bd8ad99033e3ee5fa/lz4-4.4.5-cp313-cp313t-win_arm64.whl", hash = "sha256:28ccaeb7c5222454cd5f60fcd152564205bcb801bd80e125949d2dfbadc76bbd", size = 91504, upload-time = "2025-11-03T13:02:17.313Z" },
]
[[package]]
name = "magika"
version = "0.6.2"