diff --git a/pyproject.toml b/pyproject.toml index b6e0aebb..4676e10b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,6 +70,7 @@ dependencies = [ "trafilatura", "readability-lxml", "google-genai>=1.15.0", + "datadog>=0.49.1", ] [project.scripts] diff --git a/tests/test_temporal_metrics_local.py b/tests/test_temporal_metrics_local.py new file mode 100644 index 00000000..75d98231 --- /dev/null +++ b/tests/test_temporal_metrics_local.py @@ -0,0 +1,134 @@ +""" +Local test for temporal metrics. +Run with: uv run pytest tests/test_temporal_metrics_local.py -v -s +""" + +import asyncio +import os +from unittest.mock import MagicMock, patch + +import pytest + +from letta.agents.temporal.metrics import ( + ActivityMetrics, + TemporalMetrics, + WorkerMetrics, + WorkflowMetrics, +) + + +@pytest.fixture(autouse=True) +def setup_metrics(): + """Setup metrics for testing.""" + # Force re-initialization + TemporalMetrics._initialized = False + + # Enable metrics for testing + os.environ["DD_METRICS_ENABLED"] = "true" + os.environ["DD_AGENT_HOST"] = "localhost" + os.environ["DD_DOGSTATSD_PORT"] = "8125" + os.environ["DD_ENV"] = "local-test" + os.environ["DD_SERVICE"] = "letta-temporal-test" + + yield + + # Cleanup + TemporalMetrics._initialized = False + + +@pytest.mark.asyncio +async def test_metrics_initialization(): + """Test that metrics initialize correctly.""" + TemporalMetrics.initialize() + + assert TemporalMetrics._initialized is True + print(f"\n✓ Metrics initialized: enabled={TemporalMetrics.is_enabled()}") + + +@pytest.mark.asyncio +async def test_workflow_metrics(): + """Test workflow metrics recording.""" + with patch("letta.agents.temporal.metrics.statsd") as mock_statsd: + TemporalMetrics._initialized = False + TemporalMetrics._enabled = True + TemporalMetrics._initialized = True + + # Record workflow metrics + WorkflowMetrics.record_workflow_start(workflow_type="TemporalAgentWorkflow", workflow_id="test-workflow-123") + + WorkflowMetrics.record_workflow_success( + workflow_type="TemporalAgentWorkflow", + workflow_id="test-workflow-123", + duration_ns=1_000_000_000, # 1 second + ) + + WorkflowMetrics.record_workflow_usage( + workflow_type="TemporalAgentWorkflow", + step_count=5, + completion_tokens=100, + prompt_tokens=50, + total_tokens=150, + ) + + # Verify metrics were called + assert mock_statsd.increment.called + assert mock_statsd.histogram.called + assert mock_statsd.gauge.called + + print("\n✓ Workflow metrics recorded successfully") + print(f" - increment called {mock_statsd.increment.call_count} times") + print(f" - histogram called {mock_statsd.histogram.call_count} times") + print(f" - gauge called {mock_statsd.gauge.call_count} times") + + +@pytest.mark.asyncio +async def test_activity_metrics(): + """Test activity metrics recording.""" + with patch("letta.agents.temporal.metrics.statsd") as mock_statsd: + TemporalMetrics._initialized = False + TemporalMetrics._enabled = True + TemporalMetrics._initialized = True + + # Record activity metrics + ActivityMetrics.record_activity_start("llm_request") + ActivityMetrics.record_activity_success("llm_request", duration_ms=500.0) + + # Verify metrics were called + assert mock_statsd.increment.called + assert mock_statsd.histogram.called + + print("\n✓ Activity metrics recorded successfully") + print(f" - increment called {mock_statsd.increment.call_count} times") + + +@pytest.mark.asyncio +async def test_metrics_with_real_dogstatsd(): + """ + Test metrics with real DogStatsD connection (requires Datadog agent running). + This test will skip if the agent is not available. + """ + import socket + + # Check if DogStatsD is listening + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + sock.connect(("localhost", 8125)) + dogstatsd_available = True + sock.close() + except Exception: + dogstatsd_available = False + + if not dogstatsd_available: + pytest.skip("DogStatsD not available on localhost:8125") + + # Force re-initialization with real connection + TemporalMetrics._initialized = False + TemporalMetrics.initialize() + + # Send test metrics + TemporalMetrics.increment("temporal.test.counter", value=1, tags=["test:true"]) + TemporalMetrics.gauge("temporal.test.gauge", value=42.0, tags=["test:true"]) + TemporalMetrics.histogram("temporal.test.histogram", value=100.0, tags=["test:true"]) + + print("\n✓ Real metrics sent to DogStatsD at localhost:8125") + print(" Check your Datadog UI for metrics with prefix 'temporal.test.*'") diff --git a/uv.lock b/uv.lock index 8d95e3e2..f03a1e9c 100644 --- a/uv.lock +++ b/uv.lock @@ -856,6 +856,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c3/be/d0d44e092656fe7a06b55e6103cbce807cdbdee17884a5367c68c9860853/dataclasses_json-0.6.7-py3-none-any.whl", hash = "sha256:0dbf33f26c8d5305befd61b39d2b3414e8a407bedc2834dea9b8d642666fb40a", size = 28686, upload-time = "2024-06-09T16:20:16.715Z" }, ] +[[package]] +name = "datadog" +version = "0.52.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a9/e6/ec5e4b4dbecd63cecae94009ef6dde9ab421d7d0022e6027586cc3776921/datadog-0.52.1.tar.gz", hash = "sha256:44c6deb563c4522dba206fba2e2bb93d3b04113c40191851ba3a241d82b5fd0b", size = 368037, upload-time = "2025-07-31T15:49:43.425Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/19/e0e39f10169ca3e37fa6b5be2f6d1c729c92d677f1bd21ad6d448df8bec8/datadog-0.52.1-py2.py3-none-any.whl", hash = "sha256:b8c92cd761618ee062f114171067e4c400d48c9f0dad16cb285042439d9d5d4e", size = 129952, upload-time = "2025-07-31T15:49:41.8Z" }, +] + [[package]] name = "datamodel-code-generator" version = "0.33.0" @@ -1640,6 +1652,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1f/8e/abdd3f14d735b2929290a018ecf133c901be4874b858dd1c604b9319f064/greenlet-3.2.4-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2523e5246274f54fdadbce8494458a2ebdcdbc7b802318466ac5606d3cded1f8", size = 587684, upload-time = "2025-08-07T13:18:25.164Z" }, { url = "https://files.pythonhosted.org/packages/5d/65/deb2a69c3e5996439b0176f6651e0052542bb6c8f8ec2e3fba97c9768805/greenlet-3.2.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1987de92fec508535687fb807a5cea1560f6196285a4cde35c100b8cd632cc52", size = 1116647, upload-time = "2025-08-07T13:42:38.655Z" }, { url = "https://files.pythonhosted.org/packages/3f/cc/b07000438a29ac5cfb2194bfc128151d52f333cee74dd7dfe3fb733fc16c/greenlet-3.2.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:55e9c5affaa6775e2c6b67659f3a71684de4c549b3dd9afca3bc773533d284fa", size = 1142073, upload-time = "2025-08-07T13:18:21.737Z" }, + { url = "https://files.pythonhosted.org/packages/67/24/28a5b2fa42d12b3d7e5614145f0bd89714c34c08be6aabe39c14dd52db34/greenlet-3.2.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c9c6de1940a7d828635fbd254d69db79e54619f165ee7ce32fda763a9cb6a58c", size = 1548385, upload-time = "2025-11-04T12:42:11.067Z" }, + { url = "https://files.pythonhosted.org/packages/6a/05/03f2f0bdd0b0ff9a4f7b99333d57b53a7709c27723ec8123056b084e69cd/greenlet-3.2.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:03c5136e7be905045160b1b9fdca93dd6727b180feeafda6818e6496434ed8c5", size = 1613329, upload-time = "2025-11-04T12:42:12.928Z" }, { url = "https://files.pythonhosted.org/packages/d8/0f/30aef242fcab550b0b3520b8e3561156857c94288f0332a79928c31a52cf/greenlet-3.2.4-cp311-cp311-win_amd64.whl", hash = "sha256:9c40adce87eaa9ddb593ccb0fa6a07caf34015a29bf8d344811665b573138db9", size = 299100, upload-time = "2025-08-07T13:44:12.287Z" }, { url = "https://files.pythonhosted.org/packages/44/69/9b804adb5fd0671f367781560eb5eb586c4d495277c93bde4307b9e28068/greenlet-3.2.4-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3b67ca49f54cede0186854a008109d6ee71f66bd57bb36abd6d0a0267b540cdd", size = 274079, upload-time = "2025-08-07T13:15:45.033Z" }, { url = "https://files.pythonhosted.org/packages/46/e9/d2a80c99f19a153eff70bc451ab78615583b8dac0754cfb942223d2c1a0d/greenlet-3.2.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddf9164e7a5b08e9d22511526865780a576f19ddd00d62f8a665949327fde8bb", size = 640997, upload-time = "2025-08-07T13:42:56.234Z" }, @@ -1649,6 +1663,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/19/0d/6660d55f7373b2ff8152401a83e02084956da23ae58cddbfb0b330978fe9/greenlet-3.2.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b3812d8d0c9579967815af437d96623f45c0f2ae5f04e366de62a12d83a8fb0", size = 607586, upload-time = "2025-08-07T13:18:28.544Z" }, { url = "https://files.pythonhosted.org/packages/8e/1a/c953fdedd22d81ee4629afbb38d2f9d71e37d23caace44775a3a969147d4/greenlet-3.2.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:abbf57b5a870d30c4675928c37278493044d7c14378350b3aa5d484fa65575f0", size = 1123281, upload-time = "2025-08-07T13:42:39.858Z" }, { url = "https://files.pythonhosted.org/packages/3f/c7/12381b18e21aef2c6bd3a636da1088b888b97b7a0362fac2e4de92405f97/greenlet-3.2.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:20fb936b4652b6e307b8f347665e2c615540d4b42b3b4c8a321d8286da7e520f", size = 1151142, upload-time = "2025-08-07T13:18:22.981Z" }, + { url = "https://files.pythonhosted.org/packages/27/45/80935968b53cfd3f33cf99ea5f08227f2646e044568c9b1555b58ffd61c2/greenlet-3.2.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ee7a6ec486883397d70eec05059353b8e83eca9168b9f3f9a361971e77e0bcd0", size = 1564846, upload-time = "2025-11-04T12:42:15.191Z" }, + { url = "https://files.pythonhosted.org/packages/69/02/b7c30e5e04752cb4db6202a3858b149c0710e5453b71a3b2aec5d78a1aab/greenlet-3.2.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:326d234cbf337c9c3def0676412eb7040a35a768efc92504b947b3e9cfc7543d", size = 1633814, upload-time = "2025-11-04T12:42:17.175Z" }, { url = "https://files.pythonhosted.org/packages/e9/08/b0814846b79399e585f974bbeebf5580fbe59e258ea7be64d9dfb253c84f/greenlet-3.2.4-cp312-cp312-win_amd64.whl", hash = "sha256:a7d4e128405eea3814a12cc2605e0e6aedb4035bf32697f72deca74de4105e02", size = 299899, upload-time = "2025-08-07T13:38:53.448Z" }, { url = "https://files.pythonhosted.org/packages/49/e8/58c7f85958bda41dafea50497cbd59738c5c43dbbea5ee83d651234398f4/greenlet-3.2.4-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:1a921e542453fe531144e91e1feedf12e07351b1cf6c9e8a3325ea600a715a31", size = 272814, upload-time = "2025-08-07T13:15:50.011Z" }, { url = "https://files.pythonhosted.org/packages/62/dd/b9f59862e9e257a16e4e610480cfffd29e3fae018a68c2332090b53aac3d/greenlet-3.2.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cd3c8e693bff0fff6ba55f140bf390fa92c994083f838fece0f63be121334945", size = 641073, upload-time = "2025-08-07T13:42:57.23Z" }, @@ -1658,6 +1674,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ee/43/3cecdc0349359e1a527cbf2e3e28e5f8f06d3343aaf82ca13437a9aa290f/greenlet-3.2.4-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23768528f2911bcd7e475210822ffb5254ed10d71f4028387e5a99b4c6699671", size = 610497, upload-time = "2025-08-07T13:18:31.636Z" }, { url = "https://files.pythonhosted.org/packages/b8/19/06b6cf5d604e2c382a6f31cafafd6f33d5dea706f4db7bdab184bad2b21d/greenlet-3.2.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:00fadb3fedccc447f517ee0d3fd8fe49eae949e1cd0f6a611818f4f6fb7dc83b", size = 1121662, upload-time = "2025-08-07T13:42:41.117Z" }, { url = "https://files.pythonhosted.org/packages/a2/15/0d5e4e1a66fab130d98168fe984c509249c833c1a3c16806b90f253ce7b9/greenlet-3.2.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:d25c5091190f2dc0eaa3f950252122edbbadbb682aa7b1ef2f8af0f8c0afefae", size = 1149210, upload-time = "2025-08-07T13:18:24.072Z" }, + { url = "https://files.pythonhosted.org/packages/1c/53/f9c440463b3057485b8594d7a638bed53ba531165ef0ca0e6c364b5cc807/greenlet-3.2.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6e343822feb58ac4d0a1211bd9399de2b3a04963ddeec21530fc426cc121f19b", size = 1564759, upload-time = "2025-11-04T12:42:19.395Z" }, + { url = "https://files.pythonhosted.org/packages/47/e4/3bb4240abdd0a8d23f4f88adec746a3099f0d86bfedb623f063b2e3b4df0/greenlet-3.2.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ca7f6f1f2649b89ce02f6f229d7c19f680a6238af656f61e0115b24857917929", size = 1634288, upload-time = "2025-11-04T12:42:21.174Z" }, { url = "https://files.pythonhosted.org/packages/0b/55/2321e43595e6801e105fcfdee02b34c0f996eb71e6ddffca6b10b7e1d771/greenlet-3.2.4-cp313-cp313-win_amd64.whl", hash = "sha256:554b03b6e73aaabec3745364d6239e9e012d64c68ccd0b8430c64ccc14939a8b", size = 299685, upload-time = "2025-08-07T13:24:38.824Z" }, ] @@ -2397,6 +2415,7 @@ dependencies = [ { name = "brotli" }, { name = "certifi" }, { name = "colorama" }, + { name = "datadog" }, { name = "datamodel-code-generator", extra = ["http"] }, { name = "demjson3" }, { name = "docstring-parser" }, @@ -2546,6 +2565,7 @@ requires-dist = [ { name = "brotli", specifier = ">=1.1.0" }, { name = "certifi", specifier = ">=2025.6.15" }, { name = "colorama", specifier = ">=0.4.6" }, + { name = "datadog", specifier = ">=0.49.1" }, { name = "datamodel-code-generator", extras = ["http"], specifier = ">=0.25.0" }, { name = "ddtrace", marker = "extra == 'profiling'", specifier = ">=2.18.2" }, { name = "demjson3", specifier = ">=3.0.6" },