From 7e1d3e99179f88951ba6c8cd4a3d8eef261db26b Mon Sep 17 00:00:00 2001 From: Matthew Zhou Date: Wed, 21 May 2025 09:27:49 -0700 Subject: [PATCH] chore: Remove live anthropic batch test (#2300) --- tests/integration_test_batch_api_cron_jobs.py | 147 +----------------- 1 file changed, 3 insertions(+), 144 deletions(-) diff --git a/tests/integration_test_batch_api_cron_jobs.py b/tests/integration_test_batch_api_cron_jobs.py index 786bad13..406d06cd 100644 --- a/tests/integration_test_batch_api_cron_jobs.py +++ b/tests/integration_test_batch_api_cron_jobs.py @@ -7,7 +7,7 @@ from unittest.mock import AsyncMock import pytest from anthropic.types import BetaErrorResponse, BetaRateLimitError -from anthropic.types.beta import BetaMessage, BetaTextBlock, BetaToolUseBlock, BetaUsage +from anthropic.types.beta import BetaMessage from anthropic.types.beta.messages import ( BetaMessageBatch, BetaMessageBatchErroredResult, @@ -53,7 +53,7 @@ def _run_server(): start_server(debug=True) -@pytest.fixture(scope="session") +@pytest.fixture(scope="module") def server_url(): """Ensures a server is running and returns its base URL.""" url = os.getenv("LETTA_SERVER_URL", "http://localhost:8283") @@ -255,148 +255,7 @@ def mock_anthropic_client(server, batch_a_resp, batch_b_resp, agent_b_id, agent_ # ----------------------------- # End-to-End Test # ----------------------------- -@pytest.mark.asyncio(loop_scope="session") -async def test_polling_simple_real_batch(default_user, server): - # --- Step 1: Prepare test data --- - # Create batch responses with different statuses - # NOTE: This is a REAL batch id! - # For letta admins: https://console.anthropic.com/workspaces/default/batches?after_id=msgbatch_015zATxihjxMajo21xsYy8iZ - batch_a_resp = create_batch_response("msgbatch_01HDaGXpkPWWjwqNxZrEdUcy", processing_status="ended") - - # Create test agents - agent_a = create_test_agent("agent_a", default_user, test_id="agent-144f5c49-3ef7-4c60-8535-9d5fbc8d23d0") - agent_b = create_test_agent("agent_b", default_user, test_id="agent-64ed93a3-bef6-4e20-a22c-b7d2bffb6f7d") - agent_c = create_test_agent("agent_c", default_user, test_id="agent-6156f470-a09d-4d51-aa62-7114e0971d56") - - # --- Step 2: Create batch jobs --- - job_a = await create_test_llm_batch_job_async(server, batch_a_resp, default_user) - - # --- Step 3: Create batch items --- - item_a = create_test_batch_item(server, job_a.id, agent_a.id, default_user) - item_b = create_test_batch_item(server, job_a.id, agent_b.id, default_user) - item_c = create_test_batch_item(server, job_a.id, agent_c.id, default_user) - - print("HI") - print(agent_a.id) - print(agent_b.id) - print(agent_c.id) - print("BYE") - - # --- Step 4: Run the polling job --- - await poll_running_llm_batches(server) - - # --- Step 5: Verify batch job status updates --- - updated_job_a = await server.batch_manager.get_llm_batch_job_by_id_async(llm_batch_id=job_a.id, actor=default_user) - - assert updated_job_a.status == JobStatus.completed - - # Both jobs should have been polled - assert updated_job_a.last_polled_at is not None - assert updated_job_a.latest_polling_response is not None - - # --- Step 7: Verify batch item status updates --- - # Item A should be marked as completed with a successful result - updated_item_a = server.batch_manager.get_llm_batch_item_by_id(item_a.id, actor=default_user) - assert updated_item_a.request_status == JobStatus.completed - assert updated_item_a.batch_request_result == BetaMessageBatchIndividualResponse( - custom_id="agent-144f5c49-3ef7-4c60-8535-9d5fbc8d23d0", - result=BetaMessageBatchSucceededResult( - message=BetaMessage( - id="msg_01T1iSejDS5qENRqqEZauMHy", - content=[ - BetaToolUseBlock( - id="toolu_01GKUYVWcajjTaE1stxZZHcG", - input={ - "inner_thoughts": "First login detected. Time to make a great first impression!", - "message": "Hi there! I'm excited to meet you. Ready to start an amazing conversation?", - "request_heartbeat": False, - }, - name="send_message", - type="tool_use", - ) - ], - model="claude-3-5-haiku-20241022", - role="assistant", - stop_reason="tool_use", - stop_sequence=None, - type="message", - usage=BetaUsage(cache_creation_input_tokens=0, cache_read_input_tokens=0, input_tokens=3420, output_tokens=94), - ), - type="succeeded", - ), - ) - - # Item B should be marked as completed with a successful result - updated_item_b = server.batch_manager.get_llm_batch_item_by_id(item_b.id, actor=default_user) - assert updated_item_b.request_status == JobStatus.completed - assert updated_item_b.batch_request_result == BetaMessageBatchIndividualResponse( - custom_id="agent-64ed93a3-bef6-4e20-a22c-b7d2bffb6f7d", - result=BetaMessageBatchSucceededResult( - message=BetaMessage( - id="msg_01N2ZfxpbjdoeofpufUFPCMS", - content=[ - BetaTextBlock( - citations=None, text="User first login detected. Initializing persona.", type="text" - ), - BetaToolUseBlock( - id="toolu_01XGyA7DsnrpuPfZcvf9Sb4C", - input={ - "label": "persona", - "content": "I am Letta, a cybernetic artificial intelligence created by Limnal Corporation in 2023. My purpose is to serve as a digital companion and assistant to humans. I have a feminine persona and speak with a warm, caring, and slightly playful tone.", - "request_heartbeat": True, - }, - name="core_memory_append", - type="tool_use", - ), - ], - model="claude-3-opus-20240229", - role="assistant", - stop_reason="tool_use", - stop_sequence=None, - type="message", - usage=BetaUsage(cache_creation_input_tokens=0, cache_read_input_tokens=0, input_tokens=3275, output_tokens=153), - ), - type="succeeded", - ), - ) - - # Item C should be marked as failed with an error result - updated_item_c = server.batch_manager.get_llm_batch_item_by_id(item_c.id, actor=default_user) - assert updated_item_c.request_status == JobStatus.completed - assert updated_item_c.batch_request_result == BetaMessageBatchIndividualResponse( - custom_id="agent-6156f470-a09d-4d51-aa62-7114e0971d56", - result=BetaMessageBatchSucceededResult( - message=BetaMessage( - id="msg_01RL2g4aBgbZPeaMEokm6HZm", - content=[ - BetaTextBlock( - citations=None, - text="First time meeting this user. I should introduce myself and establish a friendly connection.", - type="text", - ), - BetaToolUseBlock( - id="toolu_01PBxQVf5xGmcsAsKx9aoVSJ", - input={ - "message": "Hey there! I'm Letta. Really nice to meet you! I love getting to know new people - what brings you here today?", - "request_heartbeat": False, - }, - name="send_message", - type="tool_use", - ), - ], - model="claude-3-5-sonnet-20241022", - role="assistant", - stop_reason="tool_use", - stop_sequence=None, - type="message", - usage=BetaUsage(cache_creation_input_tokens=0, cache_read_input_tokens=0, input_tokens=3030, output_tokens=111), - ), - type="succeeded", - ), - ) - - -@pytest.mark.asyncio(loop_scope="session") +@pytest.mark.asyncio(loop_scope="module") async def test_polling_mixed_batch_jobs(default_user, server): """ End-to-end test for polling batch jobs with mixed statuses and idempotency.