From 96bdb8df1cd34e722d8385106a4a53d940205100 Mon Sep 17 00:00:00 2001 From: Matthew Zhou Date: Tue, 11 Feb 2025 10:11:41 -0800 Subject: [PATCH] chore: Add large scale many-messages test (#959) --- letta/services/agent_manager.py | 12 +- poetry.lock | 60 ++---- pyproject.toml | 1 + .../llm_model_configs/claude-3-5-sonnet.json | 8 + .../claude-3-sonnet-20240229.json | 9 - tests/manual_test_many_messages.py | 196 ++++++++++++++++++ ...manual_test_multi_agent_broadcast_large.py | 4 +- 7 files changed, 229 insertions(+), 61 deletions(-) create mode 100644 tests/configs/llm_model_configs/claude-3-5-sonnet.json delete mode 100644 tests/configs/llm_model_configs/claude-3-sonnet-20240229.json create mode 100644 tests/manual_test_many_messages.py diff --git a/letta/services/agent_manager.py b/letta/services/agent_manager.py index 3faf50fc..3c965386 100644 --- a/letta/services/agent_manager.py +++ b/letta/services/agent_manager.py @@ -477,39 +477,39 @@ class AgentManager: ) message = self.message_manager.create_message(message, actor=actor) message_ids = [message.id] + agent_state.message_ids[1:] # swap index 0 (system) - return self.set_in_context_messages(agent_id=agent_id, message_ids=message_ids, actor=actor) + return self._set_in_context_messages(agent_id=agent_id, message_ids=message_ids, actor=actor) else: return agent_state @enforce_types - def set_in_context_messages(self, agent_id: str, message_ids: List[str], actor: PydanticUser) -> PydanticAgentState: + def _set_in_context_messages(self, agent_id: str, message_ids: List[str], actor: PydanticUser) -> PydanticAgentState: return self.update_agent(agent_id=agent_id, agent_update=UpdateAgent(message_ids=message_ids), actor=actor) @enforce_types def trim_older_in_context_messages(self, num: int, agent_id: str, actor: PydanticUser) -> PydanticAgentState: message_ids = self.get_agent_by_id(agent_id=agent_id, actor=actor).message_ids new_messages = [message_ids[0]] + message_ids[num:] # 0 is system message - return self.set_in_context_messages(agent_id=agent_id, message_ids=new_messages, actor=actor) + return self._set_in_context_messages(agent_id=agent_id, message_ids=new_messages, actor=actor) @enforce_types def trim_all_in_context_messages_except_system(self, agent_id: str, actor: PydanticUser) -> PydanticAgentState: message_ids = self.get_agent_by_id(agent_id=agent_id, actor=actor).message_ids new_messages = [message_ids[0]] # 0 is system message - return self.set_in_context_messages(agent_id=agent_id, message_ids=new_messages, actor=actor) + return self._set_in_context_messages(agent_id=agent_id, message_ids=new_messages, actor=actor) @enforce_types def prepend_to_in_context_messages(self, messages: List[PydanticMessage], agent_id: str, actor: PydanticUser) -> PydanticAgentState: message_ids = self.get_agent_by_id(agent_id=agent_id, actor=actor).message_ids new_messages = self.message_manager.create_many_messages(messages, actor=actor) message_ids = [message_ids[0]] + [m.id for m in new_messages] + message_ids[1:] - return self.set_in_context_messages(agent_id=agent_id, message_ids=message_ids, actor=actor) + return self._set_in_context_messages(agent_id=agent_id, message_ids=message_ids, actor=actor) @enforce_types def append_to_in_context_messages(self, messages: List[PydanticMessage], agent_id: str, actor: PydanticUser) -> PydanticAgentState: messages = self.message_manager.create_many_messages(messages, actor=actor) message_ids = self.get_agent_by_id(agent_id=agent_id, actor=actor).message_ids or [] message_ids += [m.id for m in messages] - return self.set_in_context_messages(agent_id=agent_id, message_ids=message_ids, actor=actor) + return self._set_in_context_messages(agent_id=agent_id, message_ids=message_ids, actor=actor) @enforce_types def reset_messages(self, agent_id: str, actor: PydanticUser, add_default_initial_messages: bool = False) -> PydanticAgentState: diff --git a/poetry.lock b/poetry.lock index 8396a725..3aeab197 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -416,10 +416,6 @@ files = [ {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a37b8f0391212d29b3a91a799c8e4a2855e0576911cdfb2515487e30e322253d"}, {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:e84799f09591700a4154154cab9787452925578841a94321d5ee8fb9a9a328f0"}, {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f66b5337fa213f1da0d9000bc8dc0cb5b896b726eefd9c6046f699b169c41b9e"}, - {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5dab0844f2cf82be357a0eb11a9087f70c5430b2c241493fc122bb6f2bb0917c"}, - {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e4fe605b917c70283db7dfe5ada75e04561479075761a0b3866c081d035b01c1"}, - {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:1e9a65b5736232e7a7f91ff3d02277f11d339bf34099a56cdab6a8b3410a02b2"}, - {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:58d4b711689366d4a03ac7957ab8c28890415e267f9b6589969e74b6e42225ec"}, {file = "Brotli-1.1.0-cp310-cp310-win32.whl", hash = "sha256:be36e3d172dc816333f33520154d708a2657ea63762ec16b62ece02ab5e4daf2"}, {file = "Brotli-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:0c6244521dda65ea562d5a69b9a26120769b7a9fb3db2fe9545935ed6735b128"}, {file = "Brotli-1.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a3daabb76a78f829cafc365531c972016e4aa8d5b4bf60660ad8ecee19df7ccc"}, @@ -432,14 +428,8 @@ files = [ {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:19c116e796420b0cee3da1ccec3b764ed2952ccfcc298b55a10e5610ad7885f9"}, {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:510b5b1bfbe20e1a7b3baf5fed9e9451873559a976c1a78eebaa3b86c57b4265"}, {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a1fd8a29719ccce974d523580987b7f8229aeace506952fa9ce1d53a033873c8"}, - {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c247dd99d39e0338a604f8c2b3bc7061d5c2e9e2ac7ba9cc1be5a69cb6cd832f"}, - {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1b2c248cd517c222d89e74669a4adfa5577e06ab68771a529060cf5a156e9757"}, - {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:2a24c50840d89ded6c9a8fdc7b6ed3692ed4e86f1c4a4a938e1e92def92933e0"}, - {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f31859074d57b4639318523d6ffdca586ace54271a73ad23ad021acd807eb14b"}, {file = "Brotli-1.1.0-cp311-cp311-win32.whl", hash = "sha256:39da8adedf6942d76dc3e46653e52df937a3c4d6d18fdc94a7c29d263b1f5b50"}, {file = "Brotli-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:aac0411d20e345dc0920bdec5548e438e999ff68d77564d5e9463a7ca9d3e7b1"}, - {file = "Brotli-1.1.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:32d95b80260d79926f5fab3c41701dbb818fde1c9da590e77e571eefd14abe28"}, - {file = "Brotli-1.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b760c65308ff1e462f65d69c12e4ae085cff3b332d894637f6273a12a482d09f"}, {file = "Brotli-1.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:316cc9b17edf613ac76b1f1f305d2a748f1b976b033b049a6ecdfd5612c70409"}, {file = "Brotli-1.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:caf9ee9a5775f3111642d33b86237b05808dafcd6268faa492250e9b78046eb2"}, {file = "Brotli-1.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70051525001750221daa10907c77830bc889cb6d865cc0b813d9db7fefc21451"}, @@ -450,24 +440,8 @@ files = [ {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:4093c631e96fdd49e0377a9c167bfd75b6d0bad2ace734c6eb20b348bc3ea180"}, {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:7e4c4629ddad63006efa0ef968c8e4751c5868ff0b1c5c40f76524e894c50248"}, {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:861bf317735688269936f755fa136a99d1ed526883859f86e41a5d43c61d8966"}, - {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:87a3044c3a35055527ac75e419dfa9f4f3667a1e887ee80360589eb8c90aabb9"}, - {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c5529b34c1c9d937168297f2c1fde7ebe9ebdd5e121297ff9c043bdb2ae3d6fb"}, - {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:ca63e1890ede90b2e4454f9a65135a4d387a4585ff8282bb72964fab893f2111"}, - {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e79e6520141d792237c70bcd7a3b122d00f2613769ae0cb61c52e89fd3443839"}, {file = "Brotli-1.1.0-cp312-cp312-win32.whl", hash = "sha256:5f4d5ea15c9382135076d2fb28dde923352fe02951e66935a9efaac8f10e81b0"}, {file = "Brotli-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:906bc3a79de8c4ae5b86d3d75a8b77e44404b0f4261714306e3ad248d8ab0951"}, - {file = "Brotli-1.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8bf32b98b75c13ec7cf774164172683d6e7891088f6316e54425fde1efc276d5"}, - {file = "Brotli-1.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7bc37c4d6b87fb1017ea28c9508b36bbcb0c3d18b4260fcdf08b200c74a6aee8"}, - {file = "Brotli-1.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c0ef38c7a7014ffac184db9e04debe495d317cc9c6fb10071f7fefd93100a4f"}, - {file = "Brotli-1.1.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91d7cc2a76b5567591d12c01f019dd7afce6ba8cba6571187e21e2fc418ae648"}, - {file = "Brotli-1.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a93dde851926f4f2678e704fadeb39e16c35d8baebd5252c9fd94ce8ce68c4a0"}, - {file = "Brotli-1.1.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f0db75f47be8b8abc8d9e31bc7aad0547ca26f24a54e6fd10231d623f183d089"}, - {file = "Brotli-1.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6967ced6730aed543b8673008b5a391c3b1076d834ca438bbd70635c73775368"}, - {file = "Brotli-1.1.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:7eedaa5d036d9336c95915035fb57422054014ebdeb6f3b42eac809928e40d0c"}, - {file = "Brotli-1.1.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d487f5432bf35b60ed625d7e1b448e2dc855422e87469e3f450aa5552b0eb284"}, - {file = "Brotli-1.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:832436e59afb93e1836081a20f324cb185836c617659b07b129141a8426973c7"}, - {file = "Brotli-1.1.0-cp313-cp313-win32.whl", hash = "sha256:43395e90523f9c23a3d5bdf004733246fba087f2948f87ab28015f12359ca6a0"}, - {file = "Brotli-1.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:9011560a466d2eb3f5a6e4929cf4a09be405c64154e12df0dd72713f6500e32b"}, {file = "Brotli-1.1.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:a090ca607cbb6a34b0391776f0cb48062081f5f60ddcce5d11838e67a01928d1"}, {file = "Brotli-1.1.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2de9d02f5bda03d27ede52e8cfe7b865b066fa49258cbab568720aa5be80a47d"}, {file = "Brotli-1.1.0-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2333e30a5e00fe0fe55903c8832e08ee9c3b1382aacf4db26664a16528d51b4b"}, @@ -477,10 +451,6 @@ files = [ {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:fd5f17ff8f14003595ab414e45fce13d073e0762394f957182e69035c9f3d7c2"}, {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:069a121ac97412d1fe506da790b3e69f52254b9df4eb665cd42460c837193354"}, {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:e93dfc1a1165e385cc8239fab7c036fb2cd8093728cbd85097b284d7b99249a2"}, - {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_2_aarch64.whl", hash = "sha256:aea440a510e14e818e67bfc4027880e2fb500c2ccb20ab21c7a7c8b5b4703d75"}, - {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_2_i686.whl", hash = "sha256:6974f52a02321b36847cd19d1b8e381bf39939c21efd6ee2fc13a28b0d99348c"}, - {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_2_ppc64le.whl", hash = "sha256:a7e53012d2853a07a4a79c00643832161a910674a893d296c9f1259859a289d2"}, - {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_2_x86_64.whl", hash = "sha256:d7702622a8b40c49bffb46e1e3ba2e81268d5c04a34f460978c6b5517a34dd52"}, {file = "Brotli-1.1.0-cp36-cp36m-win32.whl", hash = "sha256:a599669fd7c47233438a56936988a2478685e74854088ef5293802123b5b2460"}, {file = "Brotli-1.1.0-cp36-cp36m-win_amd64.whl", hash = "sha256:d143fd47fad1db3d7c27a1b1d66162e855b5d50a89666af46e1679c496e8e579"}, {file = "Brotli-1.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:11d00ed0a83fa22d29bc6b64ef636c4552ebafcef57154b4ddd132f5638fbd1c"}, @@ -492,10 +462,6 @@ files = [ {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:919e32f147ae93a09fe064d77d5ebf4e35502a8df75c29fb05788528e330fe74"}, {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:23032ae55523cc7bccb4f6a0bf368cd25ad9bcdcc1990b64a647e7bbcce9cb5b"}, {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:224e57f6eac61cc449f498cc5f0e1725ba2071a3d4f48d5d9dffba42db196438"}, - {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:cb1dac1770878ade83f2ccdf7d25e494f05c9165f5246b46a621cc849341dc01"}, - {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:3ee8a80d67a4334482d9712b8e83ca6b1d9bc7e351931252ebef5d8f7335a547"}, - {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:5e55da2c8724191e5b557f8e18943b1b4839b8efc3ef60d65985bcf6f587dd38"}, - {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:d342778ef319e1026af243ed0a07c97acf3bad33b9f29e7ae6a1f68fd083e90c"}, {file = "Brotli-1.1.0-cp37-cp37m-win32.whl", hash = "sha256:587ca6d3cef6e4e868102672d3bd9dc9698c309ba56d41c2b9c85bbb903cdb95"}, {file = "Brotli-1.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:2954c1c23f81c2eaf0b0717d9380bd348578a94161a65b3a2afc62c86467dd68"}, {file = "Brotli-1.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:efa8b278894b14d6da122a72fefcebc28445f2d3f880ac59d46c90f4c13be9a3"}, @@ -508,10 +474,6 @@ files = [ {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ab4fbee0b2d9098c74f3057b2bc055a8bd92ccf02f65944a241b4349229185a"}, {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:141bd4d93984070e097521ed07e2575b46f817d08f9fa42b16b9b5f27b5ac088"}, {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fce1473f3ccc4187f75b4690cfc922628aed4d3dd013d047f95a9b3919a86596"}, - {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:d2b35ca2c7f81d173d2fadc2f4f31e88cc5f7a39ae5b6db5513cf3383b0e0ec7"}, - {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:af6fa6817889314555aede9a919612b23739395ce767fe7fcbea9a80bf140fe5"}, - {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:2feb1d960f760a575dbc5ab3b1c00504b24caaf6986e2dc2b01c09c87866a943"}, - {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:4410f84b33374409552ac9b6903507cdb31cd30d2501fc5ca13d18f73548444a"}, {file = "Brotli-1.1.0-cp38-cp38-win32.whl", hash = "sha256:db85ecf4e609a48f4b29055f1e144231b90edc90af7481aa731ba2d059226b1b"}, {file = "Brotli-1.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:3d7954194c36e304e1523f55d7042c59dc53ec20dd4e9ea9d151f1b62b4415c0"}, {file = "Brotli-1.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5fb2ce4b8045c78ebbc7b8f3c15062e435d47e7393cc57c25115cfd49883747a"}, @@ -524,10 +486,6 @@ files = [ {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:949f3b7c29912693cee0afcf09acd6ebc04c57af949d9bf77d6101ebb61e388c"}, {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:89f4988c7203739d48c6f806f1e87a1d96e0806d44f0fba61dba81392c9e474d"}, {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:de6551e370ef19f8de1807d0a9aa2cdfdce2e85ce88b122fe9f6b2b076837e59"}, - {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0737ddb3068957cf1b054899b0883830bb1fec522ec76b1098f9b6e0f02d9419"}, - {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:4f3607b129417e111e30637af1b56f24f7a49e64763253bbc275c75fa887d4b2"}, - {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:6c6e0c425f22c1c719c42670d561ad682f7bfeeef918edea971a79ac5252437f"}, - {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:494994f807ba0b92092a163a0a283961369a65f6cbe01e8891132b7a320e61eb"}, {file = "Brotli-1.1.0-cp39-cp39-win32.whl", hash = "sha256:f0d8a7a6b5983c2496e364b969f0e526647a06b075d034f3297dc66f3b360c64"}, {file = "Brotli-1.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:cdad5b9014d83ca68c25d2e9444e28e967ef16e80f6b436918c700c117a85467"}, {file = "Brotli-1.1.0.tar.gz", hash = "sha256:81de08ac11bcb85841e440c13611c00b67d3bf82698314928d0b676362546724"}, @@ -1184,6 +1142,20 @@ files = [ [package.extras] tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipython", "littleutils", "pytest", "rich"] +[[package]] +name = "faker" +version = "36.1.0" +description = "Faker is a Python package that generates fake data for you." +optional = false +python-versions = ">=3.9" +files = [ + {file = "Faker-36.1.0-py3-none-any.whl", hash = "sha256:aa0b93487d3adf7cd89953d172e3df896cb7b35d8a5222c0da873edbe2f7adf5"}, + {file = "faker-36.1.0.tar.gz", hash = "sha256:f40510350aecfe006f45cb3f8879b35e861367cf347f51a7f2ca2c0571fdcc0b"}, +] + +[package.dependencies] +tzdata = "*" + [[package]] name = "fastapi" version = "0.115.7" @@ -6448,4 +6420,4 @@ tests = ["wikipedia"] [metadata] lock-version = "2.0" python-versions = "<3.14,>=3.10" -content-hash = "2216d6724691913e968e42f1761452f241e8f5341951c744e6811293c1e8e73d" +content-hash = "e2101628603fc9585b5f082e974d9d503de7a50117ce89a518f28c783c5fe650" diff --git a/pyproject.toml b/pyproject.toml index 015fd5c2..eafe7109 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,6 +81,7 @@ e2b-code-interpreter = {version = "^1.0.3", optional = true} anthropic = "^0.43.0" letta_client = "^0.1.23" openai = "^1.60.0" +faker = "^36.1.0" [tool.poetry.extras] diff --git a/tests/configs/llm_model_configs/claude-3-5-sonnet.json b/tests/configs/llm_model_configs/claude-3-5-sonnet.json new file mode 100644 index 00000000..0a577453 --- /dev/null +++ b/tests/configs/llm_model_configs/claude-3-5-sonnet.json @@ -0,0 +1,8 @@ +{ + "model": "claude-3-5-sonnet-20241022", + "model_endpoint_type": "anthropic", + "model_endpoint": "https://api.anthropic.com/v1", + "model_wrapper": null, + "context_window": 200000, + "put_inner_thoughts_in_kwargs": true +} diff --git a/tests/configs/llm_model_configs/claude-3-sonnet-20240229.json b/tests/configs/llm_model_configs/claude-3-sonnet-20240229.json deleted file mode 100644 index 5eef194b..00000000 --- a/tests/configs/llm_model_configs/claude-3-sonnet-20240229.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "context_window": 200000, - "model": "claude-3-5-sonnet-20241022", - "model_endpoint_type": "anthropic", - "model_endpoint": "https://api.anthropic.com/v1", - "context_window": 200000, - "model_wrapper": null, - "put_inner_thoughts_in_kwargs": true -} diff --git a/tests/manual_test_many_messages.py b/tests/manual_test_many_messages.py new file mode 100644 index 00000000..0eef1764 --- /dev/null +++ b/tests/manual_test_many_messages.py @@ -0,0 +1,196 @@ +import datetime +import json +import math +import os +import random +import uuid + +import pytest +from faker import Faker +from tqdm import tqdm + +from letta import create_client +from letta.orm import Base +from letta.schemas.embedding_config import EmbeddingConfig +from letta.schemas.llm_config import LLMConfig +from letta.schemas.message import Message +from letta.services.agent_manager import AgentManager +from letta.services.message_manager import MessageManager +from tests.integration_test_summarizer import LLM_CONFIG_DIR + + +@pytest.fixture(autouse=True) +def truncate_database(): + from letta.server.server import db_context + + with db_context() as session: + for table in reversed(Base.metadata.sorted_tables): # Reverse to avoid FK issues + session.execute(table.delete()) # Truncate table + session.commit() + + +@pytest.fixture(scope="function") +def client(): + filename = os.path.join(LLM_CONFIG_DIR, "claude-3-5-sonnet.json") + config_data = json.load(open(filename, "r")) + llm_config = LLMConfig(**config_data) + client = create_client() + client.set_default_llm_config(llm_config) + client.set_default_embedding_config(EmbeddingConfig.default_config(provider="openai")) + + yield client + + +def generate_tool_call_id(): + """Generates a unique tool call ID.""" + return "toolu_" + uuid.uuid4().hex[:24] + + +def generate_timestamps(base_time): + """Creates a sequence of timestamps for user, assistant, and tool messages.""" + user_time = base_time + send_time = user_time + datetime.timedelta(seconds=random.randint(2, 5)) + tool_time = send_time + datetime.timedelta(seconds=random.randint(1, 3)) + next_group_time = tool_time + datetime.timedelta(seconds=random.randint(5, 10)) + + return user_time, send_time, tool_time, next_group_time + + +def get_conversation_pair(): + fake = Faker() + return f"Where does {fake.name()} live?", f"{fake.address()}" + + +def create_user_message(agent_id, organization_id, message_text, timestamp): + """Creates a user message dictionary.""" + return { + "role": "user", + "content": [ + { + "type": "text", + "text": json.dumps( + {"type": "user_message", "message": message_text, "time": timestamp.strftime("%Y-%m-%d %I:%M:%S %p PST-0800")}, indent=2 + ), + } + ], + "organization_id": organization_id, + "agent_id": agent_id, + "model": None, + "name": None, + "tool_calls": None, + "tool_call_id": None, + } + + +def create_send_message(agent_id, organization_id, assistant_text, tool_call_id, timestamp): + """Creates an assistant message dictionary.""" + return { + "role": "assistant", + "content": [{"type": "text", "text": f"Assistant reply generated at {timestamp.strftime('%Y-%m-%d %I:%M:%S %p PST-0800')}."}], + "organization_id": organization_id, + "agent_id": agent_id, + "model": "claude-3-5-haiku-20241022", + "name": None, + "tool_calls": [ + { + "id": tool_call_id, + "function": { + "name": "send_message", + "arguments": json.dumps( + {"message": assistant_text, "time": timestamp.strftime("%Y-%m-%d %I:%M:%S %p PST-0800")}, indent=2 + ), + }, + "type": "function", + } + ], + "tool_call_id": None, + } + + +def create_tool_message(agent_id, organization_id, tool_call_id, timestamp): + """Creates a tool response message dictionary.""" + return { + "role": "tool", + "content": [ + { + "type": "text", + "text": json.dumps( + {"status": "OK", "message": "None", "time": timestamp.strftime("%Y-%m-%d %I:%M:%S %p PST-0800")}, indent=2 + ), + } + ], + "organization_id": organization_id, + "agent_id": agent_id, + "model": "claude-3-5-haiku-20241022", + "name": "send_message", + "tool_calls": None, + "tool_call_id": tool_call_id, + } + + +@pytest.mark.parametrize("num_messages", [1000]) +def test_many_messages_performance(client, num_messages): + """Main test function to generate messages and insert them into the database.""" + message_manager = MessageManager() + agent_manager = AgentManager() + actor = client.user + + start_time = datetime.datetime.now() + last_event_time = start_time # Track last event time + + def log_event(event): + nonlocal last_event_time + now = datetime.datetime.now() + total_elapsed = (now - start_time).total_seconds() + step_elapsed = (now - last_event_time).total_seconds() + print(f"[+{total_elapsed:.3f}s | Δ{step_elapsed:.3f}s] {event}") + last_event_time = now # Update last event time + + log_event(f"Starting test with {num_messages} messages") + + agent_state = client.create_agent(name="manager") + log_event(f"Created agent with ID {agent_state.id}") + + message_group_size = 3 + num_groups = math.ceil((num_messages - 4) / message_group_size) + base_time = datetime.datetime(2025, 2, 10, 16, 3, 22) + current_time = base_time + organization_id = "org-00000000-0000-4000-8000-000000000000" + + all_messages = [] + + for _ in tqdm(range(num_groups)): + user_text, assistant_text = get_conversation_pair() + tool_call_id = generate_tool_call_id() + user_time, send_time, tool_time, current_time = generate_timestamps(current_time) + new_messages = [ + Message(**create_user_message(agent_state.id, organization_id, user_text, user_time)), + Message(**create_send_message(agent_state.id, organization_id, assistant_text, tool_call_id, send_time)), + Message(**create_tool_message(agent_state.id, organization_id, tool_call_id, tool_time)), + ] + all_messages.extend(new_messages) + + log_event(f"Finished generating {len(all_messages)} messages") + + message_manager.create_many_messages(all_messages, actor=actor) + log_event("Inserted messages into the database") + + agent_manager._set_in_context_messages( + agent_id=agent_state.id, message_ids=agent_state.message_ids + [m.id for m in all_messages], actor=client.user + ) + log_event("Updated agent context with messages") + + messages = message_manager.list_messages_for_agent(agent_id=agent_state.id, actor=client.user, limit=1000000000) + log_event(f"Retrieved {len(messages)} messages from the database") + + assert len(messages) >= num_groups * message_group_size + + response = client.send_message( + agent_id=agent_state.id, + role="user", + message="What have we been talking about?", + ) + log_event("Sent message to agent and received response") + + assert response + log_event("Test completed successfully") diff --git a/tests/manual_test_multi_agent_broadcast_large.py b/tests/manual_test_multi_agent_broadcast_large.py index 4adcfa07..2108f03a 100644 --- a/tests/manual_test_multi_agent_broadcast_large.py +++ b/tests/manual_test_multi_agent_broadcast_large.py @@ -54,7 +54,8 @@ def roll_dice_tool(client): yield tool -def test_multi_agent_large(client, roll_dice_tool): +@pytest.mark.parametrize("num_workers", [50]) +def test_multi_agent_large(client, roll_dice_tool, num_workers): manager_tags = ["manager"] worker_tags = ["helpers"] @@ -72,7 +73,6 @@ def test_multi_agent_large(client, roll_dice_tool): # Create 3 worker agents worker_agents = [] - num_workers = 50 for idx in tqdm(range(num_workers)): worker_agent_state = client.create_agent( name=f"worker-{idx}", include_multi_agent_tools=False, tags=worker_tags, tool_ids=[roll_dice_tool.id]