From 827a31d07d3224d6249d4939776af01f68ebea22 Mon Sep 17 00:00:00 2001 From: Matt Zhou Date: Sun, 29 Dec 2024 17:47:46 -0800 Subject: [PATCH 1/7] Finish --- letta/server/server.py | 12 +++++++ letta/services/tool_execution_sandbox.py | 8 ++--- tests/integration_test_composio.py | 32 +++++++++++++++++++ ...integration_test_tool_execution_sandbox.py | 6 ++++ 4 files changed, 54 insertions(+), 4 deletions(-) create mode 100644 tests/integration_test_composio.py diff --git a/letta/server/server.py b/letta/server/server.py index 03604a89..c4699eb3 100644 --- a/letta/server/server.py +++ b/letta/server/server.py @@ -53,6 +53,7 @@ from letta.schemas.memory import ArchivalMemorySummary, ContextWindowOverview, M from letta.schemas.message import Message, MessageCreate, MessageRole, MessageUpdate from letta.schemas.organization import Organization from letta.schemas.passage import Passage +from letta.schemas.sandbox_config import SandboxType, SandboxEnvironmentVariableCreate from letta.schemas.source import Source from letta.schemas.tool import Tool from letta.schemas.usage import LettaUsageStatistics @@ -298,6 +299,17 @@ class SyncServer(Server): self.block_manager.add_default_blocks(actor=self.default_user) self.tool_manager.upsert_base_tools(actor=self.default_user) + # Add composio keys to the tool sandbox env vars of the org + if tool_settings.composio_api_key: + manager = SandboxConfigManager(tool_settings) + sandbox_config = manager.get_or_create_default_sandbox_config(sandbox_type=SandboxType.LOCAL, actor=self.default_user) + + manager.create_sandbox_env_var( + SandboxEnvironmentVariableCreate(key="COMPOSIO_API_KEY", value=tool_settings.composio_api_key), + sandbox_config_id=sandbox_config.id, + actor=self.default_user, + ) + # collect providers (always has Letta as a default) self._enabled_providers: List[Provider] = [LettaProvider()] if model_settings.openai_api_key: diff --git a/letta/services/tool_execution_sandbox.py b/letta/services/tool_execution_sandbox.py index fc6e1bdd..b05b6ce2 100644 --- a/letta/services/tool_execution_sandbox.py +++ b/letta/services/tool_execution_sandbox.py @@ -127,7 +127,7 @@ class ToolExecutionSandbox: if local_configs.use_venv: return self.run_local_dir_sandbox_venv(sbx_config, env, temp_file_path) else: - return self.run_local_dir_sandbox_runpy(sbx_config, env_vars, temp_file_path) + return self.run_local_dir_sandbox_runpy(sbx_config, env, temp_file_path) except Exception as e: logger.error(f"Executing tool {self.tool_name} has an unexpected error: {e}") logger.error(f"Logging out tool {self.tool_name} auto-generated code for debugging: \n\n{code}") @@ -200,7 +200,7 @@ class ToolExecutionSandbox: logger.error(f"Executing tool {self.tool_name} has an unexpected error: {e}") raise e - def run_local_dir_sandbox_runpy(self, sbx_config: SandboxConfig, env_vars: Dict[str, str], temp_file_path: str) -> SandboxRunResult: + def run_local_dir_sandbox_runpy(self, sbx_config: SandboxConfig, env: Dict[str, str], temp_file_path: str) -> SandboxRunResult: status = "success" agent_state, stderr = None, None @@ -213,8 +213,8 @@ class ToolExecutionSandbox: try: # Execute the temp file - with self.temporary_env_vars(env_vars): - result = runpy.run_path(temp_file_path, init_globals=env_vars) + with self.temporary_env_vars(env): + result = runpy.run_path(temp_file_path, init_globals=env) # Fetch the result func_result = result.get(self.LOCAL_SANDBOX_RESULT_VAR_NAME) diff --git a/tests/integration_test_composio.py b/tests/integration_test_composio.py new file mode 100644 index 00000000..7caf3325 --- /dev/null +++ b/tests/integration_test_composio.py @@ -0,0 +1,32 @@ +import pytest +from fastapi.testclient import TestClient + +from letta.server.rest_api.app import app + + +@pytest.fixture +def client(): + return TestClient(app) + +def test_list_composio_apps(client): + response = client.get( + "/v1/tools/composio/apps" + ) + assert response.status_code == 200 + assert isinstance(response.json(), list) + +def test_list_composio_actions_by_app(client): + response = client.get( + "/v1/tools/composio/apps/github/actions" + ) + assert response.status_code == 200 + assert isinstance(response.json(), list) + +def test_add_composio_tool(client): + response = client.post( + "/v1/tools/composio/GITHUB_STAR_A_REPOSITORY_FOR_THE_AUTHENTICATED_USER" + ) + assert response.status_code == 200 + assert "id" in response.json() + assert "name" in response.json() + diff --git a/tests/integration_test_tool_execution_sandbox.py b/tests/integration_test_tool_execution_sandbox.py index 97346021..b84bcd51 100644 --- a/tests/integration_test_tool_execution_sandbox.py +++ b/tests/integration_test_tool_execution_sandbox.py @@ -351,6 +351,12 @@ def test_local_sandbox_e2e_composio_star_github(mock_e2b_api_key_none, check_com assert result.func_return["details"] == "Action executed successfully" +@pytest.mark.local_sandbox +def test_local_sandbox_e2e_composio_star_github_without_setting_db_env_vars(mock_e2b_api_key_none, check_composio_key_set, composio_github_star_tool, test_user): + result = ToolExecutionSandbox(composio_github_star_tool.name, {"owner": "letta-ai", "repo": "letta"}, user=test_user).run() + assert result.func_return["details"] == "Action executed successfully" + + @pytest.mark.local_sandbox def test_local_sandbox_external_codebase(mock_e2b_api_key_none, custom_test_sandbox_config, external_codebase_tool, test_user): # Set the args From b09ecefd73274e6d9840bb6864ffb20d1ff9a71c Mon Sep 17 00:00:00 2001 From: Matt Zhou Date: Mon, 30 Dec 2024 12:17:44 -0800 Subject: [PATCH 2/7] Run lint --- letta/server/server.py | 16 ++++++++-------- tests/integration_test_composio.py | 16 ++++++---------- tests/integration_test_tool_execution_sandbox.py | 4 +++- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/letta/server/server.py b/letta/server/server.py index c4699eb3..a619463a 100644 --- a/letta/server/server.py +++ b/letta/server/server.py @@ -53,7 +53,7 @@ from letta.schemas.memory import ArchivalMemorySummary, ContextWindowOverview, M from letta.schemas.message import Message, MessageCreate, MessageRole, MessageUpdate from letta.schemas.organization import Organization from letta.schemas.passage import Passage -from letta.schemas.sandbox_config import SandboxType, SandboxEnvironmentVariableCreate +from letta.schemas.sandbox_config import SandboxEnvironmentVariableCreate, SandboxType from letta.schemas.source import Source from letta.schemas.tool import Tool from letta.schemas.usage import LettaUsageStatistics @@ -301,14 +301,14 @@ class SyncServer(Server): # Add composio keys to the tool sandbox env vars of the org if tool_settings.composio_api_key: - manager = SandboxConfigManager(tool_settings) - sandbox_config = manager.get_or_create_default_sandbox_config(sandbox_type=SandboxType.LOCAL, actor=self.default_user) + manager = SandboxConfigManager(tool_settings) + sandbox_config = manager.get_or_create_default_sandbox_config(sandbox_type=SandboxType.LOCAL, actor=self.default_user) - manager.create_sandbox_env_var( - SandboxEnvironmentVariableCreate(key="COMPOSIO_API_KEY", value=tool_settings.composio_api_key), - sandbox_config_id=sandbox_config.id, - actor=self.default_user, - ) + manager.create_sandbox_env_var( + SandboxEnvironmentVariableCreate(key="COMPOSIO_API_KEY", value=tool_settings.composio_api_key), + sandbox_config_id=sandbox_config.id, + actor=self.default_user, + ) # collect providers (always has Letta as a default) self._enabled_providers: List[Provider] = [LettaProvider()] diff --git a/tests/integration_test_composio.py b/tests/integration_test_composio.py index 7caf3325..1b2c2e3f 100644 --- a/tests/integration_test_composio.py +++ b/tests/integration_test_composio.py @@ -8,25 +8,21 @@ from letta.server.rest_api.app import app def client(): return TestClient(app) + def test_list_composio_apps(client): - response = client.get( - "/v1/tools/composio/apps" - ) + response = client.get("/v1/tools/composio/apps") assert response.status_code == 200 assert isinstance(response.json(), list) + def test_list_composio_actions_by_app(client): - response = client.get( - "/v1/tools/composio/apps/github/actions" - ) + response = client.get("/v1/tools/composio/apps/github/actions") assert response.status_code == 200 assert isinstance(response.json(), list) + def test_add_composio_tool(client): - response = client.post( - "/v1/tools/composio/GITHUB_STAR_A_REPOSITORY_FOR_THE_AUTHENTICATED_USER" - ) + response = client.post("/v1/tools/composio/GITHUB_STAR_A_REPOSITORY_FOR_THE_AUTHENTICATED_USER") assert response.status_code == 200 assert "id" in response.json() assert "name" in response.json() - diff --git a/tests/integration_test_tool_execution_sandbox.py b/tests/integration_test_tool_execution_sandbox.py index b84bcd51..ad3d2ebb 100644 --- a/tests/integration_test_tool_execution_sandbox.py +++ b/tests/integration_test_tool_execution_sandbox.py @@ -352,7 +352,9 @@ def test_local_sandbox_e2e_composio_star_github(mock_e2b_api_key_none, check_com @pytest.mark.local_sandbox -def test_local_sandbox_e2e_composio_star_github_without_setting_db_env_vars(mock_e2b_api_key_none, check_composio_key_set, composio_github_star_tool, test_user): +def test_local_sandbox_e2e_composio_star_github_without_setting_db_env_vars( + mock_e2b_api_key_none, check_composio_key_set, composio_github_star_tool, test_user +): result = ToolExecutionSandbox(composio_github_star_tool.name, {"owner": "letta-ai", "repo": "letta"}, user=test_user).run() assert result.func_return["details"] == "Action executed successfully" From 483b71f4c1f332c3069cdc74ddc3a7c33fcaed05 Mon Sep 17 00:00:00 2001 From: Matt Zhou Date: Mon, 30 Dec 2024 14:29:35 -0800 Subject: [PATCH 3/7] Mock e2b api none for test_function_return_limit --- tests/test_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_client.py b/tests/test_client.py index cc391235..f0952bd8 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -347,7 +347,7 @@ def test_send_system_message(client: Union[LocalClient, RESTClient], agent: Agen assert send_system_message_response, "Sending message failed" -def test_function_return_limit(client: Union[LocalClient, RESTClient]): +def test_function_return_limit(mock_e2b_api_key_none, client: Union[LocalClient, RESTClient]): """Test to see if the function return limit works""" def big_return(): From bad263139649a496f7cadea885e627540e7617f0 Mon Sep 17 00:00:00 2001 From: Matt Zhou Date: Mon, 30 Dec 2024 15:04:08 -0800 Subject: [PATCH 4/7] Mock more e2b none --- tests/test_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_client.py b/tests/test_client.py index f0952bd8..e29c9116 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -385,7 +385,7 @@ def test_function_return_limit(mock_e2b_api_key_none, client: Union[LocalClient, client.delete_agent(agent_id=agent.id) -def test_function_always_error(client: Union[LocalClient, RESTClient]): +def test_function_always_error(mock_e2b_api_key_none, client: Union[LocalClient, RESTClient]): """Test to see if function that errors works correctly""" def always_error(): From 2dd7c9f2e68af67fcf98f203267d4a266dc66579 Mon Sep 17 00:00:00 2001 From: Matt Zhou Date: Mon, 30 Dec 2024 16:03:45 -0800 Subject: [PATCH 5/7] wip debug --- tests/test_client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_client.py b/tests/test_client.py index e29c9116..c3e507ae 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -42,8 +42,8 @@ def run_server(): @pytest.fixture( - params=[{"server": False}, {"server": True}], # whether to use REST API server - # params=[{"server": True}], # whether to use REST API server + # params=[{"server": False}, {"server": True}], # whether to use REST API server + params=[{"server": False}], # whether to use REST API server scope="module", ) def client(request): @@ -408,6 +408,8 @@ def test_function_always_error(mock_e2b_api_key_none, client: Union[LocalClient, assert response_message, "ToolReturnMessage message not found in response" assert response_message.status == "error" + + import ipdb;ipdb.set_trace() if isinstance(client, RESTClient): assert response_message.tool_return == "Error executing function always_error: ZeroDivisionError: division by zero" else: From f04d5df4ae449faf98a5a9c8698ffb7c7b57df45 Mon Sep 17 00:00:00 2001 From: Matt Zhou Date: Mon, 30 Dec 2024 16:27:08 -0800 Subject: [PATCH 6/7] Log out sandbox details for e2b --- letta/services/tool_execution_sandbox.py | 6 ++++++ tests/integration_test_tool_execution_sandbox.py | 2 +- tests/test_client.py | 7 +++---- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/letta/services/tool_execution_sandbox.py b/letta/services/tool_execution_sandbox.py index b05b6ce2..1060af43 100644 --- a/letta/services/tool_execution_sandbox.py +++ b/letta/services/tool_execution_sandbox.py @@ -277,6 +277,10 @@ class ToolExecutionSandbox: sbx_config = self.sandbox_config_manager.get_or_create_default_sandbox_config(sandbox_type=SandboxType.E2B, actor=self.user) sbx = self.get_running_e2b_sandbox_with_same_state(sbx_config) if not sbx or self.force_recreate: + if not sbx: + logger.info(f"No running e2b sandbox found with the same state: {sbx_config}") + else: + logger.info(f"Force recreated e2b sandbox with state: {sbx_config}") sbx = self.create_e2b_sandbox_with_metadata_hash(sandbox_config=sbx_config) # Since this sandbox was used, we extend its lifecycle by the timeout @@ -292,6 +296,8 @@ class ToolExecutionSandbox: func_return, agent_state = self.parse_best_effort(execution.results[0].text) elif execution.error: logger.error(f"Executing tool {self.tool_name} failed with {execution.error}") + logger.error(f"E2B Sandbox configurations: {sbx_config}") + logger.error(f"E2B Sandbox ID: {sbx.sandbox_id}") func_return = get_friendly_error_msg( function_name=self.tool_name, exception_name=execution.error.name, exception_message=execution.error.value ) diff --git a/tests/integration_test_tool_execution_sandbox.py b/tests/integration_test_tool_execution_sandbox.py index ad3d2ebb..3f64b287 100644 --- a/tests/integration_test_tool_execution_sandbox.py +++ b/tests/integration_test_tool_execution_sandbox.py @@ -464,7 +464,7 @@ def test_e2b_sandbox_inject_env_var_existing_sandbox(check_e2b_key_is_set, get_e config = manager.create_or_update_sandbox_config(config_create, test_user) # Run the custom sandbox once, assert nothing returns because missing env variable - sandbox = ToolExecutionSandbox(get_env_tool.name, {}, user=test_user, force_recreate=True) + sandbox = ToolExecutionSandbox(get_env_tool.name, {}, user=test_user) result = sandbox.run() # response should be None assert result.func_return is None diff --git a/tests/test_client.py b/tests/test_client.py index c3e507ae..532d8058 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -42,8 +42,8 @@ def run_server(): @pytest.fixture( - # params=[{"server": False}, {"server": True}], # whether to use REST API server - params=[{"server": False}], # whether to use REST API server + params=[{"server": False}, {"server": True}], # whether to use REST API server + # params=[{"server": False}], # whether to use REST API server scope="module", ) def client(request): @@ -385,7 +385,7 @@ def test_function_return_limit(mock_e2b_api_key_none, client: Union[LocalClient, client.delete_agent(agent_id=agent.id) -def test_function_always_error(mock_e2b_api_key_none, client: Union[LocalClient, RESTClient]): +def test_function_always_error(client: Union[LocalClient, RESTClient]): """Test to see if function that errors works correctly""" def always_error(): @@ -409,7 +409,6 @@ def test_function_always_error(mock_e2b_api_key_none, client: Union[LocalClient, assert response_message, "ToolReturnMessage message not found in response" assert response_message.status == "error" - import ipdb;ipdb.set_trace() if isinstance(client, RESTClient): assert response_message.tool_return == "Error executing function always_error: ZeroDivisionError: division by zero" else: From 172d7ac147a67f4c76a5e50b95aa1e4709198f22 Mon Sep 17 00:00:00 2001 From: Matt Zhou Date: Mon, 30 Dec 2024 16:33:59 -0800 Subject: [PATCH 7/7] Add back more e2b --- tests/test_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_client.py b/tests/test_client.py index 532d8058..5db67157 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -347,7 +347,7 @@ def test_send_system_message(client: Union[LocalClient, RESTClient], agent: Agen assert send_system_message_response, "Sending message failed" -def test_function_return_limit(mock_e2b_api_key_none, client: Union[LocalClient, RESTClient]): +def test_function_return_limit(client: Union[LocalClient, RESTClient]): """Test to see if the function return limit works""" def big_return():