From f36845b485a3ed9acacbabf9156afc2b0e484bf8 Mon Sep 17 00:00:00 2001 From: Ari Webb Date: Tue, 11 Nov 2025 17:33:24 -0600 Subject: [PATCH] feat: add create memory for archive [LET-6148] (#6110) * first hack * add to stainless * renaming field --------- Co-authored-by: Ari Webb --- fern/openapi.json | 98 ++++++++++++ letta/server/rest_api/routers/v1/archives.py | 34 ++++- letta/services/archive_manager.py | 64 +++++++- letta/services/passage_manager.py | 10 +- tests/managers/test_archive_manager.py | 150 +++++++++++++++++++ 5 files changed, 348 insertions(+), 8 deletions(-) diff --git a/fern/openapi.json b/fern/openapi.json index e8d460e0..ea050f4a 100644 --- a/fern/openapi.json +++ b/fern/openapi.json @@ -502,6 +502,63 @@ } } }, + "/v1/archives/{archive_id}/passages": { + "post": { + "tags": ["archives"], + "summary": "Create Passage In Archive", + "description": "Create a new passage in an archive.\n\nThis adds a passage to the archive and creates embeddings for vector storage.", + "operationId": "create_passage_in_archive", + "parameters": [ + { + "name": "archive_id", + "in": "path", + "required": true, + "schema": { + "type": "string", + "minLength": 44, + "maxLength": 44, + "pattern": "^archive-[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$", + "description": "The ID of the archive in the format 'archive-'", + "examples": ["archive-123e4567-e89b-42d3-8456-426614174000"], + "title": "Archive Id" + }, + "description": "The ID of the archive in the format 'archive-'" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PassageCreateRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Passage" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/v1/archives/{archive_id}/passages/{passage_id}": { "delete": { "tags": ["archives"], @@ -31256,6 +31313,47 @@ "title": "Passage", "description": "Representation of a passage, which is stored in archival memory." }, + "PassageCreateRequest": { + "properties": { + "text": { + "type": "string", + "title": "Text", + "description": "The text content of the passage" + }, + "metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Metadata", + "description": "Optional metadata for the passage" + }, + "tags": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Tags", + "description": "Optional tags for categorizing the passage" + } + }, + "type": "object", + "required": ["text"], + "title": "PassageCreateRequest", + "description": "Request model for creating a passage in an archive." + }, "PipRequirement": { "properties": { "name": { diff --git a/letta/server/rest_api/routers/v1/archives.py b/letta/server/rest_api/routers/v1/archives.py index 3bf3dcb3..6357b49f 100644 --- a/letta/server/rest_api/routers/v1/archives.py +++ b/letta/server/rest_api/routers/v1/archives.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import List, Literal, Optional +from typing import Dict, List, Literal, Optional from fastapi import APIRouter, Body, Depends, Query from pydantic import BaseModel, Field @@ -8,7 +8,7 @@ from letta import AgentState from letta.schemas.agent import AgentRelationships from letta.schemas.archive import Archive as PydanticArchive, ArchiveBase from letta.schemas.embedding_config import EmbeddingConfig -from letta.schemas.passage import Passage as PydanticPassage +from letta.schemas.passage import Passage from letta.server.rest_api.dependencies import HeaderParams, get_headers, get_letta_server from letta.server.server import SyncServer from letta.validators import AgentId, ArchiveId, PassageId @@ -37,6 +37,14 @@ class ArchiveUpdateRequest(BaseModel): description: Optional[str] = None +class PassageCreateRequest(BaseModel): + """Request model for creating a passage in an archive.""" + + text: str = Field(..., description="The text content of the passage") + metadata: Optional[Dict] = Field(default=None, description="Optional metadata for the passage") + tags: Optional[List[str]] = Field(default=None, description="Optional tags for categorizing the passage") + + @router.post("/", response_model=PydanticArchive, operation_id="create_archive") async def create_archive( archive: ArchiveCreateRequest = Body(...), @@ -179,6 +187,28 @@ async def list_agents_for_archive( ) +@router.post("/{archive_id}/passages", response_model=Passage, operation_id="create_passage_in_archive") +async def create_passage_in_archive( + archive_id: ArchiveId, + passage: PassageCreateRequest = Body(...), + server: "SyncServer" = Depends(get_letta_server), + headers: HeaderParams = Depends(get_headers), +): + """ + Create a new passage in an archive. + + This adds a passage to the archive and creates embeddings for vector storage. + """ + actor = await server.user_manager.get_actor_or_default_async(actor_id=headers.actor_id) + return await server.archive_manager.create_passage_in_archive_async( + archive_id=archive_id, + text=passage.text, + metadata=passage.metadata, + tags=passage.tags, + actor=actor, + ) + + @router.delete("/{archive_id}/passages/{passage_id}", status_code=204, operation_id="delete_passage_from_archive") async def delete_passage_from_archive( archive_id: ArchiveId, diff --git a/letta/services/archive_manager.py b/letta/services/archive_manager.py index dcf7fc3d..2758f1f9 100644 --- a/letta/services/archive_manager.py +++ b/letta/services/archive_manager.py @@ -1,6 +1,6 @@ import asyncio from datetime import datetime -from typing import List, Optional +from typing import Dict, List, Optional from sqlalchemy import delete, or_, select @@ -12,6 +12,7 @@ from letta.schemas.agent import AgentState as PydanticAgentState from letta.schemas.archive import Archive as PydanticArchive from letta.schemas.embedding_config import EmbeddingConfig from letta.schemas.enums import PrimitiveType, VectorDBProvider +from letta.schemas.passage import Passage as PydanticPassage from letta.schemas.user import User as PydanticUser from letta.server.db import db_registry from letta.services.helpers.agent_manager_helper import validate_agent_exists_async @@ -276,6 +277,67 @@ class ArchiveManager: await archive_model.hard_delete_async(session, actor=actor) logger.info(f"Deleted archive {archive_id}") + @enforce_types + @trace_method + @raise_on_invalid_id(param_name="archive_id", expected_prefix=PrimitiveType.ARCHIVE) + async def create_passage_in_archive_async( + self, + archive_id: str, + text: str, + metadata: Dict = None, + tags: List[str] = None, + actor: PydanticUser = None, + ) -> PydanticPassage: + """Create a passage in an archive. + + Args: + archive_id: ID of the archive to add the passage to + text: The text content of the passage + metadata: Optional metadata for the passage + tags: Optional tags for categorizing the passage + actor: User performing the operation + + Returns: + The created passage + + Raises: + NoResultFound: If archive not found + """ + from letta.llm_api.llm_client import LLMClient + from letta.services.passage_manager import PassageManager + + # Verify the archive exists and user has access + archive = await self.get_archive_by_id_async(archive_id=archive_id, actor=actor) + + # Generate embeddings for the text + embedding_client = LLMClient.create( + provider_type=archive.embedding_config.embedding_endpoint_type, + actor=actor, + ) + embeddings = await embedding_client.request_embeddings([text], archive.embedding_config) + embedding = embeddings[0] if embeddings else None + + # Create the passage object with embedding + passage = PydanticPassage( + text=text, + archive_id=archive_id, + organization_id=actor.organization_id, + metadata=metadata or {}, + tags=tags, + embedding_config=archive.embedding_config, + embedding=embedding, + ) + + # Use PassageManager to create the passage + passage_manager = PassageManager() + created_passage = await passage_manager.create_agent_passage_async( + pydantic_passage=passage, + actor=actor, + ) + + logger.info(f"Created passage {created_passage.id} in archive {archive_id}") + return created_passage + @enforce_types @trace_method @raise_on_invalid_id(param_name="archive_id", expected_prefix=PrimitiveType.ARCHIVE) diff --git a/letta/services/passage_manager.py b/letta/services/passage_manager.py index 42f78d8a..55f628ed 100644 --- a/letta/services/passage_manager.py +++ b/letta/services/passage_manager.py @@ -172,7 +172,7 @@ class PassageManager: "embedding": embedding, "embedding_config": data["embedding_config"], "organization_id": data["organization_id"], - "metadata_": data.get("metadata", {}), + "metadata_": data.get("metadata_", {}), "tags": tags, "is_deleted": data.get("is_deleted", False), "created_at": data.get("created_at", datetime.now(timezone.utc)), @@ -233,7 +233,7 @@ class PassageManager: "embedding": embedding, "embedding_config": data["embedding_config"], "organization_id": data["organization_id"], - "metadata_": data.get("metadata", {}), + "metadata_": data.get("metadata_", {}), "tags": tags, "is_deleted": data.get("is_deleted", False), "created_at": data.get("created_at", datetime.now(timezone.utc)), @@ -270,7 +270,7 @@ class PassageManager: "embedding": data["embedding"], "embedding_config": data["embedding_config"], "organization_id": data["organization_id"], - "metadata_": data.get("metadata", {}), + "metadata_": data.get("metadata_", {}), "tags": data.get("tags"), "is_deleted": data.get("is_deleted", False), "created_at": data.get("created_at", datetime.now(timezone.utc)), @@ -332,7 +332,7 @@ class PassageManager: "embedding": embedding, "embedding_config": data["embedding_config"], "organization_id": data["organization_id"], - "metadata_": data.get("metadata", {}), + "metadata_": data.get("metadata_", {}), "tags": data.get("tags"), "is_deleted": data.get("is_deleted", False), "created_at": data.get("created_at", datetime.now(timezone.utc)), @@ -386,7 +386,7 @@ class PassageManager: "embedding": embedding, "embedding_config": data["embedding_config"], "organization_id": data["organization_id"], - "metadata_": data.get("metadata", {}), + "metadata_": data.get("metadata_", {}), "tags": data.get("tags"), "is_deleted": data.get("is_deleted", False), "created_at": data.get("created_at", datetime.now(timezone.utc)), diff --git a/tests/managers/test_archive_manager.py b/tests/managers/test_archive_manager.py index 8b3f3437..ece11330 100644 --- a/tests/managers/test_archive_manager.py +++ b/tests/managers/test_archive_manager.py @@ -1140,3 +1140,153 @@ async def test_archive_manager_delete_passage_from_nonexistent_archive(server: S # cleanup await server.passage_manager.delete_agent_passage_by_id_async(passage.id, actor=default_user) await server.archive_manager.delete_archive_async(archive.id, actor=default_user) + + +@pytest.mark.asyncio +async def test_archive_manager_create_passage_in_archive_async(server: SyncServer, default_user): + """Test creating a passage in an archive.""" + # create archive + archive = await server.archive_manager.create_archive_async( + name="test_passage_creation_archive", + description="Archive for testing passage creation", + embedding_config=DEFAULT_EMBEDDING_CONFIG, + actor=default_user, + ) + + # create a passage in the archive + created_passage = await server.archive_manager.create_passage_in_archive_async( + archive_id=archive.id, + text="This is a test passage for creation", + actor=default_user, + ) + + # verify the passage was created + assert created_passage.id is not None + assert created_passage.text == "This is a test passage for creation" + assert created_passage.archive_id == archive.id + assert created_passage.organization_id == default_user.organization_id + + # verify we can retrieve it + retrieved_passage = await server.passage_manager.get_agent_passage_by_id_async(passage_id=created_passage.id, actor=default_user) + assert retrieved_passage.id == created_passage.id + assert retrieved_passage.text == created_passage.text + assert retrieved_passage.archive_id == archive.id + + # cleanup + await server.passage_manager.delete_agent_passage_by_id_async(created_passage.id, actor=default_user) + await server.archive_manager.delete_archive_async(archive.id, actor=default_user) + + +@pytest.mark.asyncio +async def test_archive_manager_create_passage_with_metadata_and_tags(server: SyncServer, default_user): + """Test creating a passage with metadata and tags.""" + # create archive + archive = await server.archive_manager.create_archive_async( + name="test_passage_metadata_archive", + embedding_config=DEFAULT_EMBEDDING_CONFIG, + actor=default_user, + ) + + # create passage with metadata and tags + test_metadata = {"source": "unit_test", "version": 1} + test_tags = ["test", "archive", "passage"] + + created_passage = await server.archive_manager.create_passage_in_archive_async( + archive_id=archive.id, + text="Passage with metadata and tags", + metadata=test_metadata, + tags=test_tags, + actor=default_user, + ) + + # verify metadata and tags were stored + assert created_passage.metadata == test_metadata + assert set(created_passage.tags) == set(test_tags) # Use set comparison to ignore order + + # retrieve and verify persistence + retrieved_passage = await server.passage_manager.get_agent_passage_by_id_async(passage_id=created_passage.id, actor=default_user) + assert retrieved_passage.metadata == test_metadata + assert set(retrieved_passage.tags) == set(test_tags) + + # cleanup + await server.passage_manager.delete_agent_passage_by_id_async(created_passage.id, actor=default_user) + await server.archive_manager.delete_archive_async(archive.id, actor=default_user) + + +@pytest.mark.asyncio +async def test_archive_manager_create_passage_in_nonexistent_archive(server: SyncServer, default_user): + """Test that creating a passage in a non-existent archive raises an error.""" + # attempt to create passage in non-existent archive + fake_archive_id = f"archive-{uuid.uuid4()}" + + with pytest.raises(NoResultFound): + await server.archive_manager.create_passage_in_archive_async( + archive_id=fake_archive_id, + text="This should fail", + actor=default_user, + ) + + +@pytest.mark.asyncio +async def test_archive_manager_create_passage_inherits_embedding_config(server: SyncServer, default_user): + """Test that created passages inherit the archive's embedding configuration.""" + # create archive with specific embedding config + specific_embedding_config = EmbeddingConfig.default_config(provider="openai") + + archive = await server.archive_manager.create_archive_async( + name="test_embedding_inheritance_archive", + embedding_config=specific_embedding_config, + actor=default_user, + ) + + # create passage + created_passage = await server.archive_manager.create_passage_in_archive_async( + archive_id=archive.id, + text="Test passage for embedding config inheritance", + actor=default_user, + ) + + # verify the passage inherited the archive's embedding config + assert created_passage.embedding_config is not None + assert created_passage.embedding_config.embedding_endpoint_type == specific_embedding_config.embedding_endpoint_type + assert created_passage.embedding_config.embedding_model == specific_embedding_config.embedding_model + assert created_passage.embedding_config.embedding_dim == specific_embedding_config.embedding_dim + + # cleanup + await server.passage_manager.delete_agent_passage_by_id_async(created_passage.id, actor=default_user) + await server.archive_manager.delete_archive_async(archive.id, actor=default_user) + + +@pytest.mark.asyncio +async def test_archive_manager_create_multiple_passages_in_archive(server: SyncServer, default_user): + """Test creating multiple passages in the same archive.""" + # create archive + archive = await server.archive_manager.create_archive_async( + name="test_multiple_passages_archive", + embedding_config=DEFAULT_EMBEDDING_CONFIG, + actor=default_user, + ) + + # create multiple passages + passages = [] + for i in range(3): + passage = await server.archive_manager.create_passage_in_archive_async( + archive_id=archive.id, + text=f"Test passage number {i}", + metadata={"index": i}, + tags=[f"passage_{i}"], + actor=default_user, + ) + passages.append(passage) + + # verify all passages were created with correct data + for i, passage in enumerate(passages): + assert passage.text == f"Test passage number {i}" + assert passage.metadata["index"] == i + assert f"passage_{i}" in passage.tags + assert passage.archive_id == archive.id + + # cleanup + for passage in passages: + await server.passage_manager.delete_agent_passage_by_id_async(passage.id, actor=default_user) + await server.archive_manager.delete_archive_async(archive.id, actor=default_user)