cthomas
3
.github/workflows/core-integration-tests.yml
vendored
3
.github/workflows/core-integration-tests.yml
vendored
@@ -41,8 +41,7 @@ jobs:
|
||||
"integration_test_batch_api_cron_jobs.py",
|
||||
"integration_test_builtin_tools.py",
|
||||
"integration_test_turbopuffer.py",
|
||||
"integration_test_human_in_the_loop.py",
|
||||
"integration_test_git_memory_repo_http.py"
|
||||
"integration_test_human_in_the_loop.py"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -230,15 +230,14 @@ async def lifespan(app_: FastAPI):
|
||||
global server
|
||||
await server.init_async(init_with_default_org_and_user=not settings.no_default_actor)
|
||||
|
||||
# Set server instance for git HTTP endpoints and start dulwich sidecar
|
||||
# Set server instance for git HTTP endpoints
|
||||
try:
|
||||
from letta.server.rest_api.routers.v1.git_http import set_server_instance, start_dulwich_server
|
||||
from letta.server.rest_api.routers.v1.git_http import set_server_instance
|
||||
|
||||
set_server_instance(server)
|
||||
start_dulwich_server()
|
||||
logger.info(f"[Worker {worker_id}] Git HTTP server instance set (dulwich sidecar started)")
|
||||
logger.info(f"[Worker {worker_id}] Git HTTP server instance set")
|
||||
except Exception as e:
|
||||
logger.warning(f"[Worker {worker_id}] Failed to start git HTTP sidecar: {e}")
|
||||
logger.warning(f"[Worker {worker_id}] Failed to set git HTTP server instance: {e}")
|
||||
|
||||
try:
|
||||
await start_scheduler_with_leader_election(server)
|
||||
@@ -845,8 +844,6 @@ def create_application() -> "FastAPI":
|
||||
# /api/auth endpoints
|
||||
app.include_router(setup_auth_router(server, interface, random_password), prefix=API_PREFIX)
|
||||
|
||||
# Git smart HTTP is served by a dulwich sidecar and proxied by the /v1/git router.
|
||||
|
||||
# / static files
|
||||
mount_static_files(app)
|
||||
|
||||
|
||||
@@ -1,13 +1,7 @@
|
||||
"""Git HTTP Smart Protocol endpoints via dulwich (proxied).
|
||||
"""Git HTTP Smart Protocol endpoints (proxied to memfs service).
|
||||
|
||||
## Why a separate dulwich server?
|
||||
|
||||
Dulwich's `HTTPGitApplication` is a WSGI app and relies on the WSGI `write()`
|
||||
callback pattern. Starlette's `WSGIMiddleware` does not fully support this
|
||||
pattern, which causes failures when mounting dulwich directly into FastAPI.
|
||||
|
||||
To avoid the ASGI/WSGI impedance mismatch, we run dulwich's WSGI server on a
|
||||
separate local port (default: 8284) and proxy `/v1/git/*` requests to it.
|
||||
This module proxies `/v1/git/*` requests to the external memfs service, which
|
||||
handles git smart HTTP protocol (clone, push, pull).
|
||||
|
||||
Example:
|
||||
|
||||
@@ -19,43 +13,16 @@ Routes (smart HTTP):
|
||||
GET /v1/git/{agent_id}/state.git/info/refs?service=git-receive-pack
|
||||
POST /v1/git/{agent_id}/state.git/git-receive-pack
|
||||
|
||||
The dulwich server uses `GCSBackend` to materialize repositories from GCS on
|
||||
-demand.
|
||||
|
||||
Post-push sync back to GCS/PostgreSQL is triggered from the proxy route after a
|
||||
Post-push sync to PostgreSQL is triggered from the proxy route after a
|
||||
successful `git-receive-pack`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextvars
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import threading
|
||||
from typing import Dict, Iterable, Optional
|
||||
|
||||
import httpx
|
||||
|
||||
# dulwich is an optional dependency (extra = "git-state"). CI installs don't
|
||||
# include it, so imports must be lazy/guarded.
|
||||
try:
|
||||
from dulwich.repo import Repo
|
||||
from dulwich.server import Backend
|
||||
from dulwich.web import HTTPGitApplication, make_server
|
||||
|
||||
_DULWICH_AVAILABLE = True
|
||||
except ImportError: # pragma: no cover
|
||||
Repo = None # type: ignore[assignment]
|
||||
|
||||
class Backend: # type: ignore[no-redef]
|
||||
pass
|
||||
|
||||
HTTPGitApplication = None # type: ignore[assignment]
|
||||
make_server = None # type: ignore[assignment]
|
||||
_DULWICH_AVAILABLE = False
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
from starlette.background import BackgroundTask
|
||||
@@ -72,38 +39,6 @@ router = APIRouter(prefix="/git", tags=["git"], include_in_schema=False)
|
||||
# Global storage for the server instance (set during app startup)
|
||||
_server_instance = None
|
||||
|
||||
# org_id/agent_id -> temp working tree path (repo root, with .git inside)
|
||||
_repo_cache: Dict[str, str] = {}
|
||||
_repo_locks: Dict[str, threading.Lock] = {}
|
||||
|
||||
|
||||
def _dulwich_repo_path_marker_file(cache_key: str) -> str:
|
||||
"""Path to a marker file that stores the dulwich temp repo path.
|
||||
|
||||
Dulwich runs in-process and mutates a repo materialized into a temp directory.
|
||||
We then need to locate that same temp directory after the push to persist the
|
||||
updated `.git/` contents back to object storage.
|
||||
|
||||
In production we may have multiple FastAPI workers; in-memory `_repo_cache`
|
||||
is not shared across workers, so we store the repo_path in a small file under
|
||||
/tmp as a best-effort handoff. (Longer-term, we'll likely move dulwich to its
|
||||
own service/process and remove this.)
|
||||
"""
|
||||
|
||||
safe = cache_key.replace("/", "__")
|
||||
base = os.path.join(tempfile.gettempdir(), "letta-git-http")
|
||||
os.makedirs(base, exist_ok=True)
|
||||
return os.path.join(base, f"dulwich_repo_path__{safe}.txt")
|
||||
|
||||
|
||||
# org_id for the currently-handled dulwich request (set by a WSGI wrapper).
|
||||
_current_org_id: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar("letta_git_http_org_id", default=None)
|
||||
|
||||
|
||||
# Dulwich server globals
|
||||
_dulwich_server = None
|
||||
_dulwich_thread: Optional[threading.Thread] = None
|
||||
|
||||
|
||||
def set_server_instance(server) -> None:
|
||||
"""Set the Letta server instance for git operations. Called during app startup."""
|
||||
@@ -112,301 +47,12 @@ def set_server_instance(server) -> None:
|
||||
_server_instance = server
|
||||
|
||||
|
||||
def _get_dulwich_port() -> int:
|
||||
return int(os.getenv("LETTA_GIT_HTTP_DULWICH_PORT", "8284"))
|
||||
|
||||
|
||||
def start_dulwich_server(host: str = "127.0.0.1", port: Optional[int] = None) -> None:
|
||||
"""Start a local dulwich HTTP server in a background thread.
|
||||
|
||||
This is safe to call multiple times; only the first successful call will
|
||||
start a server in the current process.
|
||||
"""
|
||||
|
||||
global _dulwich_server, _dulwich_thread
|
||||
|
||||
if not _DULWICH_AVAILABLE:
|
||||
logger.info("dulwich not installed; git smart HTTP is disabled")
|
||||
return
|
||||
|
||||
if _dulwich_thread and _dulwich_thread.is_alive():
|
||||
return
|
||||
|
||||
if port is None:
|
||||
port = _get_dulwich_port()
|
||||
|
||||
# Ensure backend can access storage through the running server.
|
||||
if _server_instance is None:
|
||||
raise RuntimeError("Server instance not set (did you call set_server_instance?)")
|
||||
|
||||
try:
|
||||
_dulwich_server = make_server(host, port, _git_wsgi_app)
|
||||
except OSError as e:
|
||||
# When running with multiple uvicorn workers, only one process can bind
|
||||
# to the configured port.
|
||||
logger.warning("Failed to bind dulwich git server on %s:%s: %s", host, port, e)
|
||||
return
|
||||
|
||||
def _run():
|
||||
logger.info("Starting dulwich git HTTP server on http://%s:%s", host, port)
|
||||
try:
|
||||
_dulwich_server.serve_forever()
|
||||
except Exception:
|
||||
logger.exception("Dulwich git HTTP server crashed")
|
||||
|
||||
_dulwich_thread = threading.Thread(target=_run, name="dulwich-git-http", daemon=True)
|
||||
_dulwich_thread.start()
|
||||
|
||||
|
||||
def stop_dulwich_server() -> None:
|
||||
"""Stop the local dulwich server (best-effort)."""
|
||||
|
||||
global _dulwich_server
|
||||
if _dulwich_server is None:
|
||||
return
|
||||
try:
|
||||
_dulwich_server.shutdown()
|
||||
except Exception:
|
||||
logger.exception("Failed to shutdown dulwich server")
|
||||
|
||||
|
||||
def _require_current_org_id() -> str:
|
||||
"""Read the org_id set by the WSGI wrapper for the current request."""
|
||||
|
||||
org_id = _current_org_id.get()
|
||||
if not org_id:
|
||||
raise RuntimeError("Missing org_id for git HTTP request")
|
||||
return org_id
|
||||
|
||||
|
||||
def _resolve_org_id_from_wsgi_environ(environ: dict) -> Optional[str]:
|
||||
"""Resolve org_id for dulwich, preferring X-Organization-Id.
|
||||
|
||||
This is used by the dulwich WSGI wrapper. If X-Organization-Id is missing,
|
||||
we fall back to resolving via the authenticated user_id header.
|
||||
|
||||
Note: dulwich is served on 127.0.0.1, so these headers should only be set by
|
||||
our trusted in-pod proxy layer.
|
||||
"""
|
||||
|
||||
org_id = environ.get("HTTP_X_ORGANIZATION_ID")
|
||||
if org_id:
|
||||
return org_id
|
||||
|
||||
user_id = environ.get("HTTP_USER_ID")
|
||||
if not user_id:
|
||||
return None
|
||||
|
||||
if _server_instance is None:
|
||||
return None
|
||||
|
||||
try:
|
||||
# We are in a dulwich WSGI thread; run async DB lookup in a fresh loop.
|
||||
actor = asyncio.run(_server_instance.user_manager.get_actor_by_id_async(user_id))
|
||||
resolved = actor.organization_id
|
||||
except Exception:
|
||||
logger.exception("Failed to resolve org_id from user_id for dulwich request (user_id=%s)", user_id)
|
||||
return None
|
||||
|
||||
return resolved
|
||||
|
||||
|
||||
class GCSBackend(Backend):
|
||||
"""Dulwich backend that materializes repos from GCS."""
|
||||
|
||||
def open_repository(self, path: str | bytes):
|
||||
"""Open a repository by path.
|
||||
|
||||
dulwich passes paths like:
|
||||
/{agent_id}/state.git
|
||||
/{agent_id}/state.git/info/refs
|
||||
/{agent_id}/state.git/git-upload-pack
|
||||
/{agent_id}/state.git/git-receive-pack
|
||||
|
||||
We map those to an on-disk repo cached in a temp dir.
|
||||
"""
|
||||
|
||||
if not _DULWICH_AVAILABLE or Repo is None:
|
||||
raise RuntimeError("dulwich not installed")
|
||||
|
||||
if isinstance(path, (bytes, bytearray)):
|
||||
path = path.decode("utf-8", errors="surrogateescape")
|
||||
|
||||
parts = path.strip("/").split("/")
|
||||
|
||||
# Supported path form: /{agent_id}/state.git[/...]
|
||||
if "state.git" not in parts:
|
||||
raise ValueError(f"Invalid repository path (missing state.git): {path}")
|
||||
|
||||
repo_idx = parts.index("state.git")
|
||||
if repo_idx != 1:
|
||||
raise ValueError(f"Invalid repository path (expected /{{agent_id}}/state.git): {path}")
|
||||
|
||||
agent_id = parts[0]
|
||||
org_id = _require_current_org_id()
|
||||
|
||||
cache_key = f"{org_id}/{agent_id}"
|
||||
logger.info("GCSBackend.open_repository: org=%s agent=%s", org_id, agent_id)
|
||||
|
||||
lock = _repo_locks.setdefault(cache_key, threading.Lock())
|
||||
with lock:
|
||||
# Always refresh from GCS to avoid serving stale refs/objects when the
|
||||
# repo is mutated through non-git code paths (e.g. git-state APIs)
|
||||
# or when multiple app workers are running.
|
||||
old_repo_path = _repo_cache.pop(cache_key, None)
|
||||
if old_repo_path:
|
||||
shutil.rmtree(os.path.dirname(old_repo_path), ignore_errors=True)
|
||||
try:
|
||||
os.unlink(_dulwich_repo_path_marker_file(cache_key))
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
repo_path = self._download_repo_sync(agent_id=agent_id, org_id=org_id)
|
||||
_repo_cache[cache_key] = repo_path
|
||||
|
||||
# Persist repo_path for cross-worker post-push sync.
|
||||
try:
|
||||
with open(_dulwich_repo_path_marker_file(cache_key), "w") as f:
|
||||
f.write(repo_path)
|
||||
except Exception:
|
||||
logger.exception("Failed to write repo_path marker for %s", cache_key)
|
||||
|
||||
repo = Repo(repo_path)
|
||||
_prune_broken_refs(repo)
|
||||
return repo
|
||||
|
||||
def _download_repo_sync(self, agent_id: str, org_id: str) -> str:
|
||||
"""Synchronously download a repo from GCS.
|
||||
|
||||
dulwich runs in a background thread (wsgiref server thread), so we should
|
||||
not assume we're on the main event loop.
|
||||
"""
|
||||
|
||||
if _server_instance is None:
|
||||
raise RuntimeError("Server instance not set (did you call set_server_instance?)")
|
||||
|
||||
# This runs in a dulwich-managed WSGI thread, not an AnyIO worker thread.
|
||||
# Use a dedicated event loop to run the async download.
|
||||
return asyncio.run(self._download_repo(agent_id, org_id))
|
||||
|
||||
async def _download_repo(self, agent_id: str, org_id: str) -> str:
|
||||
"""Download repo from GCS into a temporary working tree."""
|
||||
|
||||
storage = _server_instance.memory_repo_manager.git.storage
|
||||
storage_prefix = f"{org_id}/{agent_id}/repo.git"
|
||||
|
||||
files = await storage.list_files(storage_prefix)
|
||||
if not files:
|
||||
# Create an empty repo on-demand so clients can `git clone` immediately.
|
||||
logger.info("Repository not found for agent %s; creating empty repo", agent_id)
|
||||
await _server_instance.memory_repo_manager.git.create_repo(
|
||||
agent_id=agent_id,
|
||||
org_id=org_id,
|
||||
initial_files={},
|
||||
author_name="Letta System",
|
||||
author_email="system@letta.ai",
|
||||
)
|
||||
files = await storage.list_files(storage_prefix)
|
||||
if not files:
|
||||
raise FileNotFoundError(f"Repository not found for agent {agent_id}")
|
||||
|
||||
temp_dir = tempfile.mkdtemp(prefix="letta-git-http-")
|
||||
repo_path = os.path.join(temp_dir, "repo")
|
||||
git_dir = os.path.join(repo_path, ".git")
|
||||
os.makedirs(git_dir)
|
||||
|
||||
# Ensure required git directories exist for fetch/push even if GCS doesn't
|
||||
# have any objects packed yet.
|
||||
for subdir in [
|
||||
"objects",
|
||||
os.path.join("objects", "pack"),
|
||||
os.path.join("objects", "info"),
|
||||
"refs",
|
||||
os.path.join("refs", "heads"),
|
||||
os.path.join("refs", "tags"),
|
||||
"info",
|
||||
]:
|
||||
os.makedirs(os.path.join(git_dir, subdir), exist_ok=True)
|
||||
|
||||
async def download_file(file_path: str):
|
||||
if file_path.startswith(storage_prefix):
|
||||
rel_path = file_path[len(storage_prefix) + 1 :]
|
||||
else:
|
||||
rel_path = file_path.split("/")[-1]
|
||||
|
||||
if not rel_path:
|
||||
return
|
||||
|
||||
local_path = os.path.join(git_dir, rel_path)
|
||||
os.makedirs(os.path.dirname(local_path), exist_ok=True)
|
||||
content = await storage.download_bytes(file_path)
|
||||
with open(local_path, "wb") as f:
|
||||
f.write(content)
|
||||
|
||||
await asyncio.gather(*[download_file(f) for f in files])
|
||||
logger.info("Downloaded %s files from GCS for agent %s", len(files), agent_id)
|
||||
|
||||
return repo_path
|
||||
|
||||
|
||||
def _prune_broken_refs(repo: Repo) -> int:
|
||||
"""Remove refs that point at missing objects.
|
||||
|
||||
This can happen if a prior push partially failed after updating refs but
|
||||
before all objects were persisted to backing storage.
|
||||
|
||||
We prune these so dulwich doesn't advertise/resolve against corrupt refs,
|
||||
which can lead to `UnresolvedDeltas` during subsequent pushes.
|
||||
"""
|
||||
|
||||
removed = 0
|
||||
try:
|
||||
ref_names = list(repo.refs.keys())
|
||||
except Exception:
|
||||
logger.exception("Failed to enumerate refs for pruning")
|
||||
return 0
|
||||
|
||||
for name in ref_names:
|
||||
# HEAD is commonly symbolic; skip.
|
||||
if name in {b"HEAD", "HEAD"}:
|
||||
continue
|
||||
try:
|
||||
sha = repo.refs[name]
|
||||
except Exception:
|
||||
continue
|
||||
if not sha:
|
||||
continue
|
||||
try:
|
||||
if sha not in repo.object_store:
|
||||
logger.warning("Pruning broken ref %r -> %r", name, sha)
|
||||
try:
|
||||
repo.refs.remove_if_equals(name, sha)
|
||||
except Exception:
|
||||
# Best-effort fallback
|
||||
try:
|
||||
del repo.refs[name]
|
||||
except Exception:
|
||||
pass
|
||||
removed += 1
|
||||
except Exception:
|
||||
logger.exception("Failed while checking ref %r", name)
|
||||
|
||||
return removed
|
||||
|
||||
|
||||
async def _sync_after_push(actor_id: str, agent_id: str) -> None:
|
||||
"""Sync repo back to GCS and PostgreSQL after a successful push.
|
||||
"""Sync blocks to PostgreSQL after a successful push.
|
||||
|
||||
When using memfs service:
|
||||
- GCS sync is handled by memfs (skipped here)
|
||||
- We still sync blocks to Postgres
|
||||
|
||||
When using local dulwich:
|
||||
- Upload repo to GCS
|
||||
- Sync blocks to Postgres
|
||||
GCS sync is handled by the memfs service. This function syncs the
|
||||
block contents to PostgreSQL for caching/querying.
|
||||
"""
|
||||
from letta.settings import settings
|
||||
|
||||
if _server_instance is None:
|
||||
logger.warning("Server instance not set; cannot sync after push")
|
||||
return
|
||||
@@ -418,51 +64,6 @@ async def _sync_after_push(actor_id: str, agent_id: str) -> None:
|
||||
return
|
||||
|
||||
org_id = actor.organization_id
|
||||
using_memfs = bool(settings.memfs_service_url)
|
||||
|
||||
# When using local dulwich, we need to upload to GCS
|
||||
if not using_memfs:
|
||||
cache_key = f"{org_id}/{agent_id}"
|
||||
|
||||
repo_path = _repo_cache.get(cache_key)
|
||||
if not repo_path:
|
||||
# Cross-worker fallback: read marker file written by the dulwich process.
|
||||
try:
|
||||
with open(_dulwich_repo_path_marker_file(cache_key), "r") as f:
|
||||
repo_path = f.read().strip() or None
|
||||
except FileNotFoundError:
|
||||
repo_path = None
|
||||
|
||||
if not repo_path:
|
||||
logger.warning("No cached repo for %s after push", cache_key)
|
||||
return
|
||||
|
||||
if not os.path.exists(repo_path):
|
||||
logger.warning("Repo path %s does not exist after push", repo_path)
|
||||
return
|
||||
|
||||
logger.info("Syncing repo after push: org=%s agent=%s", org_id, agent_id)
|
||||
|
||||
storage = _server_instance.memory_repo_manager.git.storage
|
||||
storage_prefix = f"{org_id}/{agent_id}/repo.git"
|
||||
git_dir = os.path.join(repo_path, ".git")
|
||||
|
||||
upload_tasks = []
|
||||
for root, _dirs, files in os.walk(git_dir):
|
||||
for filename in files:
|
||||
local_file = os.path.join(root, filename)
|
||||
rel_path = os.path.relpath(local_file, git_dir)
|
||||
storage_path = f"{storage_prefix}/{rel_path}"
|
||||
|
||||
with open(local_file, "rb") as f:
|
||||
content = f.read()
|
||||
|
||||
upload_tasks.append(storage.upload_bytes(storage_path, content))
|
||||
|
||||
await asyncio.gather(*upload_tasks)
|
||||
logger.info("Uploaded %s files to GCS", len(upload_tasks))
|
||||
else:
|
||||
logger.info("Using memfs service; GCS sync handled by memfs (agent=%s)", agent_id)
|
||||
|
||||
# Sync blocks to Postgres (if using GitEnabledBlockManager).
|
||||
#
|
||||
@@ -471,118 +72,110 @@ async def _sync_after_push(actor_id: str, agent_id: str) -> None:
|
||||
# relying on a working tree checkout under repo_path/.
|
||||
from letta.services.block_manager_git import GitEnabledBlockManager
|
||||
|
||||
if isinstance(_server_instance.block_manager, GitEnabledBlockManager):
|
||||
# Retry with backoff to handle race condition where GCS upload is still in progress
|
||||
# after git-receive-pack returns. The webhook fires immediately but commit objects
|
||||
# may not be fully uploaded yet.
|
||||
files = {}
|
||||
max_retries = 3
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
files = await _server_instance.memory_repo_manager.git.get_files(
|
||||
agent_id=agent_id,
|
||||
org_id=org_id,
|
||||
ref="HEAD",
|
||||
)
|
||||
logger.info("get_files returned %d files (attempt %d)", len(files), attempt + 1)
|
||||
break
|
||||
except Exception as e:
|
||||
if attempt < max_retries - 1:
|
||||
wait_time = 2**attempt # 1s, 2s, 4s
|
||||
logger.warning("Failed to read repo files (attempt %d/%d), retrying in %ds: %s", attempt + 1, max_retries, wait_time, e)
|
||||
await asyncio.sleep(wait_time)
|
||||
else:
|
||||
logger.exception("Failed to read repo files after %d retries (agent=%s)", max_retries, agent_id)
|
||||
if not isinstance(_server_instance.block_manager, GitEnabledBlockManager):
|
||||
return
|
||||
|
||||
expected_labels = set()
|
||||
from letta.services.memory_repo.block_markdown import parse_block_markdown
|
||||
|
||||
md_file_paths = sorted([file_path for file_path in files if file_path.endswith(".md")])
|
||||
nested_md_file_paths = [file_path for file_path in md_file_paths if "/" in file_path[:-3]]
|
||||
logger.info(
|
||||
"Post-push sync file scan: agent=%s total_files=%d md_files=%d nested_md_files=%d sample_md_paths=%s",
|
||||
agent_id,
|
||||
len(files),
|
||||
len(md_file_paths),
|
||||
len(nested_md_file_paths),
|
||||
md_file_paths[:10],
|
||||
)
|
||||
|
||||
synced = 0
|
||||
for file_path, content in files.items():
|
||||
if not file_path.endswith(".md"):
|
||||
continue
|
||||
|
||||
label = file_path[:-3]
|
||||
expected_labels.add(label)
|
||||
|
||||
# Parse frontmatter to extract metadata alongside value
|
||||
parsed = parse_block_markdown(content)
|
||||
|
||||
try:
|
||||
await _server_instance.block_manager._sync_block_to_postgres(
|
||||
agent_id=agent_id,
|
||||
label=label,
|
||||
value=parsed["value"],
|
||||
actor=actor,
|
||||
description=parsed.get("description"),
|
||||
limit=parsed.get("limit"),
|
||||
read_only=parsed.get("read_only"),
|
||||
metadata=parsed.get("metadata"),
|
||||
)
|
||||
synced += 1
|
||||
logger.info("Synced block %s to PostgreSQL", label)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to sync block %s to PostgreSQL (agent=%s) [path=%s nested=%s]",
|
||||
label,
|
||||
agent_id,
|
||||
file_path,
|
||||
"/" in label,
|
||||
)
|
||||
|
||||
if synced == 0:
|
||||
logger.warning("No *.md files found in repo HEAD during post-push sync (agent=%s)", agent_id)
|
||||
else:
|
||||
# Detach blocks that were removed in git.
|
||||
#
|
||||
# We treat git as the source of truth for which blocks are attached to
|
||||
# this agent. If a *.md file disappears from HEAD, detach the
|
||||
# corresponding block from the agent in Postgres.
|
||||
try:
|
||||
existing_blocks = await _server_instance.agent_manager.list_agent_blocks_async(
|
||||
agent_id=agent_id,
|
||||
actor=actor,
|
||||
before=None,
|
||||
after=None,
|
||||
limit=1000,
|
||||
ascending=True,
|
||||
)
|
||||
existing_by_label = {b.label: b for b in existing_blocks}
|
||||
removed_labels = set(existing_by_label.keys()) - expected_labels
|
||||
|
||||
for label in sorted(removed_labels):
|
||||
block = existing_by_label.get(label)
|
||||
if not block:
|
||||
continue
|
||||
await _server_instance.agent_manager.detach_block_async(
|
||||
agent_id=agent_id,
|
||||
block_id=block.id,
|
||||
actor=actor,
|
||||
)
|
||||
logger.info("Detached block %s from agent (removed from git)", label)
|
||||
except Exception:
|
||||
logger.exception("Failed detaching removed blocks during post-push sync (agent=%s)", agent_id)
|
||||
|
||||
# Cleanup local cache (only relevant when using local dulwich)
|
||||
if not using_memfs:
|
||||
cache_key = f"{org_id}/{agent_id}"
|
||||
_repo_cache.pop(cache_key, None)
|
||||
# Retry with backoff to handle race condition where GCS upload is still in progress
|
||||
# after git-receive-pack returns. The webhook fires immediately but commit objects
|
||||
# may not be fully uploaded yet.
|
||||
files = {}
|
||||
max_retries = 3
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
os.unlink(_dulwich_repo_path_marker_file(cache_key))
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
shutil.rmtree(os.path.dirname(repo_path), ignore_errors=True)
|
||||
files = await _server_instance.memory_repo_manager.git.get_files(
|
||||
agent_id=agent_id,
|
||||
org_id=org_id,
|
||||
ref="HEAD",
|
||||
)
|
||||
logger.info("get_files returned %d files (attempt %d)", len(files), attempt + 1)
|
||||
break
|
||||
except Exception as e:
|
||||
if attempt < max_retries - 1:
|
||||
wait_time = 2**attempt # 1s, 2s, 4s
|
||||
logger.warning("Failed to read repo files (attempt %d/%d), retrying in %ds: %s", attempt + 1, max_retries, wait_time, e)
|
||||
await asyncio.sleep(wait_time)
|
||||
else:
|
||||
logger.exception("Failed to read repo files after %d retries (agent=%s)", max_retries, agent_id)
|
||||
|
||||
expected_labels = set()
|
||||
from letta.services.memory_repo.block_markdown import parse_block_markdown
|
||||
|
||||
md_file_paths = sorted([file_path for file_path in files if file_path.endswith(".md")])
|
||||
nested_md_file_paths = [file_path for file_path in md_file_paths if "/" in file_path[:-3]]
|
||||
logger.info(
|
||||
"Post-push sync file scan: agent=%s total_files=%d md_files=%d nested_md_files=%d sample_md_paths=%s",
|
||||
agent_id,
|
||||
len(files),
|
||||
len(md_file_paths),
|
||||
len(nested_md_file_paths),
|
||||
md_file_paths[:10],
|
||||
)
|
||||
|
||||
synced = 0
|
||||
for file_path, content in files.items():
|
||||
if not file_path.endswith(".md"):
|
||||
continue
|
||||
|
||||
label = file_path[:-3]
|
||||
expected_labels.add(label)
|
||||
|
||||
# Parse frontmatter to extract metadata alongside value
|
||||
parsed = parse_block_markdown(content)
|
||||
|
||||
try:
|
||||
await _server_instance.block_manager._sync_block_to_postgres(
|
||||
agent_id=agent_id,
|
||||
label=label,
|
||||
value=parsed["value"],
|
||||
actor=actor,
|
||||
description=parsed.get("description"),
|
||||
limit=parsed.get("limit"),
|
||||
read_only=parsed.get("read_only"),
|
||||
metadata=parsed.get("metadata"),
|
||||
)
|
||||
synced += 1
|
||||
logger.info("Synced block %s to PostgreSQL", label)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to sync block %s to PostgreSQL (agent=%s) [path=%s nested=%s]",
|
||||
label,
|
||||
agent_id,
|
||||
file_path,
|
||||
"/" in label,
|
||||
)
|
||||
|
||||
if synced == 0:
|
||||
logger.warning("No *.md files found in repo HEAD during post-push sync (agent=%s)", agent_id)
|
||||
else:
|
||||
# Detach blocks that were removed in git.
|
||||
#
|
||||
# We treat git as the source of truth for which blocks are attached to
|
||||
# this agent. If a *.md file disappears from HEAD, detach the
|
||||
# corresponding block from the agent in Postgres.
|
||||
try:
|
||||
existing_blocks = await _server_instance.agent_manager.list_agent_blocks_async(
|
||||
agent_id=agent_id,
|
||||
actor=actor,
|
||||
before=None,
|
||||
after=None,
|
||||
limit=1000,
|
||||
ascending=True,
|
||||
)
|
||||
existing_by_label = {b.label: b for b in existing_blocks}
|
||||
removed_labels = set(existing_by_label.keys()) - expected_labels
|
||||
|
||||
for label in sorted(removed_labels):
|
||||
block = existing_by_label.get(label)
|
||||
if not block:
|
||||
continue
|
||||
await _server_instance.agent_manager.detach_block_async(
|
||||
agent_id=agent_id,
|
||||
block_id=block.id,
|
||||
actor=actor,
|
||||
)
|
||||
logger.info("Detached block %s from agent (removed from git)", label)
|
||||
except Exception:
|
||||
logger.exception("Failed detaching removed blocks during post-push sync (agent=%s)", agent_id)
|
||||
|
||||
|
||||
def _parse_agent_id_from_repo_path(path: str) -> Optional[str]:
|
||||
@@ -638,40 +231,31 @@ async def proxy_git_http(
|
||||
server=Depends(get_letta_server),
|
||||
headers: HeaderParams = Depends(get_headers),
|
||||
):
|
||||
"""Proxy `/v1/git/*` requests to the git HTTP backend.
|
||||
"""Proxy `/v1/git/*` requests to the memfs service.
|
||||
|
||||
If LETTA_MEMFS_SERVICE_URL is set, proxies to the external memfs service.
|
||||
Otherwise, proxies to the local dulwich WSGI server.
|
||||
Requires LETTA_MEMFS_SERVICE_URL to be configured.
|
||||
"""
|
||||
|
||||
memfs_url = _get_memfs_service_url()
|
||||
|
||||
if memfs_url:
|
||||
# Proxy to external memfs service
|
||||
url = f"{memfs_url.rstrip('/')}/git/{path}"
|
||||
logger.info("proxy_git_http: using memfs service at %s", memfs_url)
|
||||
else:
|
||||
# Proxy to local dulwich server
|
||||
if not _DULWICH_AVAILABLE:
|
||||
return JSONResponse(
|
||||
status_code=501,
|
||||
content={
|
||||
"detail": "git smart HTTP is disabled (dulwich not installed)",
|
||||
},
|
||||
)
|
||||
if not memfs_url:
|
||||
return JSONResponse(
|
||||
status_code=501,
|
||||
content={
|
||||
"detail": "git HTTP requires memfs service (LETTA_MEMFS_SERVICE_URL not configured)",
|
||||
},
|
||||
)
|
||||
|
||||
# Ensure server is running (best-effort). We also start it during lifespan.
|
||||
start_dulwich_server()
|
||||
|
||||
port = _get_dulwich_port()
|
||||
url = f"http://127.0.0.1:{port}/{path}"
|
||||
# Proxy to external memfs service
|
||||
url = f"{memfs_url.rstrip('/')}/git/{path}"
|
||||
logger.info("proxy_git_http: using memfs service at %s", memfs_url)
|
||||
|
||||
req_headers = _filter_out_hop_by_hop_headers(request.headers.items())
|
||||
# Avoid sending FastAPI host/length; httpx will compute
|
||||
req_headers.pop("host", None)
|
||||
req_headers.pop("content-length", None)
|
||||
|
||||
# Resolve org_id from the authenticated actor + agent and forward to dulwich.
|
||||
# Resolve org_id from the authenticated actor + agent and forward to memfs.
|
||||
agent_id = _parse_agent_id_from_repo_path(path)
|
||||
if agent_id is not None:
|
||||
actor = await server.user_manager.get_actor_or_default_async(actor_id=headers.actor_id)
|
||||
@@ -738,58 +322,3 @@ async def proxy_git_http(
|
||||
media_type=upstream.headers.get("content-type"),
|
||||
background=BackgroundTask(_aclose_upstream_and_client),
|
||||
)
|
||||
|
||||
|
||||
def _org_header_middleware(app):
|
||||
"""WSGI wrapper to capture org_id from proxied requests.
|
||||
|
||||
FastAPI proxies requests to the dulwich server and injects `X-Organization-Id`.
|
||||
Dulwich itself only passes repository *paths* into the Backend, so we capture
|
||||
the org_id from the WSGI environ and stash it in a contextvar.
|
||||
|
||||
Important: WSGI apps can return iterables/generators, and the server may
|
||||
iterate the response body *after* this wrapper returns. We must therefore
|
||||
keep the contextvar set for the duration of iteration.
|
||||
|
||||
Defensive fallback: if X-Organization-Id is missing, attempt to derive org_id
|
||||
from `user_id` (set by our auth proxy layer).
|
||||
"""
|
||||
|
||||
def _wrapped(environ, start_response):
|
||||
org_id = _resolve_org_id_from_wsgi_environ(environ)
|
||||
|
||||
logger.info(
|
||||
"dulwich_wsgi: path=%s remote=%s has_x_org=%s has_user_id=%s resolved_org=%s",
|
||||
environ.get("PATH_INFO"),
|
||||
environ.get("REMOTE_ADDR"),
|
||||
bool(environ.get("HTTP_X_ORGANIZATION_ID")),
|
||||
bool(environ.get("HTTP_USER_ID")),
|
||||
org_id,
|
||||
)
|
||||
|
||||
token = _current_org_id.set(org_id)
|
||||
|
||||
try:
|
||||
app_iter = app(environ, start_response)
|
||||
except Exception:
|
||||
_current_org_id.reset(token)
|
||||
raise
|
||||
|
||||
def _iter():
|
||||
try:
|
||||
yield from app_iter
|
||||
finally:
|
||||
try:
|
||||
if hasattr(app_iter, "close"):
|
||||
app_iter.close()
|
||||
finally:
|
||||
_current_org_id.reset(token)
|
||||
|
||||
return _iter()
|
||||
|
||||
return _wrapped
|
||||
|
||||
|
||||
# dulwich WSGI app (optional)
|
||||
_backend = GCSBackend()
|
||||
_git_wsgi_app = _org_header_middleware(HTTPGitApplication(_backend)) if _DULWICH_AVAILABLE and HTTPGitApplication is not None else None
|
||||
|
||||
@@ -89,8 +89,7 @@ from letta.services.mcp.sse_client import MCP_CONFIG_TOPLEVEL_KEY
|
||||
from letta.services.mcp.stdio_client import AsyncStdioMCPClient
|
||||
from letta.services.mcp_manager import MCPManager
|
||||
from letta.services.mcp_server_manager import MCPServerManager
|
||||
from letta.services.memory_repo import MemoryRepoManager
|
||||
from letta.services.memory_repo.storage.gcs import GCSStorageBackend
|
||||
from letta.services.memory_repo import MemfsClient
|
||||
from letta.services.message_manager import MessageManager
|
||||
from letta.services.organization_manager import OrganizationManager
|
||||
from letta.services.passage_manager import PassageManager
|
||||
@@ -414,62 +413,22 @@ class SyncServer(object):
|
||||
force_recreate=True,
|
||||
)
|
||||
|
||||
def _init_memory_repo_manager(self) -> Optional[MemoryRepoManager]:
|
||||
def _init_memory_repo_manager(self) -> Optional[MemfsClient]:
|
||||
"""Initialize the memory repository manager if configured.
|
||||
|
||||
If LETTA_MEMFS_SERVICE_URL is set, uses the external memfs service.
|
||||
Otherwise, configure the object store via settings (recommended):
|
||||
|
||||
LETTA_OBJECT_STORE_URI="gs://my-bucket/repository?project=my-gcp-project"
|
||||
|
||||
Supported schemes:
|
||||
- gs:// (or gcs://) -> Google Cloud Storage
|
||||
Requires LETTA_MEMFS_SERVICE_URL to be set to the external memfs service URL.
|
||||
|
||||
Returns:
|
||||
MemoryRepoManager (or MemfsClient) if configured, None otherwise
|
||||
MemfsClient if configured, None otherwise
|
||||
"""
|
||||
|
||||
# Keep import local to avoid import/circular issues during server bootstrap.
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from letta.settings import settings
|
||||
|
||||
# Check if memfs service is configured (takes priority over local object store)
|
||||
if settings.memfs_service_url:
|
||||
from letta.services.memory_repo import MemfsClient
|
||||
|
||||
logger.info("Memory repo manager using memfs service: %s", settings.memfs_service_url)
|
||||
return MemfsClient(base_url=settings.memfs_service_url)
|
||||
|
||||
uri = settings.object_store_uri
|
||||
if not uri:
|
||||
logger.debug("Memory repo manager not configured (object_store_uri not set)")
|
||||
if not settings.memfs_service_url:
|
||||
logger.debug("Memory repo manager not configured (memfs_service_url not set)")
|
||||
return None
|
||||
|
||||
try:
|
||||
parsed = urlparse(uri)
|
||||
scheme = (parsed.scheme or "").lower()
|
||||
|
||||
if scheme in {"gs", "gcs"}:
|
||||
bucket = parsed.netloc
|
||||
if not bucket:
|
||||
raise ValueError(f"Invalid GCS object store URI (missing bucket): {uri}")
|
||||
|
||||
# URI path is treated as the storage prefix
|
||||
prefix = parsed.path.lstrip("/") or "repository"
|
||||
qs = parse_qs(parsed.query)
|
||||
|
||||
# Allow settings-level overrides (handy for templated URIs).
|
||||
project = settings.object_store_project or (qs.get("project") or [None])[0]
|
||||
|
||||
storage = GCSStorageBackend(bucket=bucket, prefix=prefix, project=project)
|
||||
logger.info("Memory repo manager initialized with object store: %s", uri)
|
||||
return MemoryRepoManager(storage=storage)
|
||||
|
||||
raise ValueError(f"Unsupported object store scheme '{scheme}' in URI: {uri}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to initialize memory repo manager: {e}")
|
||||
return None
|
||||
logger.info("Memory repo manager using memfs service: %s", settings.memfs_service_url)
|
||||
return MemfsClient(base_url=settings.memfs_service_url)
|
||||
|
||||
def _get_enabled_provider(self, provider_name: str) -> Optional[Provider]:
|
||||
"""Find and return an enabled provider by name.
|
||||
|
||||
@@ -18,7 +18,7 @@ from letta.schemas.block import Block as PydanticBlock, BlockUpdate, CreateBlock
|
||||
from letta.schemas.user import User as PydanticUser
|
||||
from letta.server.db import db_registry
|
||||
from letta.services.block_manager import BlockManager
|
||||
from letta.services.memory_repo.manager import MemoryRepoManager
|
||||
from letta.services.memory_repo import MemfsClient
|
||||
from letta.utils import enforce_types
|
||||
|
||||
logger = get_logger(__name__)
|
||||
@@ -39,7 +39,7 @@ class GitEnabledBlockManager(BlockManager):
|
||||
- Behaves exactly like the standard BlockManager
|
||||
"""
|
||||
|
||||
def __init__(self, memory_repo_manager: Optional[MemoryRepoManager] = None):
|
||||
def __init__(self, memory_repo_manager: Optional[MemfsClient] = None):
|
||||
"""Initialize the git-enabled block manager.
|
||||
|
||||
Args:
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
"""Git-based memory repository services."""
|
||||
|
||||
from letta.services.memory_repo.manager import MemoryRepoManager
|
||||
from letta.services.memory_repo.storage.base import StorageBackend
|
||||
from letta.services.memory_repo.storage.gcs import GCSStorageBackend
|
||||
from letta.services.memory_repo.storage.local import LocalStorageBackend
|
||||
|
||||
# MemfsClient: try cloud implementation first, fall back to local filesystem
|
||||
@@ -12,9 +10,7 @@ except ImportError:
|
||||
from letta.services.memory_repo.memfs_client_base import MemfsClient
|
||||
|
||||
__all__ = [
|
||||
"GCSStorageBackend",
|
||||
"LocalStorageBackend",
|
||||
"MemfsClient",
|
||||
"MemoryRepoManager",
|
||||
"StorageBackend",
|
||||
]
|
||||
|
||||
@@ -1,15 +1,14 @@
|
||||
"""Git operations for memory repositories using dulwich.
|
||||
|
||||
Dulwich is a pure-Python implementation of Git that allows us to
|
||||
manipulate git repositories without requiring libgit2 or the git CLI.
|
||||
"""Git operations for memory repositories using git CLI.
|
||||
|
||||
This module provides high-level operations for working with git repos
|
||||
stored in object storage (GCS/S3).
|
||||
stored in object storage (GCS/S3), using the git command-line tool
|
||||
instead of dulwich for better compatibility and maintenance.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import uuid
|
||||
@@ -24,6 +23,29 @@ from letta.services.memory_repo.storage.base import StorageBackend
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
def _run_git(args: List[str], cwd: str, check: bool = True) -> subprocess.CompletedProcess:
|
||||
"""Run a git command and return the result.
|
||||
|
||||
Args:
|
||||
args: Git command arguments (without 'git' prefix)
|
||||
cwd: Working directory
|
||||
check: Whether to raise on non-zero exit
|
||||
|
||||
Returns:
|
||||
CompletedProcess with stdout/stderr
|
||||
"""
|
||||
result = subprocess.run(
|
||||
["git", *args],
|
||||
cwd=cwd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
if check and result.returncode != 0:
|
||||
raise subprocess.CalledProcessError(result.returncode, ["git", *args], result.stdout, result.stderr)
|
||||
return result
|
||||
|
||||
|
||||
class GitOperations:
|
||||
"""High-level git operations for memory repositories.
|
||||
|
||||
@@ -36,7 +58,7 @@ class GitOperations:
|
||||
packfiles directly.
|
||||
|
||||
Requirements:
|
||||
pip install dulwich
|
||||
git CLI must be installed and available in PATH
|
||||
"""
|
||||
|
||||
def __init__(self, storage: StorageBackend):
|
||||
@@ -46,21 +68,25 @@ class GitOperations:
|
||||
storage: Storage backend for repo persistence
|
||||
"""
|
||||
self.storage = storage
|
||||
self._dulwich = None
|
||||
self._git_available = None
|
||||
|
||||
def _get_dulwich(self):
|
||||
"""Lazily import dulwich."""
|
||||
if self._dulwich is None:
|
||||
def _check_git(self) -> None:
|
||||
"""Check that git is available."""
|
||||
if self._git_available is None:
|
||||
try:
|
||||
import dulwich
|
||||
import dulwich.objects
|
||||
import dulwich.porcelain
|
||||
import dulwich.repo
|
||||
|
||||
self._dulwich = dulwich
|
||||
except ImportError:
|
||||
raise ImportError("dulwich is required for git operations. Install with: pip install dulwich")
|
||||
return self._dulwich
|
||||
result = subprocess.run(
|
||||
["git", "--version"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
self._git_available = True
|
||||
logger.debug(f"Git available: {result.stdout.strip()}")
|
||||
except (subprocess.CalledProcessError, FileNotFoundError):
|
||||
self._git_available = False
|
||||
raise RuntimeError("git CLI is required for git operations but was not found in PATH")
|
||||
elif not self._git_available:
|
||||
raise RuntimeError("git CLI is required for git operations but was not found in PATH")
|
||||
|
||||
def _repo_path(self, agent_id: str, org_id: str) -> str:
|
||||
"""Get the storage path for an agent's repo."""
|
||||
@@ -86,22 +112,20 @@ class GitOperations:
|
||||
Returns:
|
||||
Initial commit SHA
|
||||
"""
|
||||
dulwich = self._get_dulwich()
|
||||
self._check_git()
|
||||
|
||||
def _create():
|
||||
# Create a temporary directory for the repo
|
||||
temp_dir = tempfile.mkdtemp(prefix="letta-memrepo-")
|
||||
try:
|
||||
repo_path = os.path.join(temp_dir, "repo")
|
||||
os.makedirs(repo_path)
|
||||
|
||||
# Initialize a new repository
|
||||
dulwich.repo.Repo.init(repo_path)
|
||||
# Initialize a new repository with main as default branch
|
||||
_run_git(["init", "-b", "main"], cwd=repo_path)
|
||||
|
||||
# Use `main` as the default branch (git's modern default).
|
||||
head_path = os.path.join(repo_path, ".git", "HEAD")
|
||||
with open(head_path, "wb") as f:
|
||||
f.write(b"ref: refs/heads/main\n")
|
||||
# Configure user for this repo
|
||||
_run_git(["config", "user.name", author_name], cwd=repo_path)
|
||||
_run_git(["config", "user.email", author_email], cwd=repo_path)
|
||||
|
||||
# Add initial files if provided
|
||||
if initial_files:
|
||||
@@ -110,8 +134,7 @@ class GitOperations:
|
||||
os.makedirs(os.path.dirname(full_path), exist_ok=True)
|
||||
with open(full_path, "w", encoding="utf-8") as f:
|
||||
f.write(content)
|
||||
# Stage the file
|
||||
dulwich.porcelain.add(repo_path, paths=[file_path])
|
||||
_run_git(["add", file_path], cwd=repo_path)
|
||||
else:
|
||||
# Create an empty .letta directory to initialize
|
||||
letta_dir = os.path.join(repo_path, ".letta")
|
||||
@@ -119,18 +142,16 @@ class GitOperations:
|
||||
config_path = os.path.join(letta_dir, "config.json")
|
||||
with open(config_path, "w") as f:
|
||||
f.write('{"version": 1}')
|
||||
dulwich.porcelain.add(repo_path, paths=[".letta/config.json"])
|
||||
_run_git(["add", ".letta/config.json"], cwd=repo_path)
|
||||
|
||||
# Create initial commit using porcelain (dulwich 1.0+ API)
|
||||
commit_sha = dulwich.porcelain.commit(
|
||||
repo_path,
|
||||
message=b"Initial commit",
|
||||
committer=f"{author_name} <{author_email}>".encode(),
|
||||
author=f"{author_name} <{author_email}>".encode(),
|
||||
)
|
||||
# Create initial commit
|
||||
_run_git(["commit", "-m", "Initial commit"], cwd=repo_path)
|
||||
|
||||
# Return the repo directory and commit SHA for upload
|
||||
return repo_path, commit_sha.decode() if isinstance(commit_sha, bytes) else str(commit_sha)
|
||||
# Get commit SHA
|
||||
result = _run_git(["rev-parse", "HEAD"], cwd=repo_path)
|
||||
commit_sha = result.stdout.strip()
|
||||
|
||||
return repo_path, commit_sha
|
||||
except Exception:
|
||||
shutil.rmtree(temp_dir, ignore_errors=True)
|
||||
raise
|
||||
@@ -138,11 +159,9 @@ class GitOperations:
|
||||
repo_path, commit_sha = await asyncio.to_thread(_create)
|
||||
|
||||
try:
|
||||
# Upload the repo to storage
|
||||
await self._upload_repo(repo_path, agent_id, org_id)
|
||||
return commit_sha
|
||||
finally:
|
||||
# Clean up temp directory
|
||||
shutil.rmtree(os.path.dirname(repo_path), ignore_errors=True)
|
||||
|
||||
async def _upload_repo(self, local_repo_path: str, agent_id: str, org_id: str) -> None:
|
||||
@@ -150,7 +169,6 @@ class GitOperations:
|
||||
t_start = time.perf_counter()
|
||||
storage_prefix = self._repo_path(agent_id, org_id)
|
||||
|
||||
# Walk through the .git directory and collect all files
|
||||
git_dir = os.path.join(local_repo_path, ".git")
|
||||
upload_tasks = []
|
||||
total_bytes = 0
|
||||
@@ -170,7 +188,6 @@ class GitOperations:
|
||||
read_time = (time.perf_counter() - t0) * 1000
|
||||
logger.info(f"[GIT_PERF] _upload_repo read files took {read_time:.2f}ms files={len(upload_tasks)}")
|
||||
|
||||
# Upload all files in parallel
|
||||
t0 = time.perf_counter()
|
||||
await asyncio.gather(*[self.storage.upload_bytes(path, content) for path, content in upload_tasks])
|
||||
upload_time = (time.perf_counter() - t0) * 1000
|
||||
@@ -211,7 +228,6 @@ class GitOperations:
|
||||
for filename in files:
|
||||
local_path = os.path.join(root, filename)
|
||||
old_mtime = before_snapshot.get(local_path)
|
||||
# New file or modified since snapshot
|
||||
if old_mtime is None or os.path.getmtime(local_path) != old_mtime:
|
||||
rel_path = os.path.relpath(local_path, git_dir)
|
||||
storage_path = f"{storage_prefix}/{rel_path}"
|
||||
@@ -240,7 +256,6 @@ class GitOperations:
|
||||
t_start = time.perf_counter()
|
||||
storage_prefix = self._repo_path(agent_id, org_id)
|
||||
|
||||
# List all files in the repo
|
||||
t0 = time.perf_counter()
|
||||
files = await self.storage.list_files(storage_prefix)
|
||||
list_time = (time.perf_counter() - t0) * 1000
|
||||
@@ -249,7 +264,6 @@ class GitOperations:
|
||||
if not files:
|
||||
raise FileNotFoundError(f"No repository found for agent {agent_id}")
|
||||
|
||||
# Create temp directory
|
||||
t0 = time.perf_counter()
|
||||
temp_dir = tempfile.mkdtemp(prefix="letta-memrepo-")
|
||||
repo_path = os.path.join(temp_dir, "repo")
|
||||
@@ -258,7 +272,6 @@ class GitOperations:
|
||||
mkdir_time = (time.perf_counter() - t0) * 1000
|
||||
logger.info(f"[GIT_PERF] _download_repo tempdir creation took {mkdir_time:.2f}ms path={temp_dir}")
|
||||
|
||||
# Compute local paths and create directories first
|
||||
file_info = []
|
||||
for file_path in files:
|
||||
if file_path.startswith(storage_prefix):
|
||||
@@ -270,7 +283,6 @@ class GitOperations:
|
||||
os.makedirs(os.path.dirname(local_path), exist_ok=True)
|
||||
file_info.append((file_path, local_path))
|
||||
|
||||
# Download all files in parallel
|
||||
t0 = time.perf_counter()
|
||||
download_tasks = [self.storage.download_bytes(fp) for fp, _ in file_info]
|
||||
contents = await asyncio.gather(*download_tasks)
|
||||
@@ -278,7 +290,6 @@ class GitOperations:
|
||||
total_bytes = sum(len(c) for c in contents)
|
||||
logger.info(f"[GIT_PERF] _download_repo parallel download took {download_time:.2f}ms files={len(files)} bytes={total_bytes}")
|
||||
|
||||
# Write all files to disk
|
||||
t0 = time.perf_counter()
|
||||
for (_, local_path), content in zip(file_info, contents):
|
||||
with open(local_path, "wb") as f:
|
||||
@@ -310,54 +321,33 @@ class GitOperations:
|
||||
Returns:
|
||||
Dict mapping file paths to content
|
||||
"""
|
||||
dulwich = self._get_dulwich()
|
||||
self._check_git()
|
||||
repo_path = await self._download_repo(agent_id, org_id)
|
||||
|
||||
try:
|
||||
|
||||
def _get_files():
|
||||
repo = dulwich.repo.Repo(repo_path)
|
||||
# List all files tracked by git at the given ref
|
||||
result = _run_git(["ls-tree", "-r", "--name-only", ref], cwd=repo_path)
|
||||
file_paths = result.stdout.strip().split("\n") if result.stdout.strip() else []
|
||||
|
||||
# Resolve ref to commit
|
||||
if ref == "HEAD":
|
||||
commit_sha = repo.head()
|
||||
else:
|
||||
# Try as branch name first
|
||||
try:
|
||||
commit_sha = repo.refs[f"refs/heads/{ref}".encode()]
|
||||
except KeyError:
|
||||
# Try as commit SHA
|
||||
commit_sha = ref.encode() if isinstance(ref, str) else ref
|
||||
|
||||
commit = repo[commit_sha]
|
||||
tree = repo[commit.tree]
|
||||
|
||||
# Walk the tree and get all files
|
||||
files = {}
|
||||
self._walk_tree(repo, tree, "", files)
|
||||
for file_path in file_paths:
|
||||
if not file_path:
|
||||
continue
|
||||
# Get file content at ref
|
||||
try:
|
||||
content_result = _run_git(["show", f"{ref}:{file_path}"], cwd=repo_path)
|
||||
files[file_path] = content_result.stdout
|
||||
except subprocess.CalledProcessError:
|
||||
pass # Skip files that can't be read
|
||||
|
||||
return files
|
||||
|
||||
return await asyncio.to_thread(_get_files)
|
||||
finally:
|
||||
shutil.rmtree(os.path.dirname(repo_path), ignore_errors=True)
|
||||
|
||||
def _walk_tree(self, repo, tree, prefix: str, files: Dict[str, str]) -> None:
|
||||
"""Recursively walk a git tree and collect files."""
|
||||
dulwich = self._get_dulwich()
|
||||
for entry in tree.items():
|
||||
name = entry.path.decode() if isinstance(entry.path, bytes) else entry.path
|
||||
path = f"{prefix}/{name}" if prefix else name
|
||||
obj = repo[entry.sha]
|
||||
|
||||
if isinstance(obj, dulwich.objects.Blob):
|
||||
try:
|
||||
files[path] = obj.data.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
# Skip binary files
|
||||
pass
|
||||
elif isinstance(obj, dulwich.objects.Tree):
|
||||
self._walk_tree(repo, obj, path, files)
|
||||
|
||||
async def commit(
|
||||
self,
|
||||
agent_id: str,
|
||||
@@ -390,7 +380,6 @@ class GitOperations:
|
||||
t_start = time.perf_counter()
|
||||
logger.info(f"[GIT_PERF] GitOperations.commit START agent={agent_id} changes={len(changes)}")
|
||||
|
||||
# Acquire lock to prevent concurrent modifications
|
||||
t0 = time.perf_counter()
|
||||
redis_client = await get_redis_client()
|
||||
lock_token = f"commit:{uuid.uuid4().hex}"
|
||||
@@ -414,7 +403,6 @@ class GitOperations:
|
||||
logger.info(f"[GIT_PERF] GitOperations.commit TOTAL {total_time:.2f}ms")
|
||||
return result
|
||||
finally:
|
||||
# Release lock
|
||||
t0 = time.perf_counter()
|
||||
if lock:
|
||||
try:
|
||||
@@ -436,28 +424,36 @@ class GitOperations:
|
||||
) -> MemoryCommit:
|
||||
"""Internal commit implementation (called while holding lock)."""
|
||||
t_start = time.perf_counter()
|
||||
dulwich = self._get_dulwich()
|
||||
self._check_git()
|
||||
|
||||
# Download repo from GCS to temp dir
|
||||
t0 = time.perf_counter()
|
||||
repo_path = await self._download_repo(agent_id, org_id)
|
||||
download_time = (time.perf_counter() - t0) * 1000
|
||||
logger.info(f"[GIT_PERF] _commit_with_lock download phase took {download_time:.2f}ms")
|
||||
|
||||
try:
|
||||
# Snapshot git objects before commit for delta upload
|
||||
git_dir = os.path.join(repo_path, ".git")
|
||||
before_snapshot = self._snapshot_git_files(git_dir)
|
||||
|
||||
def _commit():
|
||||
t_git_start = time.perf_counter()
|
||||
repo = dulwich.repo.Repo(repo_path)
|
||||
|
||||
# Checkout the working directory
|
||||
# Configure user for this repo
|
||||
_run_git(["config", "user.name", author_name], cwd=repo_path)
|
||||
_run_git(["config", "user.email", author_email], cwd=repo_path)
|
||||
|
||||
# Reset to clean state
|
||||
t0_reset = time.perf_counter()
|
||||
dulwich.porcelain.reset(repo, "hard")
|
||||
_run_git(["reset", "--hard"], cwd=repo_path)
|
||||
reset_time = (time.perf_counter() - t0_reset) * 1000
|
||||
|
||||
# Get parent SHA before making changes
|
||||
try:
|
||||
parent_result = _run_git(["rev-parse", "HEAD"], cwd=repo_path, check=False)
|
||||
parent_sha = parent_result.stdout.strip() if parent_result.returncode == 0 else None
|
||||
except Exception:
|
||||
parent_sha = None
|
||||
|
||||
# Apply changes
|
||||
files_changed = []
|
||||
additions = 0
|
||||
@@ -470,17 +466,14 @@ class GitOperations:
|
||||
full_path = os.path.join(repo_path, file_path)
|
||||
|
||||
if change.change_type == "delete" or change.content is None:
|
||||
# Delete file
|
||||
if os.path.exists(full_path):
|
||||
with open(full_path, "r") as f:
|
||||
deletions += len(f.read())
|
||||
os.remove(full_path)
|
||||
dulwich.porcelain.remove(repo_path, paths=[file_path])
|
||||
_run_git(["rm", "-f", file_path], cwd=repo_path, check=False)
|
||||
else:
|
||||
# Add or modify file
|
||||
os.makedirs(os.path.dirname(full_path), exist_ok=True)
|
||||
|
||||
# Calculate additions/deletions
|
||||
if os.path.exists(full_path):
|
||||
with open(full_path, "r") as f:
|
||||
old_content = f.read()
|
||||
@@ -489,28 +482,19 @@ class GitOperations:
|
||||
|
||||
with open(full_path, "w", encoding="utf-8") as f:
|
||||
f.write(change.content)
|
||||
dulwich.porcelain.add(repo_path, paths=[file_path])
|
||||
_run_git(["add", file_path], cwd=repo_path)
|
||||
|
||||
files_changed.append(file_path)
|
||||
apply_time += (time.perf_counter() - t0_apply) * 1000
|
||||
|
||||
# Get parent SHA
|
||||
try:
|
||||
parent_sha = repo.head().decode()
|
||||
except Exception:
|
||||
parent_sha = None
|
||||
|
||||
# Create commit using porcelain (dulwich 1.0+ API)
|
||||
# Create commit
|
||||
t0_commit = time.perf_counter()
|
||||
commit_sha = dulwich.porcelain.commit(
|
||||
repo_path,
|
||||
message=message.encode(),
|
||||
committer=f"{author_name} <{author_email}>".encode(),
|
||||
author=f"{author_name} <{author_email}>".encode(),
|
||||
)
|
||||
_run_git(["commit", "-m", message], cwd=repo_path)
|
||||
commit_time = (time.perf_counter() - t0_commit) * 1000
|
||||
|
||||
sha_str = commit_sha.decode() if isinstance(commit_sha, bytes) else str(commit_sha)
|
||||
# Get new commit SHA
|
||||
result = _run_git(["rev-parse", "HEAD"], cwd=repo_path)
|
||||
sha_str = result.stdout.strip()
|
||||
|
||||
git_total = (time.perf_counter() - t_git_start) * 1000
|
||||
logger.info(
|
||||
@@ -536,7 +520,6 @@ class GitOperations:
|
||||
git_thread_time = (time.perf_counter() - t0) * 1000
|
||||
logger.info(f"[GIT_PERF] _commit_with_lock git thread took {git_thread_time:.2f}ms")
|
||||
|
||||
# Upload only new/modified objects (delta)
|
||||
t0 = time.perf_counter()
|
||||
await self._upload_delta(repo_path, agent_id, org_id, before_snapshot)
|
||||
upload_time = (time.perf_counter() - t0) * 1000
|
||||
@@ -572,37 +555,43 @@ class GitOperations:
|
||||
Returns:
|
||||
List of commits, newest first
|
||||
"""
|
||||
dulwich = self._get_dulwich()
|
||||
self._check_git()
|
||||
repo_path = await self._download_repo(agent_id, org_id)
|
||||
|
||||
try:
|
||||
|
||||
def _get_history():
|
||||
repo = dulwich.repo.Repo(repo_path)
|
||||
# Use git log with custom format for easy parsing
|
||||
# Format: SHA|parent_sha|author_name|timestamp|message
|
||||
format_str = "%H|%P|%an|%at|%s"
|
||||
args = ["log", f"--format={format_str}", f"-n{limit}"]
|
||||
if path:
|
||||
args.extend(["--", path])
|
||||
|
||||
result = _run_git(args, cwd=repo_path)
|
||||
lines = result.stdout.strip().split("\n") if result.stdout.strip() else []
|
||||
|
||||
commits = []
|
||||
for line in lines:
|
||||
if not line:
|
||||
continue
|
||||
parts = line.split("|", 4)
|
||||
if len(parts) < 5:
|
||||
continue
|
||||
|
||||
# Walk the commit history
|
||||
walker = repo.get_walker(max_entries=limit)
|
||||
|
||||
for entry in walker:
|
||||
commit = entry.commit
|
||||
sha = commit.id.decode() if isinstance(commit.id, bytes) else str(commit.id)
|
||||
parent_sha = commit.parents[0].decode() if commit.parents else None
|
||||
|
||||
# Parse author
|
||||
author_str = commit.author.decode() if isinstance(commit.author, bytes) else commit.author
|
||||
author_name = author_str.split("<")[0].strip() if "<" in author_str else author_str
|
||||
sha, parents, author_name, timestamp_str, message = parts
|
||||
parent_sha = parents.split()[0] if parents else None
|
||||
|
||||
commits.append(
|
||||
MemoryCommit(
|
||||
sha=sha,
|
||||
parent_sha=parent_sha,
|
||||
message=commit.message.decode() if isinstance(commit.message, bytes) else commit.message,
|
||||
message=message,
|
||||
author_type="system",
|
||||
author_id="",
|
||||
author_name=author_name,
|
||||
timestamp=datetime.fromtimestamp(commit.commit_time, tz=timezone.utc),
|
||||
files_changed=[], # Would need to compute diff for this
|
||||
timestamp=datetime.fromtimestamp(int(timestamp_str), tz=timezone.utc),
|
||||
files_changed=[],
|
||||
additions=0,
|
||||
deletions=0,
|
||||
)
|
||||
@@ -624,15 +613,14 @@ class GitOperations:
|
||||
Returns:
|
||||
HEAD commit SHA
|
||||
"""
|
||||
dulwich = self._get_dulwich()
|
||||
self._check_git()
|
||||
repo_path = await self._download_repo(agent_id, org_id)
|
||||
|
||||
try:
|
||||
|
||||
def _get_head():
|
||||
repo = dulwich.repo.Repo(repo_path)
|
||||
head = repo.head()
|
||||
return head.decode() if isinstance(head, bytes) else str(head)
|
||||
result = _run_git(["rev-parse", "HEAD"], cwd=repo_path)
|
||||
return result.stdout.strip()
|
||||
|
||||
return await asyncio.to_thread(_get_head)
|
||||
finally:
|
||||
|
||||
@@ -1,11 +1,9 @@
|
||||
"""Storage backends for memory repositories."""
|
||||
|
||||
from letta.services.memory_repo.storage.base import StorageBackend
|
||||
from letta.services.memory_repo.storage.gcs import GCSStorageBackend
|
||||
from letta.services.memory_repo.storage.local import LocalStorageBackend
|
||||
|
||||
__all__ = [
|
||||
"GCSStorageBackend",
|
||||
"LocalStorageBackend",
|
||||
"StorageBackend",
|
||||
]
|
||||
|
||||
@@ -114,12 +114,6 @@ bedrock = [
|
||||
"aioboto3>=14.3.0",
|
||||
]
|
||||
|
||||
# ====== Git State (git-backed memory repos) ======
|
||||
git-state = [
|
||||
"google-cloud-storage>=2.10.0",
|
||||
"dulwich>=0.22.0",
|
||||
]
|
||||
|
||||
# ====== Development ======
|
||||
dev = [
|
||||
"pytest",
|
||||
|
||||
133
uv.lock
generated
133
uv.lock
generated
@@ -1146,36 +1146,6 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl", hash = "sha256:d0013f540772d1420576855455d050a2180186c91c15779301ac2ccb3eeb68de", size = 633196, upload-time = "2025-12-18T19:00:18.077Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dulwich"
|
||||
version = "1.0.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "typing-extensions", marker = "python_full_version < '3.12'" },
|
||||
{ name = "urllib3" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/ee/df/4178b6465e118e6e74fd78774b451953dd53c09fdec18f2c4b3319dd0485/dulwich-1.0.0.tar.gz", hash = "sha256:3d07104735525f22bfec35514ac611cf328c89b7acb059316a4f6e583c8f09bc", size = 1135862, upload-time = "2026-01-17T23:44:16.357Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/66/fa/99a422ac3bca08eab07a537c86dce12b6ce20b72cf5a14bef5cdb122eddf/dulwich-1.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1213da9832621b07dfaafdb651b74edb8966481475c52be0bff8dee352d75853", size = 1336935, upload-time = "2026-01-17T23:43:38.84Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/86/54/1739af06492a4e98b5c96aa3e22d0b58fda282c10849db733ee8c52f423d/dulwich-1.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:e370e3cdd0b00c059ebee8371cc1644aa61d6de3de0ca5c2f2a5f075bf4c53d9", size = 1400229, upload-time = "2026-01-17T23:43:41.03Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0c/76/efde5050ae9422cf418bee98d3d35dc99935fb076679100e558491e691c9/dulwich-1.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:86271e17d76a667abb1d68dad83b6324422a1ab20d60be30395fd60a37b735b1", size = 1428812, upload-time = "2026-01-17T23:43:43.412Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/02/82/f166b206db70db11fb222abeb661b2879ea10f32ad86c85949e5a4fba26a/dulwich-1.0.0-cp311-cp311-win32.whl", hash = "sha256:3051007bc2792b5a72fee938842cf45b66924d6d5147d824f3e609eb75fc0322", size = 985517, upload-time = "2026-01-17T23:43:45.409Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e7/b7/3f8c0059fc8a0eba22e8bb9cec7e2b4e514bc75ede83a320570c5de17599/dulwich-1.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:5cf6e9b5620a3e842663b58ad534da29944db6a6016ba61fc9bbed24830cd85f", size = 1001981, upload-time = "2026-01-17T23:43:47.29Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cc/54/78054a9fd62aa7b1484e97673435cae494cad5d04f020d4571c47e9a2875/dulwich-1.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6736abc2ce2994e38a00a3a4c80237b2b944e7c6f4e346119debdd2592312d83", size = 1316278, upload-time = "2026-01-17T23:43:49.542Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/30/20/b2140acf9431c8c862c200cd880b9e5cce8dbe9914324bf238ed92574aea/dulwich-1.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:06514b02da1e32a077062924d2c3b20a7bc76ab9b92eeac691f72b76b14111bc", size = 1393024, upload-time = "2026-01-17T23:43:51.453Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/69/71/ad744b9802f222dc364a851bd6130c17809b3472a81a16aefd7d3196f22a/dulwich-1.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:32b6fb1205b1d9c0e43986f9e4e5e50a3670014440e61498eca2b8ab6b00129f", size = 1421022, upload-time = "2026-01-17T23:43:53.053Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/21/c0/dfcd795a6b516b9e24aa4339dcc9cdd5ceffe007ad397e5b4938f9793981/dulwich-1.0.0-cp312-cp312-win32.whl", hash = "sha256:1a6583499b915fe5a8ac5595325f1e6a6a5a456de1575e0293e8a6ebb6915f3f", size = 980617, upload-time = "2026-01-17T23:43:54.642Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7c/d4/11075795cc8ab48c771c997fdefef612775ef2582c4710a8fba6ca987500/dulwich-1.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:f90b54faeb995607c876cdb2c082c0f0af702e1ccb524c6126ce99a36536fa3f", size = 998048, upload-time = "2026-01-17T23:43:56.176Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/97/82/5ce63c7a2ac8d756bc7477298633e420632eed97ea645ecea13210e9b1a7/dulwich-1.0.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:ff94f47f0b5787d4e6a0105daf51ff9cdb4e5b9d4e9f8dd01b58ba9a5b79bbd9", size = 1417766, upload-time = "2026-01-17T23:43:57.855Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b9/71/7d4ecdf9e0da21ceec3ac05b03c2cac8cf2271a52172fd55dd65a9faa9e7/dulwich-1.0.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:1d95663441c930631d9d1765dc4f427dcc0662af45f42a0831357e60055ddb84", size = 1417760, upload-time = "2026-01-17T23:43:59.42Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/09/3d/0486cefda75c7e9ea8d8dbdeaa014d618e694bc75734f073927135b37a4b/dulwich-1.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:78542a62fabea894943a1d01c9c477a56eee5f7d58d3bdee42c7e0622ddf6893", size = 1316186, upload-time = "2026-01-17T23:44:01.334Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f7/a7/a24c6e1e9f7e5a2ee8f9e362e2c3e5d864cc2b69f04d02bedf82673f31c3/dulwich-1.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:d1c33f6456e4335dfe6f4d3917fa7d77050d6470bbbaf8054b5c5084ee8e8cd1", size = 1392530, upload-time = "2026-01-17T23:44:03.655Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d4/03/1ff9dbda655fc714528786e3fdbbe16278bbefc02b9836e91a38620aa616/dulwich-1.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:581330cf799577f194fda2b5384b7ba50e095de7ff088779c027a6de63642de2", size = 1420386, upload-time = "2026-01-17T23:44:05.844Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f0/ca/72e7cdde2ee0a4f858166ba8eb81a0d89f61762d9114bd7a358798892fc9/dulwich-1.0.0-cp313-cp313-win32.whl", hash = "sha256:276ff18ae734fe4a1be66d4267216a51d2deab0ac981d722db3d32fcc2ac4ff8", size = 981425, upload-time = "2026-01-17T23:44:07.373Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d7/27/8d4bed76ce983052e259da25255fed85b48ad30a34b4e4b7c8f518fdbc30/dulwich-1.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:cc0ab4ba7fd8617bebe20294dedaa8f713d1767ce059bfbefd971b911b702726", size = 998055, upload-time = "2026-01-17T23:44:08.908Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f9/99/4543953d2f7c1a940c1373362a70d253b85860be64b4ef8885bf8bfb340b/dulwich-1.0.0-py3-none-any.whl", hash = "sha256:221be803b71b060c928e9faae4ab3e259ff5beac6e0c251ba3c176b51b5c2ffb", size = 647950, upload-time = "2026-01-17T23:44:14.449Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "e2b"
|
||||
version = "2.0.0"
|
||||
@@ -1628,22 +1598,6 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/a8/73/2e03125170485193fcc99ef23b52749543d6c6711706d58713fe315869c4/geventhttpclient-2.3.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:5f71c75fc138331cbbe668a08951d36b641d2c26fb3677d7e497afb8419538db", size = 49011, upload-time = "2025-06-11T13:18:05.702Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "google-api-core"
|
||||
version = "2.29.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "google-auth" },
|
||||
{ name = "googleapis-common-protos" },
|
||||
{ name = "proto-plus" },
|
||||
{ name = "protobuf" },
|
||||
{ name = "requests" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/0d/10/05572d33273292bac49c2d1785925f7bc3ff2fe50e3044cf1062c1dde32e/google_api_core-2.29.0.tar.gz", hash = "sha256:84181be0f8e6b04006df75ddfe728f24489f0af57c96a529ff7cf45bc28797f7", size = 177828, upload-time = "2026-01-08T22:21:39.269Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/77/b6/85c4d21067220b9a78cfb81f516f9725ea6befc1544ec9bd2c1acd97c324/google_api_core-2.29.0-py3-none-any.whl", hash = "sha256:d30bc60980daa36e314b5d5a3e5958b0200cb44ca8fa1be2b614e932b75a3ea9", size = 173906, upload-time = "2026-01-08T22:21:36.093Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "google-auth"
|
||||
version = "2.40.3"
|
||||
@@ -1658,61 +1612,6 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/17/63/b19553b658a1692443c62bd07e5868adaa0ad746a0751ba62c59568cd45b/google_auth-2.40.3-py2.py3-none-any.whl", hash = "sha256:1370d4593e86213563547f97a92752fc658456fe4514c809544f330fed45a7ca", size = 216137, upload-time = "2025-06-04T18:04:55.573Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "google-cloud-core"
|
||||
version = "2.5.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "google-api-core" },
|
||||
{ name = "google-auth" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/a6/03/ef0bc99d0e0faf4fdbe67ac445e18cdaa74824fd93cd069e7bb6548cb52d/google_cloud_core-2.5.0.tar.gz", hash = "sha256:7c1b7ef5c92311717bd05301aa1a91ffbc565673d3b0b4163a52d8413a186963", size = 36027, upload-time = "2025-10-29T23:17:39.513Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/89/20/bfa472e327c8edee00f04beecc80baeddd2ab33ee0e86fd7654da49d45e9/google_cloud_core-2.5.0-py3-none-any.whl", hash = "sha256:67d977b41ae6c7211ee830c7912e41003ea8194bff15ae7d72fd6f51e57acabc", size = 29469, upload-time = "2025-10-29T23:17:38.548Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "google-cloud-storage"
|
||||
version = "3.8.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "google-api-core" },
|
||||
{ name = "google-auth" },
|
||||
{ name = "google-cloud-core" },
|
||||
{ name = "google-crc32c" },
|
||||
{ name = "google-resumable-media" },
|
||||
{ name = "requests" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/a8/90/4398cecc2704cb066bc7dee6111a5c93c59bcd6fb751f0541315655774a8/google_cloud_storage-3.8.0.tar.gz", hash = "sha256:cc67952dce84ebc9d44970e24647a58260630b7b64d72360cedaf422d6727f28", size = 17273792, upload-time = "2026-01-14T00:45:31.289Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/11/db/326279870d349fb9592263343dca4ad76088c17c88ba97b0f64c1088276c/google_cloud_storage-3.8.0-py3-none-any.whl", hash = "sha256:78cfeae7cac2ca9441d0d0271c2eb4ebfa21aa4c6944dd0ccac0389e81d955a7", size = 312430, upload-time = "2026-01-14T00:45:28.689Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "google-crc32c"
|
||||
version = "1.8.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/03/41/4b9c02f99e4c5fb477122cd5437403b552873f014616ac1d19ac8221a58d/google_crc32c-1.8.0.tar.gz", hash = "sha256:a428e25fb7691024de47fecfbff7ff957214da51eddded0da0ae0e0f03a2cf79", size = 14192, upload-time = "2025-12-16T00:35:25.142Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/5d/ef/21ccfaab3d5078d41efe8612e0ed0bfc9ce22475de074162a91a25f7980d/google_crc32c-1.8.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:014a7e68d623e9a4222d663931febc3033c5c7c9730785727de2a81f87d5bab8", size = 31298, upload-time = "2025-12-16T00:20:32.241Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c5/b8/f8413d3f4b676136e965e764ceedec904fe38ae8de0cdc52a12d8eb1096e/google_crc32c-1.8.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:86cfc00fe45a0ac7359e5214a1704e51a99e757d0272554874f419f79838c5f7", size = 30872, upload-time = "2025-12-16T00:33:58.785Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f6/fd/33aa4ec62b290477181c55bb1c9302c9698c58c0ce9a6ab4874abc8b0d60/google_crc32c-1.8.0-cp311-cp311-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:19b40d637a54cb71e0829179f6cb41835f0fbd9e8eb60552152a8b52c36cbe15", size = 33243, upload-time = "2025-12-16T00:40:21.46Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/71/03/4820b3bd99c9653d1a5210cb32f9ba4da9681619b4d35b6a052432df4773/google_crc32c-1.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:17446feb05abddc187e5441a45971b8394ea4c1b6efd88ab0af393fd9e0a156a", size = 33608, upload-time = "2025-12-16T00:40:22.204Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7c/43/acf61476a11437bf9733fb2f70599b1ced11ec7ed9ea760fdd9a77d0c619/google_crc32c-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:71734788a88f551fbd6a97be9668a0020698e07b2bf5b3aa26a36c10cdfb27b2", size = 34439, upload-time = "2025-12-16T00:35:20.458Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e9/5f/7307325b1198b59324c0fa9807cafb551afb65e831699f2ce211ad5c8240/google_crc32c-1.8.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:4b8286b659c1335172e39563ab0a768b8015e88e08329fa5321f774275fc3113", size = 31300, upload-time = "2025-12-16T00:21:56.723Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/21/8e/58c0d5d86e2220e6a37befe7e6a94dd2f6006044b1a33edf1ff6d9f7e319/google_crc32c-1.8.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:2a3dc3318507de089c5384cc74d54318401410f82aa65b2d9cdde9d297aca7cb", size = 30867, upload-time = "2025-12-16T00:38:31.302Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ce/a9/a780cc66f86335a6019f557a8aaca8fbb970728f0efd2430d15ff1beae0e/google_crc32c-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:14f87e04d613dfa218d6135e81b78272c3b904e2a7053b841481b38a7d901411", size = 33364, upload-time = "2025-12-16T00:40:22.96Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/21/3f/3457ea803db0198c9aaca2dd373750972ce28a26f00544b6b85088811939/google_crc32c-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb5c869c2923d56cb0c8e6bcdd73c009c36ae39b652dbe46a05eb4ef0ad01454", size = 33740, upload-time = "2025-12-16T00:40:23.96Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/df/c0/87c2073e0c72515bb8733d4eef7b21548e8d189f094b5dad20b0ecaf64f6/google_crc32c-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:3cc0c8912038065eafa603b238abf252e204accab2a704c63b9e14837a854962", size = 34437, upload-time = "2025-12-16T00:35:21.395Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d1/db/000f15b41724589b0e7bc24bc7a8967898d8d3bc8caf64c513d91ef1f6c0/google_crc32c-1.8.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:3ebb04528e83b2634857f43f9bb8ef5b2bbe7f10f140daeb01b58f972d04736b", size = 31297, upload-time = "2025-12-16T00:23:20.709Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d7/0d/8ebed0c39c53a7e838e2a486da8abb0e52de135f1b376ae2f0b160eb4c1a/google_crc32c-1.8.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:450dc98429d3e33ed2926fc99ee81001928d63460f8538f21a5d6060912a8e27", size = 30867, upload-time = "2025-12-16T00:43:14.628Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ce/42/b468aec74a0354b34c8cbf748db20d6e350a68a2b0912e128cabee49806c/google_crc32c-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3b9776774b24ba76831609ffbabce8cdf6fa2bd5e9df37b594221c7e333a81fa", size = 33344, upload-time = "2025-12-16T00:40:24.742Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1c/e8/b33784d6fc77fb5062a8a7854e43e1e618b87d5ddf610a88025e4de6226e/google_crc32c-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:89c17d53d75562edfff86679244830599ee0a48efc216200691de8b02ab6b2b8", size = 33694, upload-time = "2025-12-16T00:40:25.505Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/92/b1/d3cbd4d988afb3d8e4db94ca953df429ed6db7282ed0e700d25e6c7bfc8d/google_crc32c-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:57a50a9035b75643996fbf224d6661e386c7162d1dfdab9bc4ca790947d1007f", size = 34435, upload-time = "2025-12-16T00:35:22.107Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/52/c5/c171e4d8c44fec1422d801a6d2e5d7ddabd733eeda505c79730ee9607f07/google_crc32c-1.8.0-pp311-pypy311_pp73-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:87fa445064e7db928226b2e6f0d5304ab4cd0339e664a4e9a25029f384d9bb93", size = 28615, upload-time = "2025-12-16T00:40:29.298Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9c/97/7d75fe37a7a6ed171a2cf17117177e7aab7e6e0d115858741b41e9dd4254/google_crc32c-1.8.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f639065ea2042d5c034bf258a9f085eaa7af0cd250667c0635a3118e8f92c69c", size = 28800, upload-time = "2025-12-16T00:40:30.322Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "google-genai"
|
||||
version = "1.52.0"
|
||||
@@ -1732,18 +1631,6 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/ec/66/03f663e7bca7abe9ccfebe6cb3fe7da9a118fd723a5abb278d6117e7990e/google_genai-1.52.0-py3-none-any.whl", hash = "sha256:c8352b9f065ae14b9322b949c7debab8562982f03bf71d44130cd2b798c20743", size = 261219, upload-time = "2025-11-21T02:18:54.515Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "google-resumable-media"
|
||||
version = "2.8.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "google-crc32c" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/64/d7/520b62a35b23038ff005e334dba3ffc75fcf583bee26723f1fd8fd4b6919/google_resumable_media-2.8.0.tar.gz", hash = "sha256:f1157ed8b46994d60a1bc432544db62352043113684d4e030ee02e77ebe9a1ae", size = 2163265, upload-time = "2025-11-17T15:38:06.659Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/1f/0b/93afde9cfe012260e9fe1522f35c9b72d6ee222f316586b1f23ecf44d518/google_resumable_media-2.8.0-py3-none-any.whl", hash = "sha256:dd14a116af303845a8d932ddae161a26e86cc229645bc98b39f026f9b1717582", size = 81340, upload-time = "2025-11-17T15:38:05.594Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "googleapis-common-protos"
|
||||
version = "1.70.0"
|
||||
@@ -2745,10 +2632,6 @@ external-tools = [
|
||||
{ name = "turbopuffer" },
|
||||
{ name = "wikipedia" },
|
||||
]
|
||||
git-state = [
|
||||
{ name = "dulwich" },
|
||||
{ name = "google-cloud-storage" },
|
||||
]
|
||||
modal = [
|
||||
{ name = "modal" },
|
||||
]
|
||||
@@ -2805,7 +2688,6 @@ requires-dist = [
|
||||
{ name = "docker", marker = "extra == 'desktop'", specifier = ">=7.1.0" },
|
||||
{ name = "docker", marker = "extra == 'external-tools'", specifier = ">=7.1.0" },
|
||||
{ name = "docstring-parser", specifier = ">=0.16,<0.17" },
|
||||
{ name = "dulwich", marker = "extra == 'git-state'", specifier = ">=0.22.0" },
|
||||
{ name = "e2b-code-interpreter", marker = "extra == 'cloud-tool-sandbox'", specifier = ">=1.0.3" },
|
||||
{ name = "exa-py", specifier = ">=1.15.4" },
|
||||
{ name = "exa-py", marker = "extra == 'external-tools'", specifier = ">=1.15.4" },
|
||||
@@ -2813,7 +2695,6 @@ requires-dist = [
|
||||
{ name = "fastapi", marker = "extra == 'desktop'", specifier = ">=0.115.6" },
|
||||
{ name = "fastapi", marker = "extra == 'server'", specifier = ">=0.115.6" },
|
||||
{ name = "fastmcp", specifier = ">=2.12.5" },
|
||||
{ name = "google-cloud-storage", marker = "extra == 'git-state'", specifier = ">=2.10.0" },
|
||||
{ name = "google-genai", specifier = ">=1.52.0" },
|
||||
{ name = "granian", extras = ["uvloop", "reload"], marker = "extra == 'experimental'", specifier = ">=2.3.2" },
|
||||
{ name = "grpcio", specifier = ">=1.68.1" },
|
||||
@@ -2901,7 +2782,7 @@ requires-dist = [
|
||||
{ name = "wikipedia", marker = "extra == 'desktop'", specifier = ">=1.4.0" },
|
||||
{ name = "wikipedia", marker = "extra == 'external-tools'", specifier = ">=1.4.0" },
|
||||
]
|
||||
provides-extras = ["postgres", "redis", "pinecone", "sqlite", "experimental", "server", "bedrock", "git-state", "dev", "cloud-tool-sandbox", "modal", "external-tools", "desktop", "profiling"]
|
||||
provides-extras = ["postgres", "redis", "pinecone", "sqlite", "experimental", "server", "bedrock", "dev", "cloud-tool-sandbox", "modal", "external-tools", "desktop", "profiling"]
|
||||
|
||||
[[package]]
|
||||
name = "letta-client"
|
||||
@@ -4541,18 +4422,6 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/cc/35/cc0aaecf278bb4575b8555f2b137de5ab821595ddae9da9d3cd1da4072c7/propcache-0.3.2-py3-none-any.whl", hash = "sha256:98f1ec44fb675f5052cccc8e609c46ed23a35a1cfd18545ad4e29002d858a43f", size = 12663, upload-time = "2025-06-09T22:56:04.484Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proto-plus"
|
||||
version = "1.27.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "protobuf" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/01/89/9cbe2f4bba860e149108b683bc2efec21f14d5f7ed6e25562ad86acbc373/proto_plus-1.27.0.tar.gz", hash = "sha256:873af56dd0d7e91836aee871e5799e1c6f1bda86ac9a983e0bb9f0c266a568c4", size = 56158, upload-time = "2025-12-16T13:46:25.729Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/cd/24/3b7a0818484df9c28172857af32c2397b6d8fcd99d9468bd4684f98ebf0a/proto_plus-1.27.0-py3-none-any.whl", hash = "sha256:1baa7f81cf0f8acb8bc1f6d085008ba4171eaf669629d1b6d1673b21ed1c0a82", size = 50205, upload-time = "2025-12-16T13:46:24.76Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "protobuf"
|
||||
version = "5.29.5"
|
||||
|
||||
Reference in New Issue
Block a user