/** * Community ADE Approval System - Redis Lock Service * Distributed locking with FIFO queues, heartbeats, and deadlock detection */ import Redis from 'ioredis'; import { v4 as uuidv4 } from 'uuid'; import { AcquireLockRequest, LockResponse, LockInfoExtended, DeadlockInfo, QueueItem, LockMode } from '../schemas/approval'; // Redis key prefixes following the schema const KEY_PREFIXES = { LOCK_TASK: 'ade:lock:task', LOCK_RESOURCE: 'ade:lock:resource', LOCK_AGENT: 'ade:lock:agent', QUEUE: 'queue', INDEX_AGENT: 'ade:lock:index:agent', INDEX_RESOURCE: 'ade:lock:index:resource', REGISTRY: 'ade:lock:registry', WAITFOR: 'ade:lock:waitfor', } as const; export interface LockServiceOptions { redisUrl?: string; defaultTtlSeconds?: number; maxTtlSeconds?: number; heartbeatIntervalSeconds?: number; } export interface LockAcquisitionResult { success: boolean; lock?: LockResponse; queuePosition?: number; estimatedWaitSeconds?: number; error?: string; } export interface LockReleaseResult { success: boolean; released: boolean; nextWaiter?: { agent_id: string; mode: LockMode }; error?: string; } export class LockService { private redis: Redis; private defaultTtlSeconds: number; private maxTtlSeconds: number; private heartbeatIntervalSeconds: number; constructor(options: LockServiceOptions = {}) { this.redis = new Redis(options.redisUrl || 'redis://localhost:6379/0'); this.defaultTtlSeconds = options.defaultTtlSeconds || 30; this.maxTtlSeconds = options.maxTtlSeconds || 300; this.heartbeatIntervalSeconds = options.heartbeatIntervalSeconds || 10; } /** * Build Redis key for a lock */ private buildLockKey(resourceType: string, resourceId: string): string { const prefix = KEY_PREFIXES[`LOCK_${resourceType.toUpperCase()}` as keyof typeof KEY_PREFIXES]; return `${prefix}:${resourceId}`; } /** * Build Redis key for lock queue */ private buildQueueKey(resourceType: string, resourceId: string): string { return `${this.buildLockKey(resourceType, resourceId)}:queue`; } /** * Build Redis key for lock channel (pub/sub) */ private buildChannelKey(resourceType: string, resourceId: string): string { return `${this.buildLockKey(resourceType, resourceId)}:channel`; } /** * Check if a lock is currently held and not expired */ async isLocked(resourceType: string, resourceId: string): Promise { const key = this.buildLockKey(resourceType, resourceId); const exists = await this.redis.exists(key); return exists === 1; } /** * Get lock info */ async getLock(resourceType: string, resourceId: string): Promise { const key = this.buildLockKey(resourceType, resourceId); const lockData = await this.redis.hgetall(key); if (!lockData || Object.keys(lockData).length === 0) { return null; } const queueKey = this.buildQueueKey(resourceType, resourceId); const queueData = await this.redis.zrange(queueKey, 0, -1, 'WITHSCORES'); const queue: QueueItem[] = []; for (let i = 0; i < queueData.length; i += 2) { try { const item = JSON.parse(queueData[i]); queue.push({ agent_id: item.agent_id, mode: item.mode, requested_at: item.requested_at, priority: item.priority || 0 }); } catch { // Skip invalid entries } } return { id: lockData.id || uuidv4(), resource_type: resourceType as any, resource_id: resourceId, mode: (lockData.mode || 'exclusive') as LockMode, holder: { agent_id: lockData.holder_agent_id, acquired_at: lockData.acquired_at, expires_at: lockData.expires_at, purpose: lockData.purpose }, queue }; } /** * Acquire a lock with optional queuing */ async acquireLock( request: AcquireLockRequest, agentId: string ): Promise { const lockId = uuidv4(); const key = this.buildLockKey(request.resource_type, request.resource_id); const queueKey = this.buildQueueKey(request.resource_type, request.resource_id); const now = new Date().toISOString(); const ttlSeconds = Math.min(request.ttl_seconds || this.defaultTtlSeconds, this.maxTtlSeconds); const expiresAt = new Date(Date.now() + ttlSeconds * 1000).toISOString(); try { // Lua script for atomic lock acquisition const acquireScript = ` local key = KEYS[1] local queueKey = KEYS[2] local agentId = ARGV[1] local lockId = ARGV[2] local mode = ARGV[3] local ttl = tonumber(ARGV[4]) local purpose = ARGV[5] local acquiredAt = ARGV[6] local expiresAt = ARGV[7] local waitForAvailable = ARGV[8] == "true" local maxWait = tonumber(ARGV[9]) local queueEntry = ARGV[10] -- Check if lock exists local exists = redis.call('exists', key) if exists == 0 then -- Lock is free, acquire it redis.call('hset', key, 'id', lockId, 'holder_agent_id', agentId, 'mode', mode, 'acquired_at', acquiredAt, 'expires_at', expiresAt, 'purpose', purpose, 'heartbeat_count', 0, 'queue_length', 0 ) redis.call('expire', key, ttl) -- Add to agent's lock index local agentIndex = 'ade:lock:index:agent:' .. agentId redis.call('sadd', agentIndex, key) redis.call('expire', agentIndex, ttl) -- Add to registry redis.call('zadd', 'ade:lock:registry', ttl, key) return {1, lockId, 0, 0} -- acquired, lock_id, queue_pos, wait_time end -- Lock is held - check if we should queue if waitForAvailable then -- Add to queue local score = redis.call('time')[1] * 1000 + redis.call('time')[2] / 1000 redis.call('zadd', queueKey, score, queueEntry) local queuePos = redis.call('zrank', queueKey, queueEntry) redis.call('expire', queueKey, maxWait) -- Update queue length on lock redis.call('hincrby', key, 'queue_length', 1) return {2, nil, queuePos + 1, maxWait} -- queued, nil, position, wait_time end return {0, nil, 0, 0} -- failed `; const queueEntry = JSON.stringify({ agent_id: agentId, mode: request.mode, priority: 0, requested_at: now, max_wait_seconds: request.max_wait_seconds }); const result = await this.redis.eval( acquireScript, 2, key, queueKey, agentId, lockId, request.mode, ttlSeconds, request.purpose || '', now, expiresAt, request.wait_for_available.toString(), request.max_wait_seconds, queueEntry ) as [number, string | null, number, number]; const [status, returnedLockId, queuePosition, waitTime] = result; if (status === 1) { // Lock acquired return { success: true, lock: { id: returnedLockId!, acquired: true, resource_type: request.resource_type, resource_id: request.resource_id, mode: request.mode, holder: { agent_id: agentId, acquired_at: now, expires_at: expiresAt, purpose: request.purpose } } }; } else if (status === 2) { // Queued return { success: true, lock: { id: lockId, acquired: false, resource_type: request.resource_type, resource_id: request.resource_id, mode: request.mode, holder: { agent_id: agentId, acquired_at: now, expires_at: expiresAt, purpose: request.purpose }, queue_position: queuePosition, estimated_wait_seconds: waitTime } }; } else { // Failed to acquire return { success: false, error: 'Resource is locked by another agent' }; } } catch (err: any) { return { success: false, error: err.message }; } } /** * Extend lock TTL via heartbeat */ async heartbeat( lockId: string, agentId: string, resourceType: string, resourceId: string, ttlExtensionSeconds: number ): Promise { const key = this.buildLockKey(resourceType, resourceId); const ttl = Math.min(ttlExtensionSeconds, this.maxTtlSeconds); try { // Lua script for atomic heartbeat const heartbeatScript = ` local key = KEYS[1] local agentId = ARGV[1] local ttl = tonumber(ARGV[2]) local newExpiresAt = ARGV[3] local holder = redis.call('hget', key, 'holder_agent_id') if holder ~= agentId then return 0 -- Not the holder end redis.call('hset', key, 'expires_at', newExpiresAt) redis.call('hincrby', key, 'heartbeat_count', 1) redis.call('expire', key, ttl) return 1 `; const newExpiresAt = new Date(Date.now() + ttl * 1000).toISOString(); const result = await this.redis.eval( heartbeatScript, 1, key, agentId, ttl, newExpiresAt ) as number; return result === 1; } catch (err) { return false; } } /** * Release a lock */ async releaseLock( lockId: string, agentId: string, resourceType: string, resourceId: string, force: boolean = false, reason?: string ): Promise { const key = this.buildLockKey(resourceType, resourceId); const queueKey = this.buildQueueKey(resourceType, resourceId); try { // Lua script for atomic lock release const releaseScript = ` local key = KEYS[1] local queueKey = KEYS[2] local agentId = ARGV[1] local force = ARGV[2] == "true" local holder = redis.call('hget', key, 'holder_agent_id') if not holder then return {0, nil} -- Lock doesn't exist end if holder ~= agentId and not force then return {0, nil} -- Not the holder and not forced end -- Get next waiter before deleting local nextWaiter = redis.call('zpopmin', queueKey, 1) -- Delete the lock redis.call('del', key) redis.call('del', queueKey) -- Remove from agent's lock index local agentIndex = 'ade:lock:index:agent:' .. holder redis.call('srem', agentIndex, key) -- Remove from registry redis.call('zrem', 'ade:lock:registry', key) if nextWaiter and #nextWaiter > 0 then return {1, nextWaiter[1]} -- released, next_waiter end return {1, nil} -- released, no waiters `; const result = await this.redis.eval( releaseScript, 2, key, queueKey, agentId, force.toString() ) as [number, string | null]; const [released, nextWaiterJson] = result; if (released === 0) { return { success: false, released: false, error: 'Lock not found or not held by agent' }; } let nextWaiter: { agent_id: string; mode: LockMode } | undefined; if (nextWaiterJson) { try { const parsed = JSON.parse(nextWaiterJson); nextWaiter = { agent_id: parsed.agent_id, mode: parsed.mode }; // Notify the next waiter via pub/sub const channelKey = this.buildChannelKey(resourceType, resourceId); await this.redis.publish(channelKey, JSON.stringify({ event: 'lock:released', previous_holder: agentId, next_waiter: parsed.agent_id, reason })); } catch { // Invalid JSON, skip } } return { success: true, released: true, nextWaiter }; } catch (err: any) { return { success: false, released: false, error: err.message }; } } /** * List all active locks */ async listLocks( resourceType?: string, resourceId?: string, agentId?: string ): Promise { const locks: LockInfoExtended[] = []; try { if (agentId) { // Get locks held by specific agent const agentIndex = `${KEY_PREFIXES.INDEX_AGENT}:${agentId}`; const lockKeys = await this.redis.smembers(agentIndex); for (const key of lockKeys) { const parts = key.split(':'); if (parts.length >= 4) { const type = parts[2]; const id = parts[3]; const lock = await this.getLock(type, id); if (lock) { locks.push(lock); } } } } else if (resourceType && resourceId) { // Get specific lock const lock = await this.getLock(resourceType, resourceId); if (lock) { locks.push(lock); } } else if (resourceType) { // Get all locks of a specific resource type const indexKey = `${KEY_PREFIXES.INDEX_RESOURCE}:${resourceType}`; const lockKeys = await this.redis.smembers(indexKey); for (const key of lockKeys) { const parts = key.split(':'); if (parts.length >= 4) { const id = parts[3]; const lock = await this.getLock(resourceType, id); if (lock) { locks.push(lock); } } } } else { // Get all locks from registry const lockKeys = await this.redis.zrange(KEY_PREFIXES.REGISTRY, 0, -1); for (const key of lockKeys) { const parts = key.split(':'); if (parts.length >= 4) { const type = parts[2]; const id = parts[3]; const lock = await this.getLock(type, id); if (lock) { locks.push(lock); } } } } return locks; } catch (err) { return locks; } } /** * Detect deadlocks using wait-for graph */ async detectDeadlocks(): Promise { const deadlocks: DeadlockInfo[] = []; try { // Build wait-for graph const waitForKeys = await this.redis.keys(`${KEY_PREFIXES.WAITFOR}:*`); const graph: Map> = new Map(); for (const key of waitForKeys) { const agentId = key.split(':').pop(); if (!agentId) continue; const waitsFor = await this.redis.smembers(key); graph.set(agentId, new Set(waitsFor)); } // Detect cycles using DFS const visited = new Set(); const recStack = new Set(); const path: string[] = []; const dfs = (node: string): string[] | null => { visited.add(node); recStack.add(node); path.push(node); const neighbors = graph.get(node) || new Set(); for (const neighbor of neighbors) { if (!visited.has(neighbor)) { const cycle = dfs(neighbor); if (cycle) return cycle; } else if (recStack.has(neighbor)) { // Found cycle const cycleStart = path.indexOf(neighbor); return path.slice(cycleStart); } } path.pop(); recStack.delete(node); return null; }; for (const [agentId] of graph) { if (!visited.has(agentId)) { const cycle = dfs(agentId); if (cycle && cycle.length > 0) { // Build deadlock info from cycle const cycleInfo = cycle.map((agentId, index) => ({ agent_id: agentId, holds_lock: uuidv4(), // Placeholder - would need actual lock lookup waits_for: uuidv4() })); deadlocks.push({ detected_at: new Date().toISOString(), cycle: cycleInfo, resolution: { victim_agent_id: cycle[cycle.length - 1], // Youngest strategy: 'abort_youngest' as const, released_locks: [] } }); // Resolve the deadlock by aborting the youngest transaction await this.resolveDeadlock(cycle[cycle.length - 1]); } } } return deadlocks; } catch (err) { return deadlocks; } } /** * Resolve a deadlock by aborting a victim */ private async resolveDeadlock(victimAgentId: string): Promise { // Clear victim's wait-for entries const victimKey = `${KEY_PREFIXES.WAITFOR}:${victimAgentId}`; await this.redis.del(victimKey); // Remove victim from other agents' wait-for sets const allWaitForKeys = await this.redis.keys(`${KEY_PREFIXES.WAITFOR}:*`); for (const key of allWaitForKeys) { // Parse the key to get agent_id:lock mappings // This is simplified - full implementation would track which locks each agent waits for await this.redis.srem(key, victimAgentId); } } /** * Record that an agent is waiting for a lock */ async recordWait(agentId: string, resourceType: string, resourceId: string): Promise { const waitKey = `${KEY_PREFIXES.WAITFOR}:${agentId}`; const lockKey = this.buildLockKey(resourceType, resourceId); await this.redis.sadd(waitKey, lockKey); await this.redis.expire(waitKey, 300); // 5 minute TTL } /** * Clear wait-for record */ async clearWait(agentId: string): Promise { const waitKey = `${KEY_PREFIXES.WAITFOR}:${agentId}`; await this.redis.del(waitKey); } /** * Clean up expired locks from registry */ async cleanupExpiredLocks(): Promise { const now = Math.floor(Date.now() / 1000); const expired = await this.redis.zrangebyscore(KEY_PREFIXES.REGISTRY, 0, now); for (const key of expired) { await this.redis.del(key); await this.redis.zrem(KEY_PREFIXES.REGISTRY, key); // Also clean up agent index const parts = key.split(':'); if (parts.length >= 4) { const agentId = parts[3]; const agentIndex = `${KEY_PREFIXES.INDEX_AGENT}:${agentId}`; await this.redis.srem(agentIndex, key); } } return expired.length; } /** * Get queue position for a waiting agent */ async getQueuePosition( resourceType: string, resourceId: string, agentId: string ): Promise { const queueKey = this.buildQueueKey(resourceType, resourceId); const items = await this.redis.zrange(queueKey, 0, -1); for (let i = 0; i < items.length; i++) { try { const item = JSON.parse(items[i]); if (item.agent_id === agentId) { return i + 1; // 1-based position } } catch { continue; } } return null; } /** * Cancel a queued lock request */ async cancelQueueRequest( resourceType: string, resourceId: string, agentId: string ): Promise { const queueKey = this.buildQueueKey(resourceType, resourceId); const items = await this.redis.zrange(queueKey, 0, -1); for (const item of items) { try { const parsed = JSON.parse(item); if (parsed.agent_id === agentId) { await this.redis.zrem(queueKey, item); return true; } } catch { continue; } } return false; } /** * Get lock info by lock ID * Searches all locks to find one with matching ID */ async getLockInfoById(lockId: string): Promise<{ resource_type: string; resource_id: string } | null> { try { // Search in registry const keys = await this.redis.zrange(KEY_PREFIXES.REGISTRY, 0, -1); for (const key of keys) { const data = await this.redis.hgetall(key); if (data && data.id === lockId) { const parts = key.split(':'); if (parts.length >= 4) { return { resource_type: parts[2], resource_id: parts[3] }; } } } return null; } catch (err) { return null; } } /** * Close the Redis connection */ async close(): Promise { await this.redis.quit(); } } export default LockService;