Files
community-ade/src/services/lock.ts

749 lines
20 KiB
TypeScript

/**
* Community ADE Approval System - Redis Lock Service
* Distributed locking with FIFO queues, heartbeats, and deadlock detection
*/
import Redis from 'ioredis';
import { v4 as uuidv4 } from 'uuid';
import {
AcquireLockRequest,
LockResponse,
LockInfoExtended,
DeadlockInfo,
QueueItem,
LockMode
} from '../schemas/approval';
// Redis key prefixes following the schema
const KEY_PREFIXES = {
LOCK_TASK: 'ade:lock:task',
LOCK_RESOURCE: 'ade:lock:resource',
LOCK_AGENT: 'ade:lock:agent',
QUEUE: 'queue',
INDEX_AGENT: 'ade:lock:index:agent',
INDEX_RESOURCE: 'ade:lock:index:resource',
REGISTRY: 'ade:lock:registry',
WAITFOR: 'ade:lock:waitfor',
} as const;
export interface LockServiceOptions {
redisUrl?: string;
defaultTtlSeconds?: number;
maxTtlSeconds?: number;
heartbeatIntervalSeconds?: number;
}
export interface LockAcquisitionResult {
success: boolean;
lock?: LockResponse;
queuePosition?: number;
estimatedWaitSeconds?: number;
error?: string;
}
export interface LockReleaseResult {
success: boolean;
released: boolean;
nextWaiter?: { agent_id: string; mode: LockMode };
error?: string;
}
export class LockService {
private redis: Redis;
private defaultTtlSeconds: number;
private maxTtlSeconds: number;
private heartbeatIntervalSeconds: number;
constructor(options: LockServiceOptions = {}) {
this.redis = new Redis(options.redisUrl || 'redis://localhost:6379/0');
this.defaultTtlSeconds = options.defaultTtlSeconds || 30;
this.maxTtlSeconds = options.maxTtlSeconds || 300;
this.heartbeatIntervalSeconds = options.heartbeatIntervalSeconds || 10;
}
/**
* Build Redis key for a lock
*/
private buildLockKey(resourceType: string, resourceId: string): string {
const prefix = KEY_PREFIXES[`LOCK_${resourceType.toUpperCase()}` as keyof typeof KEY_PREFIXES];
return `${prefix}:${resourceId}`;
}
/**
* Build Redis key for lock queue
*/
private buildQueueKey(resourceType: string, resourceId: string): string {
return `${this.buildLockKey(resourceType, resourceId)}:queue`;
}
/**
* Build Redis key for lock channel (pub/sub)
*/
private buildChannelKey(resourceType: string, resourceId: string): string {
return `${this.buildLockKey(resourceType, resourceId)}:channel`;
}
/**
* Check if a lock is currently held and not expired
*/
async isLocked(resourceType: string, resourceId: string): Promise<boolean> {
const key = this.buildLockKey(resourceType, resourceId);
const exists = await this.redis.exists(key);
return exists === 1;
}
/**
* Get lock info
*/
async getLock(resourceType: string, resourceId: string): Promise<LockInfoExtended | null> {
const key = this.buildLockKey(resourceType, resourceId);
const lockData = await this.redis.hgetall(key);
if (!lockData || Object.keys(lockData).length === 0) {
return null;
}
const queueKey = this.buildQueueKey(resourceType, resourceId);
const queueData = await this.redis.zrange(queueKey, 0, -1, 'WITHSCORES');
const queue: QueueItem[] = [];
for (let i = 0; i < queueData.length; i += 2) {
try {
const item = JSON.parse(queueData[i]);
queue.push({
agent_id: item.agent_id,
mode: item.mode,
requested_at: item.requested_at,
priority: item.priority || 0
});
} catch {
// Skip invalid entries
}
}
return {
id: lockData.id || uuidv4(),
resource_type: resourceType as any,
resource_id: resourceId,
mode: (lockData.mode || 'exclusive') as LockMode,
holder: {
agent_id: lockData.holder_agent_id,
acquired_at: lockData.acquired_at,
expires_at: lockData.expires_at,
purpose: lockData.purpose
},
queue
};
}
/**
* Acquire a lock with optional queuing
*/
async acquireLock(
request: AcquireLockRequest,
agentId: string
): Promise<LockAcquisitionResult> {
const lockId = uuidv4();
const key = this.buildLockKey(request.resource_type, request.resource_id);
const queueKey = this.buildQueueKey(request.resource_type, request.resource_id);
const now = new Date().toISOString();
const ttlSeconds = Math.min(request.ttl_seconds || this.defaultTtlSeconds, this.maxTtlSeconds);
const expiresAt = new Date(Date.now() + ttlSeconds * 1000).toISOString();
try {
// Lua script for atomic lock acquisition
const acquireScript = `
local key = KEYS[1]
local queueKey = KEYS[2]
local agentId = ARGV[1]
local lockId = ARGV[2]
local mode = ARGV[3]
local ttl = tonumber(ARGV[4])
local purpose = ARGV[5]
local acquiredAt = ARGV[6]
local expiresAt = ARGV[7]
local waitForAvailable = ARGV[8] == "true"
local maxWait = tonumber(ARGV[9])
local queueEntry = ARGV[10]
-- Check if lock exists
local exists = redis.call('exists', key)
if exists == 0 then
-- Lock is free, acquire it
redis.call('hset', key,
'id', lockId,
'holder_agent_id', agentId,
'mode', mode,
'acquired_at', acquiredAt,
'expires_at', expiresAt,
'purpose', purpose,
'heartbeat_count', 0,
'queue_length', 0
)
redis.call('expire', key, ttl)
-- Add to agent's lock index
local agentIndex = 'ade:lock:index:agent:' .. agentId
redis.call('sadd', agentIndex, key)
redis.call('expire', agentIndex, ttl)
-- Add to registry
redis.call('zadd', 'ade:lock:registry', ttl, key)
return {1, lockId, 0, 0} -- acquired, lock_id, queue_pos, wait_time
end
-- Lock is held - check if we should queue
if waitForAvailable then
-- Add to queue
local score = redis.call('time')[1] * 1000 + redis.call('time')[2] / 1000
redis.call('zadd', queueKey, score, queueEntry)
local queuePos = redis.call('zrank', queueKey, queueEntry)
redis.call('expire', queueKey, maxWait)
-- Update queue length on lock
redis.call('hincrby', key, 'queue_length', 1)
return {2, nil, queuePos + 1, maxWait} -- queued, nil, position, wait_time
end
return {0, nil, 0, 0} -- failed
`;
const queueEntry = JSON.stringify({
agent_id: agentId,
mode: request.mode,
priority: 0,
requested_at: now,
max_wait_seconds: request.max_wait_seconds
});
const result = await this.redis.eval(
acquireScript,
2,
key,
queueKey,
agentId,
lockId,
request.mode,
ttlSeconds,
request.purpose || '',
now,
expiresAt,
request.wait_for_available.toString(),
request.max_wait_seconds,
queueEntry
) as [number, string | null, number, number];
const [status, returnedLockId, queuePosition, waitTime] = result;
if (status === 1) {
// Lock acquired
return {
success: true,
lock: {
id: returnedLockId!,
acquired: true,
resource_type: request.resource_type,
resource_id: request.resource_id,
mode: request.mode,
holder: {
agent_id: agentId,
acquired_at: now,
expires_at: expiresAt,
purpose: request.purpose
}
}
};
} else if (status === 2) {
// Queued
return {
success: true,
lock: {
id: lockId,
acquired: false,
resource_type: request.resource_type,
resource_id: request.resource_id,
mode: request.mode,
holder: {
agent_id: agentId,
acquired_at: now,
expires_at: expiresAt,
purpose: request.purpose
},
queue_position: queuePosition,
estimated_wait_seconds: waitTime
}
};
} else {
// Failed to acquire
return {
success: false,
error: 'Resource is locked by another agent'
};
}
} catch (err: any) {
return {
success: false,
error: err.message
};
}
}
/**
* Extend lock TTL via heartbeat
*/
async heartbeat(
lockId: string,
agentId: string,
resourceType: string,
resourceId: string,
ttlExtensionSeconds: number
): Promise<boolean> {
const key = this.buildLockKey(resourceType, resourceId);
const ttl = Math.min(ttlExtensionSeconds, this.maxTtlSeconds);
try {
// Lua script for atomic heartbeat
const heartbeatScript = `
local key = KEYS[1]
local agentId = ARGV[1]
local ttl = tonumber(ARGV[2])
local newExpiresAt = ARGV[3]
local holder = redis.call('hget', key, 'holder_agent_id')
if holder ~= agentId then
return 0 -- Not the holder
end
redis.call('hset', key, 'expires_at', newExpiresAt)
redis.call('hincrby', key, 'heartbeat_count', 1)
redis.call('expire', key, ttl)
return 1
`;
const newExpiresAt = new Date(Date.now() + ttl * 1000).toISOString();
const result = await this.redis.eval(
heartbeatScript,
1,
key,
agentId,
ttl,
newExpiresAt
) as number;
return result === 1;
} catch (err) {
return false;
}
}
/**
* Release a lock
*/
async releaseLock(
lockId: string,
agentId: string,
resourceType: string,
resourceId: string,
force: boolean = false,
reason?: string
): Promise<LockReleaseResult> {
const key = this.buildLockKey(resourceType, resourceId);
const queueKey = this.buildQueueKey(resourceType, resourceId);
try {
// Lua script for atomic lock release
const releaseScript = `
local key = KEYS[1]
local queueKey = KEYS[2]
local agentId = ARGV[1]
local force = ARGV[2] == "true"
local holder = redis.call('hget', key, 'holder_agent_id')
if not holder then
return {0, nil} -- Lock doesn't exist
end
if holder ~= agentId and not force then
return {0, nil} -- Not the holder and not forced
end
-- Get next waiter before deleting
local nextWaiter = redis.call('zpopmin', queueKey, 1)
-- Delete the lock
redis.call('del', key)
redis.call('del', queueKey)
-- Remove from agent's lock index
local agentIndex = 'ade:lock:index:agent:' .. holder
redis.call('srem', agentIndex, key)
-- Remove from registry
redis.call('zrem', 'ade:lock:registry', key)
if nextWaiter and #nextWaiter > 0 then
return {1, nextWaiter[1]} -- released, next_waiter
end
return {1, nil} -- released, no waiters
`;
const result = await this.redis.eval(
releaseScript,
2,
key,
queueKey,
agentId,
force.toString()
) as [number, string | null];
const [released, nextWaiterJson] = result;
if (released === 0) {
return {
success: false,
released: false,
error: 'Lock not found or not held by agent'
};
}
let nextWaiter: { agent_id: string; mode: LockMode } | undefined;
if (nextWaiterJson) {
try {
const parsed = JSON.parse(nextWaiterJson);
nextWaiter = {
agent_id: parsed.agent_id,
mode: parsed.mode
};
// Notify the next waiter via pub/sub
const channelKey = this.buildChannelKey(resourceType, resourceId);
await this.redis.publish(channelKey, JSON.stringify({
event: 'lock:released',
previous_holder: agentId,
next_waiter: parsed.agent_id,
reason
}));
} catch {
// Invalid JSON, skip
}
}
return {
success: true,
released: true,
nextWaiter
};
} catch (err: any) {
return {
success: false,
released: false,
error: err.message
};
}
}
/**
* List all active locks
*/
async listLocks(
resourceType?: string,
resourceId?: string,
agentId?: string
): Promise<LockInfoExtended[]> {
const locks: LockInfoExtended[] = [];
try {
if (agentId) {
// Get locks held by specific agent
const agentIndex = `${KEY_PREFIXES.INDEX_AGENT}:${agentId}`;
const lockKeys = await this.redis.smembers(agentIndex);
for (const key of lockKeys) {
const parts = key.split(':');
if (parts.length >= 4) {
const type = parts[2];
const id = parts[3];
const lock = await this.getLock(type, id);
if (lock) {
locks.push(lock);
}
}
}
} else if (resourceType && resourceId) {
// Get specific lock
const lock = await this.getLock(resourceType, resourceId);
if (lock) {
locks.push(lock);
}
} else if (resourceType) {
// Get all locks of a specific resource type
const indexKey = `${KEY_PREFIXES.INDEX_RESOURCE}:${resourceType}`;
const lockKeys = await this.redis.smembers(indexKey);
for (const key of lockKeys) {
const parts = key.split(':');
if (parts.length >= 4) {
const id = parts[3];
const lock = await this.getLock(resourceType, id);
if (lock) {
locks.push(lock);
}
}
}
} else {
// Get all locks from registry
const lockKeys = await this.redis.zrange(KEY_PREFIXES.REGISTRY, 0, -1);
for (const key of lockKeys) {
const parts = key.split(':');
if (parts.length >= 4) {
const type = parts[2];
const id = parts[3];
const lock = await this.getLock(type, id);
if (lock) {
locks.push(lock);
}
}
}
}
return locks;
} catch (err) {
return locks;
}
}
/**
* Detect deadlocks using wait-for graph
*/
async detectDeadlocks(): Promise<DeadlockInfo[]> {
const deadlocks: DeadlockInfo[] = [];
try {
// Build wait-for graph
const waitForKeys = await this.redis.keys(`${KEY_PREFIXES.WAITFOR}:*`);
const graph: Map<string, Set<string>> = new Map();
for (const key of waitForKeys) {
const agentId = key.split(':').pop();
if (!agentId) continue;
const waitsFor = await this.redis.smembers(key);
graph.set(agentId, new Set(waitsFor));
}
// Detect cycles using DFS
const visited = new Set<string>();
const recStack = new Set<string>();
const path: string[] = [];
const dfs = (node: string): string[] | null => {
visited.add(node);
recStack.add(node);
path.push(node);
const neighbors = graph.get(node) || new Set();
for (const neighbor of neighbors) {
if (!visited.has(neighbor)) {
const cycle = dfs(neighbor);
if (cycle) return cycle;
} else if (recStack.has(neighbor)) {
// Found cycle
const cycleStart = path.indexOf(neighbor);
return path.slice(cycleStart);
}
}
path.pop();
recStack.delete(node);
return null;
};
for (const [agentId] of graph) {
if (!visited.has(agentId)) {
const cycle = dfs(agentId);
if (cycle && cycle.length > 0) {
// Build deadlock info from cycle
const cycleInfo = cycle.map((agentId, index) => ({
agent_id: agentId,
holds_lock: uuidv4(), // Placeholder - would need actual lock lookup
waits_for: uuidv4()
}));
deadlocks.push({
detected_at: new Date().toISOString(),
cycle: cycleInfo,
resolution: {
victim_agent_id: cycle[cycle.length - 1], // Youngest
strategy: 'abort_youngest' as const,
released_locks: []
}
});
// Resolve the deadlock by aborting the youngest transaction
await this.resolveDeadlock(cycle[cycle.length - 1]);
}
}
}
return deadlocks;
} catch (err) {
return deadlocks;
}
}
/**
* Resolve a deadlock by aborting a victim
*/
private async resolveDeadlock(victimAgentId: string): Promise<void> {
// Clear victim's wait-for entries
const victimKey = `${KEY_PREFIXES.WAITFOR}:${victimAgentId}`;
await this.redis.del(victimKey);
// Remove victim from other agents' wait-for sets
const allWaitForKeys = await this.redis.keys(`${KEY_PREFIXES.WAITFOR}:*`);
for (const key of allWaitForKeys) {
// Parse the key to get agent_id:lock mappings
// This is simplified - full implementation would track which locks each agent waits for
await this.redis.srem(key, victimAgentId);
}
}
/**
* Record that an agent is waiting for a lock
*/
async recordWait(agentId: string, resourceType: string, resourceId: string): Promise<void> {
const waitKey = `${KEY_PREFIXES.WAITFOR}:${agentId}`;
const lockKey = this.buildLockKey(resourceType, resourceId);
await this.redis.sadd(waitKey, lockKey);
await this.redis.expire(waitKey, 300); // 5 minute TTL
}
/**
* Clear wait-for record
*/
async clearWait(agentId: string): Promise<void> {
const waitKey = `${KEY_PREFIXES.WAITFOR}:${agentId}`;
await this.redis.del(waitKey);
}
/**
* Clean up expired locks from registry
*/
async cleanupExpiredLocks(): Promise<number> {
const now = Math.floor(Date.now() / 1000);
const expired = await this.redis.zrangebyscore(KEY_PREFIXES.REGISTRY, 0, now);
for (const key of expired) {
await this.redis.del(key);
await this.redis.zrem(KEY_PREFIXES.REGISTRY, key);
// Also clean up agent index
const parts = key.split(':');
if (parts.length >= 4) {
const agentId = parts[3];
const agentIndex = `${KEY_PREFIXES.INDEX_AGENT}:${agentId}`;
await this.redis.srem(agentIndex, key);
}
}
return expired.length;
}
/**
* Get queue position for a waiting agent
*/
async getQueuePosition(
resourceType: string,
resourceId: string,
agentId: string
): Promise<number | null> {
const queueKey = this.buildQueueKey(resourceType, resourceId);
const items = await this.redis.zrange(queueKey, 0, -1);
for (let i = 0; i < items.length; i++) {
try {
const item = JSON.parse(items[i]);
if (item.agent_id === agentId) {
return i + 1; // 1-based position
}
} catch {
continue;
}
}
return null;
}
/**
* Cancel a queued lock request
*/
async cancelQueueRequest(
resourceType: string,
resourceId: string,
agentId: string
): Promise<boolean> {
const queueKey = this.buildQueueKey(resourceType, resourceId);
const items = await this.redis.zrange(queueKey, 0, -1);
for (const item of items) {
try {
const parsed = JSON.parse(item);
if (parsed.agent_id === agentId) {
await this.redis.zrem(queueKey, item);
return true;
}
} catch {
continue;
}
}
return false;
}
/**
* Get lock info by lock ID
* Searches all locks to find one with matching ID
*/
async getLockInfoById(lockId: string): Promise<{ resource_type: string; resource_id: string } | null> {
try {
// Search in registry
const keys = await this.redis.zrange(KEY_PREFIXES.REGISTRY, 0, -1);
for (const key of keys) {
const data = await this.redis.hgetall(key);
if (data && data.id === lockId) {
const parts = key.split(':');
if (parts.length >= 4) {
return {
resource_type: parts[2],
resource_id: parts[3]
};
}
}
}
return null;
} catch (err) {
return null;
}
}
/**
* Close the Redis connection
*/
async close(): Promise<void> {
await this.redis.quit();
}
}
export default LockService;