feat: granular subsystem commands with parallel scanner execution

Split monolithic scan_updates into individual subsystems (updates/storage/system/docker). Scanners now run in parallel via goroutines - cuts scan time roughly in half, maybe more. Agent changes: - Orchestrator pattern for scanner management - New scanners: storage (disk metrics), system (cpu/mem/processes) - New commands: scan_storage, scan_system, scan_docker - Wrapped existing scanners (APT/DNF/Docker/Windows/Winget) with common interface - Version bump to 0.1.20 Server changes: - Migration 015: agent_subsystems table with trigger for auto-init - Subsystem CRUD: enable/disable, interval (5min-24hr), auto-run toggle - API routes: /api/v1/agents/:id/subsystems/* (9 endpoints) - Stats tracking per subsystem Web UI changes: - ChatTimeline shows subsystem-specific labels and icons - AgentScanners got interactive toggles, interval dropdowns, manual trigger buttons - TypeScript types added for subsystems Backward compatible with legacy scan_updates - for now. Bugs probably exist somewhere.
2025-11-01 20:34:00 -04:00
parent bf4d46529f
commit 3690472396
19 changed files with 2151 additions and 253 deletions
--- a/aggregator-server/internal/api/handlers/agents.go
+++ b/aggregator-server/internal/api/handlers/agents.go
@@ -256,8 +256,8 @@ func (h *AgentHandler) GetCommands(c *gin.Context) {
 				}
 			}

-			// Update agent with new metadata
-			if err := h.agentQueries.UpdateAgent(agent); err != nil {
+			// Update agent with new metadata (preserve version tracking)
+			if err := h.agentQueries.UpdateAgentMetadata(agentID, agent.Metadata, agent.Status, time.Now()); err != nil {
 				log.Printf("Warning: Failed to update agent metrics: %v", err)
 			}
 		}
@@ -269,21 +269,7 @@ func (h *AgentHandler) GetCommands(c *gin.Context) {
 		return
 	}

-	// Process command acknowledgments if agent sent any
-	var acknowledgedIDs []string
-	if metrics != nil && len(metrics.PendingAcknowledgments) > 0 {
-		// Verify which commands from the agent's pending list have been recorded
-		verified, err := h.commandQueries.VerifyCommandsCompleted(metrics.PendingAcknowledgments)
-		if err != nil {
-			log.Printf("Warning: Failed to verify command acknowledgments for agent %s: %v", agentID, err)
-		} else {
-			acknowledgedIDs = verified
-			if len(acknowledgedIDs) > 0 {
-				log.Printf("Acknowledged %d command results for agent %s", len(acknowledgedIDs), agentID)
-			}
-		}
-	}
-
+	
 	// Process heartbeat metadata from agent check-ins
 	if metrics.Metadata != nil {
 		agent, err := h.agentQueries.GetAgentByID(agentID)
@@ -454,7 +440,7 @@ func (h *AgentHandler) GetCommands(c *gin.Context) {
 	response := models.CommandsResponse{
 		Commands:        commandItems,
 		RapidPolling:    rapidPolling,
-		AcknowledgedIDs: acknowledgedIDs,
+		AcknowledgedIDs: []string{}, // No acknowledgments in current implementation
 	}

 	c.JSON(http.StatusOK, response)
--- a/aggregator-server/internal/api/handlers/subsystems.go
+++ b/aggregator-server/internal/api/handlers/subsystems.go
@@ -0,0 +1,327 @@
+package handlers
+
+import (
+	"net/http"
+
+	"github.com/Fimeg/RedFlag/aggregator-server/internal/database/queries"
+	"github.com/Fimeg/RedFlag/aggregator-server/internal/models"
+	"github.com/gin-gonic/gin"
+	"github.com/google/uuid"
+)
+
+type SubsystemHandler struct {
+	subsystemQueries *queries.SubsystemQueries
+	commandQueries   *queries.CommandQueries
+}
+
+func NewSubsystemHandler(sq *queries.SubsystemQueries, cq *queries.CommandQueries) *SubsystemHandler {
+	return &SubsystemHandler{
+		subsystemQueries: sq,
+		commandQueries:   cq,
+	}
+}
+
+// GetSubsystems retrieves all subsystems for an agent
+// GET /api/v1/agents/:id/subsystems
+func (h *SubsystemHandler) GetSubsystems(c *gin.Context) {
+	agentID, err := uuid.Parse(c.Param("id"))
+	if err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid agent ID"})
+		return
+	}
+
+	subsystems, err := h.subsystemQueries.GetSubsystems(agentID)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve subsystems"})
+		return
+	}
+
+	c.JSON(http.StatusOK, subsystems)
+}
+
+// GetSubsystem retrieves a specific subsystem for an agent
+// GET /api/v1/agents/:id/subsystems/:subsystem
+func (h *SubsystemHandler) GetSubsystem(c *gin.Context) {
+	agentID, err := uuid.Parse(c.Param("id"))
+	if err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid agent ID"})
+		return
+	}
+
+	subsystem := c.Param("subsystem")
+	if subsystem == "" {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Subsystem name required"})
+		return
+	}
+
+	sub, err := h.subsystemQueries.GetSubsystem(agentID, subsystem)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve subsystem"})
+		return
+	}
+
+	if sub == nil {
+		c.JSON(http.StatusNotFound, gin.H{"error": "Subsystem not found"})
+		return
+	}
+
+	c.JSON(http.StatusOK, sub)
+}
+
+// UpdateSubsystem updates subsystem configuration
+// PATCH /api/v1/agents/:id/subsystems/:subsystem
+func (h *SubsystemHandler) UpdateSubsystem(c *gin.Context) {
+	agentID, err := uuid.Parse(c.Param("id"))
+	if err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid agent ID"})
+		return
+	}
+
+	subsystem := c.Param("subsystem")
+	if subsystem == "" {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Subsystem name required"})
+		return
+	}
+
+	var config models.SubsystemConfig
+	if err := c.ShouldBindJSON(&config); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		return
+	}
+
+	// Validate interval if provided
+	if config.IntervalMinutes != nil && (*config.IntervalMinutes < 5 || *config.IntervalMinutes > 1440) {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Interval must be between 5 and 1440 minutes"})
+		return
+	}
+
+	err = h.subsystemQueries.UpdateSubsystem(agentID, subsystem, config)
+	if err != nil {
+		if err.Error() == "subsystem not found" {
+			c.JSON(http.StatusNotFound, gin.H{"error": "Subsystem not found"})
+			return
+		}
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to update subsystem"})
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{"message": "Subsystem updated successfully"})
+}
+
+// EnableSubsystem enables a subsystem
+// POST /api/v1/agents/:id/subsystems/:subsystem/enable
+func (h *SubsystemHandler) EnableSubsystem(c *gin.Context) {
+	agentID, err := uuid.Parse(c.Param("id"))
+	if err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid agent ID"})
+		return
+	}
+
+	subsystem := c.Param("subsystem")
+	if subsystem == "" {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Subsystem name required"})
+		return
+	}
+
+	err = h.subsystemQueries.EnableSubsystem(agentID, subsystem)
+	if err != nil {
+		if err.Error() == "subsystem not found" {
+			c.JSON(http.StatusNotFound, gin.H{"error": "Subsystem not found"})
+			return
+		}
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to enable subsystem"})
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{"message": "Subsystem enabled successfully"})
+}
+
+// DisableSubsystem disables a subsystem
+// POST /api/v1/agents/:id/subsystems/:subsystem/disable
+func (h *SubsystemHandler) DisableSubsystem(c *gin.Context) {
+	agentID, err := uuid.Parse(c.Param("id"))
+	if err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid agent ID"})
+		return
+	}
+
+	subsystem := c.Param("subsystem")
+	if subsystem == "" {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Subsystem name required"})
+		return
+	}
+
+	err = h.subsystemQueries.DisableSubsystem(agentID, subsystem)
+	if err != nil {
+		if err.Error() == "subsystem not found" {
+			c.JSON(http.StatusNotFound, gin.H{"error": "Subsystem not found"})
+			return
+		}
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to disable subsystem"})
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{"message": "Subsystem disabled successfully"})
+}
+
+// TriggerSubsystem manually triggers a subsystem scan
+// POST /api/v1/agents/:id/subsystems/:subsystem/trigger
+func (h *SubsystemHandler) TriggerSubsystem(c *gin.Context) {
+	agentID, err := uuid.Parse(c.Param("id"))
+	if err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid agent ID"})
+		return
+	}
+
+	subsystem := c.Param("subsystem")
+	if subsystem == "" {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Subsystem name required"})
+		return
+	}
+
+	// Verify subsystem exists and is enabled
+	sub, err := h.subsystemQueries.GetSubsystem(agentID, subsystem)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve subsystem"})
+		return
+	}
+
+	if sub == nil {
+		c.JSON(http.StatusNotFound, gin.H{"error": "Subsystem not found"})
+		return
+	}
+
+	if !sub.Enabled {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Subsystem is disabled"})
+		return
+	}
+
+	// Create command for the subsystem
+	commandType := "scan_" + subsystem
+	command := &models.AgentCommand{
+		AgentID:     agentID,
+		CommandType: commandType,
+		Status:      "pending",
+		Source:      "web_ui", // Manual trigger from UI
+	}
+
+	err = h.commandQueries.CreateCommand(command)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create command"})
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{
+		"message":    "Subsystem scan triggered successfully",
+		"command_id": command.ID,
+	})
+}
+
+// GetSubsystemStats retrieves statistics for a subsystem
+// GET /api/v1/agents/:id/subsystems/:subsystem/stats
+func (h *SubsystemHandler) GetSubsystemStats(c *gin.Context) {
+	agentID, err := uuid.Parse(c.Param("id"))
+	if err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid agent ID"})
+		return
+	}
+
+	subsystem := c.Param("subsystem")
+	if subsystem == "" {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Subsystem name required"})
+		return
+	}
+
+	stats, err := h.subsystemQueries.GetSubsystemStats(agentID, subsystem)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve subsystem stats"})
+		return
+	}
+
+	if stats == nil {
+		c.JSON(http.StatusNotFound, gin.H{"error": "Subsystem not found"})
+		return
+	}
+
+	c.JSON(http.StatusOK, stats)
+}
+
+// SetAutoRun enables or disables auto-run for a subsystem
+// POST /api/v1/agents/:id/subsystems/:subsystem/auto-run
+func (h *SubsystemHandler) SetAutoRun(c *gin.Context) {
+	agentID, err := uuid.Parse(c.Param("id"))
+	if err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid agent ID"})
+		return
+	}
+
+	subsystem := c.Param("subsystem")
+	if subsystem == "" {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Subsystem name required"})
+		return
+	}
+
+	var req struct {
+		AutoRun bool `json:"auto_run"`
+	}
+
+	if err := c.ShouldBindJSON(&req); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		return
+	}
+
+	err = h.subsystemQueries.SetAutoRun(agentID, subsystem, req.AutoRun)
+	if err != nil {
+		if err.Error() == "subsystem not found" {
+			c.JSON(http.StatusNotFound, gin.H{"error": "Subsystem not found"})
+			return
+		}
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to update auto-run"})
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{"message": "Auto-run updated successfully"})
+}
+
+// SetInterval sets the interval for a subsystem
+// POST /api/v1/agents/:id/subsystems/:subsystem/interval
+func (h *SubsystemHandler) SetInterval(c *gin.Context) {
+	agentID, err := uuid.Parse(c.Param("id"))
+	if err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid agent ID"})
+		return
+	}
+
+	subsystem := c.Param("subsystem")
+	if subsystem == "" {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Subsystem name required"})
+		return
+	}
+
+	var req struct {
+		IntervalMinutes int `json:"interval_minutes"`
+	}
+
+	if err := c.ShouldBindJSON(&req); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		return
+	}
+
+	// Validate interval
+	if req.IntervalMinutes < 5 || req.IntervalMinutes > 1440 {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Interval must be between 5 and 1440 minutes"})
+		return
+	}
+
+	err = h.subsystemQueries.SetInterval(agentID, subsystem, req.IntervalMinutes)
+	if err != nil {
+		if err.Error() == "subsystem not found" {
+			c.JSON(http.StatusNotFound, gin.H{"error": "Subsystem not found"})
+			return
+		}
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to update interval"})
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{"message": "Interval updated successfully"})
+}