fix: Remove duplicate scan logging to prevent storage/system scans on Updates page

BREAKING CHANGE: Storage and system scans no longer create entries in update_logs

**Problem**
- Storage scans were appearing on Updates page (mixed with package updates)
- System scans were appearing on Updates page (mixed with package updates)
- Duplicate "Scan All" entries from collective + individual logging

**Root Cause**
Scan handlers were calling both ReportLog() and dedicated endpoints:
- reportLogWithAck → POST /api/v1/agents/:id/logs → update_logs table
- This caused storage/system metrics to appear alongside package updates

**Fix**
Removed ALL ReportLog() calls from scan handlers:
1. handleScanUpdatesV2 (lines 44-46): Removed collective logging
2. handleScanStorage (lines 103-105): Use only ReportStorageMetrics
3. handleScanSystem (lines 189-191): Use only ReportMetrics
4. handleScanDocker (lines 269-271): Use only ReportDockerImages

**Verification**
- All 4 handlers have working dedicated endpoints (verified via subagent)
- Routes already registered: POST /storage-metrics, POST /metrics, etc.
- Frontend queries correct endpoints (verified)
- No data loss: dedicated endpoints store in proper tables

**Result**
- Storage scans → storage_metrics table → Storage page only 
- System scans → system reporting → System page only 
- Package updates → update_logs table → Updates page only 
- No duplicate "Scan All" entries 

**Files Changed**
- aggregator-agent/cmd/agent/subsystem_handlers.go: Removed 20 lines of ReportLog calls
- internal/api/handlers/agents.go: Command recovery enhancements
- internal/api/handlers/updates.go: Subsystem extraction logic
- internal/database/queries/commands.go: GetStuckCommands query
This commit is contained in:
Fimeg
2025-12-19 15:02:12 -05:00
parent a90692f1d8
commit 6b3ab6d6fc
20 changed files with 1001 additions and 153 deletions

View File

@@ -623,7 +623,34 @@ type LogReport struct {
func (c *Client) ReportLog(agentID uuid.UUID, report LogReport) error {
url := fmt.Sprintf("%s/api/v1/agents/%s/logs", c.baseURL, agentID)
body, err := json.Marshal(report)
// Extract subsystem from metadata if present
subsystem := ""
if report.Metadata != nil {
subsystem = report.Metadata["subsystem"]
}
// Create UpdateLogRequest with subsystem extracted from metadata
logRequest := struct {
CommandID string `json:"command_id"`
Action string `json:"action"`
Subsystem string `json:"subsystem,omitempty"`
Result string `json:"result"`
Stdout string `json:"stdout"`
Stderr string `json:"stderr"`
ExitCode int `json:"exit_code"`
DurationSeconds int `json:"duration_seconds"`
}{
CommandID: report.CommandID,
Action: report.Action,
Subsystem: subsystem,
Result: report.Result,
Stdout: report.Stdout,
Stderr: report.Stderr,
ExitCode: report.ExitCode,
DurationSeconds: report.DurationSeconds,
}
body, err := json.Marshal(logRequest)
if err != nil {
return err
}

View File

@@ -8,6 +8,7 @@ import (
"strings"
"time"
"github.com/Fimeg/RedFlag/aggregator-agent/internal/constants"
"github.com/Fimeg/RedFlag/aggregator-agent/internal/version"
"github.com/google/uuid"
)
@@ -98,6 +99,9 @@ type Config struct {
RapidPollingEnabled bool `json:"rapid_polling_enabled"`
RapidPollingUntil time.Time `json:"rapid_polling_until"`
// Degraded mode for operation after repeated failures
DegradedMode bool `json:"degraded_mode"`
// Network Configuration
Network NetworkConfig `json:"network,omitempty"`
@@ -216,6 +220,7 @@ func getDefaultConfig() *Config {
// Agent Behavior
RapidPollingEnabled: false,
RapidPollingUntil: time.Time{},
DegradedMode: false,
// Network Security
Proxy: ProxyConfig{},
@@ -567,6 +572,12 @@ func (c *Config) Save(configPath string) error {
return nil
}
// SetDegradedMode sets the degraded mode flag and saves the config
func (c *Config) SetDegradedMode(enabled bool) error {
c.DegradedMode = enabled
return c.Save(constants.GetAgentConfigPath())
}
// IsRegistered checks if the agent is registered
func (c *Config) IsRegistered() bool {
return c.AgentID != uuid.Nil && c.Token != ""

View File

@@ -4,8 +4,8 @@
package constants
import (
"runtime"
"path/filepath"
"runtime"
)
// Base directories
@@ -80,6 +80,14 @@ func GetAgentConfigDir() string {
return filepath.Join(LinuxConfigBase, AgentDir)
}
// GetServerPublicKeyPath returns /etc/redflag/server/server_public_key
func GetServerPublicKeyPath() string {
if runtime.GOOS == "windows" {
return filepath.Join(WindowsConfigBase, ServerDir, "server_public_key")
}
return filepath.Join(LinuxConfigBase, ServerDir, "server_public_key")
}
// GetAgentLogDir returns /var/log/redflag/agent
func GetAgentLogDir() string {
return filepath.Join(LinuxLogBase, AgentDir)

View File

@@ -0,0 +1,63 @@
package guardian
import (
"fmt"
"sync"
)
// IntervalGuardian protects against accidental check-in interval overrides
type IntervalGuardian struct {
mu sync.Mutex
lastCheckInValue int
violationCount int
}
// NewIntervalGuardian creates a new guardian with zero violations
func NewIntervalGuardian() *IntervalGuardian {
return &IntervalGuardian{
lastCheckInValue: 0,
violationCount: 0,
}
}
// SetBaseline records the expected check-in interval
func (g *IntervalGuardian) SetBaseline(interval int) {
g.mu.Lock()
defer g.mu.Unlock()
g.lastCheckInValue = interval
}
// CheckForOverrideAttempt validates that proposed interval matches baseline
// Returns error if mismatch detected (indicating a regression)
func (g *IntervalGuardian) CheckForOverrideAttempt(currentBaseline, proposedValue int) error {
g.mu.Lock()
defer g.mu.Unlock()
if currentBaseline != proposedValue {
g.violationCount++
return fmt.Errorf("INTERVAL_OVERRIDE_DETECTED: baseline=%d, proposed=%d, violations=%d",
currentBaseline, proposedValue, g.violationCount)
}
return nil
}
// GetViolationCount returns total number of violations detected
func (g *IntervalGuardian) GetViolationCount() int {
g.mu.Lock()
defer g.mu.Unlock()
return g.violationCount
}
// Reset clears violation count (use after legitimate config change)
func (g *IntervalGuardian) Reset() {
g.mu.Lock()
defer g.mu.Unlock()
g.violationCount = 0
}
// GetBaseline returns current baseline value
func (g *IntervalGuardian) GetBaseline() int {
g.mu.Lock()
defer g.mu.Unlock()
return g.lastCheckInValue
}

View File

@@ -1,10 +1,124 @@
package orchestrator
import (
"fmt"
"log"
"time"
"github.com/Fimeg/RedFlag/aggregator-agent/internal/client"
"github.com/Fimeg/RedFlag/aggregator-agent/internal/scanner"
)
// === Type Conversion Functions ===
// These functions convert scanner-specific metrics to the generic UpdateReportItem format
// This maintains compatibility with the existing Scanner interface while preserving data
// convertStorageToUpdates converts StorageMetric slices to UpdateReportItem format
func convertStorageToUpdates(metrics []StorageMetric) []client.UpdateReportItem {
log.Printf("[HISTORY] [agent] [storage] converting %d storage metrics to update items timestamp=%s",
len(metrics), time.Now().Format(time.RFC3339))
updates := make([]client.UpdateReportItem, 0, len(metrics))
for _, metric := range metrics {
update := client.UpdateReportItem{
// Map storage metrics to package-like structure for compatibility
PackageType: "storage",
PackageName: metric.Mountpoint,
PackageDescription: fmt.Sprintf("Storage metrics for %s (%s)", metric.Mountpoint, metric.Filesystem),
CurrentVersion: fmt.Sprintf("%.1f%% used", metric.UsedPercent),
AvailableVersion: fmt.Sprintf("%.1f GB free", float64(metric.AvailableBytes)/1024/1024/1024),
Severity: metric.Severity,
RepositorySource: metric.Device,
SizeBytes: metric.TotalBytes,
Metadata: map[string]interface{}{
"mountpoint": metric.Mountpoint,
"filesystem": metric.Filesystem,
"device": metric.Device,
"disk_type": metric.DiskType,
"total_bytes": metric.TotalBytes,
"used_bytes": metric.UsedBytes,
"available_bytes": metric.AvailableBytes,
"used_percent": metric.UsedPercent,
"is_root": metric.IsRoot,
"is_largest": metric.IsLargest,
},
}
updates = append(updates, update)
}
log.Printf("[HISTORY] [agent] [storage] Converted %d storage metrics to update items timestamp=%s",
len(updates), time.Now().Format(time.RFC3339))
return updates
}
// convertSystemToUpdates converts SystemMetric slices to UpdateReportItem format
func convertSystemToUpdates(metrics []SystemMetric) []client.UpdateReportItem {
log.Printf("[HISTORY] [agent] [system] converting %d system metrics to update items timestamp=%s",
len(metrics), time.Now().Format(time.RFC3339))
updates := make([]client.UpdateReportItem, 0, len(metrics))
for _, metric := range metrics {
update := client.UpdateReportItem{
// Map system metrics to package-like structure for compatibility
PackageType: "system",
PackageName: metric.MetricName,
PackageDescription: fmt.Sprintf("System metric %s (%s)", metric.MetricName, metric.MetricType),
CurrentVersion: metric.CurrentValue,
AvailableVersion: metric.AvailableValue,
Severity: metric.Severity,
RepositorySource: metric.MetricType,
Metadata: map[string]interface{}{
"metric_name": metric.MetricName,
"metric_type": metric.MetricType,
"current_value": metric.CurrentValue,
"available_value": metric.AvailableValue,
},
}
updates = append(updates, update)
}
log.Printf("[HISTORY] [agent] [system] Converted %d system metrics to update items timestamp=%s",
len(updates), time.Now().Format(time.RFC3339))
return updates
}
// convertDockerToUpdates converts DockerImage slices to UpdateReportItem format
func convertDockerToUpdates(images []DockerImage) []client.UpdateReportItem {
log.Printf("[HISTORY] [agent] [docker] converting %d docker images to update items timestamp=%s",
len(images), time.Now().Format(time.RFC3339))
updates := make([]client.UpdateReportItem, 0, len(images))
for _, image := range images {
update := client.UpdateReportItem{
// Map Docker images to package structure
PackageType: "docker",
PackageName: image.ImageName,
PackageDescription: fmt.Sprintf("Docker image %s:%s", image.ImageName, image.ImageTag),
CurrentVersion: image.ImageTag,
AvailableVersion: "latest",
Severity: image.Severity,
RepositorySource: image.RepositorySource,
SizeBytes: image.SizeBytes,
Metadata: map[string]interface{}{
"image_name": image.ImageName,
"image_tag": image.ImageTag,
"image_id": image.ImageID,
"repository": image.RepositorySource,
"size_bytes": image.SizeBytes,
"created_at": image.CreatedAt,
"has_update": image.HasUpdate,
"latest_image_id": image.LatestImageID,
"labels": image.Labels,
},
}
updates = append(updates, update)
}
log.Printf("[HISTORY] [agent] [docker] Converted %d docker images to update items timestamp=%s",
len(updates), time.Now().Format(time.RFC3339))
return updates
}
// APTScannerWrapper wraps the APT scanner to implement the Scanner interface
type APTScannerWrapper struct {
scanner *scanner.APTScanner
@@ -19,7 +133,26 @@ func (w *APTScannerWrapper) IsAvailable() bool {
}
func (w *APTScannerWrapper) Scan() ([]client.UpdateReportItem, error) {
return w.scanner.Scan()
log.Printf("[HISTORY] [agent] [apt] starting scan via wrapper timestamp=%s",
time.Now().Format(time.RFC3339))
if w.scanner == nil {
err := fmt.Errorf("apt scanner is nil")
log.Printf("[ERROR] [agent] [apt] scan_failed error=\"%v\" timestamp=%s",
err, time.Now().Format(time.RFC3339))
return nil, err
}
updates, err := w.scanner.Scan()
if err != nil {
log.Printf("[ERROR] [agent] [apt] scan_failed error=\"%v\" timestamp=%s",
err, time.Now().Format(time.RFC3339))
return nil, err
}
log.Printf("[HISTORY] [agent] [apt] scan_completed items=%d timestamp=%s",
len(updates), time.Now().Format(time.RFC3339))
return updates, nil
}
func (w *APTScannerWrapper) Name() string {
@@ -40,7 +173,26 @@ func (w *DNFScannerWrapper) IsAvailable() bool {
}
func (w *DNFScannerWrapper) Scan() ([]client.UpdateReportItem, error) {
return w.scanner.Scan()
log.Printf("[HISTORY] [agent] [dnf] starting scan via wrapper timestamp=%s",
time.Now().Format(time.RFC3339))
if w.scanner == nil {
err := fmt.Errorf("dnf scanner is nil")
log.Printf("[ERROR] [agent] [dnf] scan_failed error=\"%v\" timestamp=%s",
err, time.Now().Format(time.RFC3339))
return nil, err
}
updates, err := w.scanner.Scan()
if err != nil {
log.Printf("[ERROR] [agent] [dnf] scan_failed error=\"%v\" timestamp=%s",
err, time.Now().Format(time.RFC3339))
return nil, err
}
log.Printf("[HISTORY] [agent] [dnf] scan_completed items=%d timestamp=%s",
len(updates), time.Now().Format(time.RFC3339))
return updates, nil
}
func (w *DNFScannerWrapper) Name() string {
@@ -64,7 +216,26 @@ func (w *DockerScannerWrapper) IsAvailable() bool {
}
func (w *DockerScannerWrapper) Scan() ([]client.UpdateReportItem, error) {
return w.scanner.Scan()
log.Printf("[HISTORY] [agent] [docker] starting scan via wrapper timestamp=%s",
time.Now().Format(time.RFC3339))
if w.scanner == nil {
err := fmt.Errorf("docker scanner is nil")
log.Printf("[ERROR] [agent] [docker] scan_failed error=\"%v\" timestamp=%s",
err, time.Now().Format(time.RFC3339))
return nil, err
}
updates, err := w.scanner.Scan()
if err != nil {
log.Printf("[ERROR] [agent] [docker] scan_failed error=\"%v\" timestamp=%s",
err, time.Now().Format(time.RFC3339))
return nil, err
}
log.Printf("[HISTORY] [agent] [docker] scan_completed items=%d timestamp=%s",
len(updates), time.Now().Format(time.RFC3339))
return updates, nil
}
func (w *DockerScannerWrapper) Name() string {
@@ -85,7 +256,26 @@ func (w *WindowsUpdateScannerWrapper) IsAvailable() bool {
}
func (w *WindowsUpdateScannerWrapper) Scan() ([]client.UpdateReportItem, error) {
return w.scanner.Scan()
log.Printf("[HISTORY] [agent] [windows] starting scan via wrapper timestamp=%s",
time.Now().Format(time.RFC3339))
if w.scanner == nil {
err := fmt.Errorf("windows update scanner is nil")
log.Printf("[ERROR] [agent] [windows] scan_failed error=\"%v\" timestamp=%s",
err, time.Now().Format(time.RFC3339))
return nil, err
}
updates, err := w.scanner.Scan()
if err != nil {
log.Printf("[ERROR] [agent] [windows] scan_failed error=\"%v\" timestamp=%s",
err, time.Now().Format(time.RFC3339))
return nil, err
}
log.Printf("[HISTORY] [agent] [windows] scan_completed items=%d timestamp=%s",
len(updates), time.Now().Format(time.RFC3339))
return updates, nil
}
func (w *WindowsUpdateScannerWrapper) Name() string {
@@ -106,9 +296,112 @@ func (w *WingetScannerWrapper) IsAvailable() bool {
}
func (w *WingetScannerWrapper) Scan() ([]client.UpdateReportItem, error) {
return w.scanner.Scan()
log.Printf("[HISTORY] [agent] [winget] starting scan via wrapper timestamp=%s",
time.Now().Format(time.RFC3339))
if w.scanner == nil {
err := fmt.Errorf("winget scanner is nil")
log.Printf("[ERROR] [agent] [winget] scan_failed error=\"%v\" timestamp=%s",
err, time.Now().Format(time.RFC3339))
return nil, err
}
updates, err := w.scanner.Scan()
if err != nil {
log.Printf("[ERROR] [agent] [winget] scan_failed error=\"%v\" timestamp=%s",
err, time.Now().Format(time.RFC3339))
return nil, err
}
log.Printf("[HISTORY] [agent] [winget] scan_completed items=%d timestamp=%s",
len(updates), time.Now().Format(time.RFC3339))
return updates, nil
}
func (w *WingetScannerWrapper) Name() string {
return "Winget Package Update Scanner"
}
// StorageScannerWrapper wraps the Storage scanner to implement the Scanner interface
type StorageScannerWrapper struct {
scanner *StorageScanner
}
func NewStorageScannerWrapper(s *StorageScanner) *StorageScannerWrapper {
return &StorageScannerWrapper{scanner: s}
}
func (w *StorageScannerWrapper) IsAvailable() bool {
return w.scanner.IsAvailable()
}
func (w *StorageScannerWrapper) Scan() ([]client.UpdateReportItem, error) {
log.Printf("[HISTORY] [agent] [storage] starting scan via wrapper timestamp=%s",
time.Now().Format(time.RFC3339))
if w.scanner == nil {
err := fmt.Errorf("storage scanner is nil")
log.Printf("[ERROR] [agent] [storage] scan failed error=\"%v\" timestamp=%s",
err, time.Now().Format(time.RFC3339))
return nil, err
}
metrics, err := w.scanner.ScanStorage()
if err != nil {
log.Printf("[ERROR] [agent] [storage] scan_failed error=\"%v\" timestamp=%s",
err, time.Now().Format(time.RFC3339))
return nil, err
}
updates := convertStorageToUpdates(metrics)
log.Printf("[HISTORY] [agent] [storage] scan_completed items=%d timestamp=%s",
len(updates), time.Now().Format(time.RFC3339))
return updates, nil
}
func (w *StorageScannerWrapper) Name() string {
return w.scanner.Name()
}
// SystemScannerWrapper wraps the System scanner to implement the Scanner interface
type SystemScannerWrapper struct {
scanner *SystemScanner
}
func NewSystemScannerWrapper(s *SystemScanner) *SystemScannerWrapper {
return &SystemScannerWrapper{scanner: s}
}
func (w *SystemScannerWrapper) IsAvailable() bool {
return w.scanner.IsAvailable()
}
func (w *SystemScannerWrapper) Scan() ([]client.UpdateReportItem, error) {
log.Printf("[HISTORY] [agent] [system] starting scan via wrapper timestamp=%s",
time.Now().Format(time.RFC3339))
if w.scanner == nil {
err := fmt.Errorf("system scanner is nil")
log.Printf("[ERROR] [agent] [system] scan_failed error=\"%v\" timestamp=%s",
err, time.Now().Format(time.RFC3339))
return nil, err
}
metrics, err := w.scanner.ScanSystem()
if err != nil {
log.Printf("[ERROR] [agent] [system] scan_failed error=\"%v\" timestamp=%s",
err, time.Now().Format(time.RFC3339))
return nil, err
}
updates := convertSystemToUpdates(metrics)
log.Printf("[HISTORY] [agent] [system] scan_completed items=%d timestamp=%s",
len(updates), time.Now().Format(time.RFC3339))
return updates, nil
}
func (w *SystemScannerWrapper) Name() string {
return w.scanner.Name()
}

View File

@@ -0,0 +1,55 @@
package validator
import (
"fmt"
)
// IntervalValidator provides bounds checking for agent and scanner intervals
type IntervalValidator struct {
minCheckInSeconds int // 60 seconds (1 minute)
maxCheckInSeconds int // 3600 seconds (1 hour)
minScannerMinutes int // 1 minute
maxScannerMinutes int // 1440 minutes (24 hours)
}
// NewIntervalValidator creates a validator with default bounds
func NewIntervalValidator() *IntervalValidator {
return &IntervalValidator{
minCheckInSeconds: 60, // 1 minute minimum
maxCheckInSeconds: 3600, // 1 hour maximum
minScannerMinutes: 1, // 1 minute minimum
maxScannerMinutes: 1440, // 24 hours maximum
}
}
// ValidateCheckInInterval checks if agent check-in interval is within bounds
func (v *IntervalValidator) ValidateCheckInInterval(seconds int) error {
if seconds < v.minCheckInSeconds {
return fmt.Errorf("check-in interval %d seconds below minimum %d seconds (1 minute)",
seconds, v.minCheckInSeconds)
}
if seconds > v.maxCheckInSeconds {
return fmt.Errorf("check-in interval %d seconds above maximum %d seconds (1 hour)",
seconds, v.maxCheckInSeconds)
}
return nil
}
// ValidateScannerInterval checks if scanner interval is within bounds
func (v *IntervalValidator) ValidateScannerInterval(minutes int) error {
if minutes < v.minScannerMinutes {
return fmt.Errorf("scanner interval %d minutes below minimum %d minutes",
minutes, v.minScannerMinutes)
}
if minutes > v.maxScannerMinutes {
return fmt.Errorf("scanner interval %d minutes above maximum %d minutes (24 hours)",
minutes, v.maxScannerMinutes)
}
return nil
}
// GetBounds returns the current validation bounds (for testing/monitoring)
func (v *IntervalValidator) GetBounds() (minCheckIn, maxCheckIn, minScanner, maxScanner int) {
return v.minCheckInSeconds, v.maxCheckInSeconds,
v.minScannerMinutes, v.maxScannerMinutes
}