Fimeg/Redflag
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: separate data classification architecture

Browse Source 
- Create separate scanner interfaces for storage, system, and docker data
- Add dedicated endpoints for metrics and docker images instead of misclassifying as updates
- Implement proper database tables for storage metrics and docker images
- Fix storage/system metrics appearing incorrectly as package updates
- Add scanner types with proper data structures for each subsystem
- Update agent handlers to use correct endpoints for each data type
This commit is contained in:
Fimeg
2025-11-03 21:44:48 -05:00
parent 57be3754c6
commit eccc38d7c9
16 changed files with 2183 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
aggregator-agent/cmd/agent/subsystem_handlers.go
View File

@@ -121,19 +121,43 @@ func handleScanStorage(apiClient *client.Client, cfg *config.Config, ackTracker
log.Printf("Failed to report scan log: %v\n", err)
}
// Report "updates" (disk info) to server
if len(result.Updates) > 0 {
report := client.UpdateReport{
CommandID: commandID,
Timestamp: time.Now(),
Updates: result.Updates,
// Report storage metrics to server using dedicated endpoint
// Get storage scanner and use proper interface
storageScanner := orchestrator.NewStorageScanner("unknown") // TODO: Get actual agent version
if storageScanner.IsAvailable() {
metrics, err := storageScanner.ScanStorage()
if err != nil {
return fmt.Errorf("failed to scan storage metrics: %w", err)
}
if err := apiClient.ReportUpdates(cfg.AgentID, report); err != nil {
return fmt.Errorf("failed to report storage metrics: %w", err)
}
if len(metrics) > 0 {
// Convert StorageMetric to MetricsReportItem for API call
metricItems := make([]client.MetricsReportItem, 0, len(metrics))
for _, metric := range metrics {
item := client.MetricsReportItem{
PackageType: "storage",
PackageName: metric.Mountpoint,
CurrentVersion: fmt.Sprintf("%d bytes used", metric.UsedBytes),
AvailableVersion: fmt.Sprintf("%d bytes total", metric.TotalBytes),
Severity: metric.Severity,
RepositorySource: metric.Filesystem,
Metadata: metric.Metadata,
}
metricItems = append(metricItems, item)
}
log.Printf("✓ Reported %d disk mount points to server\n", len(result.Updates))
report := client.MetricsReport{
CommandID: commandID,
Timestamp: time.Now(),
Metrics: metricItems,
}
if err := apiClient.ReportMetrics(cfg.AgentID, report); err != nil {
return fmt.Errorf("failed to report storage metrics: %w", err)
}
log.Printf("✓ Reported %d storage metrics to server\n", len(metrics))
}
}
return nil
@@ -179,19 +203,43 @@ func handleScanSystem(apiClient *client.Client, cfg *config.Config, ackTracker *
log.Printf("Failed to report scan log: %v\n", err)
}
// Report "updates" (system metrics) to server
if len(result.Updates) > 0 {
report := client.UpdateReport{
CommandID: commandID,
Timestamp: time.Now(),
Updates: result.Updates,
// Report system metrics to server using dedicated endpoint
// Get system scanner and use proper interface
systemScanner := orchestrator.NewSystemScanner("unknown") // TODO: Get actual agent version
if systemScanner.IsAvailable() {
metrics, err := systemScanner.ScanSystem()
if err != nil {
return fmt.Errorf("failed to scan system metrics: %w", err)
}
if err := apiClient.ReportUpdates(cfg.AgentID, report); err != nil {
return fmt.Errorf("failed to report system metrics: %w", err)
}
if len(metrics) > 0 {
// Convert SystemMetric to MetricsReportItem for API call
metricItems := make([]client.MetricsReportItem, 0, len(metrics))
for _, metric := range metrics {
item := client.MetricsReportItem{
PackageType: "system",
PackageName: metric.MetricName,
CurrentVersion: metric.CurrentValue,
AvailableVersion: metric.AvailableValue,
Severity: metric.Severity,
RepositorySource: metric.MetricType,
Metadata: metric.Metadata,
}
metricItems = append(metricItems, item)
}
log.Printf("✓ Reported system metrics to server\n")
report := client.MetricsReport{
CommandID: commandID,
Timestamp: time.Now(),
Metrics: metricItems,
}
if err := apiClient.ReportMetrics(cfg.AgentID, report); err != nil {
return fmt.Errorf("failed to report system metrics: %w", err)
}
log.Printf("✓ Reported %d system metrics to server\n", len(metrics))
}
}
return nil

98
aggregator-agent/internal/client/client.go
View File

@@ -334,6 +334,104 @@ func (c *Client) ReportUpdates(agentID uuid.UUID, report UpdateReport) error {
return nil
}
// MetricsReport represents metrics data (storage, system, CPU, memory)
type MetricsReport struct {
CommandID string `json:"command_id"`
Timestamp time.Time `json:"timestamp"`
Metrics []MetricsReportItem `json:"metrics"`
}
// MetricsReportItem represents a single metric
type MetricsReportItem struct {
PackageType string `json:"package_type"`
PackageName string `json:"package_name"`
CurrentVersion string `json:"current_version"`
AvailableVersion string `json:"available_version"`
Severity string `json:"severity"`
RepositorySource string `json:"repository_source"`
Metadata map[string]interface{} `json:"metadata"`
}
// ReportMetrics sends metrics data to the server
func (c *Client) ReportMetrics(agentID uuid.UUID, report MetricsReport) error {
url := fmt.Sprintf("%s/api/v1/agents/%s/metrics", c.baseURL, agentID)
body, err := json.Marshal(report)
if err != nil {
return err
}
req, err := http.NewRequest("POST", url, bytes.NewBuffer(body))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", "Bearer "+c.token)
c.addMachineIDHeader(req) // Security: Validate machine binding (v0.1.22+)
resp, err := c.http.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
bodyBytes, _ := io.ReadAll(resp.Body)
return fmt.Errorf("failed to report metrics: %s - %s", resp.Status, string(bodyBytes))
}
return nil
}
// DockerReport represents Docker image information
type DockerReport struct {
CommandID string `json:"command_id"`
Timestamp time.Time `json:"timestamp"`
Images []DockerReportItem `json:"images"`
}
// DockerReportItem represents a single Docker image
type DockerReportItem struct {
PackageType string `json:"package_type"`
PackageName string `json:"package_name"`
CurrentVersion string `json:"current_version"`
AvailableVersion string `json:"available_version"`
Severity string `json:"severity"`
RepositorySource string `json:"repository_source"`
Metadata map[string]interface{} `json:"metadata"`
}
// ReportDockerImages sends Docker image information to the server
func (c *Client) ReportDockerImages(agentID uuid.UUID, report DockerReport) error {
url := fmt.Sprintf("%s/api/v1/agents/%s/docker-images", c.baseURL, agentID)
body, err := json.Marshal(report)
if err != nil {
return err
}
req, err := http.NewRequest("POST", url, bytes.NewBuffer(body))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", "Bearer "+c.token)
c.addMachineIDHeader(req) // Security: Validate machine binding (v0.1.22+)
resp, err := c.http.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
bodyBytes, _ := io.ReadAll(resp.Body)
return fmt.Errorf("failed to report docker images: %s - %s", resp.Status, string(bodyBytes))
}
return nil
}
// LogReport represents an execution log
type LogReport struct {
CommandID string `json:"command_id"`

113
aggregator-agent/internal/orchestrator/scanner_types.go Normal file
View File

@@ -0,0 +1,113 @@
package orchestrator
import (
"github.com/Fimeg/RedFlag/aggregator-agent/internal/client"
"github.com/Fimeg/RedFlag/aggregator-agent/internal/system"
)
// StorageMetric represents a single storage/disk metric
type StorageMetric struct {
Mountpoint string `json:"mountpoint"`
Filesystem string `json:"filesystem"`
Device string `json:"device"`
DiskType string `json:"disk_type"`
TotalBytes int64 `json:"total_bytes"`
UsedBytes int64 `json:"used_bytes"`
AvailableBytes int64 `json:"available_bytes"`
UsedPercent float64 `json:"used_percent"`
IsRoot bool `json:"is_root"`
IsLargest bool `json:"is_largest"`
Severity string `json:"severity"`
Metadata map[string]interface{} `json:"metadata"`
}
// SystemMetric represents a single system metric (CPU, memory, etc.)
type SystemMetric struct {
MetricName string `json:"metric_name"`
MetricType string `json:"metric_type"` // "cpu", "memory", "processes", "uptime", etc.
CurrentValue string `json:"current_value"`
AvailableValue string `json:"available_value"`
Severity string `json:"severity"`
Metadata map[string]interface{} `json:"metadata"`
}
// DockerImage represents a single Docker image
type DockerImage struct {
ImageName string `json:"image_name"`
ImageTag string `json:"image_tag"`
ImageID string `json:"image_id"`
RepositorySource string `json:"repository_source"`
SizeBytes int64 `json:"size_bytes"`
CreatedAt string `json:"created_at"`
HasUpdate bool `json:"has_update"`
LatestImageID string `json:"latest_image_id"`
Severity string `json:"severity"`
Labels map[string]string `json:"labels"`
Metadata map[string]interface{} `json:"metadata"`
}
// PackageUpdate represents an actual software package update (legacy, for package scanners only)
type PackageUpdate = client.UpdateReportItem
// --- Scanner Interfaces ---
// StorageScannerInterface handles storage/disk metrics scanning
type StorageScannerInterface interface {
IsAvailable() bool
ScanStorage() ([]StorageMetric, error)
Name() string
}
// SystemScannerInterface handles system metrics scanning
type SystemScannerInterface interface {
IsAvailable() bool
ScanSystem() ([]SystemMetric, error)
Name() string
}
// DockerScannerInterface handles Docker image scanning
type DockerScannerInterface interface {
IsAvailable() bool
ScanDocker() ([]DockerImage, error)
Name() string
}
// PackageScannerInterface handles package update scanning (legacy)
type PackageScannerInterface interface {
IsAvailable() bool
ScanPackages() ([]PackageUpdate, error)
Name() string
}
// --- Unified Scanner Types for Backwards Compatibility ---
// ScannerType represents the type of data a scanner returns
type ScannerType string
const (
ScannerTypeStorage ScannerType = "storage"
ScannerTypeSystem ScannerType = "system"
ScannerTypeDocker ScannerType = "docker"
ScannerTypePackage ScannerType = "package"
)
// TypedScannerResult represents the result of any type of scanner
type TypedScannerResult struct {
ScannerName string
ScannerType ScannerType
StorageData []StorageMetric
SystemData []SystemMetric
DockerData []DockerImage
PackageData []PackageUpdate
Error error
Duration int64 // milliseconds
Status string
}
// TypedScanner is a unified interface that can return any type of data
type TypedScanner interface {
IsAvailable() bool
GetType() ScannerType
Scan() (TypedScannerResult, error)
Name() string
}

111
aggregator-agent/internal/orchestrator/storage_scanner.go
View File

@@ -2,6 +2,7 @@ package orchestrator
import (
"fmt"
"time"
"github.com/Fimeg/RedFlag/aggregator-agent/internal/client"
"github.com/Fimeg/RedFlag/aggregator-agent/internal/system"
@@ -24,8 +25,8 @@ func (s *StorageScanner) IsAvailable() bool {
return true
}
// Scan collects disk usage information and returns it as "updates" for reporting
func (s *StorageScanner) Scan() ([]client.UpdateReportItem, error) {
// ScanStorage collects disk usage information and returns proper storage metrics
func (s *StorageScanner) ScanStorage() ([]StorageMetric, error) {
sysInfo, err := system.GetSystemInfo(s.agentVersion)
if err != nil {
return nil, fmt.Errorf("failed to get system info: %w", err)
@@ -35,41 +36,99 @@ func (s *StorageScanner) Scan() ([]client.UpdateReportItem, error) {
return nil, fmt.Errorf("no disk information available")
}
// Convert disk info to UpdateReportItem format for reporting
// This is a bit unconventional but allows us to use the existing reporting infrastructure
var items []client.UpdateReportItem
// Convert disk info to proper StorageMetric format
var metrics []StorageMetric
for _, disk := range sysInfo.DiskInfo {
// Create a pseudo-update item for each disk
item := client.UpdateReportItem{
PackageName: fmt.Sprintf("disk-%s", disk.Mountpoint),
CurrentVersion: fmt.Sprintf("%.1f%% used", disk.UsedPercent),
AvailableVersion: fmt.Sprintf("%d GB available", disk.Available/(1024*1024*1024)),
PackageType: "storage",
Severity: determineDiskSeverity(disk.UsedPercent),
PackageDescription: fmt.Sprintf("Disk: %s (%s) - %s", disk.Mountpoint, disk.Filesystem, disk.Device),
metric := StorageMetric{
Mountpoint: disk.Mountpoint,
Filesystem: disk.Filesystem,
Device: disk.Device,
DiskType: disk.DiskType,
TotalBytes: disk.Total,
UsedBytes: disk.Used,
AvailableBytes: disk.Available,
UsedPercent: disk.UsedPercent,
IsRoot: disk.IsRoot,
IsLargest: disk.IsLargest,
Severity: determineDiskSeverity(disk.UsedPercent),
Metadata: map[string]interface{}{
"mountpoint": disk.Mountpoint,
"filesystem": disk.Filesystem,
"device": disk.Device,
"disk_type": disk.DiskType,
"total_bytes": disk.Total,
"used_bytes": disk.Used,
"available_bytes": disk.Available,
"used_percent": disk.UsedPercent,
"is_root": disk.IsRoot,
"is_largest": disk.IsLargest,
"agent_version": s.agentVersion,
"collected_at": sysInfo.Timestamp,
},
}
metrics = append(metrics, metric)
}
return metrics, nil
}
// Name returns the scanner name
func (s *StorageScanner) Name() string {
return "Disk Usage Reporter"
}
// --- Legacy Compatibility Methods ---
// Scan collects disk usage information and returns it as "updates" for reporting (LEGACY)
// This method is kept for backwards compatibility with the old Scanner interface
func (s *StorageScanner) Scan() ([]client.UpdateReportItem, error) {
metrics, err := s.ScanStorage()
if err != nil {
return nil, err
}
// Convert proper StorageMetric back to legacy UpdateReportItem format
var items []client.UpdateReportItem
for _, metric := range metrics {
item := client.UpdateReportItem{
PackageName: fmt.Sprintf("disk-%s", metric.Mountpoint),
CurrentVersion: fmt.Sprintf("%.1f%% used", metric.UsedPercent),
AvailableVersion: fmt.Sprintf("%d GB available", metric.AvailableBytes/(1024*1024*1024)),
PackageType: "storage",
Severity: metric.Severity,
PackageDescription: fmt.Sprintf("Disk: %s (%s) - %s", metric.Mountpoint, metric.Filesystem, metric.Device),
Metadata: metric.Metadata,
}
items = append(items, item)
}
return items, nil
}
// Name returns the scanner name
func (s *StorageScanner) Name() string {
return "Disk Usage Reporter"
// --- Typed Scanner Implementation ---
// GetType returns the scanner type
func (s *StorageScanner) GetType() ScannerType {
return ScannerTypeStorage
}
// ScanTyped returns typed results (new implementation)
func (s *StorageScanner) ScanTyped() (TypedScannerResult, error) {
startTime := time.Now()
defer func() {
// Duration will be set at the end
}()
metrics, err := s.ScanStorage()
if err != nil {
return TypedScannerResult{
ScannerName: s.Name(),
ScannerType: ScannerTypeStorage,
Error: err,
Status: "failed",
Duration: time.Since(startTime).Milliseconds(),
}, err
}
return TypedScannerResult{
ScannerName: s.Name(),
ScannerType: ScannerTypeStorage,
StorageData: metrics,
Status: "success",
Duration: time.Since(startTime).Milliseconds(),
}, nil
}
// determineDiskSeverity returns severity based on disk usage percentage

158
aggregator-agent/internal/orchestrator/system_scanner.go
View File

@@ -2,6 +2,7 @@ package orchestrator
import (
"fmt"
"time"
"github.com/Fimeg/RedFlag/aggregator-agent/internal/client"
"github.com/Fimeg/RedFlag/aggregator-agent/internal/system"
@@ -24,42 +25,38 @@ func (s *SystemScanner) IsAvailable() bool {
return true
}
// Scan collects system information and returns it as "updates" for reporting
func (s *SystemScanner) Scan() ([]client.UpdateReportItem, error) {
// ScanSystem collects system information and returns proper system metrics
func (s *SystemScanner) ScanSystem() ([]SystemMetric, error) {
sysInfo, err := system.GetSystemInfo(s.agentVersion)
if err != nil {
return nil, fmt.Errorf("failed to get system info: %w", err)
}
// Convert system info to UpdateReportItem format for reporting
var items []client.UpdateReportItem
// Convert system info to proper SystemMetric format
var metrics []SystemMetric
// CPU info item
cpuItem := client.UpdateReportItem{
PackageName: "system-cpu",
CurrentVersion: fmt.Sprintf("%d cores, %d threads", sysInfo.CPUInfo.Cores, sysInfo.CPUInfo.Threads),
AvailableVersion: sysInfo.CPUInfo.ModelName,
PackageType: "system",
Severity: "low",
PackageDescription: fmt.Sprintf("CPU: %s", sysInfo.CPUInfo.ModelName),
// CPU info metric
cpuMetric := SystemMetric{
MetricName: "system-cpu",
MetricType: "cpu",
CurrentValue: fmt.Sprintf("%d cores, %d threads", sysInfo.CPUInfo.Cores, sysInfo.CPUInfo.Threads),
AvailableValue: sysInfo.CPUInfo.ModelName,
Severity: "low",
Metadata: map[string]interface{}{
"cpu_model": sysInfo.CPUInfo.ModelName,
"cpu_cores": sysInfo.CPUInfo.Cores,
"cpu_threads": sysInfo.CPUInfo.Threads,
},
}
items = append(items, cpuItem)
metrics = append(metrics, cpuMetric)
// Memory info item
memItem := client.UpdateReportItem{
PackageName: "system-memory",
CurrentVersion: fmt.Sprintf("%.1f%% used", sysInfo.MemoryInfo.UsedPercent),
AvailableVersion: fmt.Sprintf("%d GB total", sysInfo.MemoryInfo.Total/(1024*1024*1024)),
PackageType: "system",
Severity: determineMemorySeverity(sysInfo.MemoryInfo.UsedPercent),
PackageDescription: fmt.Sprintf("Memory: %.1f GB / %.1f GB used",
float64(sysInfo.MemoryInfo.Used)/(1024*1024*1024),
float64(sysInfo.MemoryInfo.Total)/(1024*1024*1024)),
// Memory info metric
memMetric := SystemMetric{
MetricName: "system-memory",
MetricType: "memory",
CurrentValue: fmt.Sprintf("%.1f%% used", sysInfo.MemoryInfo.UsedPercent),
AvailableValue: fmt.Sprintf("%d GB total", sysInfo.MemoryInfo.Total/(1024*1024*1024)),
Severity: determineMemorySeverity(sysInfo.MemoryInfo.UsedPercent),
Metadata: map[string]interface{}{
"memory_total": sysInfo.MemoryInfo.Total,
"memory_used": sysInfo.MemoryInfo.Used,
@@ -67,54 +64,51 @@ func (s *SystemScanner) Scan() ([]client.UpdateReportItem, error) {
"memory_used_percent": sysInfo.MemoryInfo.UsedPercent,
},
}
items = append(items, memItem)
metrics = append(metrics, memMetric)
// Process count item
processItem := client.UpdateReportItem{
PackageName: "system-processes",
CurrentVersion: fmt.Sprintf("%d processes", sysInfo.RunningProcesses),
AvailableVersion: "n/a",
PackageType: "system",
Severity: "low",
PackageDescription: fmt.Sprintf("Running Processes: %d", sysInfo.RunningProcesses),
// Process count metric
processMetric := SystemMetric{
MetricName: "system-processes",
MetricType: "processes",
CurrentValue: fmt.Sprintf("%d processes", sysInfo.RunningProcesses),
AvailableValue: "n/a",
Severity: "low",
Metadata: map[string]interface{}{
"process_count": sysInfo.RunningProcesses,
},
}
items = append(items, processItem)
metrics = append(metrics, processMetric)
// Uptime item
uptimeItem := client.UpdateReportItem{
PackageName: "system-uptime",
CurrentVersion: sysInfo.Uptime,
AvailableVersion: "n/a",
PackageType: "system",
Severity: "low",
PackageDescription: fmt.Sprintf("System Uptime: %s", sysInfo.Uptime),
// Uptime metric
uptimeMetric := SystemMetric{
MetricName: "system-uptime",
MetricType: "uptime",
CurrentValue: sysInfo.Uptime,
AvailableValue: "n/a",
Severity: "low",
Metadata: map[string]interface{}{
"uptime": sysInfo.Uptime,
},
}
items = append(items, uptimeItem)
metrics = append(metrics, uptimeMetric)
// Reboot required item (if applicable)
// Reboot required metric (if applicable)
if sysInfo.RebootRequired {
rebootItem := client.UpdateReportItem{
PackageName: "system-reboot",
CurrentVersion: "required",
AvailableVersion: "n/a",
PackageType: "system",
Severity: "important",
PackageDescription: fmt.Sprintf("Reboot Required: %s", sysInfo.RebootReason),
rebootMetric := SystemMetric{
MetricName: "system-reboot",
MetricType: "reboot",
CurrentValue: "required",
AvailableValue: "n/a",
Severity: "important",
Metadata: map[string]interface{}{
"reboot_required": true,
"reboot_reason": sysInfo.RebootReason,
},
}
items = append(items, rebootItem)
metrics = append(metrics, rebootMetric)
}
return items, nil
return metrics, nil
}
// Name returns the scanner name
@@ -122,6 +116,66 @@ func (s *SystemScanner) Name() string {
return "System Metrics Reporter"
}
// --- Legacy Compatibility Methods ---
// Scan collects system information and returns it as "updates" for reporting (LEGACY)
// This method is kept for backwards compatibility with the old Scanner interface
func (s *SystemScanner) Scan() ([]client.UpdateReportItem, error) {
metrics, err := s.ScanSystem()
if err != nil {
return nil, err
}
// Convert proper SystemMetric back to legacy UpdateReportItem format
var items []client.UpdateReportItem
for _, metric := range metrics {
item := client.UpdateReportItem{
PackageName: metric.MetricName,
CurrentVersion: metric.CurrentValue,
AvailableVersion: metric.AvailableValue,
PackageType: "system",
Severity: metric.Severity,
PackageDescription: fmt.Sprintf("System %s: %s", metric.MetricType, metric.MetricName),
Metadata: metric.Metadata,
}
items = append(items, item)
}
return items, nil
}
// --- Typed Scanner Implementation ---
// GetType returns the scanner type
func (s *SystemScanner) GetType() ScannerType {
return ScannerTypeSystem
}
// ScanTyped returns typed results (new implementation)
func (s *SystemScanner) ScanTyped() (TypedScannerResult, error) {
startTime := time.Now()
metrics, err := s.ScanSystem()
if err != nil {
return TypedScannerResult{
ScannerName: s.Name(),
ScannerType: ScannerTypeSystem,
Error: err,
Status: "failed",
Duration: time.Since(startTime).Milliseconds(),
}, err
}
return TypedScannerResult{
ScannerName: s.Name(),
ScannerType: ScannerTypeSystem,
SystemData: metrics,
Status: "success",
Duration: time.Since(startTime).Milliseconds(),
}, nil
}
// determineMemorySeverity returns severity based on memory usage percentage
func determineMemorySeverity(usedPercent float64) string {
switch {