new-api/common/system_monitor.go

package common

import (
	"fmt"
	"sync/atomic"
	"time"

	"github.com/shirou/gopsutil/cpu"
	"github.com/shirou/gopsutil/mem"
)

// DiskSpaceInfo 磁盘空间信息
type DiskSpaceInfo struct {
	// 总空间（字节）
	Total uint64 `json:"total"`
	// 可用空间（字节）
	Free uint64 `json:"free"`
	// 已用空间（字节）
	Used uint64 `json:"used"`
	// 使用百分比
	UsedPercent float64 `json:"used_percent"`
}

// SystemStatus 系统状态信息
type SystemStatus struct {
	CPUUsage    float64
	MemoryUsage float64
	DiskUsage   float64
	SampledAt   time.Time
	// 过载判定带连续性要求：单一采样尖峰不再直接熔断，需连续多次超阈值。
	CPUOverloaded    bool
	MemoryOverloaded bool
	DiskOverloaded   bool
}

// SystemOverload 是哨兵据以拒绝请求的过载项，Error 文本带当前值与生效阈值。
type SystemOverload struct {
	Code      string
	Resource  string
	Usage     float64
	Threshold int
}

func (o *SystemOverload) Error() string {
	return fmt.Sprintf("system %s overloaded (current: %.1f%%, threshold: %d%%)", o.Resource, o.Usage, o.Threshold)
}

var latestSystemStatus atomic.Value

const (
	systemMonitorInterval = 5 * time.Second
	// 连续 3 次采样（约 15 秒）仍高于阈值才判定过载，
	// 任一采样回落到阈值以下即恢复，避免瞬时尖峰打满重试。
	overloadConsecutiveSamples = 3
	// 采样超过该时长未刷新（监控未运行或被阻塞）时，哨兵不据此熔断。
	systemStatusMaxAge = 30 * time.Second
)

func init() {
	latestSystemStatus.Store(SystemStatus{})
}

// StartSystemMonitor 启动系统监控
func StartSystemMonitor() {
	go func() {
		usage := newContainerUsage()
		var detector overloadDetector
		for {
			config := GetPerformanceMonitorConfig()
			if !config.Enabled {
				detector = overloadDetector{}
				latestSystemStatus.Store(SystemStatus{SampledAt: time.Now()})
				time.Sleep(systemMonitorInterval)
				continue
			}

			status := collectSystemStatus(usage)
			detector.mark(&status, config)
			latestSystemStatus.Store(status)

			time.Sleep(systemMonitorInterval)
		}
	}()
}

// overloadDetector 为每项资源累计连续超阈值的采样次数：连续 overloadConsecutiveSamples 次
// 才标记过载，任一次回落立即清零并重新累计。
type overloadDetector struct {
	cpuSamples    int
	memorySamples int
	diskSamples   int
}

func (d *overloadDetector) mark(status *SystemStatus, config PerformanceMonitorConfig) {
	status.CPUOverloaded = countOverloadSample(&d.cpuSamples, exceedsThreshold(status.CPUUsage, config.CPUThreshold))
	status.MemoryOverloaded = countOverloadSample(&d.memorySamples, exceedsThreshold(status.MemoryUsage, config.MemoryThreshold))
	status.DiskOverloaded = countOverloadSample(&d.diskSamples, exceedsThreshold(status.DiskUsage, config.DiskThreshold))
}

func countOverloadSample(samples *int, over bool) bool {
	if !over {
		*samples = 0
		return false
	}
	*samples = min(*samples+1, overloadConsecutiveSamples)
	return *samples == overloadConsecutiveSamples
}

// exceedsThreshold 判断采样值是否超过阈值；阈值 <= 0 表示不检查该项。
func exceedsThreshold(usage float64, threshold int) bool {
	return threshold > 0 && int(usage) > threshold
}

func collectSystemStatus(usage *containerUsage) SystemStatus {
	now := time.Now()
	status := SystemStatus{SampledAt: now}

	// CPU / 内存优先按容器 cgroup 配额计算；容器口径不可用时回退宿主机口径，
	// 保持对未限制资源或被挂载为宿主 cgroup 的环境的兼容。
	if saturation, ok := usage.cpuSaturation(now); ok {
		status.CPUUsage = saturation
	} else if percents, err := cpu.Percent(0, false); err == nil && len(percents) > 0 {
		status.CPUUsage = percents[0]
	}
	if containerMemory, ok := usage.memoryUsage(); ok {
		status.MemoryUsage = containerMemory
	} else if memInfo, err := mem.VirtualMemory(); err == nil {
		status.MemoryUsage = memInfo.UsedPercent
	}

	diskInfo := GetDiskSpaceInfo()
	if diskInfo.Total > 0 {
		status.DiskUsage = diskInfo.UsedPercent
	}
	return status
}

// GetSystemStatus 获取当前系统状态
func GetSystemStatus() SystemStatus {
	return latestSystemStatus.Load().(SystemStatus)
}

// CheckSystemOverload 返回哨兵应据以熔断的过载项。监控关闭、采样缺失或已超过
// systemStatusMaxAge 时返回 nil。过载标记来自连续采样，阈值按当前配置复核，
// 运行时关闭或调高阈值后立即放行，不必等待下一次采样。
func CheckSystemOverload(status SystemStatus, config PerformanceMonitorConfig, now time.Time) *SystemOverload {
	if !config.Enabled || status.SampledAt.IsZero() || now.Sub(status.SampledAt) > systemStatusMaxAge {
		return nil
	}
	if status.CPUOverloaded && exceedsThreshold(status.CPUUsage, config.CPUThreshold) {
		return &SystemOverload{Code: "system_cpu_overloaded", Resource: "cpu", Usage: status.CPUUsage, Threshold: config.CPUThreshold}
	}
	if status.MemoryOverloaded && exceedsThreshold(status.MemoryUsage, config.MemoryThreshold) {
		return &SystemOverload{Code: "system_memory_overloaded", Resource: "memory", Usage: status.MemoryUsage, Threshold: config.MemoryThreshold}
	}
	if status.DiskOverloaded && exceedsThreshold(status.DiskUsage, config.DiskThreshold) {
		return &SystemOverload{Code: "system_disk_overloaded", Resource: "disk", Usage: status.DiskUsage, Threshold: config.DiskThreshold}
	}
	return nil
}