feat: 添加过载保护、背压和熔断机制

internal/scheduler/backpressure.go

@@ -0,0 +1,92 @@
+package scheduler
+
+import (
+	"sync/atomic"
+)
+
+// BackpressureManager tracks system load and provides admission control.
+// Level 1 (70%): accept but log warning
+// Level 2 (85%): accept only high priority (P0-P1)
+// Level 3 (95%): reject all new requests
+type BackpressureManager struct {
+	maxRunning  int64
+	maxQueued   int64
+	level1Threshold float64
+	level2Threshold float64
+	level3Threshold float64
+
+	runningCount int64 // atomic
+	queuedCount  int64 // atomic
+}
+
+// NewBackpressureManager creates a new backpressure manager.
+func NewBackpressureManager(maxRunning, maxQueued int, l1, l2, l3 float64) *BackpressureManager {
+	return &BackpressureManager{
+		maxRunning:      int64(maxRunning),
+		maxQueued:       int64(maxQueued),
+		level1Threshold: l1,
+		level2Threshold: l2,
+		level3Threshold: l3,
+	}
+}
+
+// Update sets the current running and queued counts.
+func (bp *BackpressureManager) Update(running, queued int) {
+	atomic.StoreInt64(&bp.runningCount, int64(running))
+	atomic.StoreInt64(&bp.queuedCount, int64(queued))
+}
+
+// LoadRatio returns the current system load ratio (0.0 to 1.0).
+func (bp *BackpressureManager) LoadRatio() float64 {
+	running := atomic.LoadInt64(&bp.runningCount)
+	queued := atomic.LoadInt64(&bp.queuedCount)
+	total := running + queued
+	capacity := bp.maxRunning + bp.maxQueued
+	if capacity == 0 {
+		return 0
+	}
+	return float64(total) / float64(capacity)
+}
+
+// Level returns the current backpressure level (0=normal, 1=warning, 2=restricted, 3=reject).
+func (bp *BackpressureManager) Level() int {
+	load := bp.LoadRatio()
+	if load >= bp.level3Threshold {
+		return 3
+	}
+	if load >= bp.level2Threshold {
+		return 2
+	}
+	if load >= bp.level1Threshold {
+		return 1
+	}
+	return 0
+}
+
+// ShouldAccept decides whether to accept a request based on priority and load.
+// priority: 0=P0(highest) to 4=P4(lowest)
+func (bp *BackpressureManager) ShouldAccept(priority int) bool {
+	level := bp.Level()
+	switch level {
+	case 0, 1:
+		return true
+	case 2:
+		// Only accept P0 and P1
+		return priority <= 1
+	case 3:
+		return false
+	}
+	return true
+}
+
+// RejectReason returns a human-readable reason if the request should be rejected.
+func (bp *BackpressureManager) RejectReason(priority int) string {
+	level := bp.Level()
+	if level == 3 {
+		return "system overloaded, please retry later"
+	}
+	if level == 2 && priority > 1 {
+		return "backpressure active, only high-priority requests accepted"
+	}
+	return ""
+}

internal/scheduler/circuit_breaker.go

@@ -0,0 +1,145 @@
+package scheduler
+
+import (
+	"sync"
+	"time"
+)
+
+// CircuitBreaker implements a sliding-window circuit breaker for adapter health.
+type CircuitBreaker struct {
+	mu                sync.Mutex
+	errorRateThreshold float64
+	minRequests       int
+	windowSeconds     int
+	openDuration      time.Duration
+	halfOpenMax       int
+
+	// sliding window state
+	requests  []time.Time
+	errors   []time.Time
+
+	// breaker state
+	state       breakerState
+	openedAt    time.Time
+	halfOpenCount int
+}
+
+type breakerState int
+
+const (
+	breakerClosed   breakerState = iota
+	breakerOpen
+	breakerHalfOpen
+)
+
+// NewCircuitBreaker creates a new circuit breaker from config.
+func NewCircuitBreaker(errorRate float64, minRequests, windowSec, openSec, halfOpenMax int) *CircuitBreaker {
+	return &CircuitBreaker{
+		errorRateThreshold: errorRate,
+		minRequests:        minRequests,
+		windowSeconds:      windowSec,
+		openDuration:       time.Duration(openSec) * time.Second,
+		halfOpenMax:        halfOpenMax,
+		state:              breakerClosed,
+	}
+}
+
+// AllowRequest checks if a request should be allowed through.
+func (cb *CircuitBreaker) AllowRequest() bool {
+	cb.mu.Lock()
+	defer cb.mu.Unlock()
+
+	now := time.Now()
+	cb.prune(now)
+
+	switch cb.state {
+	case breakerClosed:
+		return true
+	case breakerOpen:
+		if now.Sub(cb.openedAt) >= cb.openDuration {
+			cb.state = breakerHalfOpen
+			cb.halfOpenCount = 0
+			return true
+		}
+		return false
+	case breakerHalfOpen:
+		if cb.halfOpenCount < cb.halfOpenMax {
+			cb.halfOpenCount++
+			return true
+		}
+		return false
+	}
+	return true
+}
+
+// RecordSuccess records a successful request.
+func (cb *CircuitBreaker) RecordSuccess() {
+	cb.mu.Lock()
+	defer cb.mu.Unlock()
+
+	now := time.Now()
+	cb.requests = append(cb.requests, now)
+
+	if cb.state == breakerHalfOpen {
+		cb.state = breakerClosed
+		cb.requests = nil
+		cb.errors = nil
+	}
+}
+
+// RecordError records a failed request and may trip the breaker.
+func (cb *CircuitBreaker) RecordError() {
+	cb.mu.Lock()
+	defer cb.mu.Unlock()
+
+	now := time.Now()
+	cb.requests = append(cb.requests, now)
+	cb.errors = append(cb.errors, now)
+
+	if cb.state == breakerHalfOpen {
+		cb.state = breakerOpen
+		cb.openedAt = now
+		return
+	}
+
+	if cb.state == breakerClosed && len(cb.requests) >= cb.minRequests {
+		errorRate := float64(len(cb.errors)) / float64(len(cb.requests))
+		if errorRate >= cb.errorRateThreshold {
+			cb.state = breakerOpen
+			cb.openedAt = now
+		}
+	}
+}
+
+// State returns the current breaker state name.
+func (cb *CircuitBreaker) State() string {
+	cb.mu.Lock()
+	defer cb.mu.Unlock()
+	switch cb.state {
+	case breakerClosed:
+		return "closed"
+	case breakerOpen:
+		return "open"
+	case breakerHalfOpen:
+		return "half_open"
+	}
+	return "unknown"
+}
+
+// prune removes entries outside the sliding window.
+func (cb *CircuitBreaker) prune(now time.Time) {
+	cutoff := now.Add(-time.Duration(cb.windowSeconds) * time.Second)
+	cb.requests = pruneBefore(cb.requests, cutoff)
+	cb.errors = pruneBefore(cb.errors, cutoff)
+}
+
+func pruneBefore(times []time.Time, cutoff time.Time) []time.Time {
+	idx := 0
+	for idx < len(times) && times[idx].Before(cutoff) {
+		idx++
+	}
+	if idx > 0 {
+		times = times[idx:]
+	}
+	return times
+}

internal/scheduler/scheduler.go

@@ -14,15 +14,15 @@ import (
 
 // Scheduler manages task queuing and execution with priority-based scheduling.
 type Scheduler struct {
-	mu              sync.Mutex
-	queue           *priorityQueue
-	running         map[string]*task.Task
-	maxRunning      int
-	maxQueued       int
-	notifyCh        chan struct{}
-	logger          *observability.Logger
-	ctx             context.Context
-	cancel          context.CancelFunc
+	mu         sync.Mutex
+	queue      *priorityQueue
+	running    map[string]*task.Task
+	maxRunning int
+	maxQueued  int
+	notifyCh   chan struct{}
+	logger     *observability.Logger
+	ctx        context.Context
+	cancel     context.CancelFunc
 }
 
 // NewScheduler creates a new scheduler.
@@ -114,6 +114,22 @@ func (s *Scheduler) RunningCount() int {
 	return len(s.running)
 }
 
+// GetTask returns a running task by ID, or a queued task by ID.
+func (s *Scheduler) GetTask(taskID string) (*task.Task, bool) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if t, ok := s.running[taskID]; ok {
+		return t, true
+	}
+	for i := 0; i < s.queue.Len(); i++ {
+		t := (*s.queue)[i]
+		if t.ID == taskID {
+			return t, true
+		}
+	}
+	return nil, false
+}
+
 // Stop shuts down the scheduler.
 func (s *Scheduler) Stop() {
 	s.cancel()