selfrelease
da9c8334d8
- SSE Keepalive Ping (15s心跳防止代理断连) - Timing HTTP 头 (X-Timing-Queue/Inference/Total-Ms) - Adapter Request-ID 传播到后端 - Session 清理日志回调 - Server 安全加固 (ReadHeaderTimeout/MaxHeaderBytes 防 slowloris) - Usage Tracker 数据保留清理 (retentionDays + 定期清理) - Config Reload 后 Adapter Registry 更新 (RegisterIfAbsent + RWMutex) - Rate Limiter 空闲 Bucket 清理 (30分钟过期) - Shutdown Drain 超时可配置 (ShutdownDrainSeconds) - Config 模型字段校验增强 (provider/endpoint/actual_model) - Auth 过期 Key 自动清理 (5分钟扫描) - Admin API Rate Limiting - Adapter Health Check 独立超时 (每个 adapter 3s) - TCP 连接阶段超时 (DialContext 5s + KeepAlive 30s) - 幂等键缓存、审计日志、Gzip 中间件、CORS Expose Headers - Backpressure 响应头、熔断器 Prometheus 指标 - 连接池优化、Trace-ID 全链路传播
123 lines
3.0 KiB
Go
123 lines
3.0 KiB
Go
package adapter
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"sync"
|
|
|
|
"github.com/edgeai/gateway/pkg/api"
|
|
)
|
|
|
|
// ModelAdapter is the interface that all inference engine adapters must implement.
|
|
type ModelAdapter interface {
|
|
// Name returns the adapter name (e.g., "ollama", "vllm").
|
|
Name() string
|
|
|
|
// ChatCompletion sends a non-streaming chat completion request.
|
|
ChatCompletion(ctx context.Context, req *ChatRequest) (*ChatResponse, error)
|
|
|
|
// ChatCompletionStream sends a streaming chat completion request.
|
|
ChatCompletionStream(ctx context.Context, req *ChatRequest) (<-chan StreamChunk, error)
|
|
|
|
// ListModels returns available models from the engine.
|
|
ListModels(ctx context.Context) ([]ModelInfo, error)
|
|
|
|
// HealthCheck checks if the engine is reachable.
|
|
HealthCheck(ctx context.Context) error
|
|
|
|
// Cancel cancels an in-progress request by request ID.
|
|
Cancel(requestID string) error
|
|
}
|
|
|
|
// ChatRequest is the internal request sent to an adapter.
|
|
type ChatRequest struct {
|
|
RequestID string
|
|
Model string // actual model name
|
|
Messages []api.Message
|
|
MaxTokens int
|
|
Temperature *float64
|
|
TopP *float64
|
|
Stream bool
|
|
CancelCh <-chan struct{}
|
|
}
|
|
|
|
// ChatResponse is the internal response from an adapter.
|
|
type ChatResponse struct {
|
|
Content string
|
|
FinishReason string
|
|
InputTokens int
|
|
OutputTokens int
|
|
ActualModel string
|
|
}
|
|
|
|
// StreamChunk represents a single chunk in a streaming response.
|
|
type StreamChunk struct {
|
|
Delta string
|
|
FinishReason string
|
|
InputTokens int
|
|
OutputTokens int
|
|
Error error
|
|
Done bool
|
|
}
|
|
|
|
// ModelInfo describes a model available in the engine.
|
|
type ModelInfo struct {
|
|
Name string
|
|
ContextWindow int
|
|
}
|
|
|
|
// Registry manages model adapters by provider name.
|
|
type Registry struct {
|
|
mu sync.RWMutex
|
|
adapters map[string]ModelAdapter
|
|
}
|
|
|
|
func NewRegistry() *Registry {
|
|
return &Registry{adapters: make(map[string]ModelAdapter)}
|
|
}
|
|
|
|
// Register 注册一个 adapter，如果同名 adapter 已存在则覆盖。
|
|
func (r *Registry) Register(name string, adapter ModelAdapter) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
r.adapters[name] = adapter
|
|
}
|
|
|
|
// RegisterIfAbsent 注册一个 adapter，仅当该 provider 尚未注册时才创建。
|
|
// 返回 true 表示新注册，false 表示已存在。
|
|
func (r *Registry) RegisterIfAbsent(name string, factory func() ModelAdapter) bool {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
if _, exists := r.adapters[name]; exists {
|
|
return false
|
|
}
|
|
r.adapters[name] = factory()
|
|
return true
|
|
}
|
|
|
|
// Get 返回指定名称的 adapter。
|
|
func (r *Registry) Get(name string) (ModelAdapter, error) {
|
|
r.mu.RLock()
|
|
defer r.mu.RUnlock()
|
|
a, ok := r.adapters[name]
|
|
if !ok {
|
|
return nil, fmt.Errorf("adapter not found: %s", name)
|
|
}
|
|
return a, nil
|
|
}
|
|
|
|
// Names 返回所有已注册 adapter 的名称列表。
|
|
func (r *Registry) Names() []string {
|
|
r.mu.RLock()
|
|
defer r.mu.RUnlock()
|
|
names := make([]string, 0, len(r.adapters))
|
|
for n := range r.adapters {
|
|
names = append(names, n)
|
|
}
|
|
return names
|
|
}
|
|
|
|
// Ensure io is imported for future use (streaming readers).
|
|
var _ = io.EOF
|