feat: 十轮网关优化 - 安全加固/可观测性/性能/可靠性
- SSE Keepalive Ping (15s心跳防止代理断连) - Timing HTTP 头 (X-Timing-Queue/Inference/Total-Ms) - Adapter Request-ID 传播到后端 - Session 清理日志回调 - Server 安全加固 (ReadHeaderTimeout/MaxHeaderBytes 防 slowloris) - Usage Tracker 数据保留清理 (retentionDays + 定期清理) - Config Reload 后 Adapter Registry 更新 (RegisterIfAbsent + RWMutex) - Rate Limiter 空闲 Bucket 清理 (30分钟过期) - Shutdown Drain 超时可配置 (ShutdownDrainSeconds) - Config 模型字段校验增强 (provider/endpoint/actual_model) - Auth 过期 Key 自动清理 (5分钟扫描) - Admin API Rate Limiting - Adapter Health Check 独立超时 (每个 adapter 3s) - TCP 连接阶段超时 (DialContext 5s + KeepAlive 30s) - 幂等键缓存、审计日志、Gzip 中间件、CORS Expose Headers - Backpressure 响应头、熔断器 Prometheus 指标 - 连接池优化、Trace-ID 全链路传播
This commit is contained in:
+42
-19
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"sync"
|
||||
|
||||
"github.com/edgeai/gateway/pkg/api"
|
||||
)
|
||||
@@ -31,33 +32,33 @@ type ModelAdapter interface {
|
||||
|
||||
// ChatRequest is the internal request sent to an adapter.
|
||||
type ChatRequest struct {
|
||||
RequestID string
|
||||
Model string // actual model name
|
||||
Messages []api.Message
|
||||
MaxTokens int
|
||||
Temperature *float64
|
||||
TopP *float64
|
||||
Stream bool
|
||||
CancelCh <-chan struct{}
|
||||
RequestID string
|
||||
Model string // actual model name
|
||||
Messages []api.Message
|
||||
MaxTokens int
|
||||
Temperature *float64
|
||||
TopP *float64
|
||||
Stream bool
|
||||
CancelCh <-chan struct{}
|
||||
}
|
||||
|
||||
// ChatResponse is the internal response from an adapter.
|
||||
type ChatResponse struct {
|
||||
Content string
|
||||
FinishReason string
|
||||
InputTokens int
|
||||
OutputTokens int
|
||||
ActualModel string
|
||||
Content string
|
||||
FinishReason string
|
||||
InputTokens int
|
||||
OutputTokens int
|
||||
ActualModel string
|
||||
}
|
||||
|
||||
// StreamChunk represents a single chunk in a streaming response.
|
||||
type StreamChunk struct {
|
||||
Delta string
|
||||
FinishReason string
|
||||
InputTokens int
|
||||
OutputTokens int
|
||||
Error error
|
||||
Done bool
|
||||
Delta string
|
||||
FinishReason string
|
||||
InputTokens int
|
||||
OutputTokens int
|
||||
Error error
|
||||
Done bool
|
||||
}
|
||||
|
||||
// ModelInfo describes a model available in the engine.
|
||||
@@ -68,6 +69,7 @@ type ModelInfo struct {
|
||||
|
||||
// Registry manages model adapters by provider name.
|
||||
type Registry struct {
|
||||
mu sync.RWMutex
|
||||
adapters map[string]ModelAdapter
|
||||
}
|
||||
|
||||
@@ -75,11 +77,29 @@ func NewRegistry() *Registry {
|
||||
return &Registry{adapters: make(map[string]ModelAdapter)}
|
||||
}
|
||||
|
||||
// Register 注册一个 adapter,如果同名 adapter 已存在则覆盖。
|
||||
func (r *Registry) Register(name string, adapter ModelAdapter) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.adapters[name] = adapter
|
||||
}
|
||||
|
||||
// RegisterIfAbsent 注册一个 adapter,仅当该 provider 尚未注册时才创建。
|
||||
// 返回 true 表示新注册,false 表示已存在。
|
||||
func (r *Registry) RegisterIfAbsent(name string, factory func() ModelAdapter) bool {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if _, exists := r.adapters[name]; exists {
|
||||
return false
|
||||
}
|
||||
r.adapters[name] = factory()
|
||||
return true
|
||||
}
|
||||
|
||||
// Get 返回指定名称的 adapter。
|
||||
func (r *Registry) Get(name string) (ModelAdapter, error) {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
a, ok := r.adapters[name]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("adapter not found: %s", name)
|
||||
@@ -87,7 +107,10 @@ func (r *Registry) Get(name string) (ModelAdapter, error) {
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// Names 返回所有已注册 adapter 的名称列表。
|
||||
func (r *Registry) Names() []string {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
names := make([]string, 0, len(r.adapters))
|
||||
for n := range r.adapters {
|
||||
names = append(names, n)
|
||||
|
||||
+37
-14
@@ -6,11 +6,30 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// newHTTPClient 创建带连接池优化的 HTTP 客户端,复用 TCP 连接以减少延迟。
|
||||
// timeout 为整体请求超时,connectTimeout 为 TCP 连接阶段超时。
|
||||
func newHTTPClient(timeout time.Duration) *http.Client {
|
||||
return &http.Client{
|
||||
Timeout: timeout,
|
||||
Transport: &http.Transport{
|
||||
MaxIdleConns: 100,
|
||||
MaxIdleConnsPerHost: 20,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
MaxConnsPerHost: 0, // 不限制
|
||||
DialContext: (&net.Dialer{
|
||||
Timeout: 5 * time.Second,
|
||||
KeepAlive: 30 * time.Second,
|
||||
}).DialContext,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// OllamaAdapter implements ModelAdapter for Ollama inference engine.
|
||||
type OllamaAdapter struct {
|
||||
endpoint string
|
||||
@@ -20,10 +39,8 @@ type OllamaAdapter struct {
|
||||
// NewOllamaAdapter creates a new Ollama adapter.
|
||||
func NewOllamaAdapter(endpoint string) *OllamaAdapter {
|
||||
return &OllamaAdapter{
|
||||
endpoint: strings.TrimRight(endpoint, "/"),
|
||||
httpClient: &http.Client{
|
||||
Timeout: 120 * time.Second,
|
||||
},
|
||||
endpoint: strings.TrimRight(endpoint, "/"),
|
||||
httpClient: newHTTPClient(120 * time.Second),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,20 +69,20 @@ type ollamaOptions struct {
|
||||
|
||||
// ollamaChatResponse is the Ollama /api/chat non-streaming response.
|
||||
type ollamaChatResponse struct {
|
||||
Model string `json:"model"`
|
||||
Message ollamaMsg `json:"message"`
|
||||
Done bool `json:"done"`
|
||||
PromptEvalCount int `json:"prompt_eval_count"`
|
||||
EvalCount int `json:"eval_count"`
|
||||
Model string `json:"model"`
|
||||
Message ollamaMsg `json:"message"`
|
||||
Done bool `json:"done"`
|
||||
PromptEvalCount int `json:"prompt_eval_count"`
|
||||
EvalCount int `json:"eval_count"`
|
||||
}
|
||||
|
||||
// ollamaChatStreamResponse is a single chunk in Ollama streaming response.
|
||||
type ollamaChatStreamResponse struct {
|
||||
Model string `json:"model"`
|
||||
Message ollamaMsg `json:"message"`
|
||||
Done bool `json:"done"`
|
||||
PromptEvalCount int `json:"prompt_eval_count,omitempty"`
|
||||
EvalCount int `json:"eval_count,omitempty"`
|
||||
Model string `json:"model"`
|
||||
Message ollamaMsg `json:"message"`
|
||||
Done bool `json:"done"`
|
||||
PromptEvalCount int `json:"prompt_eval_count,omitempty"`
|
||||
EvalCount int `json:"eval_count,omitempty"`
|
||||
}
|
||||
|
||||
func (a *OllamaAdapter) ChatCompletion(ctx context.Context, req *ChatRequest) (*ChatResponse, error) {
|
||||
@@ -81,6 +98,9 @@ func (a *OllamaAdapter) ChatCompletion(ctx context.Context, req *ChatRequest) (*
|
||||
return nil, fmt.Errorf("create ollama request: %w", err)
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
if req.RequestID != "" {
|
||||
httpReq.Header.Set("X-Request-ID", req.RequestID)
|
||||
}
|
||||
|
||||
resp, err := a.httpClient.Do(httpReq)
|
||||
if err != nil {
|
||||
@@ -120,6 +140,9 @@ func (a *OllamaAdapter) ChatCompletionStream(ctx context.Context, req *ChatReque
|
||||
return nil, fmt.Errorf("create ollama stream request: %w", err)
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
if req.RequestID != "" {
|
||||
httpReq.Header.Set("X-Request-ID", req.RequestID)
|
||||
}
|
||||
|
||||
resp, err := a.httpClient.Do(httpReq)
|
||||
if err != nil {
|
||||
|
||||
+17
-13
@@ -24,10 +24,8 @@ type VLLMAdapter struct {
|
||||
// NewVLLMAdapter creates a new vLLM adapter.
|
||||
func NewVLLMAdapter(endpoint string) *VLLMAdapter {
|
||||
return &VLLMAdapter{
|
||||
endpoint: strings.TrimRight(endpoint, "/"),
|
||||
httpClient: &http.Client{
|
||||
Timeout: 120 * time.Second,
|
||||
},
|
||||
endpoint: strings.TrimRight(endpoint, "/"),
|
||||
httpClient: newHTTPClient(120 * time.Second),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -37,12 +35,12 @@ func (a *VLLMAdapter) Name() string {
|
||||
|
||||
// vllmChatRequest is the OpenAI-compatible chat request for vLLM.
|
||||
type vllmChatRequest struct {
|
||||
Model string `json:"model"`
|
||||
Messages []vllmMsg `json:"messages"`
|
||||
Stream bool `json:"stream"`
|
||||
MaxTokens int `json:"max_tokens,omitempty"`
|
||||
Temperature *float64 `json:"temperature,omitempty"`
|
||||
TopP *float64 `json:"top_p,omitempty"`
|
||||
Model string `json:"model"`
|
||||
Messages []vllmMsg `json:"messages"`
|
||||
Stream bool `json:"stream"`
|
||||
MaxTokens int `json:"max_tokens,omitempty"`
|
||||
Temperature *float64 `json:"temperature,omitempty"`
|
||||
TopP *float64 `json:"top_p,omitempty"`
|
||||
}
|
||||
|
||||
type vllmMsg struct {
|
||||
@@ -55,9 +53,9 @@ type vllmChatResponse struct {
|
||||
ID string `json:"id"`
|
||||
Model string `json:"model"`
|
||||
Choices []struct {
|
||||
Index int `json:"index"`
|
||||
Index int `json:"index"`
|
||||
Message vllmMsg `json:"message"`
|
||||
FinishReason string `json:"finish_reason"`
|
||||
FinishReason string `json:"finish_reason"`
|
||||
} `json:"choices"`
|
||||
Usage struct {
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
@@ -71,7 +69,7 @@ type vllmStreamChunk struct {
|
||||
ID string `json:"id"`
|
||||
Model string `json:"model"`
|
||||
Choices []struct {
|
||||
Index int `json:"index"`
|
||||
Index int `json:"index"`
|
||||
Delta vllmMsg `json:"delta"`
|
||||
FinishReason *string `json:"finish_reason"`
|
||||
} `json:"choices"`
|
||||
@@ -95,6 +93,9 @@ func (a *VLLMAdapter) ChatCompletion(ctx context.Context, req *ChatRequest) (*Ch
|
||||
return nil, fmt.Errorf("create vllm request: %w", err)
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
if req.RequestID != "" {
|
||||
httpReq.Header.Set("X-Request-ID", req.RequestID)
|
||||
}
|
||||
|
||||
resp, err := a.httpClient.Do(httpReq)
|
||||
if err != nil {
|
||||
@@ -145,6 +146,9 @@ func (a *VLLMAdapter) ChatCompletionStream(ctx context.Context, req *ChatRequest
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
httpReq.Header.Set("Accept", "text/event-stream")
|
||||
if req.RequestID != "" {
|
||||
httpReq.Header.Set("X-Request-ID", req.RequestID)
|
||||
}
|
||||
|
||||
resp, err := a.httpClient.Do(httpReq)
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user