feat: 十轮网关优化 - 安全加固/可观测性/性能/可靠性
- SSE Keepalive Ping (15s心跳防止代理断连) - Timing HTTP 头 (X-Timing-Queue/Inference/Total-Ms) - Adapter Request-ID 传播到后端 - Session 清理日志回调 - Server 安全加固 (ReadHeaderTimeout/MaxHeaderBytes 防 slowloris) - Usage Tracker 数据保留清理 (retentionDays + 定期清理) - Config Reload 后 Adapter Registry 更新 (RegisterIfAbsent + RWMutex) - Rate Limiter 空闲 Bucket 清理 (30分钟过期) - Shutdown Drain 超时可配置 (ShutdownDrainSeconds) - Config 模型字段校验增强 (provider/endpoint/actual_model) - Auth 过期 Key 自动清理 (5分钟扫描) - Admin API Rate Limiting - Adapter Health Check 独立超时 (每个 adapter 3s) - TCP 连接阶段超时 (DialContext 5s + KeepAlive 30s) - 幂等键缓存、审计日志、Gzip 中间件、CORS Expose Headers - Backpressure 响应头、熔断器 Prometheus 指标 - 连接池优化、Trace-ID 全链路传播
This commit is contained in:
+85
-50
@@ -4,6 +4,8 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/edgeai/gateway/internal/adapter"
|
||||
"github.com/edgeai/gateway/pkg/api"
|
||||
@@ -47,77 +49,110 @@ func (s *SSEWriter) WriteDone() {
|
||||
s.flusher.Flush()
|
||||
}
|
||||
|
||||
// WritePing 发送 SSE 注释行作为心跳,防止代理/负载均衡器因空闲超时断开连接。
|
||||
// 注释行以冒号开头,客户端会忽略,不影响事件流。
|
||||
func (s *SSEWriter) WritePing() {
|
||||
fmt.Fprintf(s.w, ": keepalive\n\n")
|
||||
s.flusher.Flush()
|
||||
}
|
||||
|
||||
// StreamChatCompletion streams chunks from an adapter to the client in OpenAI SSE format.
|
||||
func StreamChatCompletion(sse *SSEWriter, ch <-chan adapter.StreamChunk, requestID, taskID, model string) (int, int, error) {
|
||||
// Returns input tokens, output tokens, full content string, and error.
|
||||
// onFirstToken is called when the first content chunk is received (may be nil).
|
||||
// 每 15 秒发送一次 keepalive ping,防止连接被代理断开。
|
||||
func StreamChatCompletion(sse *SSEWriter, ch <-chan adapter.StreamChunk, requestID, taskID, model string, onFirstToken func()) (int, int, string, error) {
|
||||
inputTokens := 0
|
||||
outputTokens := 0
|
||||
var contentBuilder strings.Builder
|
||||
firstTokenSent := false
|
||||
|
||||
for chunk := range ch {
|
||||
if chunk.Error != nil {
|
||||
return inputTokens, outputTokens, chunk.Error
|
||||
}
|
||||
// 启动 keepalive 心跳定时器
|
||||
keepalive := time.NewTicker(15 * time.Second)
|
||||
defer keepalive.Stop()
|
||||
|
||||
if chunk.Done {
|
||||
if chunk.InputTokens > 0 {
|
||||
inputTokens = chunk.InputTokens
|
||||
for {
|
||||
select {
|
||||
case chunk, ok := <-ch:
|
||||
if !ok {
|
||||
return inputTokens, outputTokens, contentBuilder.String(), nil
|
||||
}
|
||||
if chunk.OutputTokens > 0 {
|
||||
outputTokens = chunk.OutputTokens
|
||||
if chunk.Error != nil {
|
||||
return inputTokens, outputTokens, contentBuilder.String(), chunk.Error
|
||||
}
|
||||
|
||||
// Write final chunk with finish_reason
|
||||
sseChunk := map[string]any{
|
||||
"id": requestID,
|
||||
"object": "chat.completion.chunk",
|
||||
"model": model,
|
||||
"choices": []map[string]any{
|
||||
{
|
||||
"index": 0,
|
||||
"delta": map[string]any{},
|
||||
"finish_reason": chunk.FinishReason,
|
||||
if chunk.Done {
|
||||
if chunk.InputTokens > 0 {
|
||||
inputTokens = chunk.InputTokens
|
||||
}
|
||||
if chunk.OutputTokens > 0 {
|
||||
outputTokens = chunk.OutputTokens
|
||||
}
|
||||
|
||||
// Write final chunk with finish_reason
|
||||
sseChunk := map[string]any{
|
||||
"id": requestID,
|
||||
"object": "chat.completion.chunk",
|
||||
"model": model,
|
||||
"choices": []map[string]any{
|
||||
{
|
||||
"index": 0,
|
||||
"delta": map[string]any{},
|
||||
"finish_reason": chunk.FinishReason,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
if inputTokens > 0 || outputTokens > 0 {
|
||||
sseChunk["usage"] = map[string]int{
|
||||
"input_tokens": inputTokens,
|
||||
"output_tokens": outputTokens,
|
||||
"total_tokens": inputTokens + outputTokens,
|
||||
}
|
||||
}
|
||||
sse.WriteChunk(sseChunk)
|
||||
sse.WriteDone()
|
||||
return inputTokens, outputTokens, contentBuilder.String(), nil
|
||||
}
|
||||
if inputTokens > 0 || outputTokens > 0 {
|
||||
sseChunk["usage"] = map[string]int{
|
||||
"input_tokens": inputTokens,
|
||||
"output_tokens": outputTokens,
|
||||
"total_tokens": inputTokens + outputTokens,
|
||||
|
||||
// 首 token 延迟回调
|
||||
if !firstTokenSent && chunk.Delta != "" {
|
||||
firstTokenSent = true
|
||||
if onFirstToken != nil {
|
||||
onFirstToken()
|
||||
}
|
||||
}
|
||||
sse.WriteChunk(sseChunk)
|
||||
sse.WriteDone()
|
||||
return inputTokens, outputTokens, nil
|
||||
}
|
||||
|
||||
// Write content delta
|
||||
sseChunk := map[string]any{
|
||||
"id": requestID,
|
||||
"object": "chat.completion.chunk",
|
||||
"model": model,
|
||||
"choices": []map[string]any{
|
||||
{
|
||||
"index": 0,
|
||||
"delta": map[string]any{
|
||||
"content": chunk.Delta,
|
||||
// Write content delta
|
||||
sseChunk := map[string]any{
|
||||
"id": requestID,
|
||||
"object": "chat.completion.chunk",
|
||||
"model": model,
|
||||
"choices": []map[string]any{
|
||||
{
|
||||
"index": 0,
|
||||
"delta": map[string]any{
|
||||
"content": chunk.Delta,
|
||||
},
|
||||
"finish_reason": nil,
|
||||
},
|
||||
"finish_reason": nil,
|
||||
},
|
||||
},
|
||||
}
|
||||
sse.WriteChunk(sseChunk)
|
||||
}
|
||||
}
|
||||
sse.WriteChunk(sseChunk)
|
||||
contentBuilder.WriteString(chunk.Delta)
|
||||
|
||||
return inputTokens, outputTokens, nil
|
||||
case <-keepalive.C:
|
||||
// 发送心跳,保持连接活跃
|
||||
sse.WritePing()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// BuildChatResponse creates a non-streaming ChatResponse from adapter result.
|
||||
func BuildChatResponse(requestID, taskID, logicalModel string, resp *adapter.ChatResponse) api.ChatResponse {
|
||||
return api.ChatResponse{
|
||||
RequestID: requestID,
|
||||
TaskID: taskID,
|
||||
Status: "completed",
|
||||
Model: logicalModel,
|
||||
RequestID: requestID,
|
||||
TaskID: taskID,
|
||||
Status: "completed",
|
||||
Model: logicalModel,
|
||||
Choices: []api.Choice{
|
||||
{
|
||||
Index: 0,
|
||||
|
||||
Reference in New Issue
Block a user