freedakgmail
283 lines
6.8 KiB
Go
283 lines
6.8 KiB
Go
package chaos
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/edgeai/gateway/internal/auth"
|
|
"github.com/edgeai/gateway/internal/config"
|
|
"github.com/edgeai/gateway/internal/observability"
|
|
"github.com/edgeai/gateway/internal/server"
|
|
"github.com/edgeai/gateway/internal/task"
|
|
)
|
|
|
|
var chaosServer *httptest.Server
|
|
var chaosClient *http.Client
|
|
|
|
func TestMain(m *testing.M) {
|
|
cfg := &config.Config{
|
|
Server: config.ServerConfig{Host: "127.0.0.1", Port: 0, MaxRequestBodyMB: 5},
|
|
Auth: config.AuthConfig{Enabled: true, Methods: []string{"api_key"}},
|
|
Scheduler: config.SchedulerConfig{
|
|
MaxRunningTasks: 4, MaxQueuedTasks: 50,
|
|
},
|
|
Timeouts: config.TimeoutConfig{
|
|
DefaultQueueMs: 2000, DefaultInferenceMs: 10000, DefaultTotalMs: 15000,
|
|
},
|
|
Context: config.ContextConfig{SafetyMarginRatio: 0.1, DefaultPolicy: "recent_only"},
|
|
Models: map[string]config.ModelConfig{
|
|
"test-chat": {
|
|
Provider: "ollama", ActualModel: "qwen2.5:0.5b",
|
|
Endpoint: "http://127.0.0.1:11434", ContextWindow: 4096,
|
|
MaxOutputTokens: 256, CancelSupported: true,
|
|
},
|
|
},
|
|
Observability: config.ObservabilityConfig{MetricsPath: "/metrics", LogLevel: "error"},
|
|
Storage: config.StorageConfig{
|
|
SessionDB: "sqlite:///tmp/edgeai-chaos/sessions.db",
|
|
TaskState: "sqlite:///tmp/edgeai-chaos/tasks.db",
|
|
},
|
|
}
|
|
|
|
os.MkdirAll("/tmp/edgeai-chaos", 0755)
|
|
defer os.RemoveAll("/tmp/edgeai-chaos")
|
|
|
|
logger := observability.NewLogger(observability.LevelError, os.Stderr, "metadata_only")
|
|
srv, err := server.New(cfg, logger)
|
|
if err != nil {
|
|
panic("failed to create chaos test server: " + err.Error())
|
|
}
|
|
|
|
// Add a test API key for authenticated tests
|
|
srv.Authenticator().AddKey("test-key", &auth.AppIdentity{
|
|
AppID: "test-app",
|
|
TenantID: "test-tenant",
|
|
Name: "test",
|
|
IsAdmin: true,
|
|
})
|
|
|
|
chaosServer = httptest.NewServer(srv.HTTPSrv.Handler)
|
|
defer chaosServer.Close()
|
|
|
|
chaosClient = &http.Client{
|
|
Transport: &http.Transport{
|
|
MaxIdleConns: 200,
|
|
MaxIdleConnsPerHost: 200,
|
|
IdleConnTimeout: 30 * time.Second,
|
|
},
|
|
Timeout: 5 * time.Second,
|
|
}
|
|
|
|
m.Run()
|
|
}
|
|
|
|
// CHAOS-001: Server survives rapid connect/disconnect
|
|
func TestChaos001_RapidConnectDisconnect(t *testing.T) {
|
|
var wg sync.WaitGroup
|
|
for i := 0; i < 50; i++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond)
|
|
defer cancel()
|
|
req, _ := http.NewRequestWithContext(ctx, "GET", chaosServer.URL+"/health", nil)
|
|
resp, err := chaosClient.Do(req)
|
|
if err == nil {
|
|
resp.Body.Close()
|
|
}
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
|
|
// Verify server still responds
|
|
resp, err := chaosClient.Get(chaosServer.URL + "/health")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if resp.StatusCode != 200 {
|
|
t.Errorf("server unhealthy after rapid connect/disconnect: %d", resp.StatusCode)
|
|
}
|
|
resp.Body.Close()
|
|
}
|
|
|
|
// CHAOS-002: Server handles concurrent load without crash
|
|
func TestChaos002_ConcurrentLoad(t *testing.T) {
|
|
var wg sync.WaitGroup
|
|
errors := make(chan error, 100)
|
|
|
|
for i := 0; i < 30; i++ {
|
|
wg.Add(1)
|
|
go func(idx int) {
|
|
defer wg.Done()
|
|
resp, err := chaosClient.Get(chaosServer.URL + "/health")
|
|
if err != nil {
|
|
errors <- err
|
|
return
|
|
}
|
|
if resp.StatusCode != 200 {
|
|
errors <- &chaosError{idx, resp.StatusCode}
|
|
}
|
|
resp.Body.Close()
|
|
}(i)
|
|
}
|
|
|
|
wg.Wait()
|
|
close(errors)
|
|
|
|
errorCount := 0
|
|
for err := range errors {
|
|
errorCount++
|
|
t.Logf("error: %v", err)
|
|
}
|
|
if errorCount > 0 {
|
|
t.Errorf("%d errors out of 100 requests", errorCount)
|
|
}
|
|
}
|
|
|
|
// CHAOS-003: Task state machine handles invalid transitions gracefully
|
|
func TestChaos003_InvalidTransitions(t *testing.T) {
|
|
// Try many invalid transitions
|
|
invalidTransitions := []struct {
|
|
from task.TaskState
|
|
to task.TaskState
|
|
}{
|
|
{task.StateQueued, task.StateCompleted},
|
|
{task.StateQueued, task.StateStreaming},
|
|
{task.StateCompleted, task.StateRunning},
|
|
{task.StateCompleted, task.StateFailed},
|
|
{task.StateFailed, task.StateCompleted},
|
|
{task.StateCancelled, task.StateRunning},
|
|
}
|
|
|
|
for _, tc := range invalidTransitions {
|
|
tk2 := task.NewTask("chaos-t", "req-t", "app", "tenant", "model", task.PriorityNormal, false)
|
|
tk2.State = tc.from
|
|
err := tk2.Transition(tc.to)
|
|
if err == nil {
|
|
t.Errorf("expected error for %s -> %s", tc.from, tc.to)
|
|
}
|
|
}
|
|
}
|
|
|
|
// CHAOS-004: Double cancel is safe
|
|
func TestChaos004_DoubleCancel(t *testing.T) {
|
|
tk := task.NewTask("chaos-2", "req-2", "app", "tenant", "model", task.PriorityNormal, false)
|
|
|
|
tk.Cancel("first")
|
|
err := tk.Cancel("second")
|
|
if err == nil {
|
|
t.Error("expected error on double cancel")
|
|
}
|
|
if tk.GetState() != task.StateCancelled {
|
|
t.Errorf("expected CANCELLED, got %s", tk.GetState())
|
|
}
|
|
}
|
|
|
|
// CHAOS-005: Server survives cancelled client requests
|
|
func TestChaos005_CancelledClientRequests(t *testing.T) {
|
|
var wg sync.WaitGroup
|
|
for i := 0; i < 20; i++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
go func() {
|
|
time.Sleep(5 * time.Millisecond)
|
|
cancel()
|
|
}()
|
|
req, _ := http.NewRequestWithContext(ctx, "GET", chaosServer.URL+"/health", nil)
|
|
resp, err := chaosClient.Do(req)
|
|
if err == nil {
|
|
resp.Body.Close()
|
|
}
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
|
|
// Server should still be healthy
|
|
resp, err := chaosClient.Get(chaosServer.URL + "/health")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
resp.Body.Close()
|
|
if resp.StatusCode != 200 {
|
|
t.Error("server not healthy after cancelled requests")
|
|
}
|
|
}
|
|
|
|
// CHAOS-006: Sustained load for 3 seconds
|
|
func TestChaos006_SustainedLoad(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
|
defer cancel()
|
|
|
|
count := 0
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
t.Logf("completed %d requests in 3s", count)
|
|
return
|
|
default:
|
|
resp, err := chaosClient.Get(chaosServer.URL + "/health")
|
|
if err == nil {
|
|
resp.Body.Close()
|
|
}
|
|
count++
|
|
if count%50 == 0 {
|
|
time.Sleep(20 * time.Millisecond)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// CHAOS-007: Malformed JSON doesn't crash server
|
|
func TestChaos007_MalformedJSON(t *testing.T) {
|
|
malformed := []string{
|
|
"{",
|
|
"}",
|
|
"{\"model\":}",
|
|
"{\"model\":\"test\"}",
|
|
"null",
|
|
"[]",
|
|
"\"string\"",
|
|
"",
|
|
"{\"messages\":[{\"role\":\"user\",\"content\":null}]}",
|
|
}
|
|
|
|
for _, body := range malformed {
|
|
resp, err := chaosClient.Post(chaosServer.URL+"/v1/chat/completions",
|
|
"application/json",
|
|
strings.NewReader(body))
|
|
if err != nil {
|
|
t.Logf("request error for %q: %v", body, err)
|
|
continue
|
|
}
|
|
resp.Body.Close()
|
|
if resp.StatusCode == 500 {
|
|
t.Errorf("server returned 500 for malformed JSON: %q", body)
|
|
}
|
|
}
|
|
|
|
// Server should still be healthy
|
|
resp, err := chaosClient.Get(chaosServer.URL + "/health")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
resp.Body.Close()
|
|
}
|
|
|
|
type chaosError struct {
|
|
idx int
|
|
status int
|
|
}
|
|
|
|
func (e *chaosError) Error() string {
|
|
return fmt.Sprintf("request %d: status %d", e.idx, e.status)
|
|
}
|