package chaos import ( "context" "fmt" "net/http" "net/http/httptest" "os" "strings" "sync" "testing" "time" "github.com/edgeai/gateway/internal/auth" "github.com/edgeai/gateway/internal/config" "github.com/edgeai/gateway/internal/observability" "github.com/edgeai/gateway/internal/server" "github.com/edgeai/gateway/internal/task" ) var chaosServer *httptest.Server var chaosClient *http.Client func TestMain(m *testing.M) { cfg := &config.Config{ Server: config.ServerConfig{Host: "127.0.0.1", Port: 0, MaxRequestBodyMB: 5}, Auth: config.AuthConfig{Enabled: true, Methods: []string{"api_key"}}, Scheduler: config.SchedulerConfig{ MaxRunningTasks: 4, MaxQueuedTasks: 50, }, Timeouts: config.TimeoutConfig{ DefaultQueueMs: 2000, DefaultInferenceMs: 10000, DefaultTotalMs: 15000, }, Context: config.ContextConfig{SafetyMarginRatio: 0.1, DefaultPolicy: "recent_only"}, Models: map[string]config.ModelConfig{ "test-chat": { Provider: "ollama", ActualModel: "qwen2.5:0.5b", Endpoint: "http://127.0.0.1:11434", ContextWindow: 4096, MaxOutputTokens: 256, CancelSupported: true, }, }, Observability: config.ObservabilityConfig{MetricsPath: "/metrics", LogLevel: "error"}, Storage: config.StorageConfig{ SessionDB: "sqlite:///tmp/edgeai-chaos/sessions.db", TaskState: "sqlite:///tmp/edgeai-chaos/tasks.db", }, } os.MkdirAll("/tmp/edgeai-chaos", 0755) defer os.RemoveAll("/tmp/edgeai-chaos") logger := observability.NewLogger(observability.LevelError, os.Stderr, "metadata_only") srv, err := server.New(cfg, logger) if err != nil { panic("failed to create chaos test server: " + err.Error()) } // Add a test API key for authenticated tests srv.Authenticator().AddKey("test-key", &auth.AppIdentity{ AppID: "test-app", TenantID: "test-tenant", Name: "test", IsAdmin: true, }) chaosServer = httptest.NewServer(srv.HTTPSrv.Handler) defer chaosServer.Close() chaosClient = &http.Client{ Transport: &http.Transport{ MaxIdleConns: 200, MaxIdleConnsPerHost: 200, IdleConnTimeout: 30 * time.Second, }, Timeout: 5 * time.Second, } m.Run() } // CHAOS-001: Server survives rapid connect/disconnect func TestChaos001_RapidConnectDisconnect(t *testing.T) { var wg sync.WaitGroup for i := 0; i < 50; i++ { wg.Add(1) go func() { defer wg.Done() ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) defer cancel() req, _ := http.NewRequestWithContext(ctx, "GET", chaosServer.URL+"/health", nil) resp, err := chaosClient.Do(req) if err == nil { resp.Body.Close() } }() } wg.Wait() // Verify server still responds resp, err := chaosClient.Get(chaosServer.URL + "/health") if err != nil { t.Fatal(err) } if resp.StatusCode != 200 { t.Errorf("server unhealthy after rapid connect/disconnect: %d", resp.StatusCode) } resp.Body.Close() } // CHAOS-002: Server handles concurrent load without crash func TestChaos002_ConcurrentLoad(t *testing.T) { var wg sync.WaitGroup errors := make(chan error, 100) for i := 0; i < 30; i++ { wg.Add(1) go func(idx int) { defer wg.Done() resp, err := chaosClient.Get(chaosServer.URL + "/health") if err != nil { errors <- err return } if resp.StatusCode != 200 { errors <- &chaosError{idx, resp.StatusCode} } resp.Body.Close() }(i) } wg.Wait() close(errors) errorCount := 0 for err := range errors { errorCount++ t.Logf("error: %v", err) } if errorCount > 0 { t.Errorf("%d errors out of 100 requests", errorCount) } } // CHAOS-003: Task state machine handles invalid transitions gracefully func TestChaos003_InvalidTransitions(t *testing.T) { // Try many invalid transitions invalidTransitions := []struct { from task.TaskState to task.TaskState }{ {task.StateQueued, task.StateCompleted}, {task.StateQueued, task.StateStreaming}, {task.StateCompleted, task.StateRunning}, {task.StateCompleted, task.StateFailed}, {task.StateFailed, task.StateCompleted}, {task.StateCancelled, task.StateRunning}, } for _, tc := range invalidTransitions { tk2 := task.NewTask("chaos-t", "req-t", "app", "tenant", "model", task.PriorityNormal, false) tk2.State = tc.from err := tk2.Transition(tc.to) if err == nil { t.Errorf("expected error for %s -> %s", tc.from, tc.to) } } } // CHAOS-004: Double cancel is safe func TestChaos004_DoubleCancel(t *testing.T) { tk := task.NewTask("chaos-2", "req-2", "app", "tenant", "model", task.PriorityNormal, false) tk.Cancel("first") err := tk.Cancel("second") if err == nil { t.Error("expected error on double cancel") } if tk.GetState() != task.StateCancelled { t.Errorf("expected CANCELLED, got %s", tk.GetState()) } } // CHAOS-005: Server survives cancelled client requests func TestChaos005_CancelledClientRequests(t *testing.T) { var wg sync.WaitGroup for i := 0; i < 20; i++ { wg.Add(1) go func() { defer wg.Done() ctx, cancel := context.WithCancel(context.Background()) go func() { time.Sleep(5 * time.Millisecond) cancel() }() req, _ := http.NewRequestWithContext(ctx, "GET", chaosServer.URL+"/health", nil) resp, err := chaosClient.Do(req) if err == nil { resp.Body.Close() } }() } wg.Wait() // Server should still be healthy resp, err := chaosClient.Get(chaosServer.URL + "/health") if err != nil { t.Fatal(err) } resp.Body.Close() if resp.StatusCode != 200 { t.Error("server not healthy after cancelled requests") } } // CHAOS-006: Sustained load for 3 seconds func TestChaos006_SustainedLoad(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() count := 0 for { select { case <-ctx.Done(): t.Logf("completed %d requests in 3s", count) return default: resp, err := chaosClient.Get(chaosServer.URL + "/health") if err == nil { resp.Body.Close() } count++ if count%50 == 0 { time.Sleep(20 * time.Millisecond) } } } } // CHAOS-007: Malformed JSON doesn't crash server func TestChaos007_MalformedJSON(t *testing.T) { malformed := []string{ "{", "}", "{\"model\":}", "{\"model\":\"test\"}", "null", "[]", "\"string\"", "", "{\"messages\":[{\"role\":\"user\",\"content\":null}]}", } for _, body := range malformed { resp, err := chaosClient.Post(chaosServer.URL+"/v1/chat/completions", "application/json", strings.NewReader(body)) if err != nil { t.Logf("request error for %q: %v", body, err) continue } resp.Body.Close() if resp.StatusCode == 500 { t.Errorf("server returned 500 for malformed JSON: %q", body) } } // Server should still be healthy resp, err := chaosClient.Get(chaosServer.URL + "/health") if err != nil { t.Fatal(err) } resp.Body.Close() } type chaosError struct { idx int status int } func (e *chaosError) Error() string { return fmt.Sprintf("request %d: status %d", e.idx, e.status) }