package qaopsmirror import ( "bytes" "context" "encoding/json" "errors" "fmt" "github.com/jackc/pgx/v5/pgxpool" "io" "knowledge-governance/internal/dobotaiops" pg "knowledge-governance/internal/platform/postgres" "knowledge-governance/internal/qaopsai" "net/http" "net/http/httptest" "os" "path/filepath" "strings" "sync" "sync/atomic" "testing" "time" ) func TestAIBatchManualRetryKeepsCumulativeAttemptsAndFixedDenominator(t *testing.T) { var attempts atomic.Int32 m := newAIMatrix(t, "fixture-ai-retry", func(w http.ResponseWriter, r *http.Request) { if attempts.Add(1) <= 3 { w.WriteHeader(http.StatusTooManyRequests) return } writeAICompletion(w, `{"question_chinese":"MG400 如何连接?"}`) }) ctx := context.Background() batch := m.batch(t, "r") advance := func() { t.Helper() if _, err := m.pool.Exec(ctx, `UPDATE qa_ops_ai_runtime SET next_request_at=clock_timestamp()-interval '1 second' WHERE tenant_key=$1`, m.tenant); err != nil { t.Fatal(err) } if _, err := m.pool.Exec(ctx, `UPDATE qa_ops_ai_work SET available_at=clock_timestamp()-interval '1 second' WHERE tenant_key=$1`, m.tenant); err != nil { t.Fatal(err) } } for i := 0; i < 3; i++ { advance() if done, err := m.ai.ProcessOne(ctx); err != nil || !done { t.Fatal("retryable attempt did not settle", err) } } failed, err := m.ai.GetBatch(ctx, batch.ID) if err != nil || failed.Total != 1 || failed.Failed != 1 || failed.Retry != 0 || m.calls.Load() != 3 { t.Fatal("automatic retry limit or denominator is wrong", err) } nonce := strings.Repeat("m", 24) if _, err = m.ai.ManageBatch(ctx, batch.ID, "retry", nonce, "fixture-admin"); err != nil { t.Fatal("explicit manual retry unavailable", err) } if _, err = m.ai.ManageBatch(ctx, batch.ID, "retry", nonce, "fixture-admin"); err != nil { t.Fatal("retry action replay failed", err) } advance() if done, err := m.ai.ProcessOne(ctx); err != nil || !done { t.Fatal("manual retry did not execute", err) } current, err := m.ai.GetBatch(ctx, batch.ID) if err != nil || current.Total != 1 || current.Succeeded != 1 || current.Failed != 0 || m.calls.Load() != 4 { t.Fatal("manual retry duplicated denominator or calls", err) } var count int if err = m.pool.QueryRow(ctx, `SELECT attempt FROM qa_ops_ai_work WHERE tenant_key=$1`, m.tenant).Scan(&count); err != nil || count != 4 { t.Fatal("manual retry erased previous request attempts", err) } if _, err = m.ai.ManageBatch(ctx, batch.ID, "pause", nonce, "fixture-admin"); !errors.Is(err, qaopsai.ErrConflict) { t.Fatal("same nonce accepted changed action") } } func TestAIConcurrentStartsAndClaimsKeepOneDurableEffect(t *testing.T) { m := newAIMatrix(t, "fixture-ai-concurrent", nil) ctx := context.Background() preview, err := m.ai.Preview(ctx, qaopsai.Scope{Kind: "translation", IDs: []string{"matrix-record"}}, "fixture-admin") if err != nil { t.Fatal(err) } var group sync.WaitGroup ids := make(chan string, 12) failures := make(chan error, 24) for i := 0; i < 12; i++ { group.Add(1) go func() { defer group.Done() batch, e := m.ai.Start(ctx, preview.ID, strings.Repeat("c", 24), "fixture-admin") if e != nil { failures <- e return } ids <- batch.ID }() } group.Wait() close(ids) unique := map[string]bool{} for id := range ids { unique[id] = true } if len(unique) != 1 { t.Fatal("concurrent nonce created different batches") } for i := 0; i < 12; i++ { group.Add(1) go func() { defer group.Done() _, e := m.ai.ProcessOne(ctx) if e != nil { failures <- e } }() } group.Wait() close(failures) for e := range failures { t.Fatal("concurrent operation failed", e) } var batches, works, outputs int if m.pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM qa_ops_ai_batches WHERE tenant_key=$1),(SELECT count(*) FROM qa_ops_ai_work WHERE tenant_key=$1),(SELECT count(*) FROM qa_ops_ai_outputs WHERE tenant_key=$1)`, m.tenant).Scan(&batches, &works, &outputs) != nil || batches != 1 || works != 1 || outputs != 1 || m.calls.Load() != 1 { t.Fatal("concurrent start/claim created duplicate rows or outbound requests") } } func TestAIInFlightInputEditNeverAppliesObsoleteTranslation(t *testing.T) { entered := make(chan struct{}) release := make(chan struct{}) var once sync.Once m := newAIMatrix(t, "fixture-ai-drift", func(w http.ResponseWriter, r *http.Request) { close(entered) <-release writeAICompletion(w, `{"question_chinese":"MG400 如何连接?"}`) }) defer once.Do(func() { close(release) }) ctx := context.Background() batch := m.batch(t, "d") done := make(chan error, 1) go func() { _, err := m.ai.ProcessOne(ctx); done <- err }() <-entered if _, err := m.pool.Exec(ctx, `UPDATE qa_ops_master_records SET payload=payload||jsonb_build_object('question','已人工修改的问题') WHERE tenant_key=$1 AND id='matrix-record'`, m.tenant); err != nil { once.Do(func() { close(release) }) t.Fatal(err) } once.Do(func() { close(release) }) if err := <-done; err != nil { t.Fatal(err) } current, err := m.ai.GetBatch(ctx, batch.ID) if err != nil || current.Skipped != 1 || current.Succeeded != 0 { t.Fatal("in-flight input drift was not skipped", err) } var applied []byte if m.pool.QueryRow(ctx, `SELECT applied_fields FROM qa_ops_ai_outputs WHERE tenant_key=$1`, m.tenant).Scan(&applied) != nil || string(applied) != "{}" { t.Fatal("obsolete output was applied") } record, err := m.master.GetMasterRecord(ctx, m.tenant, "matrix-record") if err != nil || record.Question != "已人工修改的问题" || record.MasterFields["question_chinese"] != "已人工修改的问题" { t.Fatal("concurrent original/Chinese edit was overwritten", err) } } func TestAIExpiredLeaseFencesLateModelResult(t *testing.T) { entered := make(chan struct{}) release := make(chan struct{}) var calls atomic.Int32 var once sync.Once m := newAIMatrix(t, "fixture-ai-lease", func(w http.ResponseWriter, r *http.Request) { if calls.Add(1) == 1 { close(entered) <-release writeAICompletion(w, `{"question_chinese":"MG400 旧的连接问题?"}`) return } writeAICompletion(w, `{"question_chinese":"MG400 新的连接问题?"}`) }) defer once.Do(func() { close(release) }) ctx := context.Background() _ = m.batch(t, "l") done := make(chan error, 1) go func() { _, err := m.ai.ProcessOne(ctx); done <- err }() <-entered if _, err := m.pool.Exec(ctx, `UPDATE qa_ops_ai_runtime SET lease_until=clock_timestamp()-interval '1 second',next_request_at=clock_timestamp()-interval '1 second' WHERE tenant_key=$1`, m.tenant); err != nil { once.Do(func() { close(release) }) t.Fatal(err) } if _, err := m.pool.Exec(ctx, `UPDATE qa_ops_ai_work SET lease_until=clock_timestamp()-interval '1 second' WHERE tenant_key=$1`, m.tenant); err != nil { once.Do(func() { close(release) }) t.Fatal(err) } if worked, err := m.ai.ProcessOne(ctx); err != nil || !worked { once.Do(func() { close(release) }) t.Fatal("expired lease not recovered", err) } once.Do(func() { close(release) }) if err := <-done; !errors.Is(err, qaopsai.ErrConflict) { t.Fatal("stale worker result not fenced", err) } var count int if m.pool.QueryRow(ctx, `SELECT count(*) FROM qa_ops_ai_outputs WHERE tenant_key=$1`, m.tenant).Scan(&count) != nil || count != 1 { t.Fatal("late worker duplicated evidence") } record, err := m.master.GetMasterRecord(ctx, m.tenant, "matrix-record") if err != nil || record.MasterFields["question_chinese"] != "MG400 新的连接问题?" { t.Fatal("late lease overwrote replacement result", err) } } type aiMatrix struct { pool *pgxpool.Pool ai *qaopsai.Store master *Store tenant string calls *atomic.Int32 } func newAIMatrix(t *testing.T, tenant string, handler func(http.ResponseWriter, *http.Request)) aiMatrix { t.Helper() pool := mirrorFixture(t) ctx := context.Background() keyFile := filepath.Join(t.TempDir(), "master-key") if err := os.WriteFile(keyFile, []byte(strings.Repeat("a", 32)), 0600); err != nil { t.Fatal(err) } calls := &atomic.Int32{} server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var body struct { Messages []struct{ Content string } `json:"messages"` } rawBody, readErr := io.ReadAll(r.Body) if readErr != nil || json.Unmarshal(rawBody, &body) != nil || len(body.Messages) != 2 { t.Error("invalid synthetic request") w.WriteHeader(400) return } if strings.Contains(body.Messages[0].Content, "connection probe") { writeAICompletion(w, `{"ok":true}`) return } calls.Add(1) r.Body = io.NopCloser(bytes.NewReader(rawBody)) if handler != nil { handler(w, r) return } writeAICompletion(w, `{"question_chinese":"MG400 如何连接?"}`) })) t.Cleanup(server.Close) ai := qaopsai.NewStore(pool, tenant, qaopsai.Options{KeyFile: keyFile, AllowPrivate: true}) if _, err := ai.SaveSettings(ctx, qaopsai.SaveSettings{BaseURL: server.URL + "/v1", Model: "fixture", KeyAction: "replace", APIKey: "fixture-only"}, "fixture-admin"); err != nil { t.Fatal(err) } master := NewStore(pool, tenant) entry := fixtureEntry("matrix-record", "fixture answer") var wire map[string]any _ = json.Unmarshal(entry.Payload, &wire) wire["question"] = "How do I connect MG400?" entry.Payload, _ = json.Marshal(wire) job := claimMirrorJob(t, master, pg.NewStore(pool), "matrix-seed") if err := master.HandleSync(ctx, *job, func(context.Context) (Source, error) { return &fixtureSource{rows: []dobotaiops.MirrorRecord{entry}}, nil }); err != nil { t.Fatal(err) } return aiMatrix{pool, ai, master, tenant, calls} } func writeAICompletion(w http.ResponseWriter, content string) { w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]any{"model": "fixture-actual", "choices": []any{map[string]any{"message": map[string]string{"content": content}, "finish_reason": "stop"}}}) } func (m aiMatrix) batch(t *testing.T, nonce string) qaopsai.Batch { t.Helper() preview, err := m.ai.Preview(context.Background(), qaopsai.Scope{Kind: "translation", IDs: []string{"matrix-record"}}, "fixture-admin") if err != nil || preview.Eligible != 1 { t.Fatal("matrix preview failed", err) } batch, err := m.ai.Start(context.Background(), preview.ID, strings.Repeat(nonce, 24), "fixture-admin") if err != nil { t.Fatal(err) } return batch } func TestAIRunnerActuallyDispatchesFourConcurrentRequests(t *testing.T) { entered := make(chan struct{}, 8) release := make(chan struct{}) var once sync.Once m := newAIMatrix(t, "fixture-ai-runner-slots", func(w http.ResponseWriter, r *http.Request) { entered <- struct{}{} <-release writeAICompletion(w, `{"question_chinese":"MG400 如何连接?"}`) }) seedParallelAIWork(t, m) ctx, cancel := context.WithCancel(context.Background()) done := make(chan struct{}) go func() { m.ai.Run(ctx, nil); close(done) }() defer func() { once.Do(func() { close(release) }); cancel(); <-done }() count := 0 timer := time.NewTimer(2 * time.Second) defer timer.Stop() loop: for count < 4 { select { case <-entered: count++ case <-timer.C: break loop } } if count != 4 { t.Fatalf("actual worker runner is still serial: concurrent=%d", count) } } func seedParallelAIWork(t *testing.T, m aiMatrix) { t.Helper() ctx := context.Background() entries := []dobotaiops.MirrorRecord{} ids := []string{} for i, word := range []string{"Alpha", "Bravo", "Charlie", "Delta", "Echo"} { id := fmt.Sprintf("parallel-record-%d", i) entry := fixtureEntry(id, "fixture answer") var wire map[string]any _ = json.Unmarshal(entry.Payload, &wire) wire["question"] = "How do I connect MG400? " + word entry.Payload, _ = json.Marshal(wire) entries = append(entries, entry) ids = append(ids, id) } var seedJob string if m.pool.QueryRow(ctx, `SELECT id FROM jobs WHERE type=$1 AND state='running'`, m.master.SyncJobType()).Scan(&seedJob) != nil { t.Fatal("seed lease unavailable") } if e := pg.NewStore(m.pool).Complete(ctx, seedJob, "fixture-sync-worker", time.Now()); e != nil { t.Fatal(e) } job := claimMirrorJob(t, m.master, pg.NewStore(m.pool), "parallel-seed") if e := m.master.HandleSync(ctx, *job, func(context.Context) (Source, error) { return &fixtureSource{rows: entries}, nil }); e != nil { t.Fatal(e) } preview, e := m.ai.Preview(ctx, qaopsai.Scope{Kind: "translation", IDs: ids}, "fixture-admin") if e != nil || preview.Eligible != 5 { t.Fatal("five unique inputs unavailable", e, preview.Eligible) } if _, e = m.ai.Start(ctx, preview.ID, strings.Repeat("parallel", 4), "fixture-admin"); e != nil { t.Fatal(e) } } func TestAIParallelClaimsUseFourSlotsWithoutDuplicatingOrExceedingLimit(t *testing.T) { entered := make(chan struct{}, 8) release := make(chan struct{}) var once sync.Once m := newAIMatrix(t, "fixture-ai-four-slots", func(w http.ResponseWriter, r *http.Request) { entered <- struct{}{} <-release writeAICompletion(w, `{"question_chinese":"MG400 如何连接?"}`) }) defer once.Do(func() { close(release) }) ctx := context.Background() seedParallelAIWork(t, m) var group sync.WaitGroup failures := make(chan error, 5) for i := 0; i < 5; i++ { group.Add(1) go func() { defer group.Done() _, err := m.ai.ProcessOne(ctx) if err != nil { failures <- err } }() } seen := 0 timeout := time.NewTimer(1500 * time.Millisecond) loop: for seen < 4 { select { case <-entered: seen++ case <-timeout.C: break loop } } timeout.Stop() if seen != 4 { t.Errorf("wanted four concurrent model requests before any completes; got %d", seen) } select { case <-entered: t.Error("exceeded four global slots") case <-time.After(100 * time.Millisecond): } once.Do(func() { close(release) }) group.Wait() close(failures) for err := range failures { t.Error(err) } if seen == 4 { var running, queued, outputs int if m.pool.QueryRow(ctx, `SELECT (SELECT count(*)FROM qa_ops_ai_work WHERE tenant_key=$1 AND state='running'),(SELECT count(*)FROM qa_ops_ai_work WHERE tenant_key=$1 AND state='queued'),(SELECT count(*)FROM qa_ops_ai_outputs WHERE tenant_key=$1)`, m.tenant).Scan(&running, &queued, &outputs) != nil || running != 0 || queued != 1 || outputs != 4 { t.Fatal("slot completion lost/duplicated work", running, queued, outputs) } } } func TestAITruncationRetriesExpandBudgetAndStopAfterThreeAttempts(t *testing.T) { budgets := []int{} m := newAIMatrix(t, "fixture-ai-truncation-budget", func(w http.ResponseWriter, r *http.Request) { var req struct { Max int `json:"max_tokens"` } if json.NewDecoder(r.Body).Decode(&req) != nil { t.Error("request decode") } budgets = append(budgets, req.Max) if len(budgets) < 3 { _ = json.NewEncoder(w).Encode(map[string]any{"model": "fixture", "choices": []any{map[string]any{"finish_reason": "length", "message": map[string]string{"content": ""}}}}) return } writeAICompletion(w, `{"question_chinese":"MG400 如何连接?"}`) }) batch := m.batch(t, "t") ctx := context.Background() for i := 0; i < 3; i++ { if _, err := m.pool.Exec(ctx, `UPDATE qa_ops_ai_work SET available_at=clock_timestamp()-interval '1 second'WHERE tenant_key=$1`, m.tenant); err != nil { t.Fatal(err) } if worked, err := m.ai.ProcessOne(ctx); err != nil || !worked { t.Fatal("bounded retry did not execute", err) } } if fmt.Sprint(budgets) != "[4096 8192 16384]" { t.Fatal("truncation budget did not grow", budgets) } current, err := m.ai.GetBatch(ctx, batch.ID) if err != nil || current.Succeeded != 1 || current.Total != 1 || m.calls.Load() != 3 { t.Fatal("retry denominator or completion", err) } } func TestAIParallelSuccessCannotClearAnotherRequestsRateLimit(t *testing.T) { entered := make(chan struct{}, 4) release := make(chan struct{}) var once sync.Once var calls atomic.Int32 m := newAIMatrix(t, "fixture-ai-shared-cooldown", func(w http.ResponseWriter, r *http.Request) { n := calls.Add(1) entered <- struct{}{} if n == 4 { w.Header().Set("Retry-After", "45") w.WriteHeader(429) return } <-release writeAICompletion(w, `{"question_chinese":"MG400 如何连接?"}`) }) defer once.Do(func() { close(release) }) seedParallelAIWork(t, m) ctx := context.Background() var group sync.WaitGroup errs := make(chan error, 4) for i := 0; i < 4; i++ { group.Add(1) go func() { defer group.Done(); _, err := m.ai.ProcessOne(ctx); errs <- err }() } for i := 0; i < 4; i++ { select { case <-entered: case <-time.After(2 * time.Second): t.Fatal("four parallel slots not reached") } } deadline := time.Now().Add(2 * time.Second) remaining := 0.0 for time.Now().Before(deadline) { if m.pool.QueryRow(ctx, `SELECT EXTRACT(EPOCH FROM next_request_at-clock_timestamp())::double precision FROM qa_ops_ai_runtime WHERE tenant_key=$1`, m.tenant).Scan(&remaining) == nil && remaining > 40 { break } time.Sleep(time.Millisecond * 10) } once.Do(func() { close(release) }) group.Wait() close(errs) for err := range errs { if err != nil { t.Fatal(err) } } if m.pool.QueryRow(ctx, `SELECT EXTRACT(EPOCH FROM next_request_at-clock_timestamp())::double precision FROM qa_ops_ai_runtime WHERE tenant_key=$1`, m.tenant).Scan(&remaining) != nil || remaining < 40 { t.Fatal("success cleared shared provider cooldown", remaining) } if worked, err := m.ai.ProcessOne(ctx); err != nil || worked || calls.Load() != 4 { t.Fatal("new request bypassed shared rate limit", err) } } func TestAIManualBatchPauseResumeKeepsSharedWorkAndActionReplaysSafe(t *testing.T) { m := newAIMatrix(t, "fixture-ai-pause", nil) ctx := context.Background() first := m.batch(t, "a") second := m.batch(t, "b") var count int if err := m.pool.QueryRow(ctx, `SELECT count(*) FROM qa_ops_ai_work WHERE tenant_key=$1`, m.tenant).Scan(&count); err != nil || count != 1 { t.Fatal("same input/config was duplicated across batches", err) } if _, err := m.ai.ManageBatch(ctx, first.ID, "pause", strings.Repeat("p", 24), "fixture-admin"); err != nil { t.Fatal(err) } var state string if err := m.pool.QueryRow(ctx, `SELECT state FROM qa_ops_ai_work WHERE tenant_key=$1`, m.tenant).Scan(&state); err != nil || state != "queued" { t.Fatal("pausing one batch stopped work still needed by another active batch", err) } if _, err := m.ai.ManageBatch(ctx, second.ID, "pause", strings.Repeat("q", 24), "fixture-admin"); err != nil { t.Fatal(err) } if worked, err := m.ai.ProcessOne(ctx); err != nil || worked || m.calls.Load() != 0 { t.Fatal("paused batch still sent a request", err) } if _, err := m.ai.ManageBatch(ctx, first.ID, "resume", strings.Repeat("r", 24), "fixture-admin"); err != nil { t.Fatal(err) } if _, err := m.ai.ManageBatch(ctx, first.ID, "pause", strings.Repeat("p", 24), "fixture-admin"); err != nil { t.Fatal(err) } current, err := m.ai.GetBatch(ctx, first.ID) if err != nil || current.State != "active" { t.Fatal("replayed old action applied again instead of reading current state", err) } if worked, err := m.ai.ProcessOne(ctx); err != nil || !worked || m.calls.Load() != 1 { t.Fatal("resumed persistent batch did not execute exactly once", err) } }