Files
knowledge-governance/internal/qaopsmirror/ai_matrix_integration_test.go
T
姚章浩 9470a127f7
build / build (push) Successful in 1m11s
feat(qaops): 完成流水表六项体验与AI执行修正
统一AI分析入口及双任务安全确认,采用全量载入的可见行渲染,保留完整原文。
右侧非模态详情固定五项质量字段,清理冗余展示并对齐查询和同步按钮。
默认4并发由数据库准入与逐任务租约保护,保留共享限流冷却、截断归因和有界重试。
兼容真实引用标题字段,不伪造证据或准确率,不修改源生产问答与公司区域规则。

验证:完整make verify退出0,710项Go通过、124项已声明可选跳过且必需0跳过;
另全量mirror隔离PostgreSQL race退出0,AI单元race退出0。
真实本地浏览器通过六项交互、390px内滚动及显式AI部分失败/零缺口故障夹具。
治理原生备份已真实恢复,schema57、72表计数与29张QA内容哈希一致。
候选镜像、精确提交CI与正式上线验收待本次提交后执行,不宣称已部署。
2026-10-19 22:09:32 +08:00

539 lines
19 KiB
Go

package qaopsmirror
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"github.com/jackc/pgx/v5/pgxpool"
"io"
"knowledge-governance/internal/dobotaiops"
pg "knowledge-governance/internal/platform/postgres"
"knowledge-governance/internal/qaopsai"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
)
func TestAIBatchManualRetryKeepsCumulativeAttemptsAndFixedDenominator(t *testing.T) {
var attempts atomic.Int32
m := newAIMatrix(t, "fixture-ai-retry", func(w http.ResponseWriter, r *http.Request) {
if attempts.Add(1) <= 3 {
w.WriteHeader(http.StatusTooManyRequests)
return
}
writeAICompletion(w, `{"question_chinese":"MG400 如何连接?"}`)
})
ctx := context.Background()
batch := m.batch(t, "r")
advance := func() {
t.Helper()
if _, err := m.pool.Exec(ctx, `UPDATE qa_ops_ai_runtime SET next_request_at=clock_timestamp()-interval '1 second' WHERE tenant_key=$1`, m.tenant); err != nil {
t.Fatal(err)
}
if _, err := m.pool.Exec(ctx, `UPDATE qa_ops_ai_work SET available_at=clock_timestamp()-interval '1 second' WHERE tenant_key=$1`, m.tenant); err != nil {
t.Fatal(err)
}
}
for i := 0; i < 3; i++ {
advance()
if done, err := m.ai.ProcessOne(ctx); err != nil || !done {
t.Fatal("retryable attempt did not settle", err)
}
}
failed, err := m.ai.GetBatch(ctx, batch.ID)
if err != nil || failed.Total != 1 || failed.Failed != 1 || failed.Retry != 0 || m.calls.Load() != 3 {
t.Fatal("automatic retry limit or denominator is wrong", err)
}
nonce := strings.Repeat("m", 24)
if _, err = m.ai.ManageBatch(ctx, batch.ID, "retry", nonce, "fixture-admin"); err != nil {
t.Fatal("explicit manual retry unavailable", err)
}
if _, err = m.ai.ManageBatch(ctx, batch.ID, "retry", nonce, "fixture-admin"); err != nil {
t.Fatal("retry action replay failed", err)
}
advance()
if done, err := m.ai.ProcessOne(ctx); err != nil || !done {
t.Fatal("manual retry did not execute", err)
}
current, err := m.ai.GetBatch(ctx, batch.ID)
if err != nil || current.Total != 1 || current.Succeeded != 1 || current.Failed != 0 || m.calls.Load() != 4 {
t.Fatal("manual retry duplicated denominator or calls", err)
}
var count int
if err = m.pool.QueryRow(ctx, `SELECT attempt FROM qa_ops_ai_work WHERE tenant_key=$1`, m.tenant).Scan(&count); err != nil || count != 4 {
t.Fatal("manual retry erased previous request attempts", err)
}
if _, err = m.ai.ManageBatch(ctx, batch.ID, "pause", nonce, "fixture-admin"); !errors.Is(err, qaopsai.ErrConflict) {
t.Fatal("same nonce accepted changed action")
}
}
func TestAIConcurrentStartsAndClaimsKeepOneDurableEffect(t *testing.T) {
m := newAIMatrix(t, "fixture-ai-concurrent", nil)
ctx := context.Background()
preview, err := m.ai.Preview(ctx, qaopsai.Scope{Kind: "translation", IDs: []string{"matrix-record"}}, "fixture-admin")
if err != nil {
t.Fatal(err)
}
var group sync.WaitGroup
ids := make(chan string, 12)
failures := make(chan error, 24)
for i := 0; i < 12; i++ {
group.Add(1)
go func() {
defer group.Done()
batch, e := m.ai.Start(ctx, preview.ID, strings.Repeat("c", 24), "fixture-admin")
if e != nil {
failures <- e
return
}
ids <- batch.ID
}()
}
group.Wait()
close(ids)
unique := map[string]bool{}
for id := range ids {
unique[id] = true
}
if len(unique) != 1 {
t.Fatal("concurrent nonce created different batches")
}
for i := 0; i < 12; i++ {
group.Add(1)
go func() {
defer group.Done()
_, e := m.ai.ProcessOne(ctx)
if e != nil {
failures <- e
}
}()
}
group.Wait()
close(failures)
for e := range failures {
t.Fatal("concurrent operation failed", e)
}
var batches, works, outputs int
if m.pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM qa_ops_ai_batches WHERE tenant_key=$1),(SELECT count(*) FROM qa_ops_ai_work WHERE tenant_key=$1),(SELECT count(*) FROM qa_ops_ai_outputs WHERE tenant_key=$1)`, m.tenant).Scan(&batches, &works, &outputs) != nil || batches != 1 || works != 1 || outputs != 1 || m.calls.Load() != 1 {
t.Fatal("concurrent start/claim created duplicate rows or outbound requests")
}
}
func TestAIInFlightInputEditNeverAppliesObsoleteTranslation(t *testing.T) {
entered := make(chan struct{})
release := make(chan struct{})
var once sync.Once
m := newAIMatrix(t, "fixture-ai-drift", func(w http.ResponseWriter, r *http.Request) {
close(entered)
<-release
writeAICompletion(w, `{"question_chinese":"MG400 如何连接?"}`)
})
defer once.Do(func() { close(release) })
ctx := context.Background()
batch := m.batch(t, "d")
done := make(chan error, 1)
go func() { _, err := m.ai.ProcessOne(ctx); done <- err }()
<-entered
if _, err := m.pool.Exec(ctx, `UPDATE qa_ops_master_records SET payload=payload||jsonb_build_object('question','已人工修改的问题') WHERE tenant_key=$1 AND id='matrix-record'`, m.tenant); err != nil {
once.Do(func() { close(release) })
t.Fatal(err)
}
once.Do(func() { close(release) })
if err := <-done; err != nil {
t.Fatal(err)
}
current, err := m.ai.GetBatch(ctx, batch.ID)
if err != nil || current.Skipped != 1 || current.Succeeded != 0 {
t.Fatal("in-flight input drift was not skipped", err)
}
var applied []byte
if m.pool.QueryRow(ctx, `SELECT applied_fields FROM qa_ops_ai_outputs WHERE tenant_key=$1`, m.tenant).Scan(&applied) != nil || string(applied) != "{}" {
t.Fatal("obsolete output was applied")
}
record, err := m.master.GetMasterRecord(ctx, m.tenant, "matrix-record")
if err != nil || record.Question != "已人工修改的问题" || record.MasterFields["question_chinese"] != "已人工修改的问题" {
t.Fatal("concurrent original/Chinese edit was overwritten", err)
}
}
func TestAIExpiredLeaseFencesLateModelResult(t *testing.T) {
entered := make(chan struct{})
release := make(chan struct{})
var calls atomic.Int32
var once sync.Once
m := newAIMatrix(t, "fixture-ai-lease", func(w http.ResponseWriter, r *http.Request) {
if calls.Add(1) == 1 {
close(entered)
<-release
writeAICompletion(w, `{"question_chinese":"MG400 旧的连接问题?"}`)
return
}
writeAICompletion(w, `{"question_chinese":"MG400 新的连接问题?"}`)
})
defer once.Do(func() { close(release) })
ctx := context.Background()
_ = m.batch(t, "l")
done := make(chan error, 1)
go func() { _, err := m.ai.ProcessOne(ctx); done <- err }()
<-entered
if _, err := m.pool.Exec(ctx, `UPDATE qa_ops_ai_runtime SET lease_until=clock_timestamp()-interval '1 second',next_request_at=clock_timestamp()-interval '1 second' WHERE tenant_key=$1`, m.tenant); err != nil {
once.Do(func() { close(release) })
t.Fatal(err)
}
if _, err := m.pool.Exec(ctx, `UPDATE qa_ops_ai_work SET lease_until=clock_timestamp()-interval '1 second' WHERE tenant_key=$1`, m.tenant); err != nil {
once.Do(func() { close(release) })
t.Fatal(err)
}
if worked, err := m.ai.ProcessOne(ctx); err != nil || !worked {
once.Do(func() { close(release) })
t.Fatal("expired lease not recovered", err)
}
once.Do(func() { close(release) })
if err := <-done; !errors.Is(err, qaopsai.ErrConflict) {
t.Fatal("stale worker result not fenced", err)
}
var count int
if m.pool.QueryRow(ctx, `SELECT count(*) FROM qa_ops_ai_outputs WHERE tenant_key=$1`, m.tenant).Scan(&count) != nil || count != 1 {
t.Fatal("late worker duplicated evidence")
}
record, err := m.master.GetMasterRecord(ctx, m.tenant, "matrix-record")
if err != nil || record.MasterFields["question_chinese"] != "MG400 新的连接问题?" {
t.Fatal("late lease overwrote replacement result", err)
}
}
type aiMatrix struct {
pool *pgxpool.Pool
ai *qaopsai.Store
master *Store
tenant string
calls *atomic.Int32
}
func newAIMatrix(t *testing.T, tenant string, handler func(http.ResponseWriter, *http.Request)) aiMatrix {
t.Helper()
pool := mirrorFixture(t)
ctx := context.Background()
keyFile := filepath.Join(t.TempDir(), "master-key")
if err := os.WriteFile(keyFile, []byte(strings.Repeat("a", 32)), 0600); err != nil {
t.Fatal(err)
}
calls := &atomic.Int32{}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var body struct {
Messages []struct{ Content string } `json:"messages"`
}
rawBody, readErr := io.ReadAll(r.Body)
if readErr != nil || json.Unmarshal(rawBody, &body) != nil || len(body.Messages) != 2 {
t.Error("invalid synthetic request")
w.WriteHeader(400)
return
}
if strings.Contains(body.Messages[0].Content, "connection probe") {
writeAICompletion(w, `{"ok":true}`)
return
}
calls.Add(1)
r.Body = io.NopCloser(bytes.NewReader(rawBody))
if handler != nil {
handler(w, r)
return
}
writeAICompletion(w, `{"question_chinese":"MG400 如何连接?"}`)
}))
t.Cleanup(server.Close)
ai := qaopsai.NewStore(pool, tenant, qaopsai.Options{KeyFile: keyFile, AllowPrivate: true})
if _, err := ai.SaveSettings(ctx, qaopsai.SaveSettings{BaseURL: server.URL + "/v1", Model: "fixture", KeyAction: "replace", APIKey: "fixture-only"}, "fixture-admin"); err != nil {
t.Fatal(err)
}
master := NewStore(pool, tenant)
entry := fixtureEntry("matrix-record", "fixture answer")
var wire map[string]any
_ = json.Unmarshal(entry.Payload, &wire)
wire["question"] = "How do I connect MG400?"
entry.Payload, _ = json.Marshal(wire)
job := claimMirrorJob(t, master, pg.NewStore(pool), "matrix-seed")
if err := master.HandleSync(ctx, *job, func(context.Context) (Source, error) {
return &fixtureSource{rows: []dobotaiops.MirrorRecord{entry}}, nil
}); err != nil {
t.Fatal(err)
}
return aiMatrix{pool, ai, master, tenant, calls}
}
func writeAICompletion(w http.ResponseWriter, content string) {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{"model": "fixture-actual", "choices": []any{map[string]any{"message": map[string]string{"content": content}, "finish_reason": "stop"}}})
}
func (m aiMatrix) batch(t *testing.T, nonce string) qaopsai.Batch {
t.Helper()
preview, err := m.ai.Preview(context.Background(), qaopsai.Scope{Kind: "translation", IDs: []string{"matrix-record"}}, "fixture-admin")
if err != nil || preview.Eligible != 1 {
t.Fatal("matrix preview failed", err)
}
batch, err := m.ai.Start(context.Background(), preview.ID, strings.Repeat(nonce, 24), "fixture-admin")
if err != nil {
t.Fatal(err)
}
return batch
}
func TestAIRunnerActuallyDispatchesFourConcurrentRequests(t *testing.T) {
entered := make(chan struct{}, 8)
release := make(chan struct{})
var once sync.Once
m := newAIMatrix(t, "fixture-ai-runner-slots", func(w http.ResponseWriter, r *http.Request) {
entered <- struct{}{}
<-release
writeAICompletion(w, `{"question_chinese":"MG400 如何连接?"}`)
})
seedParallelAIWork(t, m)
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() { m.ai.Run(ctx, nil); close(done) }()
defer func() { once.Do(func() { close(release) }); cancel(); <-done }()
count := 0
timer := time.NewTimer(2 * time.Second)
defer timer.Stop()
loop:
for count < 4 {
select {
case <-entered:
count++
case <-timer.C:
break loop
}
}
if count != 4 {
t.Fatalf("actual worker runner is still serial: concurrent=%d", count)
}
}
func seedParallelAIWork(t *testing.T, m aiMatrix) {
t.Helper()
ctx := context.Background()
entries := []dobotaiops.MirrorRecord{}
ids := []string{}
for i, word := range []string{"Alpha", "Bravo", "Charlie", "Delta", "Echo"} {
id := fmt.Sprintf("parallel-record-%d", i)
entry := fixtureEntry(id, "fixture answer")
var wire map[string]any
_ = json.Unmarshal(entry.Payload, &wire)
wire["question"] = "How do I connect MG400? " + word
entry.Payload, _ = json.Marshal(wire)
entries = append(entries, entry)
ids = append(ids, id)
}
var seedJob string
if m.pool.QueryRow(ctx, `SELECT id FROM jobs WHERE type=$1 AND state='running'`, m.master.SyncJobType()).Scan(&seedJob) != nil {
t.Fatal("seed lease unavailable")
}
if e := pg.NewStore(m.pool).Complete(ctx, seedJob, "fixture-sync-worker", time.Now()); e != nil {
t.Fatal(e)
}
job := claimMirrorJob(t, m.master, pg.NewStore(m.pool), "parallel-seed")
if e := m.master.HandleSync(ctx, *job, func(context.Context) (Source, error) { return &fixtureSource{rows: entries}, nil }); e != nil {
t.Fatal(e)
}
preview, e := m.ai.Preview(ctx, qaopsai.Scope{Kind: "translation", IDs: ids}, "fixture-admin")
if e != nil || preview.Eligible != 5 {
t.Fatal("five unique inputs unavailable", e, preview.Eligible)
}
if _, e = m.ai.Start(ctx, preview.ID, strings.Repeat("parallel", 4), "fixture-admin"); e != nil {
t.Fatal(e)
}
}
func TestAIParallelClaimsUseFourSlotsWithoutDuplicatingOrExceedingLimit(t *testing.T) {
entered := make(chan struct{}, 8)
release := make(chan struct{})
var once sync.Once
m := newAIMatrix(t, "fixture-ai-four-slots", func(w http.ResponseWriter, r *http.Request) {
entered <- struct{}{}
<-release
writeAICompletion(w, `{"question_chinese":"MG400 如何连接?"}`)
})
defer once.Do(func() { close(release) })
ctx := context.Background()
seedParallelAIWork(t, m)
var group sync.WaitGroup
failures := make(chan error, 5)
for i := 0; i < 5; i++ {
group.Add(1)
go func() {
defer group.Done()
_, err := m.ai.ProcessOne(ctx)
if err != nil {
failures <- err
}
}()
}
seen := 0
timeout := time.NewTimer(1500 * time.Millisecond)
loop:
for seen < 4 {
select {
case <-entered:
seen++
case <-timeout.C:
break loop
}
}
timeout.Stop()
if seen != 4 {
t.Errorf("wanted four concurrent model requests before any completes; got %d", seen)
}
select {
case <-entered:
t.Error("exceeded four global slots")
case <-time.After(100 * time.Millisecond):
}
once.Do(func() { close(release) })
group.Wait()
close(failures)
for err := range failures {
t.Error(err)
}
if seen == 4 {
var running, queued, outputs int
if m.pool.QueryRow(ctx, `SELECT (SELECT count(*)FROM qa_ops_ai_work WHERE tenant_key=$1 AND state='running'),(SELECT count(*)FROM qa_ops_ai_work WHERE tenant_key=$1 AND state='queued'),(SELECT count(*)FROM qa_ops_ai_outputs WHERE tenant_key=$1)`, m.tenant).Scan(&running, &queued, &outputs) != nil || running != 0 || queued != 1 || outputs != 4 {
t.Fatal("slot completion lost/duplicated work", running, queued, outputs)
}
}
}
func TestAITruncationRetriesExpandBudgetAndStopAfterThreeAttempts(t *testing.T) {
budgets := []int{}
m := newAIMatrix(t, "fixture-ai-truncation-budget", func(w http.ResponseWriter, r *http.Request) {
var req struct {
Max int `json:"max_tokens"`
}
if json.NewDecoder(r.Body).Decode(&req) != nil {
t.Error("request decode")
}
budgets = append(budgets, req.Max)
if len(budgets) < 3 {
_ = json.NewEncoder(w).Encode(map[string]any{"model": "fixture", "choices": []any{map[string]any{"finish_reason": "length", "message": map[string]string{"content": ""}}}})
return
}
writeAICompletion(w, `{"question_chinese":"MG400 如何连接?"}`)
})
batch := m.batch(t, "t")
ctx := context.Background()
for i := 0; i < 3; i++ {
if _, err := m.pool.Exec(ctx, `UPDATE qa_ops_ai_work SET available_at=clock_timestamp()-interval '1 second'WHERE tenant_key=$1`, m.tenant); err != nil {
t.Fatal(err)
}
if worked, err := m.ai.ProcessOne(ctx); err != nil || !worked {
t.Fatal("bounded retry did not execute", err)
}
}
if fmt.Sprint(budgets) != "[4096 8192 16384]" {
t.Fatal("truncation budget did not grow", budgets)
}
current, err := m.ai.GetBatch(ctx, batch.ID)
if err != nil || current.Succeeded != 1 || current.Total != 1 || m.calls.Load() != 3 {
t.Fatal("retry denominator or completion", err)
}
}
func TestAIParallelSuccessCannotClearAnotherRequestsRateLimit(t *testing.T) {
entered := make(chan struct{}, 4)
release := make(chan struct{})
var once sync.Once
var calls atomic.Int32
m := newAIMatrix(t, "fixture-ai-shared-cooldown", func(w http.ResponseWriter, r *http.Request) {
n := calls.Add(1)
entered <- struct{}{}
if n == 4 {
w.Header().Set("Retry-After", "45")
w.WriteHeader(429)
return
}
<-release
writeAICompletion(w, `{"question_chinese":"MG400 如何连接?"}`)
})
defer once.Do(func() { close(release) })
seedParallelAIWork(t, m)
ctx := context.Background()
var group sync.WaitGroup
errs := make(chan error, 4)
for i := 0; i < 4; i++ {
group.Add(1)
go func() { defer group.Done(); _, err := m.ai.ProcessOne(ctx); errs <- err }()
}
for i := 0; i < 4; i++ {
select {
case <-entered:
case <-time.After(2 * time.Second):
t.Fatal("four parallel slots not reached")
}
}
deadline := time.Now().Add(2 * time.Second)
remaining := 0.0
for time.Now().Before(deadline) {
if m.pool.QueryRow(ctx, `SELECT EXTRACT(EPOCH FROM next_request_at-clock_timestamp())::double precision FROM qa_ops_ai_runtime WHERE tenant_key=$1`, m.tenant).Scan(&remaining) == nil && remaining > 40 {
break
}
time.Sleep(time.Millisecond * 10)
}
once.Do(func() { close(release) })
group.Wait()
close(errs)
for err := range errs {
if err != nil {
t.Fatal(err)
}
}
if m.pool.QueryRow(ctx, `SELECT EXTRACT(EPOCH FROM next_request_at-clock_timestamp())::double precision FROM qa_ops_ai_runtime WHERE tenant_key=$1`, m.tenant).Scan(&remaining) != nil || remaining < 40 {
t.Fatal("success cleared shared provider cooldown", remaining)
}
if worked, err := m.ai.ProcessOne(ctx); err != nil || worked || calls.Load() != 4 {
t.Fatal("new request bypassed shared rate limit", err)
}
}
func TestAIManualBatchPauseResumeKeepsSharedWorkAndActionReplaysSafe(t *testing.T) {
m := newAIMatrix(t, "fixture-ai-pause", nil)
ctx := context.Background()
first := m.batch(t, "a")
second := m.batch(t, "b")
var count int
if err := m.pool.QueryRow(ctx, `SELECT count(*) FROM qa_ops_ai_work WHERE tenant_key=$1`, m.tenant).Scan(&count); err != nil || count != 1 {
t.Fatal("same input/config was duplicated across batches", err)
}
if _, err := m.ai.ManageBatch(ctx, first.ID, "pause", strings.Repeat("p", 24), "fixture-admin"); err != nil {
t.Fatal(err)
}
var state string
if err := m.pool.QueryRow(ctx, `SELECT state FROM qa_ops_ai_work WHERE tenant_key=$1`, m.tenant).Scan(&state); err != nil || state != "queued" {
t.Fatal("pausing one batch stopped work still needed by another active batch", err)
}
if _, err := m.ai.ManageBatch(ctx, second.ID, "pause", strings.Repeat("q", 24), "fixture-admin"); err != nil {
t.Fatal(err)
}
if worked, err := m.ai.ProcessOne(ctx); err != nil || worked || m.calls.Load() != 0 {
t.Fatal("paused batch still sent a request", err)
}
if _, err := m.ai.ManageBatch(ctx, first.ID, "resume", strings.Repeat("r", 24), "fixture-admin"); err != nil {
t.Fatal(err)
}
if _, err := m.ai.ManageBatch(ctx, first.ID, "pause", strings.Repeat("p", 24), "fixture-admin"); err != nil {
t.Fatal(err)
}
current, err := m.ai.GetBatch(ctx, first.ID)
if err != nil || current.State != "active" {
t.Fatal("replayed old action applied again instead of reading current state", err)
}
if worked, err := m.ai.ProcessOne(ctx); err != nil || !worked || m.calls.Load() != 1 {
t.Fatal("resumed persistent batch did not execute exactly once", err)
}
}