Files
knowledge-governance/internal/qaopsmirror/ai_matrix_integration_test.go
T
姚章浩 533db535f1
build / build (push) Successful in 1m12s
feat(qaops): 接入真实模型补缺任务与总表看板
按已确认需求提供独立安全模型配置、真实探测、正式字典字段评估与原问题译中,持久批次支持只补空白、暂停恢复、有限重试、幂等和逐字段依据。覆盖固定规则、本地语言、组织关联与主表有效看板共用治理侧读模型,保留源生产和冻结历史。

验证:完整 make verify 退出0,702项Go与329项前端通过;默认118项外部/隔离集成明确跳过,另跑完整镜像及组织隔离PG竞态。12并发、输入漂移、过期租约、暂停/恢复及有限重试通过。用户授权的百炼 deepseek-v4.1-flash 完成真实后台与浏览器评估/翻译;552行、16列、11块看板及完整PNG、独立配置页测试/保存、重开恢复和零JS异常通过。仅非业务样本外发,自动处理关闭,原payload不变。

上线前治理库已做导出快照原生备份并真实恢复,60表行数与17个QA表内容哈希一致;不修改DobotAI生产或隧道。本提交时尚未部署,线上验收须独立完成。
2026-10-19 14:29:49 +08:00

318 lines
12 KiB
Go

package qaopsmirror
import (
"context"
"encoding/json"
"errors"
"github.com/jackc/pgx/v5/pgxpool"
"knowledge-governance/internal/dobotaiops"
pg "knowledge-governance/internal/platform/postgres"
"knowledge-governance/internal/qaopsai"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"testing"
)
func TestAIBatchManualRetryKeepsCumulativeAttemptsAndFixedDenominator(t *testing.T) {
var attempts atomic.Int32
m := newAIMatrix(t, "fixture-ai-retry", func(w http.ResponseWriter, r *http.Request) {
if attempts.Add(1) <= 3 {
w.WriteHeader(http.StatusTooManyRequests)
return
}
writeAICompletion(w, `{"question_chinese":"MG400 如何连接?"}`)
})
ctx := context.Background()
batch := m.batch(t, "r")
advance := func() {
t.Helper()
if _, err := m.pool.Exec(ctx, `UPDATE qa_ops_ai_runtime SET next_request_at=clock_timestamp()-interval '1 second' WHERE tenant_key=$1`, m.tenant); err != nil {
t.Fatal(err)
}
if _, err := m.pool.Exec(ctx, `UPDATE qa_ops_ai_work SET available_at=clock_timestamp()-interval '1 second' WHERE tenant_key=$1`, m.tenant); err != nil {
t.Fatal(err)
}
}
for i := 0; i < 3; i++ {
advance()
if done, err := m.ai.ProcessOne(ctx); err != nil || !done {
t.Fatal("retryable attempt did not settle", err)
}
}
failed, err := m.ai.GetBatch(ctx, batch.ID)
if err != nil || failed.Total != 1 || failed.Failed != 1 || failed.Retry != 0 || m.calls.Load() != 3 {
t.Fatal("automatic retry limit or denominator is wrong", err)
}
nonce := strings.Repeat("m", 24)
if _, err = m.ai.ManageBatch(ctx, batch.ID, "retry", nonce, "fixture-admin"); err != nil {
t.Fatal("explicit manual retry unavailable", err)
}
if _, err = m.ai.ManageBatch(ctx, batch.ID, "retry", nonce, "fixture-admin"); err != nil {
t.Fatal("retry action replay failed", err)
}
advance()
if done, err := m.ai.ProcessOne(ctx); err != nil || !done {
t.Fatal("manual retry did not execute", err)
}
current, err := m.ai.GetBatch(ctx, batch.ID)
if err != nil || current.Total != 1 || current.Succeeded != 1 || current.Failed != 0 || m.calls.Load() != 4 {
t.Fatal("manual retry duplicated denominator or calls", err)
}
var count int
if err = m.pool.QueryRow(ctx, `SELECT attempt FROM qa_ops_ai_work WHERE tenant_key=$1`, m.tenant).Scan(&count); err != nil || count != 4 {
t.Fatal("manual retry erased previous request attempts", err)
}
if _, err = m.ai.ManageBatch(ctx, batch.ID, "pause", nonce, "fixture-admin"); !errors.Is(err, qaopsai.ErrConflict) {
t.Fatal("same nonce accepted changed action")
}
}
func TestAIConcurrentStartsAndClaimsKeepOneDurableEffect(t *testing.T) {
m := newAIMatrix(t, "fixture-ai-concurrent", nil)
ctx := context.Background()
preview, err := m.ai.Preview(ctx, qaopsai.Scope{Kind: "translation", IDs: []string{"matrix-record"}}, "fixture-admin")
if err != nil {
t.Fatal(err)
}
var group sync.WaitGroup
ids := make(chan string, 12)
failures := make(chan error, 24)
for i := 0; i < 12; i++ {
group.Add(1)
go func() {
defer group.Done()
batch, e := m.ai.Start(ctx, preview.ID, strings.Repeat("c", 24), "fixture-admin")
if e != nil {
failures <- e
return
}
ids <- batch.ID
}()
}
group.Wait()
close(ids)
unique := map[string]bool{}
for id := range ids {
unique[id] = true
}
if len(unique) != 1 {
t.Fatal("concurrent nonce created different batches")
}
for i := 0; i < 12; i++ {
group.Add(1)
go func() {
defer group.Done()
_, e := m.ai.ProcessOne(ctx)
if e != nil {
failures <- e
}
}()
}
group.Wait()
close(failures)
for e := range failures {
t.Fatal("concurrent operation failed", e)
}
var batches, works, outputs int
if m.pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM qa_ops_ai_batches WHERE tenant_key=$1),(SELECT count(*) FROM qa_ops_ai_work WHERE tenant_key=$1),(SELECT count(*) FROM qa_ops_ai_outputs WHERE tenant_key=$1)`, m.tenant).Scan(&batches, &works, &outputs) != nil || batches != 1 || works != 1 || outputs != 1 || m.calls.Load() != 1 {
t.Fatal("concurrent start/claim created duplicate rows or outbound requests")
}
}
func TestAIInFlightInputEditNeverAppliesObsoleteTranslation(t *testing.T) {
entered := make(chan struct{})
release := make(chan struct{})
var once sync.Once
m := newAIMatrix(t, "fixture-ai-drift", func(w http.ResponseWriter, r *http.Request) {
close(entered)
<-release
writeAICompletion(w, `{"question_chinese":"MG400 如何连接?"}`)
})
defer once.Do(func() { close(release) })
ctx := context.Background()
batch := m.batch(t, "d")
done := make(chan error, 1)
go func() { _, err := m.ai.ProcessOne(ctx); done <- err }()
<-entered
if _, err := m.pool.Exec(ctx, `UPDATE qa_ops_master_records SET payload=payload||jsonb_build_object('question','已人工修改的问题') WHERE tenant_key=$1 AND id='matrix-record'`, m.tenant); err != nil {
once.Do(func() { close(release) })
t.Fatal(err)
}
once.Do(func() { close(release) })
if err := <-done; err != nil {
t.Fatal(err)
}
current, err := m.ai.GetBatch(ctx, batch.ID)
if err != nil || current.Skipped != 1 || current.Succeeded != 0 {
t.Fatal("in-flight input drift was not skipped", err)
}
var applied []byte
if m.pool.QueryRow(ctx, `SELECT applied_fields FROM qa_ops_ai_outputs WHERE tenant_key=$1`, m.tenant).Scan(&applied) != nil || string(applied) != "{}" {
t.Fatal("obsolete output was applied")
}
record, err := m.master.GetMasterRecord(ctx, m.tenant, "matrix-record")
if err != nil || record.Question != "已人工修改的问题" || record.MasterFields["question_chinese"] != "已人工修改的问题" {
t.Fatal("concurrent original/Chinese edit was overwritten", err)
}
}
func TestAIExpiredLeaseFencesLateModelResult(t *testing.T) {
entered := make(chan struct{})
release := make(chan struct{})
var calls atomic.Int32
var once sync.Once
m := newAIMatrix(t, "fixture-ai-lease", func(w http.ResponseWriter, r *http.Request) {
if calls.Add(1) == 1 {
close(entered)
<-release
writeAICompletion(w, `{"question_chinese":"MG400 旧的连接问题?"}`)
return
}
writeAICompletion(w, `{"question_chinese":"MG400 新的连接问题?"}`)
})
defer once.Do(func() { close(release) })
ctx := context.Background()
_ = m.batch(t, "l")
done := make(chan error, 1)
go func() { _, err := m.ai.ProcessOne(ctx); done <- err }()
<-entered
if _, err := m.pool.Exec(ctx, `UPDATE qa_ops_ai_runtime SET lease_until=clock_timestamp()-interval '1 second',next_request_at=clock_timestamp()-interval '1 second' WHERE tenant_key=$1`, m.tenant); err != nil {
once.Do(func() { close(release) })
t.Fatal(err)
}
if _, err := m.pool.Exec(ctx, `UPDATE qa_ops_ai_work SET lease_until=clock_timestamp()-interval '1 second' WHERE tenant_key=$1`, m.tenant); err != nil {
once.Do(func() { close(release) })
t.Fatal(err)
}
if worked, err := m.ai.ProcessOne(ctx); err != nil || !worked {
once.Do(func() { close(release) })
t.Fatal("expired lease not recovered", err)
}
once.Do(func() { close(release) })
if err := <-done; !errors.Is(err, qaopsai.ErrConflict) {
t.Fatal("stale worker result not fenced", err)
}
var count int
if m.pool.QueryRow(ctx, `SELECT count(*) FROM qa_ops_ai_outputs WHERE tenant_key=$1`, m.tenant).Scan(&count) != nil || count != 1 {
t.Fatal("late worker duplicated evidence")
}
record, err := m.master.GetMasterRecord(ctx, m.tenant, "matrix-record")
if err != nil || record.MasterFields["question_chinese"] != "MG400 新的连接问题?" {
t.Fatal("late lease overwrote replacement result", err)
}
}
type aiMatrix struct {
pool *pgxpool.Pool
ai *qaopsai.Store
master *Store
tenant string
calls *atomic.Int32
}
func newAIMatrix(t *testing.T, tenant string, handler func(http.ResponseWriter, *http.Request)) aiMatrix {
t.Helper()
pool := mirrorFixture(t)
ctx := context.Background()
keyFile := filepath.Join(t.TempDir(), "master-key")
if err := os.WriteFile(keyFile, []byte(strings.Repeat("a", 32)), 0600); err != nil {
t.Fatal(err)
}
calls := &atomic.Int32{}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var body struct {
Messages []struct{ Content string } `json:"messages"`
}
if json.NewDecoder(r.Body).Decode(&body) != nil || len(body.Messages) != 2 {
t.Error("invalid synthetic request")
w.WriteHeader(400)
return
}
if strings.Contains(body.Messages[0].Content, "connection probe") {
writeAICompletion(w, `{"ok":true}`)
return
}
calls.Add(1)
if handler != nil {
handler(w, r)
return
}
writeAICompletion(w, `{"question_chinese":"MG400 如何连接?"}`)
}))
t.Cleanup(server.Close)
ai := qaopsai.NewStore(pool, tenant, qaopsai.Options{KeyFile: keyFile, AllowPrivate: true})
if _, err := ai.SaveSettings(ctx, qaopsai.SaveSettings{BaseURL: server.URL + "/v1", Model: "fixture", KeyAction: "replace", APIKey: "fixture-only"}, "fixture-admin"); err != nil {
t.Fatal(err)
}
master := NewStore(pool, tenant)
entry := fixtureEntry("matrix-record", "fixture answer")
var wire map[string]any
_ = json.Unmarshal(entry.Payload, &wire)
wire["question"] = "How do I connect MG400?"
entry.Payload, _ = json.Marshal(wire)
job := claimMirrorJob(t, master, pg.NewStore(pool), "matrix-seed")
if err := master.HandleSync(ctx, *job, func(context.Context) (Source, error) {
return &fixtureSource{rows: []dobotaiops.MirrorRecord{entry}}, nil
}); err != nil {
t.Fatal(err)
}
return aiMatrix{pool, ai, master, tenant, calls}
}
func writeAICompletion(w http.ResponseWriter, content string) {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{"model": "fixture-actual", "choices": []any{map[string]any{"message": map[string]string{"content": content}, "finish_reason": "stop"}}})
}
func (m aiMatrix) batch(t *testing.T, nonce string) qaopsai.Batch {
t.Helper()
preview, err := m.ai.Preview(context.Background(), qaopsai.Scope{Kind: "translation", IDs: []string{"matrix-record"}}, "fixture-admin")
if err != nil || preview.Eligible != 1 {
t.Fatal("matrix preview failed", err)
}
batch, err := m.ai.Start(context.Background(), preview.ID, strings.Repeat(nonce, 24), "fixture-admin")
if err != nil {
t.Fatal(err)
}
return batch
}
func TestAIManualBatchPauseResumeKeepsSharedWorkAndActionReplaysSafe(t *testing.T) {
m := newAIMatrix(t, "fixture-ai-pause", nil)
ctx := context.Background()
first := m.batch(t, "a")
second := m.batch(t, "b")
var count int
if err := m.pool.QueryRow(ctx, `SELECT count(*) FROM qa_ops_ai_work WHERE tenant_key=$1`, m.tenant).Scan(&count); err != nil || count != 1 {
t.Fatal("same input/config was duplicated across batches", err)
}
if _, err := m.ai.ManageBatch(ctx, first.ID, "pause", strings.Repeat("p", 24), "fixture-admin"); err != nil {
t.Fatal(err)
}
var state string
if err := m.pool.QueryRow(ctx, `SELECT state FROM qa_ops_ai_work WHERE tenant_key=$1`, m.tenant).Scan(&state); err != nil || state != "queued" {
t.Fatal("pausing one batch stopped work still needed by another active batch", err)
}
if _, err := m.ai.ManageBatch(ctx, second.ID, "pause", strings.Repeat("q", 24), "fixture-admin"); err != nil {
t.Fatal(err)
}
if worked, err := m.ai.ProcessOne(ctx); err != nil || worked || m.calls.Load() != 0 {
t.Fatal("paused batch still sent a request", err)
}
if _, err := m.ai.ManageBatch(ctx, first.ID, "resume", strings.Repeat("r", 24), "fixture-admin"); err != nil {
t.Fatal(err)
}
if _, err := m.ai.ManageBatch(ctx, first.ID, "pause", strings.Repeat("p", 24), "fixture-admin"); err != nil {
t.Fatal(err)
}
current, err := m.ai.GetBatch(ctx, first.ID)
if err != nil || current.State != "active" {
t.Fatal("replayed old action applied again instead of reading current state", err)
}
if worked, err := m.ai.ProcessOne(ctx); err != nil || !worked || m.calls.Load() != 1 {
t.Fatal("resumed persistent batch did not execute exactly once", err)
}
}