533db535f1
build / build (push) Successful in 1m12s
按已确认需求提供独立安全模型配置、真实探测、正式字典字段评估与原问题译中,持久批次支持只补空白、暂停恢复、有限重试、幂等和逐字段依据。覆盖固定规则、本地语言、组织关联与主表有效看板共用治理侧读模型,保留源生产和冻结历史。 验证:完整 make verify 退出0,702项Go与329项前端通过;默认118项外部/隔离集成明确跳过,另跑完整镜像及组织隔离PG竞态。12并发、输入漂移、过期租约、暂停/恢复及有限重试通过。用户授权的百炼 deepseek-v4.1-flash 完成真实后台与浏览器评估/翻译;552行、16列、11块看板及完整PNG、独立配置页测试/保存、重开恢复和零JS异常通过。仅非业务样本外发,自动处理关闭,原payload不变。 上线前治理库已做导出快照原生备份并真实恢复,60表行数与17个QA表内容哈希一致;不修改DobotAI生产或隧道。本提交时尚未部署,线上验收须独立完成。
318 lines
12 KiB
Go
318 lines
12 KiB
Go
package qaopsmirror
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
"knowledge-governance/internal/dobotaiops"
|
|
pg "knowledge-governance/internal/platform/postgres"
|
|
"knowledge-governance/internal/qaopsai"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
)
|
|
|
|
func TestAIBatchManualRetryKeepsCumulativeAttemptsAndFixedDenominator(t *testing.T) {
|
|
var attempts atomic.Int32
|
|
m := newAIMatrix(t, "fixture-ai-retry", func(w http.ResponseWriter, r *http.Request) {
|
|
if attempts.Add(1) <= 3 {
|
|
w.WriteHeader(http.StatusTooManyRequests)
|
|
return
|
|
}
|
|
writeAICompletion(w, `{"question_chinese":"MG400 如何连接?"}`)
|
|
})
|
|
ctx := context.Background()
|
|
batch := m.batch(t, "r")
|
|
advance := func() {
|
|
t.Helper()
|
|
if _, err := m.pool.Exec(ctx, `UPDATE qa_ops_ai_runtime SET next_request_at=clock_timestamp()-interval '1 second' WHERE tenant_key=$1`, m.tenant); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := m.pool.Exec(ctx, `UPDATE qa_ops_ai_work SET available_at=clock_timestamp()-interval '1 second' WHERE tenant_key=$1`, m.tenant); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for i := 0; i < 3; i++ {
|
|
advance()
|
|
if done, err := m.ai.ProcessOne(ctx); err != nil || !done {
|
|
t.Fatal("retryable attempt did not settle", err)
|
|
}
|
|
}
|
|
failed, err := m.ai.GetBatch(ctx, batch.ID)
|
|
if err != nil || failed.Total != 1 || failed.Failed != 1 || failed.Retry != 0 || m.calls.Load() != 3 {
|
|
t.Fatal("automatic retry limit or denominator is wrong", err)
|
|
}
|
|
nonce := strings.Repeat("m", 24)
|
|
if _, err = m.ai.ManageBatch(ctx, batch.ID, "retry", nonce, "fixture-admin"); err != nil {
|
|
t.Fatal("explicit manual retry unavailable", err)
|
|
}
|
|
if _, err = m.ai.ManageBatch(ctx, batch.ID, "retry", nonce, "fixture-admin"); err != nil {
|
|
t.Fatal("retry action replay failed", err)
|
|
}
|
|
advance()
|
|
if done, err := m.ai.ProcessOne(ctx); err != nil || !done {
|
|
t.Fatal("manual retry did not execute", err)
|
|
}
|
|
current, err := m.ai.GetBatch(ctx, batch.ID)
|
|
if err != nil || current.Total != 1 || current.Succeeded != 1 || current.Failed != 0 || m.calls.Load() != 4 {
|
|
t.Fatal("manual retry duplicated denominator or calls", err)
|
|
}
|
|
var count int
|
|
if err = m.pool.QueryRow(ctx, `SELECT attempt FROM qa_ops_ai_work WHERE tenant_key=$1`, m.tenant).Scan(&count); err != nil || count != 4 {
|
|
t.Fatal("manual retry erased previous request attempts", err)
|
|
}
|
|
if _, err = m.ai.ManageBatch(ctx, batch.ID, "pause", nonce, "fixture-admin"); !errors.Is(err, qaopsai.ErrConflict) {
|
|
t.Fatal("same nonce accepted changed action")
|
|
}
|
|
}
|
|
|
|
func TestAIConcurrentStartsAndClaimsKeepOneDurableEffect(t *testing.T) {
|
|
m := newAIMatrix(t, "fixture-ai-concurrent", nil)
|
|
ctx := context.Background()
|
|
preview, err := m.ai.Preview(ctx, qaopsai.Scope{Kind: "translation", IDs: []string{"matrix-record"}}, "fixture-admin")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var group sync.WaitGroup
|
|
ids := make(chan string, 12)
|
|
failures := make(chan error, 24)
|
|
for i := 0; i < 12; i++ {
|
|
group.Add(1)
|
|
go func() {
|
|
defer group.Done()
|
|
batch, e := m.ai.Start(ctx, preview.ID, strings.Repeat("c", 24), "fixture-admin")
|
|
if e != nil {
|
|
failures <- e
|
|
return
|
|
}
|
|
ids <- batch.ID
|
|
}()
|
|
}
|
|
group.Wait()
|
|
close(ids)
|
|
unique := map[string]bool{}
|
|
for id := range ids {
|
|
unique[id] = true
|
|
}
|
|
if len(unique) != 1 {
|
|
t.Fatal("concurrent nonce created different batches")
|
|
}
|
|
for i := 0; i < 12; i++ {
|
|
group.Add(1)
|
|
go func() {
|
|
defer group.Done()
|
|
_, e := m.ai.ProcessOne(ctx)
|
|
if e != nil {
|
|
failures <- e
|
|
}
|
|
}()
|
|
}
|
|
group.Wait()
|
|
close(failures)
|
|
for e := range failures {
|
|
t.Fatal("concurrent operation failed", e)
|
|
}
|
|
var batches, works, outputs int
|
|
if m.pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM qa_ops_ai_batches WHERE tenant_key=$1),(SELECT count(*) FROM qa_ops_ai_work WHERE tenant_key=$1),(SELECT count(*) FROM qa_ops_ai_outputs WHERE tenant_key=$1)`, m.tenant).Scan(&batches, &works, &outputs) != nil || batches != 1 || works != 1 || outputs != 1 || m.calls.Load() != 1 {
|
|
t.Fatal("concurrent start/claim created duplicate rows or outbound requests")
|
|
}
|
|
}
|
|
|
|
func TestAIInFlightInputEditNeverAppliesObsoleteTranslation(t *testing.T) {
|
|
entered := make(chan struct{})
|
|
release := make(chan struct{})
|
|
var once sync.Once
|
|
m := newAIMatrix(t, "fixture-ai-drift", func(w http.ResponseWriter, r *http.Request) {
|
|
close(entered)
|
|
<-release
|
|
writeAICompletion(w, `{"question_chinese":"MG400 如何连接?"}`)
|
|
})
|
|
defer once.Do(func() { close(release) })
|
|
ctx := context.Background()
|
|
batch := m.batch(t, "d")
|
|
done := make(chan error, 1)
|
|
go func() { _, err := m.ai.ProcessOne(ctx); done <- err }()
|
|
<-entered
|
|
if _, err := m.pool.Exec(ctx, `UPDATE qa_ops_master_records SET payload=payload||jsonb_build_object('question','已人工修改的问题') WHERE tenant_key=$1 AND id='matrix-record'`, m.tenant); err != nil {
|
|
once.Do(func() { close(release) })
|
|
t.Fatal(err)
|
|
}
|
|
once.Do(func() { close(release) })
|
|
if err := <-done; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
current, err := m.ai.GetBatch(ctx, batch.ID)
|
|
if err != nil || current.Skipped != 1 || current.Succeeded != 0 {
|
|
t.Fatal("in-flight input drift was not skipped", err)
|
|
}
|
|
var applied []byte
|
|
if m.pool.QueryRow(ctx, `SELECT applied_fields FROM qa_ops_ai_outputs WHERE tenant_key=$1`, m.tenant).Scan(&applied) != nil || string(applied) != "{}" {
|
|
t.Fatal("obsolete output was applied")
|
|
}
|
|
record, err := m.master.GetMasterRecord(ctx, m.tenant, "matrix-record")
|
|
if err != nil || record.Question != "已人工修改的问题" || record.MasterFields["question_chinese"] != "已人工修改的问题" {
|
|
t.Fatal("concurrent original/Chinese edit was overwritten", err)
|
|
}
|
|
}
|
|
|
|
func TestAIExpiredLeaseFencesLateModelResult(t *testing.T) {
|
|
entered := make(chan struct{})
|
|
release := make(chan struct{})
|
|
var calls atomic.Int32
|
|
var once sync.Once
|
|
m := newAIMatrix(t, "fixture-ai-lease", func(w http.ResponseWriter, r *http.Request) {
|
|
if calls.Add(1) == 1 {
|
|
close(entered)
|
|
<-release
|
|
writeAICompletion(w, `{"question_chinese":"MG400 旧的连接问题?"}`)
|
|
return
|
|
}
|
|
writeAICompletion(w, `{"question_chinese":"MG400 新的连接问题?"}`)
|
|
})
|
|
defer once.Do(func() { close(release) })
|
|
ctx := context.Background()
|
|
_ = m.batch(t, "l")
|
|
done := make(chan error, 1)
|
|
go func() { _, err := m.ai.ProcessOne(ctx); done <- err }()
|
|
<-entered
|
|
if _, err := m.pool.Exec(ctx, `UPDATE qa_ops_ai_runtime SET lease_until=clock_timestamp()-interval '1 second',next_request_at=clock_timestamp()-interval '1 second' WHERE tenant_key=$1`, m.tenant); err != nil {
|
|
once.Do(func() { close(release) })
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := m.pool.Exec(ctx, `UPDATE qa_ops_ai_work SET lease_until=clock_timestamp()-interval '1 second' WHERE tenant_key=$1`, m.tenant); err != nil {
|
|
once.Do(func() { close(release) })
|
|
t.Fatal(err)
|
|
}
|
|
if worked, err := m.ai.ProcessOne(ctx); err != nil || !worked {
|
|
once.Do(func() { close(release) })
|
|
t.Fatal("expired lease not recovered", err)
|
|
}
|
|
once.Do(func() { close(release) })
|
|
if err := <-done; !errors.Is(err, qaopsai.ErrConflict) {
|
|
t.Fatal("stale worker result not fenced", err)
|
|
}
|
|
var count int
|
|
if m.pool.QueryRow(ctx, `SELECT count(*) FROM qa_ops_ai_outputs WHERE tenant_key=$1`, m.tenant).Scan(&count) != nil || count != 1 {
|
|
t.Fatal("late worker duplicated evidence")
|
|
}
|
|
record, err := m.master.GetMasterRecord(ctx, m.tenant, "matrix-record")
|
|
if err != nil || record.MasterFields["question_chinese"] != "MG400 新的连接问题?" {
|
|
t.Fatal("late lease overwrote replacement result", err)
|
|
}
|
|
}
|
|
|
|
type aiMatrix struct {
|
|
pool *pgxpool.Pool
|
|
ai *qaopsai.Store
|
|
master *Store
|
|
tenant string
|
|
calls *atomic.Int32
|
|
}
|
|
|
|
func newAIMatrix(t *testing.T, tenant string, handler func(http.ResponseWriter, *http.Request)) aiMatrix {
|
|
t.Helper()
|
|
pool := mirrorFixture(t)
|
|
ctx := context.Background()
|
|
keyFile := filepath.Join(t.TempDir(), "master-key")
|
|
if err := os.WriteFile(keyFile, []byte(strings.Repeat("a", 32)), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
calls := &atomic.Int32{}
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
var body struct {
|
|
Messages []struct{ Content string } `json:"messages"`
|
|
}
|
|
if json.NewDecoder(r.Body).Decode(&body) != nil || len(body.Messages) != 2 {
|
|
t.Error("invalid synthetic request")
|
|
w.WriteHeader(400)
|
|
return
|
|
}
|
|
if strings.Contains(body.Messages[0].Content, "connection probe") {
|
|
writeAICompletion(w, `{"ok":true}`)
|
|
return
|
|
}
|
|
calls.Add(1)
|
|
if handler != nil {
|
|
handler(w, r)
|
|
return
|
|
}
|
|
writeAICompletion(w, `{"question_chinese":"MG400 如何连接?"}`)
|
|
}))
|
|
t.Cleanup(server.Close)
|
|
ai := qaopsai.NewStore(pool, tenant, qaopsai.Options{KeyFile: keyFile, AllowPrivate: true})
|
|
if _, err := ai.SaveSettings(ctx, qaopsai.SaveSettings{BaseURL: server.URL + "/v1", Model: "fixture", KeyAction: "replace", APIKey: "fixture-only"}, "fixture-admin"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
master := NewStore(pool, tenant)
|
|
entry := fixtureEntry("matrix-record", "fixture answer")
|
|
var wire map[string]any
|
|
_ = json.Unmarshal(entry.Payload, &wire)
|
|
wire["question"] = "How do I connect MG400?"
|
|
entry.Payload, _ = json.Marshal(wire)
|
|
job := claimMirrorJob(t, master, pg.NewStore(pool), "matrix-seed")
|
|
if err := master.HandleSync(ctx, *job, func(context.Context) (Source, error) {
|
|
return &fixtureSource{rows: []dobotaiops.MirrorRecord{entry}}, nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return aiMatrix{pool, ai, master, tenant, calls}
|
|
}
|
|
func writeAICompletion(w http.ResponseWriter, content string) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]any{"model": "fixture-actual", "choices": []any{map[string]any{"message": map[string]string{"content": content}, "finish_reason": "stop"}}})
|
|
}
|
|
func (m aiMatrix) batch(t *testing.T, nonce string) qaopsai.Batch {
|
|
t.Helper()
|
|
preview, err := m.ai.Preview(context.Background(), qaopsai.Scope{Kind: "translation", IDs: []string{"matrix-record"}}, "fixture-admin")
|
|
if err != nil || preview.Eligible != 1 {
|
|
t.Fatal("matrix preview failed", err)
|
|
}
|
|
batch, err := m.ai.Start(context.Background(), preview.ID, strings.Repeat(nonce, 24), "fixture-admin")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return batch
|
|
}
|
|
func TestAIManualBatchPauseResumeKeepsSharedWorkAndActionReplaysSafe(t *testing.T) {
|
|
m := newAIMatrix(t, "fixture-ai-pause", nil)
|
|
ctx := context.Background()
|
|
first := m.batch(t, "a")
|
|
second := m.batch(t, "b")
|
|
var count int
|
|
if err := m.pool.QueryRow(ctx, `SELECT count(*) FROM qa_ops_ai_work WHERE tenant_key=$1`, m.tenant).Scan(&count); err != nil || count != 1 {
|
|
t.Fatal("same input/config was duplicated across batches", err)
|
|
}
|
|
if _, err := m.ai.ManageBatch(ctx, first.ID, "pause", strings.Repeat("p", 24), "fixture-admin"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var state string
|
|
if err := m.pool.QueryRow(ctx, `SELECT state FROM qa_ops_ai_work WHERE tenant_key=$1`, m.tenant).Scan(&state); err != nil || state != "queued" {
|
|
t.Fatal("pausing one batch stopped work still needed by another active batch", err)
|
|
}
|
|
if _, err := m.ai.ManageBatch(ctx, second.ID, "pause", strings.Repeat("q", 24), "fixture-admin"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if worked, err := m.ai.ProcessOne(ctx); err != nil || worked || m.calls.Load() != 0 {
|
|
t.Fatal("paused batch still sent a request", err)
|
|
}
|
|
if _, err := m.ai.ManageBatch(ctx, first.ID, "resume", strings.Repeat("r", 24), "fixture-admin"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := m.ai.ManageBatch(ctx, first.ID, "pause", strings.Repeat("p", 24), "fixture-admin"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
current, err := m.ai.GetBatch(ctx, first.ID)
|
|
if err != nil || current.State != "active" {
|
|
t.Fatal("replayed old action applied again instead of reading current state", err)
|
|
}
|
|
if worked, err := m.ai.ProcessOne(ctx); err != nil || !worked || m.calls.Load() != 1 {
|
|
t.Fatal("resumed persistent batch did not execute exactly once", err)
|
|
}
|
|
}
|