9470a127f7
build / build (push) Successful in 1m11s
统一AI分析入口及双任务安全确认,采用全量载入的可见行渲染,保留完整原文。 右侧非模态详情固定五项质量字段,清理冗余展示并对齐查询和同步按钮。 默认4并发由数据库准入与逐任务租约保护,保留共享限流冷却、截断归因和有界重试。 兼容真实引用标题字段,不伪造证据或准确率,不修改源生产问答与公司区域规则。 验证:完整make verify退出0,710项Go通过、124项已声明可选跳过且必需0跳过; 另全量mirror隔离PostgreSQL race退出0,AI单元race退出0。 真实本地浏览器通过六项交互、390px内滚动及显式AI部分失败/零缺口故障夹具。 治理原生备份已真实恢复,schema57、72表计数与29张QA内容哈希一致。 候选镜像、精确提交CI与正式上线验收待本次提交后执行,不宣称已部署。
539 lines
19 KiB
Go
539 lines
19 KiB
Go
package qaopsmirror
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
"io"
|
|
"knowledge-governance/internal/dobotaiops"
|
|
pg "knowledge-governance/internal/platform/postgres"
|
|
"knowledge-governance/internal/qaopsai"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestAIBatchManualRetryKeepsCumulativeAttemptsAndFixedDenominator(t *testing.T) {
|
|
var attempts atomic.Int32
|
|
m := newAIMatrix(t, "fixture-ai-retry", func(w http.ResponseWriter, r *http.Request) {
|
|
if attempts.Add(1) <= 3 {
|
|
w.WriteHeader(http.StatusTooManyRequests)
|
|
return
|
|
}
|
|
writeAICompletion(w, `{"question_chinese":"MG400 如何连接?"}`)
|
|
})
|
|
ctx := context.Background()
|
|
batch := m.batch(t, "r")
|
|
advance := func() {
|
|
t.Helper()
|
|
if _, err := m.pool.Exec(ctx, `UPDATE qa_ops_ai_runtime SET next_request_at=clock_timestamp()-interval '1 second' WHERE tenant_key=$1`, m.tenant); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := m.pool.Exec(ctx, `UPDATE qa_ops_ai_work SET available_at=clock_timestamp()-interval '1 second' WHERE tenant_key=$1`, m.tenant); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for i := 0; i < 3; i++ {
|
|
advance()
|
|
if done, err := m.ai.ProcessOne(ctx); err != nil || !done {
|
|
t.Fatal("retryable attempt did not settle", err)
|
|
}
|
|
}
|
|
failed, err := m.ai.GetBatch(ctx, batch.ID)
|
|
if err != nil || failed.Total != 1 || failed.Failed != 1 || failed.Retry != 0 || m.calls.Load() != 3 {
|
|
t.Fatal("automatic retry limit or denominator is wrong", err)
|
|
}
|
|
nonce := strings.Repeat("m", 24)
|
|
if _, err = m.ai.ManageBatch(ctx, batch.ID, "retry", nonce, "fixture-admin"); err != nil {
|
|
t.Fatal("explicit manual retry unavailable", err)
|
|
}
|
|
if _, err = m.ai.ManageBatch(ctx, batch.ID, "retry", nonce, "fixture-admin"); err != nil {
|
|
t.Fatal("retry action replay failed", err)
|
|
}
|
|
advance()
|
|
if done, err := m.ai.ProcessOne(ctx); err != nil || !done {
|
|
t.Fatal("manual retry did not execute", err)
|
|
}
|
|
current, err := m.ai.GetBatch(ctx, batch.ID)
|
|
if err != nil || current.Total != 1 || current.Succeeded != 1 || current.Failed != 0 || m.calls.Load() != 4 {
|
|
t.Fatal("manual retry duplicated denominator or calls", err)
|
|
}
|
|
var count int
|
|
if err = m.pool.QueryRow(ctx, `SELECT attempt FROM qa_ops_ai_work WHERE tenant_key=$1`, m.tenant).Scan(&count); err != nil || count != 4 {
|
|
t.Fatal("manual retry erased previous request attempts", err)
|
|
}
|
|
if _, err = m.ai.ManageBatch(ctx, batch.ID, "pause", nonce, "fixture-admin"); !errors.Is(err, qaopsai.ErrConflict) {
|
|
t.Fatal("same nonce accepted changed action")
|
|
}
|
|
}
|
|
|
|
func TestAIConcurrentStartsAndClaimsKeepOneDurableEffect(t *testing.T) {
|
|
m := newAIMatrix(t, "fixture-ai-concurrent", nil)
|
|
ctx := context.Background()
|
|
preview, err := m.ai.Preview(ctx, qaopsai.Scope{Kind: "translation", IDs: []string{"matrix-record"}}, "fixture-admin")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var group sync.WaitGroup
|
|
ids := make(chan string, 12)
|
|
failures := make(chan error, 24)
|
|
for i := 0; i < 12; i++ {
|
|
group.Add(1)
|
|
go func() {
|
|
defer group.Done()
|
|
batch, e := m.ai.Start(ctx, preview.ID, strings.Repeat("c", 24), "fixture-admin")
|
|
if e != nil {
|
|
failures <- e
|
|
return
|
|
}
|
|
ids <- batch.ID
|
|
}()
|
|
}
|
|
group.Wait()
|
|
close(ids)
|
|
unique := map[string]bool{}
|
|
for id := range ids {
|
|
unique[id] = true
|
|
}
|
|
if len(unique) != 1 {
|
|
t.Fatal("concurrent nonce created different batches")
|
|
}
|
|
for i := 0; i < 12; i++ {
|
|
group.Add(1)
|
|
go func() {
|
|
defer group.Done()
|
|
_, e := m.ai.ProcessOne(ctx)
|
|
if e != nil {
|
|
failures <- e
|
|
}
|
|
}()
|
|
}
|
|
group.Wait()
|
|
close(failures)
|
|
for e := range failures {
|
|
t.Fatal("concurrent operation failed", e)
|
|
}
|
|
var batches, works, outputs int
|
|
if m.pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM qa_ops_ai_batches WHERE tenant_key=$1),(SELECT count(*) FROM qa_ops_ai_work WHERE tenant_key=$1),(SELECT count(*) FROM qa_ops_ai_outputs WHERE tenant_key=$1)`, m.tenant).Scan(&batches, &works, &outputs) != nil || batches != 1 || works != 1 || outputs != 1 || m.calls.Load() != 1 {
|
|
t.Fatal("concurrent start/claim created duplicate rows or outbound requests")
|
|
}
|
|
}
|
|
|
|
func TestAIInFlightInputEditNeverAppliesObsoleteTranslation(t *testing.T) {
|
|
entered := make(chan struct{})
|
|
release := make(chan struct{})
|
|
var once sync.Once
|
|
m := newAIMatrix(t, "fixture-ai-drift", func(w http.ResponseWriter, r *http.Request) {
|
|
close(entered)
|
|
<-release
|
|
writeAICompletion(w, `{"question_chinese":"MG400 如何连接?"}`)
|
|
})
|
|
defer once.Do(func() { close(release) })
|
|
ctx := context.Background()
|
|
batch := m.batch(t, "d")
|
|
done := make(chan error, 1)
|
|
go func() { _, err := m.ai.ProcessOne(ctx); done <- err }()
|
|
<-entered
|
|
if _, err := m.pool.Exec(ctx, `UPDATE qa_ops_master_records SET payload=payload||jsonb_build_object('question','已人工修改的问题') WHERE tenant_key=$1 AND id='matrix-record'`, m.tenant); err != nil {
|
|
once.Do(func() { close(release) })
|
|
t.Fatal(err)
|
|
}
|
|
once.Do(func() { close(release) })
|
|
if err := <-done; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
current, err := m.ai.GetBatch(ctx, batch.ID)
|
|
if err != nil || current.Skipped != 1 || current.Succeeded != 0 {
|
|
t.Fatal("in-flight input drift was not skipped", err)
|
|
}
|
|
var applied []byte
|
|
if m.pool.QueryRow(ctx, `SELECT applied_fields FROM qa_ops_ai_outputs WHERE tenant_key=$1`, m.tenant).Scan(&applied) != nil || string(applied) != "{}" {
|
|
t.Fatal("obsolete output was applied")
|
|
}
|
|
record, err := m.master.GetMasterRecord(ctx, m.tenant, "matrix-record")
|
|
if err != nil || record.Question != "已人工修改的问题" || record.MasterFields["question_chinese"] != "已人工修改的问题" {
|
|
t.Fatal("concurrent original/Chinese edit was overwritten", err)
|
|
}
|
|
}
|
|
|
|
func TestAIExpiredLeaseFencesLateModelResult(t *testing.T) {
|
|
entered := make(chan struct{})
|
|
release := make(chan struct{})
|
|
var calls atomic.Int32
|
|
var once sync.Once
|
|
m := newAIMatrix(t, "fixture-ai-lease", func(w http.ResponseWriter, r *http.Request) {
|
|
if calls.Add(1) == 1 {
|
|
close(entered)
|
|
<-release
|
|
writeAICompletion(w, `{"question_chinese":"MG400 旧的连接问题?"}`)
|
|
return
|
|
}
|
|
writeAICompletion(w, `{"question_chinese":"MG400 新的连接问题?"}`)
|
|
})
|
|
defer once.Do(func() { close(release) })
|
|
ctx := context.Background()
|
|
_ = m.batch(t, "l")
|
|
done := make(chan error, 1)
|
|
go func() { _, err := m.ai.ProcessOne(ctx); done <- err }()
|
|
<-entered
|
|
if _, err := m.pool.Exec(ctx, `UPDATE qa_ops_ai_runtime SET lease_until=clock_timestamp()-interval '1 second',next_request_at=clock_timestamp()-interval '1 second' WHERE tenant_key=$1`, m.tenant); err != nil {
|
|
once.Do(func() { close(release) })
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := m.pool.Exec(ctx, `UPDATE qa_ops_ai_work SET lease_until=clock_timestamp()-interval '1 second' WHERE tenant_key=$1`, m.tenant); err != nil {
|
|
once.Do(func() { close(release) })
|
|
t.Fatal(err)
|
|
}
|
|
if worked, err := m.ai.ProcessOne(ctx); err != nil || !worked {
|
|
once.Do(func() { close(release) })
|
|
t.Fatal("expired lease not recovered", err)
|
|
}
|
|
once.Do(func() { close(release) })
|
|
if err := <-done; !errors.Is(err, qaopsai.ErrConflict) {
|
|
t.Fatal("stale worker result not fenced", err)
|
|
}
|
|
var count int
|
|
if m.pool.QueryRow(ctx, `SELECT count(*) FROM qa_ops_ai_outputs WHERE tenant_key=$1`, m.tenant).Scan(&count) != nil || count != 1 {
|
|
t.Fatal("late worker duplicated evidence")
|
|
}
|
|
record, err := m.master.GetMasterRecord(ctx, m.tenant, "matrix-record")
|
|
if err != nil || record.MasterFields["question_chinese"] != "MG400 新的连接问题?" {
|
|
t.Fatal("late lease overwrote replacement result", err)
|
|
}
|
|
}
|
|
|
|
type aiMatrix struct {
|
|
pool *pgxpool.Pool
|
|
ai *qaopsai.Store
|
|
master *Store
|
|
tenant string
|
|
calls *atomic.Int32
|
|
}
|
|
|
|
func newAIMatrix(t *testing.T, tenant string, handler func(http.ResponseWriter, *http.Request)) aiMatrix {
|
|
t.Helper()
|
|
pool := mirrorFixture(t)
|
|
ctx := context.Background()
|
|
keyFile := filepath.Join(t.TempDir(), "master-key")
|
|
if err := os.WriteFile(keyFile, []byte(strings.Repeat("a", 32)), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
calls := &atomic.Int32{}
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
var body struct {
|
|
Messages []struct{ Content string } `json:"messages"`
|
|
}
|
|
rawBody, readErr := io.ReadAll(r.Body)
|
|
if readErr != nil || json.Unmarshal(rawBody, &body) != nil || len(body.Messages) != 2 {
|
|
t.Error("invalid synthetic request")
|
|
w.WriteHeader(400)
|
|
return
|
|
}
|
|
if strings.Contains(body.Messages[0].Content, "connection probe") {
|
|
writeAICompletion(w, `{"ok":true}`)
|
|
return
|
|
}
|
|
calls.Add(1)
|
|
r.Body = io.NopCloser(bytes.NewReader(rawBody))
|
|
if handler != nil {
|
|
handler(w, r)
|
|
return
|
|
}
|
|
writeAICompletion(w, `{"question_chinese":"MG400 如何连接?"}`)
|
|
}))
|
|
t.Cleanup(server.Close)
|
|
ai := qaopsai.NewStore(pool, tenant, qaopsai.Options{KeyFile: keyFile, AllowPrivate: true})
|
|
if _, err := ai.SaveSettings(ctx, qaopsai.SaveSettings{BaseURL: server.URL + "/v1", Model: "fixture", KeyAction: "replace", APIKey: "fixture-only"}, "fixture-admin"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
master := NewStore(pool, tenant)
|
|
entry := fixtureEntry("matrix-record", "fixture answer")
|
|
var wire map[string]any
|
|
_ = json.Unmarshal(entry.Payload, &wire)
|
|
wire["question"] = "How do I connect MG400?"
|
|
entry.Payload, _ = json.Marshal(wire)
|
|
job := claimMirrorJob(t, master, pg.NewStore(pool), "matrix-seed")
|
|
if err := master.HandleSync(ctx, *job, func(context.Context) (Source, error) {
|
|
return &fixtureSource{rows: []dobotaiops.MirrorRecord{entry}}, nil
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return aiMatrix{pool, ai, master, tenant, calls}
|
|
}
|
|
func writeAICompletion(w http.ResponseWriter, content string) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(map[string]any{"model": "fixture-actual", "choices": []any{map[string]any{"message": map[string]string{"content": content}, "finish_reason": "stop"}}})
|
|
}
|
|
func (m aiMatrix) batch(t *testing.T, nonce string) qaopsai.Batch {
|
|
t.Helper()
|
|
preview, err := m.ai.Preview(context.Background(), qaopsai.Scope{Kind: "translation", IDs: []string{"matrix-record"}}, "fixture-admin")
|
|
if err != nil || preview.Eligible != 1 {
|
|
t.Fatal("matrix preview failed", err)
|
|
}
|
|
batch, err := m.ai.Start(context.Background(), preview.ID, strings.Repeat(nonce, 24), "fixture-admin")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return batch
|
|
}
|
|
func TestAIRunnerActuallyDispatchesFourConcurrentRequests(t *testing.T) {
|
|
entered := make(chan struct{}, 8)
|
|
release := make(chan struct{})
|
|
var once sync.Once
|
|
m := newAIMatrix(t, "fixture-ai-runner-slots", func(w http.ResponseWriter, r *http.Request) {
|
|
entered <- struct{}{}
|
|
<-release
|
|
writeAICompletion(w, `{"question_chinese":"MG400 如何连接?"}`)
|
|
})
|
|
seedParallelAIWork(t, m)
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
done := make(chan struct{})
|
|
go func() { m.ai.Run(ctx, nil); close(done) }()
|
|
defer func() { once.Do(func() { close(release) }); cancel(); <-done }()
|
|
count := 0
|
|
timer := time.NewTimer(2 * time.Second)
|
|
defer timer.Stop()
|
|
loop:
|
|
for count < 4 {
|
|
select {
|
|
case <-entered:
|
|
count++
|
|
case <-timer.C:
|
|
break loop
|
|
}
|
|
}
|
|
if count != 4 {
|
|
t.Fatalf("actual worker runner is still serial: concurrent=%d", count)
|
|
}
|
|
}
|
|
|
|
func seedParallelAIWork(t *testing.T, m aiMatrix) {
|
|
t.Helper()
|
|
ctx := context.Background()
|
|
entries := []dobotaiops.MirrorRecord{}
|
|
ids := []string{}
|
|
for i, word := range []string{"Alpha", "Bravo", "Charlie", "Delta", "Echo"} {
|
|
id := fmt.Sprintf("parallel-record-%d", i)
|
|
entry := fixtureEntry(id, "fixture answer")
|
|
var wire map[string]any
|
|
_ = json.Unmarshal(entry.Payload, &wire)
|
|
wire["question"] = "How do I connect MG400? " + word
|
|
entry.Payload, _ = json.Marshal(wire)
|
|
entries = append(entries, entry)
|
|
ids = append(ids, id)
|
|
}
|
|
var seedJob string
|
|
if m.pool.QueryRow(ctx, `SELECT id FROM jobs WHERE type=$1 AND state='running'`, m.master.SyncJobType()).Scan(&seedJob) != nil {
|
|
t.Fatal("seed lease unavailable")
|
|
}
|
|
if e := pg.NewStore(m.pool).Complete(ctx, seedJob, "fixture-sync-worker", time.Now()); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
job := claimMirrorJob(t, m.master, pg.NewStore(m.pool), "parallel-seed")
|
|
if e := m.master.HandleSync(ctx, *job, func(context.Context) (Source, error) { return &fixtureSource{rows: entries}, nil }); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
preview, e := m.ai.Preview(ctx, qaopsai.Scope{Kind: "translation", IDs: ids}, "fixture-admin")
|
|
if e != nil || preview.Eligible != 5 {
|
|
t.Fatal("five unique inputs unavailable", e, preview.Eligible)
|
|
}
|
|
if _, e = m.ai.Start(ctx, preview.ID, strings.Repeat("parallel", 4), "fixture-admin"); e != nil {
|
|
t.Fatal(e)
|
|
}
|
|
}
|
|
|
|
func TestAIParallelClaimsUseFourSlotsWithoutDuplicatingOrExceedingLimit(t *testing.T) {
|
|
entered := make(chan struct{}, 8)
|
|
release := make(chan struct{})
|
|
var once sync.Once
|
|
m := newAIMatrix(t, "fixture-ai-four-slots", func(w http.ResponseWriter, r *http.Request) {
|
|
entered <- struct{}{}
|
|
<-release
|
|
writeAICompletion(w, `{"question_chinese":"MG400 如何连接?"}`)
|
|
})
|
|
defer once.Do(func() { close(release) })
|
|
ctx := context.Background()
|
|
seedParallelAIWork(t, m)
|
|
var group sync.WaitGroup
|
|
failures := make(chan error, 5)
|
|
for i := 0; i < 5; i++ {
|
|
group.Add(1)
|
|
go func() {
|
|
defer group.Done()
|
|
_, err := m.ai.ProcessOne(ctx)
|
|
if err != nil {
|
|
failures <- err
|
|
}
|
|
}()
|
|
}
|
|
seen := 0
|
|
timeout := time.NewTimer(1500 * time.Millisecond)
|
|
loop:
|
|
for seen < 4 {
|
|
select {
|
|
case <-entered:
|
|
seen++
|
|
case <-timeout.C:
|
|
break loop
|
|
}
|
|
}
|
|
timeout.Stop()
|
|
if seen != 4 {
|
|
t.Errorf("wanted four concurrent model requests before any completes; got %d", seen)
|
|
}
|
|
select {
|
|
case <-entered:
|
|
t.Error("exceeded four global slots")
|
|
case <-time.After(100 * time.Millisecond):
|
|
}
|
|
once.Do(func() { close(release) })
|
|
group.Wait()
|
|
close(failures)
|
|
for err := range failures {
|
|
t.Error(err)
|
|
}
|
|
if seen == 4 {
|
|
var running, queued, outputs int
|
|
if m.pool.QueryRow(ctx, `SELECT (SELECT count(*)FROM qa_ops_ai_work WHERE tenant_key=$1 AND state='running'),(SELECT count(*)FROM qa_ops_ai_work WHERE tenant_key=$1 AND state='queued'),(SELECT count(*)FROM qa_ops_ai_outputs WHERE tenant_key=$1)`, m.tenant).Scan(&running, &queued, &outputs) != nil || running != 0 || queued != 1 || outputs != 4 {
|
|
t.Fatal("slot completion lost/duplicated work", running, queued, outputs)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAITruncationRetriesExpandBudgetAndStopAfterThreeAttempts(t *testing.T) {
|
|
budgets := []int{}
|
|
m := newAIMatrix(t, "fixture-ai-truncation-budget", func(w http.ResponseWriter, r *http.Request) {
|
|
var req struct {
|
|
Max int `json:"max_tokens"`
|
|
}
|
|
if json.NewDecoder(r.Body).Decode(&req) != nil {
|
|
t.Error("request decode")
|
|
}
|
|
budgets = append(budgets, req.Max)
|
|
if len(budgets) < 3 {
|
|
_ = json.NewEncoder(w).Encode(map[string]any{"model": "fixture", "choices": []any{map[string]any{"finish_reason": "length", "message": map[string]string{"content": ""}}}})
|
|
return
|
|
}
|
|
writeAICompletion(w, `{"question_chinese":"MG400 如何连接?"}`)
|
|
})
|
|
batch := m.batch(t, "t")
|
|
ctx := context.Background()
|
|
for i := 0; i < 3; i++ {
|
|
if _, err := m.pool.Exec(ctx, `UPDATE qa_ops_ai_work SET available_at=clock_timestamp()-interval '1 second'WHERE tenant_key=$1`, m.tenant); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if worked, err := m.ai.ProcessOne(ctx); err != nil || !worked {
|
|
t.Fatal("bounded retry did not execute", err)
|
|
}
|
|
}
|
|
if fmt.Sprint(budgets) != "[4096 8192 16384]" {
|
|
t.Fatal("truncation budget did not grow", budgets)
|
|
}
|
|
current, err := m.ai.GetBatch(ctx, batch.ID)
|
|
if err != nil || current.Succeeded != 1 || current.Total != 1 || m.calls.Load() != 3 {
|
|
t.Fatal("retry denominator or completion", err)
|
|
}
|
|
}
|
|
|
|
func TestAIParallelSuccessCannotClearAnotherRequestsRateLimit(t *testing.T) {
|
|
entered := make(chan struct{}, 4)
|
|
release := make(chan struct{})
|
|
var once sync.Once
|
|
var calls atomic.Int32
|
|
m := newAIMatrix(t, "fixture-ai-shared-cooldown", func(w http.ResponseWriter, r *http.Request) {
|
|
n := calls.Add(1)
|
|
entered <- struct{}{}
|
|
if n == 4 {
|
|
w.Header().Set("Retry-After", "45")
|
|
w.WriteHeader(429)
|
|
return
|
|
}
|
|
<-release
|
|
writeAICompletion(w, `{"question_chinese":"MG400 如何连接?"}`)
|
|
})
|
|
defer once.Do(func() { close(release) })
|
|
seedParallelAIWork(t, m)
|
|
ctx := context.Background()
|
|
var group sync.WaitGroup
|
|
errs := make(chan error, 4)
|
|
for i := 0; i < 4; i++ {
|
|
group.Add(1)
|
|
go func() { defer group.Done(); _, err := m.ai.ProcessOne(ctx); errs <- err }()
|
|
}
|
|
for i := 0; i < 4; i++ {
|
|
select {
|
|
case <-entered:
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("four parallel slots not reached")
|
|
}
|
|
}
|
|
deadline := time.Now().Add(2 * time.Second)
|
|
remaining := 0.0
|
|
for time.Now().Before(deadline) {
|
|
if m.pool.QueryRow(ctx, `SELECT EXTRACT(EPOCH FROM next_request_at-clock_timestamp())::double precision FROM qa_ops_ai_runtime WHERE tenant_key=$1`, m.tenant).Scan(&remaining) == nil && remaining > 40 {
|
|
break
|
|
}
|
|
time.Sleep(time.Millisecond * 10)
|
|
}
|
|
once.Do(func() { close(release) })
|
|
group.Wait()
|
|
close(errs)
|
|
for err := range errs {
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if m.pool.QueryRow(ctx, `SELECT EXTRACT(EPOCH FROM next_request_at-clock_timestamp())::double precision FROM qa_ops_ai_runtime WHERE tenant_key=$1`, m.tenant).Scan(&remaining) != nil || remaining < 40 {
|
|
t.Fatal("success cleared shared provider cooldown", remaining)
|
|
}
|
|
if worked, err := m.ai.ProcessOne(ctx); err != nil || worked || calls.Load() != 4 {
|
|
t.Fatal("new request bypassed shared rate limit", err)
|
|
}
|
|
}
|
|
|
|
func TestAIManualBatchPauseResumeKeepsSharedWorkAndActionReplaysSafe(t *testing.T) {
|
|
m := newAIMatrix(t, "fixture-ai-pause", nil)
|
|
ctx := context.Background()
|
|
first := m.batch(t, "a")
|
|
second := m.batch(t, "b")
|
|
var count int
|
|
if err := m.pool.QueryRow(ctx, `SELECT count(*) FROM qa_ops_ai_work WHERE tenant_key=$1`, m.tenant).Scan(&count); err != nil || count != 1 {
|
|
t.Fatal("same input/config was duplicated across batches", err)
|
|
}
|
|
if _, err := m.ai.ManageBatch(ctx, first.ID, "pause", strings.Repeat("p", 24), "fixture-admin"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var state string
|
|
if err := m.pool.QueryRow(ctx, `SELECT state FROM qa_ops_ai_work WHERE tenant_key=$1`, m.tenant).Scan(&state); err != nil || state != "queued" {
|
|
t.Fatal("pausing one batch stopped work still needed by another active batch", err)
|
|
}
|
|
if _, err := m.ai.ManageBatch(ctx, second.ID, "pause", strings.Repeat("q", 24), "fixture-admin"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if worked, err := m.ai.ProcessOne(ctx); err != nil || worked || m.calls.Load() != 0 {
|
|
t.Fatal("paused batch still sent a request", err)
|
|
}
|
|
if _, err := m.ai.ManageBatch(ctx, first.ID, "resume", strings.Repeat("r", 24), "fixture-admin"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := m.ai.ManageBatch(ctx, first.ID, "pause", strings.Repeat("p", 24), "fixture-admin"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
current, err := m.ai.GetBatch(ctx, first.ID)
|
|
if err != nil || current.State != "active" {
|
|
t.Fatal("replayed old action applied again instead of reading current state", err)
|
|
}
|
|
if worked, err := m.ai.ProcessOne(ctx); err != nil || !worked || m.calls.Load() != 1 {
|
|
t.Fatal("resumed persistent batch did not execute exactly once", err)
|
|
}
|
|
}
|