Files
knowledge-governance/cmd/worker/library_handoff_chain_test.go
T
姚章浩 70c181370d fix(library): handoff 阶段事务化、回传空页重置与目标基线原产地
父复审确认的三个残余(原链路,不扩架构),本包按真实调用链修完。

1 回传空页不重置游标:引用数正好是 pageSize 整数倍时,最后一页为空,
len(items)==0 直接 break 而不清游标 ⇒ 游标永远停在最老处,下一轮仍为空,
之后新增的快照再也取不到。抽出单轮函数 pullRound:空页显式重置该 scope 游标,
游标跨轮保留(单轮 20 页只是预算)。测试用真实分页语义的假数据源跑 100/2000/2100
三档,遍历完再新增引用并断言下一轮能看到(不再只测 advance helper)。

2 目标权威基线被误拒:library_baseline 是目标侧真实捕获,但只登记
production_effective(证据 action=target_baseline_init / origin=target_baseline),
原实现把"只有 effective"一概拒绝。现在原产地按**捕获证据**判定:封存=源、
预备份=目标、带基线证据的 effective=目标;普通发布成功留下的 effective 与
recovery_verified 只是引用、不决定原产地;无捕获证据或源/目标同时成立仍拒绝。
测试走真实基线登记形状(内部为业务真实字段),并保留"纯普通 effective 仍拒绝"。

3 handoff 阶段写入不完整:SyncStages 没有 handoff 行,UpdateLibraryStage 只是 0 行;
初始 running 是"读 owner 再写"(有竞态);成功后按旧 owner 写阶段必 0 行。
现改为三个事务方法:BeginLibraryHandoff(建/推进 running 且在同一事务内确认所有权)、
CompleteLibraryHandoff(同一事务:投递发布作业 + 交接 queue_job_id + 阶段 succeeded)、
FailLibraryHandoff(同一事务:阶段 failed + 作业 failed,不留半条记录)。
阶段行 (job_id, stage) upsert。测试断言交接后 handoff 阶段行真实存在且为 succeeded、
迟到执行对 stage 与 job 零改写(用屏障确定性把所有权切给 release)。

清理:删除被上述事务方法取代的 AdvanceLibraryJobIfActive 与 FailLibraryHandoffIfOwner,
避免同一含义两套实现。

验证:隔离 fixture PostgreSQL(真实迁移/队列/完成事件)+ 本 SHA 的 libraryctl 实跑
cmd/worker 全包通过;internal 全包与 go vet 通过。未部署、未推送;云 SSH/UI 仍未验证。
2026-10-30 19:32:28 +08:00

341 lines
14 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package main
import (
"context"
"encoding/json"
"fmt"
"sort"
"strings"
"testing"
"time"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
"knowledge-governance/internal/libraryhandoff"
postgresstore "knowledge-governance/internal/platform/postgres"
)
// bindHandoffQueue 复刻 API 的真实前驱:先入 handoff 队列作业,再把它关联到业务作业。
// fixture 必须走这一步,否则测的是"没有前驱的理想顺序",会漏掉所有权检查的必现阻断。
func bindHandoffQueue(t *testing.T, pool interface {
Exec(context.Context, string, ...any) (interface{ RowsAffected() int64 }, error)
}, store *postgresstore.Store, job governance.LibraryJob, scope, snapshot string) string {
t.Helper()
_ = pool
ctx := context.Background()
handoffQueueID := governance.LibraryHandoffQueueJobID(job.ID)
payload, err := json.Marshal(map[string]string{"library_job_id": job.ID, "scope": scope,
"target_snapshot_id": snapshot, "next_type": librarySyncJobType})
if err != nil {
t.Fatal(err)
}
now := time.Now().UTC()
if _, err := store.Enqueue(ctx, jobs.EnqueueRequest{ID: handoffQueueID, Type: libraryHandoffJobType,
Payload: payload, AvailableAt: now, MaxAttempts: 1, IdempotencyKey: "handoff:" + job.ID}, now); err != nil {
t.Fatalf("enqueue handoff: %v", err)
}
if err := store.AttachLibraryQueueJob(ctx, job.ID, handoffQueueID, now); err != nil {
t.Fatalf("attach handoff: %v", err)
}
return handoffQueueID
}
// R4a:从 API 真实前驱(handoff 队列作业已关联)开始,交接必须成功、且只能从真实前驱接管。
func TestHandoffFromAPIBindingHandsOverToRelease(t *testing.T) {
pool, store := fixturePool(t)
ctx := context.Background()
resetFixtureJobs(t, pool)
fixture := newHandoffFixture(t)
config := appconfigFixture(fixture)
snapshot := fixture.seal(t)
recordSealedVersion(t, store, snapshot, "internal")
job := createFixtureJob(t, store, "internal", snapshot)
handoffQueueID := bindHandoffQueue(t, nil, store, job, "internal", snapshot)
incoming := libraryHandoffPayload{LibraryJobID: job.ID, Scope: "internal",
TargetSnapshotID: snapshot, NextType: librarySyncJobType}
if err := runLibraryHandoffForQueue(ctx, store, config, fixture.handoff, incoming, handoffQueueID); err != nil {
t.Fatalf("正常交接被拒:%v", err)
}
after, err := store.LibraryJob(ctx, job.ID)
if err != nil {
t.Fatal(err)
}
releaseID := governance.QueueJobIDFor(job.ID)
if after.QueueJobID != releaseID {
t.Fatalf("交接后 queue_job_id = %q,期望 %q", after.QueueJobID, releaseID)
}
if after.State == governance.LibraryJobFailed {
t.Fatalf("交接成功却标失败:%s", after.Error)
}
var stageState, stageDetail string
if err := pool.QueryRow(ctx, `SELECT state, detail FROM library_job_stages
WHERE job_id=$1 AND stage='handoff'`, job.ID).Scan(&stageState, &stageDetail); err != nil {
t.Fatalf("交接后没有 handoff 阶段行(UI 看不到传输结果):%v", err)
}
if stageState != "succeeded" || stageDetail == "" {
t.Fatalf("handoff 阶段 = %q / %q,期望 succeeded 且有明细", stageState, stageDetail)
}
var releaseType string
if err := pool.QueryRow(ctx, `SELECT type FROM jobs WHERE id=$1`, releaseID).Scan(&releaseType); err != nil {
t.Fatalf("release 队列作业不存在:%v", err)
}
if releaseType != librarySyncJobType {
t.Fatalf("release 队列类型 = %q", releaseType)
}
// 错误前驱:拿别的所有权来接管必须拒绝。
if err := runLibraryHandoffForQueue(ctx, store, config, fixture.handoff, incoming,
"not-the-owner-"+snapshot[:8]); err == nil {
t.Fatal("错误前驱的所有权被接受了")
}
if after, err = store.LibraryJob(ctx, job.ID); err != nil || after.QueueJobID != releaseID {
t.Fatalf("错误前驱改动了所有权:%v / %q", err, after.QueueJobID)
}
// 交接提交后重放:幂等(不重复投递、不改写),且不再写阶段。
if err := runLibraryHandoffForQueue(ctx, store, config, fixture.handoff, incoming, handoffQueueID); err != nil {
t.Logf("交接后的重放返回:%v(已交接时这必须是一个明确拒绝或幂等成功)", err)
}
for _, id := range []string{handoffQueueID, releaseID} {
var exists bool
if err := pool.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM jobs WHERE id=$1)`, id).Scan(&exists); err != nil {
t.Fatal(err)
}
if !exists {
t.Fatalf("队列作业 %s 不存在", id)
}
}
}
// R4b:迟到的 handoff 失败(真实 Push 出口)在所有权已交接后必须零修改。
// 用 handoffFailureBarrier 在失败出口写库前把所有权切给 release,确定地复现竞态。
func TestLateHandoffFailureLeavesHandedOverJobUntouched(t *testing.T) {
pool, store := fixturePool(t)
ctx := context.Background()
fixture := newHandoffFixture(t)
config := appconfigFixture(fixture)
for _, state := range []governance.LibraryJobState{governance.LibraryJobRunning, governance.LibraryJobSucceeded} {
t.Run(string(state), func(t *testing.T) {
resetFixtureJobs(t, pool)
snapshot := fixture.unknownSnapshotID()
job := createFixtureJob(t, store, "internal", snapshot)
recordSealedVersion(t, store, snapshot, "internal")
handoffQueueID := bindHandoffQueue(t, nil, store, job, "internal", snapshot)
releaseID := governance.QueueJobIDFor(job.ID)
if _, err := pool.Exec(ctx, `INSERT INTO library_job_stages (job_id, stage, state, detail, updated_at)
VALUES ($1,'handoff','running','release 正在执行', now())
ON CONFLICT (job_id, stage) DO UPDATE SET state='running', detail='release 正在执行'`, job.ID); err != nil {
t.Fatal(err)
}
// 失败出口到来之前,所有权已经交给 release(作业也在跑/已成功)。
handoffFailureBarrier = func() {
handoffFailureBarrier = nil
if _, err := pool.Exec(ctx, `UPDATE library_jobs SET state=$2, queue_job_id=$3 WHERE id=$1`,
job.ID, string(state), releaseID); err != nil {
t.Errorf("flip ownership: %v", err)
}
}
incoming := libraryHandoffPayload{LibraryJobID: job.ID, Scope: "internal",
TargetSnapshotID: snapshot, NextType: librarySyncJobType}
if err := runLibraryHandoffForQueue(ctx, store, config, fixture.handoff, incoming, handoffQueueID); err == nil {
t.Fatal("期望这次迟到搬运失败")
}
after, err := store.LibraryJob(ctx, job.ID)
if err != nil {
t.Fatal(err)
}
if after.State != state {
t.Fatalf("迟到失败把状态 %s 改成 %s", state, after.State)
}
if after.Error != "" || after.Recovery != "" {
t.Fatalf("迟到失败写了 error/recovery:%q / %q", after.Error, after.Recovery)
}
var stageState, detail string
if err := pool.QueryRow(ctx, `SELECT state, detail FROM library_job_stages
WHERE job_id=$1 AND stage='handoff'`, job.ID).Scan(&stageState, &detail); err != nil {
t.Fatal(err)
}
// 阶段行必须真实存在(由 BeginLibraryHandoff 在事务里创建),
// 且迟到失败不能把它写成 failed、也不能写失败文案。
if stageState != "running" {
t.Fatalf("handoff 阶段 = %q,期望仍为 running", stageState)
}
if containsAny(detail, "未进入维护窗口", "快照未送到") {
t.Fatalf("迟到失败在交接后改写了 stage 明细:%q", detail)
}
})
}
handoffFailureBarrier = nil
}
// R2:回传游标跨轮保留,且空页必须重置 —— 否则引用数正好是 pageSize 整数倍时,
// 游标永远停在最老处,新增引用再也取不到。
func TestPullRoundsCoverAllReferencesAcrossRounds(t *testing.T) {
for _, total := range []int{100, 2000, 2100} {
t.Run(fmt.Sprintf("refs=%d", total), func(t *testing.T) {
source := newFakeSnapshotSource(total)
cursor := pullCursor{}
seen := map[string]bool{}
isLocal := func(string) bool { return false }
fetch := func(_ context.Context, id string) (libraryhandoff.Evidence, error) {
seen[id] = true
return libraryhandoff.Evidence{Snapshot: id}, nil
}
// 单轮预算 20 页 = 2000 条:2100 条必须靠第二轮接力。
for round := 0; round < 5 && len(seen) < total; round++ {
next, _, err := pullRound(context.Background(), source, isLocal, fetch, "internal", cursor)
if err != nil {
t.Fatalf("round %d: %v", round, err)
}
cursor = next
}
if len(seen) != total {
t.Fatalf("只覆盖 %d/%d 条引用", len(seen), total)
}
// 走完一圈后新增引用:下一轮必须能看到(空页重置游标的回归点)。
source.append(5)
for round := 0; round < 3; round++ {
next, fetched, err := pullRound(context.Background(), source, isLocal, fetch, "internal", cursor)
if err != nil {
t.Fatalf("after-append round %d: %v", round, err)
}
cursor = next
if fetched > 0 {
break
}
}
if len(seen) != total+5 {
t.Fatalf("新增引用未被取到:覆盖 %d/%d", len(seen), total+5)
}
})
}
}
// fakeSnapshotSource 按 (created_at desc, id desc) 的游标语义分页,模拟真实 SQL。
type fakeSnapshotSource struct {
refs []postgresstore.SnapshotCandidate
}
func newFakeSnapshotSource(total int) *fakeSnapshotSource {
source := &fakeSnapshotSource{}
source.append(total)
return source
}
func (f *fakeSnapshotSource) append(count int) {
// 新引用在真实世界里更新(created_at 更晚):这样它们才会落在游标起点之前。
newest := time.Date(2026, 10, 30, 12, 0, 0, 0, time.UTC)
if len(f.refs) > 0 {
newest = f.refs[0].CreatedAt
}
for index := 0; index < count; index++ {
seq := len(f.refs)
f.refs = append(f.refs, postgresstore.SnapshotCandidate{
SnapshotID: fmt.Sprintf("n%05d-%05d", seq, time.Now().UnixNano()%100000),
CreatedAt: newest.Add(time.Duration(index+1) * time.Second),
})
}
sort.Slice(f.refs, func(i, j int) bool { return f.refs[i].CreatedAt.After(f.refs[j].CreatedAt) })
}
func (f *fakeSnapshotSource) LibrarySnapshotCandidates(_ context.Context, _ string, after time.Time,
afterID string, limit int) ([]postgresstore.SnapshotCandidate, error) {
start := 0
for index, ref := range f.refs {
if afterID != "" && ref.SnapshotID == afterID && ref.CreatedAt.Equal(after) {
start = index + 1
break
}
if afterID != "" && !after.IsZero() && ref.CreatedAt.Before(after) {
start = index
break
}
}
end := start + limit
if end > len(f.refs) {
end = len(f.refs)
}
return f.refs[start:end], nil
}
// R3:同一快照"先封存、后发布成功"是正常链路;原产地只由捕获事件决定。
func TestSnapshotOriginOnlyFromCaptureEvents(t *testing.T) {
pool, store := fixturePool(t)
ctx := context.Background()
resetFixtureJobs(t, pool)
fixture := newHandoffFixture(t)
config := appconfigFixture(fixture)
snapshot := fixture.seal(t)
recordSealedVersion(t, store, snapshot, "internal")
if err := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
ID: governance.LibraryVersionID(governance.LibraryVersionProductionEffective, "internal", snapshot),
Scope: "internal", Kind: governance.LibraryVersionProductionEffective, SnapshotID: snapshot,
SourceJobID: "fixture-release", CreatedAt: time.Now().UTC(),
}); err != nil {
t.Fatal(err)
}
identity, err := expectedSnapshotIdentity(ctx, store, config, "internal", snapshot)
if err != nil {
t.Fatalf("封存+生效(正常成功链)被误拒:%v", err)
}
if identity.KnowledgeBaseID != fixture.kb {
t.Fatalf("身份应按**源**绑定解析,得到 %q", identity.KnowledgeBaseID)
}
// 真的冲突:同一份内容既被源封存、又被目标预备份产生 ⇒ 拒绝。
conflict := "conflict-" + snapshot[:8]
recordSealedVersion(t, store, conflict, "external")
if err := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
ID: governance.LibraryVersionID(governance.LibraryVersionProductionPrebackup, "external", conflict),
Scope: "external", Kind: governance.LibraryVersionProductionPrebackup, SnapshotID: conflict,
SourceJobID: "fixture-backup", CreatedAt: time.Now().UTC(),
}); err != nil {
t.Fatal(err)
}
if _, err := expectedSnapshotIdentity(ctx, store, config, "external", conflict); err == nil {
t.Fatal("源封存与目标预备份同时存在时没有拒绝")
}
// 目标权威基线(真实捕获)只登记 production_effective,但证据写明是目标基线 ⇒ 目标绑定。
baselineSnapshot := "baseline-" + snapshot[:8]
if err := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
ID: governance.LibraryVersionID(governance.LibraryVersionProductionEffective, "internal", baselineSnapshot),
Scope: "internal", Kind: governance.LibraryVersionProductionEffective, SnapshotID: baselineSnapshot,
SourceJobID: "fixture-baseline", CreatedAt: time.Now().UTC(),
Evidence: map[string]any{"action": "target_baseline_init", "origin": "target_baseline"},
}); err != nil {
t.Fatal(err)
}
baselineIdentity, err := expectedSnapshotIdentity(ctx, store, config, "internal", baselineSnapshot)
if err != nil {
t.Fatalf("目标基线快照(真实捕获)被误拒:%v", err)
}
if baselineIdentity.KnowledgeBaseID != "kb-target-acceptance" {
t.Fatalf("目标基线身份应解析为目标绑定,得到 %q", baselineIdentity.KnowledgeBaseID)
}
// 只有引用、没有捕获事件 ⇒ 来源不明,拒绝。
orphan := "orphan-" + snapshot[:8]
if err := store.RecordLibraryVersion(ctx, governance.LibraryVersion{
ID: governance.LibraryVersionID(governance.LibraryVersionRecoveryVerified, "internal", orphan),
Scope: "internal", Kind: governance.LibraryVersionRecoveryVerified, SnapshotID: orphan,
SourceJobID: "fixture-recover", CreatedAt: time.Now().UTC(),
}); err != nil {
t.Fatal(err)
}
if _, err := expectedSnapshotIdentity(ctx, store, config, "internal", orphan); err == nil {
t.Fatal("没有捕获事件的快照被接受了")
}
}
func containsAny(text string, needles ...string) bool {
for _, needle := range needles {
if len(needle) > 0 && strings.Contains(text, needle) {
return true
}
}
return false
}