package main import ( "context" "encoding/json" "fmt" "log" "time" "knowledge-governance/internal/appconfig" "knowledge-governance/internal/governance" "knowledge-governance/internal/jobs" "knowledge-governance/internal/libraryhandoff" "knowledge-governance/internal/librarysnapshot" postgresstore "knowledge-governance/internal/platform/postgres" ) // libraryHandoffJobType 与 API 侧常量同值:API 用它把"发布/恢复"拆成 // "先搬运、后发布"。搬运作业由 capture 角色执行(只有本机有源归档与 SSH 凭据), // release-executor 永不注册它——云端没有源归档,也没有回本机的通道。 const libraryHandoffJobType = "library.snapshot.handoff" // 回传分页:每次 100 条、最多 20 页(单轮最多核对 2000 份引用), // 游标按 (created_at, snapshot_id) 推进,下一轮从最新处重新开始。 const ( pullPageSize = 100 // pullMaxPages 只是**单轮**工作量预算:一轮最多翻 20 页,避免一次扫描拖住 worker。 // 游标跨轮保留,所以预算不会变成"永远只取前 2000 条"。 pullMaxPages = 20 ) // pullCursor 是回传枚举的位置。跨轮保留:每轮结束把游标留在这一轮读到的最后一项, // 下一轮从那里继续;读到不足一页说明这一圈走完了,游标回到起点(从最新处重新核对)。 type pullCursor struct { created time.Time id string } func (c pullCursor) advance(last postgresstore.SnapshotCandidate, count, pageSize int) pullCursor { if count < pageSize { return pullCursor{} } return pullCursor{created: last.CreatedAt, id: last.SnapshotID} } // libraryHandoffStage 是搬运在作业阶段表里的名字(页面能看出卡在搬运还是发布)。 const libraryHandoffStage = "handoff" type libraryHandoffPayload struct { LibraryJobID string `json:"library_job_id"` Scope string `json:"scope"` TargetSnapshotID string `json:"target_snapshot_id"` NextType string `json:"next_type"` } // registerLibraryHandoffHandlers 注册"先把快照送到执行器"的作业。 // // 顺序是这条链的安全边界:push 成功(云端逐 blob 校验通过、清单摘要自证、身份一致) // 之后才投递真正的发布作业。push 失败时发布作业根本不存在,executor 不会开维护窗口、 // 不会写任何知识数据。 func registerLibraryHandoffHandlers(handlers map[string]jobs.Handler, store *postgresstore.Store, config appconfig.Config, handoff *libraryhandoff.Handoff) { handlers[libraryHandoffJobType] = func(ctx context.Context, job jobs.Job) error { var payload libraryHandoffPayload if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.LibraryJobID == "" || payload.TargetSnapshotID == "" || (payload.Scope != "internal" && payload.Scope != "external") { return fmt.Errorf("invalid library handoff payload") } return runLibraryHandoffForQueue(ctx, store, config, handoff, payload, job.ID) } } // runLibraryHandoffForQueue 是 handoff 的唯一入口。queueJobID 是这次执行的**所有权凭据**: // 业务作业的 queue_job_id 必须还是它,才允许写阶段/终态;交接给 release 之后, // 迟到的旧 handoff 一次写入都不该落下去。 func runLibraryHandoffForQueue(ctx context.Context, store *postgresstore.Store, config appconfig.Config, handoff *libraryhandoff.Handoff, payload libraryHandoffPayload, queueJobID string) error { now := time.Now().UTC() // 作业身份必须与载荷逐项一致:载荷来自队列,业务作业才是治理库的事实。 record, err := store.LibraryJob(ctx, payload.LibraryJobID) if err != nil { return fmt.Errorf("读不到业务作业 %s:%w", payload.LibraryJobID, err) } if record.Scope != payload.Scope || record.TargetSnapshotID != payload.TargetSnapshotID { return failLibraryHandoffOwned(ctx, store, payload, queueJobID, "队列载荷与业务作业不一致(scope/快照),拒绝搬运") } switch payload.NextType { case librarySyncJobType: if record.Kind != governance.LibraryJobSync { return failLibraryHandoffOwned(ctx, store, payload, queueJobID, "作业类型是恢复,但搬运要求投递同步,拒绝") } case libraryRestoreJobType: if record.Kind != governance.LibraryJobRestore { return failLibraryHandoffOwned(ctx, store, payload, queueJobID, "作业类型是同步,但搬运要求投递恢复,拒绝") } default: return failLibraryHandoffOwned(ctx, store, payload, queueJobID, "未知的后续作业类型,拒绝搬运") } if record.State == governance.LibraryJobFailed || record.State == governance.LibraryJobCancelled || record.State == governance.LibraryJobSucceeded { return failLibraryHandoffOwned(ctx, store, payload, queueJobID, "业务作业已终态,不再搬运") } // 期望身份来自**独立注入的绑定** + 版本台账的登记类型,绝不来自快照自报: // 封存快照的清单写的是源 tenant/KB,目标知识库本来就不是它(云测试验收用专用 KB)。 want, identityErr := expectedSnapshotIdentity(ctx, store, config, payload.Scope, payload.TargetSnapshotID) if identityErr != nil { return failLibraryHandoffOwned(ctx, store, payload, queueJobID, "无法确定快照的可信归属,拒绝搬运:"+identityErr.Error()) } if record.QueueJobID == governance.QueueJobIDFor(payload.LibraryJobID) && queueJobID == governance.LibraryHandoffQueueJobID(payload.LibraryJobID) { log.Printf("library handoff replay job=%s: 已交接给 release,幂等跳过", payload.LibraryJobID) return nil } // 阶段与所有权在同一事务里建立:不存在"先读 owner 再写 stage"的竞态窗口。 // 阶段写一律只发生在 Begin/Complete/Fail 三个持有 job 行锁的事务方法里: // 这里若再补一次无条件 UpdateLibraryStage,既重复事务里的写,又会在 // Begin 返回与这次写之间被交接抢走——那正是上一轮退回的原因。 started, beginErr := store.BeginLibraryHandoff(ctx, payload.LibraryJobID, queueJobID, "正在把快照送到发布执行器并校验", now) if beginErr != nil { return fmt.Errorf("开始搬运阶段失败:%w", beginErr) } if !started { return fmt.Errorf("业务作业 %s 当前不由本次 handoff 持有(%q),未做任何修改", payload.LibraryJobID, record.QueueJobID) } evidence, err := handoff.Push(ctx, want) if err != nil { message := "快照未送到发布执行器:" + err.Error() return failLibraryHandoffOwned(ctx, store, payload, queueJobID, message) } // 只有搬运校验通过后,发布作业才存在。 payloadRaw, marshalErr := json.Marshal(map[string]string{ "library_job_id": payload.LibraryJobID, "scope": payload.Scope, "target_snapshot_id": payload.TargetSnapshotID, }) if marshalErr != nil { return failLibraryHandoffOwned(ctx, store, payload, queueJobID, "无法构造发布作业载荷") } // 投递、交接、阶段成功三件事在同一个事务里完成:不会出现"作业已交给 release // 但阶段还停在 running",也不会出现"阶段成功却没交接"。 detail := fmt.Sprintf("快照已在校验后送达执行器(parts=%d bytes=%d reused=%t)", evidence.Parts, evidence.Bytes, evidence.Reused) handed, handErr := store.CompleteLibraryHandoff(ctx, payload.LibraryJobID, queueJobID, jobs.EnqueueRequest{ ID: governance.QueueJobIDFor(payload.LibraryJobID), Type: payload.NextType, Payload: payloadRaw, AvailableAt: time.Now().UTC(), MaxAttempts: 1, IdempotencyKey: payload.NextType + ".queue:" + payload.LibraryJobID, }, detail, time.Now().UTC()) if handErr != nil { // 事务已回滚:所有权仍在本 handoff 手里,失败文案可信。 return failLibraryHandoffOwned(ctx, store, payload, queueJobID, "发布作业投递或关联失败(同一事务,未产生半完成状态):"+handErr.Error()) } if !handed { log.Printf("library handoff skip-handover job=%s: 所有权在搬运期间已转移,未做任何修改", payload.LibraryJobID) return nil } log.Printf("library handoff completed job=%s scope=%s snapshot=%s parts=%d reused=%t", payload.LibraryJobID, payload.Scope, payload.TargetSnapshotID, evidence.Parts, evidence.Reused) return nil } // handoffFailureBarrier 只给测试用:在失败出口写库之前被调用,让测试能把所有权 // 确定地切给 release,从而复现"失败出口到来时所有权已交接"的竞态。生产恒为 nil。 var handoffFailureBarrier func() func failLibraryHandoffOwned(ctx context.Context, store *postgresstore.Store, payload libraryHandoffPayload, queueJobID, message string) error { if handoffFailureBarrier != nil { handoffFailureBarrier() } // 阶段失败与作业失败同一个事务:要么都写,要么都不写。 changed, err := store.FailLibraryHandoff(ctx, payload.LibraryJobID, queueJobID, message, time.Now().UTC()) if err != nil { return fmt.Errorf("%s(写失败状态出错:%v)", message, err) } if !changed { log.Printf("library handoff fail-skipped job=%s: 所有权已交接或作业已终态,未修改任何状态", payload.LibraryJobID) } return fmt.Errorf("%s", message) } // runLibraryArchivePullLoop 把云端产生的快照(预备份等)取回本机归档。 // // 为什么要回传而不是"页面直接读云端":API 的所有归档消费点(列表/详情/差异/存储/ // 容量/恢复寻址)都读本机 LIBRARY_ARCHIVE_ROOT,回传后它们读到的就是真实字节, // 不需要给每条读路径加远端分支,也不会把两端路径当成全局身份(身份始终是 snapshot ID)。 // // 失败只记日志:本机没有这份版本时页面按"无此版本/不可读"展示,不会被误报成可恢复。 func runLibraryArchivePullLoop(ctx context.Context, store *postgresstore.Store, handoff *libraryhandoff.Handoff, archiveRoot string, interval time.Duration, report func(error)) { if interval <= 0 { interval = 5 * time.Minute } cursors := map[string]pullCursor{} pull := func() { for _, scope := range []string{"internal", "external"} { next, fetched, err := pullRound(ctx, store, func(id string) bool { archive, openErr := librarysnapshot.Open(archiveRoot, 256*1024*1024) if openErr != nil { return false } defer archive.Close() _, loadErr := archive.Load(id) return loadErr == nil }, func(fetchCtx context.Context, id string) (libraryhandoff.Evidence, error) { return handoff.Fetch(fetchCtx, id) }, scope, cursors[scope]) if err != nil { report(err) } cursors[scope] = next if fetched > 0 { log.Printf("library archive pull round scope=%s fetched=%d", scope, fetched) } } } pull() ticker := time.NewTicker(interval) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: pull() } } } // expectedSnapshotIdentity 决定"这份快照的清单身份应该等于谁"。 // // 两类快照的身份来源不同,混用会把合法的跨 KB 发布判成非法(或反过来放行错数据): // - 封存版本(test_sealed):清单里是**源**的 tenant/KB → 用部署注入的源绑定核对; // - 目标状态版本(预备份/生效/恢复):清单里是**目标**的 tenant/KB → 用目标绑定核对。 // // 没有登记过版本、或两类同时出现,一律拒绝:来源不明的快照不允许进入发布链路。 func expectedSnapshotIdentity(ctx context.Context, store *postgresstore.Store, config appconfig.Config, scope, snapshotID string) (libraryhandoff.Identity, error) { rows, err := store.LibraryVersionsBySnapshot(ctx, scope, snapshotID) if err != nil { return libraryhandoff.Identity{}, err } if len(rows) == 0 { return libraryhandoff.Identity{}, fmt.Errorf("快照 %s 在该 scope 下没有登记过任何版本", snapshotID) } // 原产地只由**捕获**决定,且必须读证据: // - test_sealed:源端封存 → 源绑定; // - production_prebackup:目标端预备份 → 目标绑定; // - production_effective 且证据 action=target_baseline_init / origin=target_baseline: // 目标权威基线(library_baseline 的真实整库捕获)→ 目标绑定; // - 普通 production_effective(发布成功后的引用)与 recovery_verified 不是捕获,不决定原产地。 sourceOrigin, targetOrigin := false, false for _, row := range rows { switch row.Kind { case governance.LibraryVersionTestSealed: sourceOrigin = true case governance.LibraryVersionProductionPrebackup: targetOrigin = true case governance.LibraryVersionProductionEffective: if isTargetBaselineEvidence(row.Evidence) { targetOrigin = true } } } if sourceOrigin && targetOrigin { return libraryhandoff.Identity{}, fmt.Errorf("快照 %s 既由源封存、又由目标捕获(预备份/基线):原产地冲突", snapshotID) } if !sourceOrigin && !targetOrigin { return libraryhandoff.Identity{}, fmt.Errorf("快照 %s 没有捕获证据(封存/预备份/目标基线),来源不明", snapshotID) } sealedState := sourceOrigin tenant, knowledgeBase := config.LibrarySourceTenantID, config.LibrarySourceInternalKB label := "源绑定" if !sealedState { tenant, knowledgeBase = config.LibraryProductionTenantID, config.LibraryProductionInternalKB label = "目标绑定" } if scope == "external" { if sealedState { knowledgeBase = config.LibrarySourceExternalKB } else { knowledgeBase = config.LibraryProductionExternalKB } } if tenant == 0 || knowledgeBase == "" { return libraryhandoff.Identity{}, fmt.Errorf("未配置%s(tenant/知识库),无法核对快照身份", label) } return libraryhandoff.Identity{SnapshotID: snapshotID, Scope: scope, TenantID: tenant, KnowledgeBaseID: knowledgeBase}, nil } // snapshotLister 是回传需要的存储面(便于用真实分页语义做单测)。 type snapshotLister interface { LibrarySnapshotCandidates(ctx context.Context, scope string, afterCreated time.Time, afterID string, limit int) ([]postgresstore.SnapshotCandidate, error) } // pullRound 是**单轮回传**:最多 pullMaxPages 页,返回下一轮的游标与取回条数。 // // 两个细节都是真实缺陷的教训: // - 空页必须把游标**重置**(否则引用数正好是 pageSize 整数倍时,游标永远停在最老处, // 下一轮还是空,之后新增的快照永远取不到); // - 游标跨轮保留(单轮预算不等于总量上限)。 func pullRound(ctx context.Context, source snapshotLister, isLocal func(string) bool, fetch func(context.Context, string) (libraryhandoff.Evidence, error), scope string, cursor pullCursor) (pullCursor, int, error) { fetched := 0 for page := 0; page < pullMaxPages; page++ { items, err := source.LibrarySnapshotCandidates(ctx, scope, cursor.created, cursor.id, pullPageSize) if err != nil { return cursor, fetched, fmt.Errorf("读不到 %s 的快照引用:%w", scope, err) } if len(items) == 0 { // 这一圈走完了:回到起点,下一轮从最新处重新核对(新增引用才会被看到)。 return pullCursor{}, fetched, nil } for _, item := range items { if err := ctx.Err(); err != nil { return cursor, fetched, err } if isLocal(item.SnapshotID) { continue } evidence, fetchErr := fetch(ctx, item.SnapshotID) if fetchErr != nil { // 单份失败不阻断整轮:下一轮会再试(游标仍推进,靠的是跨轮再核对)。 continue } fetched++ log.Printf("library archive pull snapshot=%s scope=%s parts=%d bytes=%d reused=%t", evidence.Snapshot, evidence.Scope, evidence.Parts, evidence.Bytes, evidence.Reused) } cursor = cursor.advance(items[len(items)-1], len(items), pullPageSize) if cursor == (pullCursor{}) { return cursor, fetched, nil } } return cursor, fetched, nil } // isTargetBaselineEvidence 判定一条 production_effective 是不是"目标权威基线"的真实捕获。 // 只认基线登记时写下的字段;普通发布成功的 effective 事件没有这些字段,不算原产地。 func isTargetBaselineEvidence(evidence map[string]any) bool { if len(evidence) == 0 { return false } if action, _ := evidence["action"].(string); action == "target_baseline_init" { return true } if origin, _ := evidence["origin"].(string); origin == "target_baseline" { return true } return false }