diff --git a/CHANGELOG.md b/CHANGELOG.md index 183f60c..0d333ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,31 @@ # Changelog +## 开发快照(发布管理统一修复:执行身份 · 终态收尾 · 准入与数据身份) + +### Changed + +- **发布意图与执行代次分开**:`library.sync` / `library.restore` / 续作改用 operation(一次明确意图)+ attempt(执行代次)。同一个 `operation_key` 重发命中同一条意图(不重复执行),不带或换键就是新意图——A→B→A 的第三次发布不再拿到第一次的历史成功作业。请求体新增可选 `operation_key`。 +- 版本台账改为**不可变事件 + current 指针**:不同 operation 引用同一快照是两条事件,`source_job_id`/`evidence`/`created_at` 不再被后来的执行覆盖;"谁是当前"由单调次序的条件更新决定。治理库新增迁移 76(`library_operations` / `library_attempts` / `library_current_versions`),不触碰 WeKnora Schema、源码或镜像。 +- 按预备份恢复成功后登记**恢复事实**并推进 current:页面、差异与后续作业看到的是恢复出来的内容身份,而不是继续宣称已失败的新版本生效;原失败作业与失败版本保持不变。 +- 接管或恢复时,执行目标消费已持久化写集合里的复用决定;**读不懂计划的实现直接拒绝写入**(不再"账本说没动共享对象、实际又重写一遍")。 +- 预备份合同改为目标实时状态(schema/模型/映射),不再把来源快照的声明当目标合同。 + +### Fixed + +- 半写窗口接管时,前置检查(入口/后台静默/停写者)任一失败都不再重新开放目标:持久 `ContentRisk` 优先于"本进程还没写"。 +- 服务启动或健康校验失败时不再调用开入口:入口保持关闭,交恢复流程处理。 +- journal 已是 `succeeded` / `recovered` 的终态重放可从持久事实重建收尾凭据,不再永久卡住维护锁。 +- 心跳失租会立刻取消 handler;业务状态、阶段进度、版本事件与维护锁释放都带执行代次(epoch)守卫。 +- 迟到的旧代次不能推进 current,也不能释放/改写另一个代次的窗口。 +- 续作沿用原 operation 与首次锚点,不另造丢谱系的新意图。 +- 收尾续作(finalize-only)与正常出口共用同一证据构造:缺少真实字节数或写集合摘要即拒绝登记生效版本。 +- 通用 `POST /api/v1/jobs` 拒绝 `library.*` 保留类型;worker 在动维护前核对业务行、kind/scope 与队列关联。 +- 全库搬运改写定位符时重算 `location_hash = sha256(最终 physical_path)`,与文档级导入共用同一实现。 +- 整库发布在副作用前核对目标**实时**模型契约:同维不同模型、模型不可读、dirty schema 一律拒绝。 +- 容量门禁接到 worker 执行前(预算不足或读不到即拒绝);存储预览按全量依赖判定保护,保护依据读不到时拒绝生成可删候选,备份类型只认显式登记。 +- 发布页:状态读取核 HTTP 状态与响应结构;按预备份恢复纳入轮询并在维护关闭后读回终态;Wiki 差异覆盖标题/摘要/父级变化。 +- CI 版本契约门同时核进程退出码、必需测试真 PASS 与负例失败归因(编译/网络错误不算预期断言生效)。 + ## 开发快照(接管复用预备份不再抹掉写集合证据) ### Fixed diff --git a/cmd/worker/library_admission.go b/cmd/worker/library_admission.go new file mode 100644 index 0000000..44a6860 --- /dev/null +++ b/cmd/worker/library_admission.go @@ -0,0 +1,51 @@ +package main + +import ( + "context" + "errors" + "fmt" + + "knowledge-governance/internal/governance" + "knowledge-governance/internal/jobs" + postgresstore "knowledge-governance/internal/platform/postgres" +) + +// 准入:worker 在动维护/目标之前,必须自己证明"这个队列作业确实是这条业务作业"。 +// +// 为什么不能只靠"通用入口拒绝 library.*":那只堵住了一个入口。队列里仍可能因为历史 +// 残留、手工 SQL、别的服务投递而出现畸形作业;而 worker 拿到没有业务行的作业时会照常 +// 取维护锁、停写入者、开备份窗口(F14 的真实现场)。所以准入必须在**副作用之前**, +// 并且同时核三件事:业务行存在、kind/scope 与载荷一致、队列关联不可伪造。 +func admitLibraryJob(ctx context.Context, store *postgresstore.Store, queueJob jobs.Job, + libraryJobID, scope string, kind governance.LibraryJobKind, requireQueueMatch bool) (governance.LibraryJob, error) { + if libraryJobID == "" { + return governance.LibraryJob{}, errors.New("载荷缺少业务作业号:拒绝执行") + } + if scope != "internal" && scope != "external" { + return governance.LibraryJob{}, fmt.Errorf("载荷 scope 非法(%q):拒绝执行", scope) + } + business, err := store.LibraryJob(ctx, libraryJobID) + if err != nil { + // 没有业务行 ⇒ 这次执行没有准入依据(闸口/授权/维护归属都无从核对)。 + return governance.LibraryJob{}, fmt.Errorf("队列作业 %s 没有对应的业务作业 %s:拒绝在任何维护动作前继续", + queueJob.ID, libraryJobID) + } + if business.Scope != scope { + return governance.LibraryJob{}, fmt.Errorf("业务作业 %s 属于 scope=%s,载荷声称 %s:拒绝", libraryJobID, business.Scope, scope) + } + if kind != "" && business.Kind != kind { + return governance.LibraryJob{}, fmt.Errorf("业务作业 %s 的 kind=%s,与执行器 %s 不一致:拒绝", libraryJobID, business.Kind, kind) + } + if requireQueueMatch { + // 队列关联由 API 在建作业后写入;为空说明投递没有落定,不匹配说明这个队列作业 + // 不是为它准备的。两种情况都不能执行。 + if business.QueueJobID == "" { + return governance.LibraryJob{}, fmt.Errorf("业务作业 %s 没有队列关联:拒绝执行未落定的投递", libraryJobID) + } + if business.QueueJobID != queueJob.ID { + return governance.LibraryJob{}, fmt.Errorf("队列作业 %s 不是业务作业 %s 的关联作业(%s):拒绝", + queueJob.ID, libraryJobID, business.QueueJobID) + } + } + return business, nil +} diff --git a/cmd/worker/library_baseline.go b/cmd/worker/library_baseline.go index 156db1c..53c7622 100644 --- a/cmd/worker/library_baseline.go +++ b/cmd/worker/library_baseline.go @@ -100,14 +100,25 @@ func settleBaselineWindow(ctx context.Context, store libraryMaintenanceStore, he // 复核这份快照再登记; // - 登记后、收尾前被杀:重入识别出"本作业+同一快照"已生效,只做收尾 // (恢复服务/开入口/释放占用),不重复捕获、不重写版本;别人的版本仍然拒绝。 -func runLibraryBaseline(ctx context.Context, store *postgresstore.Store, config appconfig.Config, job jobs.Job) error { +func runLibraryBaseline(ctx context.Context, base *postgresstore.Store, config appconfig.Config, job jobs.Job) error { var payload libraryBaselinePayload if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.LibraryJobID == "" || (payload.Scope != "internal" && payload.Scope != "external") { return errors.New("invalid library baseline payload") } + // 准入与代次:基线也会停写入者、取整库快照,同样是"动目标"的执行, + // 必须和 sync/restore 用同一套准入与 fencing(F14/F13)。 + if _, admitErr := admitLibraryJob(ctx, base, job, payload.LibraryJobID, payload.Scope, governance.LibraryJobBaseline, true); admitErr != nil { + return admitErr + } + writer, attemptErr := beginLibraryWriter(ctx, base, payload.LibraryJobID, libraryQueueOwner(job, payload.LibraryJobID)) + if attemptErr != nil { + return attemptErr + } + store := writer now := time.Now().UTC() - recorder := libraryStageRecorder{store: store, jobID: payload.LibraryJobID, scope: payload.Scope} + recorder := libraryStageRecorder{store: store, jobID: payload.LibraryJobID, scope: payload.Scope, + operationID: writer.attempt.OperationID} fail := func(message, stage string) error { return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, message, stage) } @@ -254,7 +265,7 @@ func runLibraryBaseline(ctx context.Context, store *postgresstore.Store, config // 版本登记放在窗口内:捕获、校验、登记三件事要么一起成立,要么目标原样恢复。 recorder.Stage(ctx, "finalize", "running", "登记目标权威基线") if recordErr := store.RecordLibraryVersion(ctx, governance.LibraryVersion{ - ID: governance.LibraryVersionID(governance.LibraryVersionProductionEffective, payload.Scope, baselineID), + ID: governance.LibraryVersionEventID(governance.LibraryVersionProductionEffective, payload.Scope, baselineID, writer.attempt.OperationID), Scope: payload.Scope, Kind: governance.LibraryVersionProductionEffective, SnapshotID: baselineID, SourceJobID: payload.LibraryJobID, CreatedBy: actor, Evidence: map[string]any{ diff --git a/cmd/worker/library_capacity.go b/cmd/worker/library_capacity.go new file mode 100644 index 0000000..03b1a6c --- /dev/null +++ b/cmd/worker/library_capacity.go @@ -0,0 +1,67 @@ +package main + +import ( + "context" + "errors" + "fmt" + "math" + + "knowledge-governance/internal/appconfig" + "knowledge-governance/internal/libraryretention" + "knowledge-governance/internal/librarysnapshot" +) + +// libraryWriteFootprint 是本次写入的容量需求下界:内容字节 + 等量恢复余量。 +// +// 为什么要算两份:写入前必须留出"目标被写坏后按预备份恢复"的空间。只按本次内容算, +// 预算刚好够写、不够恢复,故障时就地无法回滚——那比不发布更糟。 +func libraryWriteFootprint(contentBytes int64) (int64, error) { + if contentBytes < 0 { + return 0, errors.New("内容字节为负:容量口径异常") + } + if contentBytes > math.MaxInt64/2 { + return 0, errors.New("内容字节过大:容量需求溢出,拒绝放行") + } + return contentBytes * 2, nil +} + +// checkLibraryCapacity 是**执行前**的容量门禁(F11)。 +// +// 旧实现只在存储页 GET 里调用 CheckCapacity:那个数字是给人看的,worker 从来不读它, +// 所以"预算为正但明显不够"照样会进入维护、拍备份、写目标。这里把它接到真正要写之前。 +// +// 读不到容量/预算时不放行:无法证明装得下就不该动生产。 +func checkLibraryCapacity(ctx context.Context, config appconfig.Config, snapshotID string) error { + if config.LibraryCapacityBudgetBytes <= 0 { + return errors.New("容量预算未设置(LIBRARY_CAPACITY_BUDGET_BYTES):无法证明装得下,拒绝进入维护") + } + archive, err := librarysnapshot.Open(config.LibraryArchiveRoot, 256*1024*1024) + if err != nil { + return fmt.Errorf("归档不可用,无法核算容量:%w", err) + } + defer archive.Close() + usage, usageErr := archive.ContentUsage() + if usageErr != nil { + return fmt.Errorf("读不到当前归档内容占用,拒绝放行:%w", usageErr) + } + manifest, loadErr := archive.Load(snapshotID) + if loadErr != nil { + return fmt.Errorf("读不到待写入快照的清单,无法核算容量:%w", loadErr) + } + var incoming int64 + for _, part := range manifest.Parts { + if part.Bytes < 0 || incoming > math.MaxInt64-part.Bytes { + return errors.New("快照分片字节异常:拒绝放行") + } + incoming += part.Bytes + } + required, footprintErr := libraryWriteFootprint(incoming) + if footprintErr != nil { + return footprintErr + } + capacity := libraryretention.CheckCapacity(usage.ContentBytes, required, config.LibraryCapacityBudgetBytes) + if !capacity.OK { + return fmt.Errorf("容量门禁拒绝,未打开维护窗口:%s", capacity.Reason) + } + return nil +} diff --git a/cmd/worker/library_capacity_test.go b/cmd/worker/library_capacity_test.go new file mode 100644 index 0000000..4b16215 --- /dev/null +++ b/cmd/worker/library_capacity_test.go @@ -0,0 +1,22 @@ +package main + +import ( + "math" + "testing" +) + +// F11:容量需求必须是"内容 + 等量恢复余量",且溢出/负值要拒绝而不是绕回。 +func TestLibraryWriteFootprint(t *testing.T) { + if got, err := libraryWriteFootprint(1000); err != nil || got != 2000 { + t.Fatalf("footprint = %d, %v(写入 + 恢复余量)", got, err) + } + if _, err := libraryWriteFootprint(-1); err == nil { + t.Fatal("负字节必须拒绝") + } + if _, err := libraryWriteFootprint(math.MaxInt64/2 + 1); err == nil { + t.Fatal("溢出必须拒绝,不能算出负数后放行") + } + if got, err := libraryWriteFootprint(0); err != nil || got != 0 { + t.Fatalf("空内容 = %d, %v", got, err) + } +} diff --git a/cmd/worker/library_completion.go b/cmd/worker/library_completion.go new file mode 100644 index 0000000..dd7ca73 --- /dev/null +++ b/cmd/worker/library_completion.go @@ -0,0 +1,152 @@ +package main + +import ( + "context" + "errors" + "fmt" + "time" + + "knowledge-governance/internal/governance" + "knowledge-governance/internal/libraryrelease" + postgresstore "knowledge-governance/internal/platform/postgres" +) + +// libraryFailStore 是"失败出口"需要的写面:带代次守卫的失败收尾。 +// 抽成接口是因为 writer 与裸 Store 都要能传进来(测试里常用裸 Store)。 +type libraryFailStore interface { + AdvanceLibraryJob(ctx context.Context, jobID string, state governance.LibraryJobState, stage, message, recovery, snapshotID, targetSnapshotID string, now time.Time) error + UpdateLibraryMaintenanceState(ctx context.Context, scope, ownerJobID, state, note string, now time.Time) error +} + +// prebackupAnchors 读"这次操作第一次拍下的那份额度":字节数与写集合摘要。 +// +// 为什么必须读它而不是重新计算:接管/收尾续作都没有重新备份,手上只有已登记的事实; +// 重新用当前状态算一遍就等于把"这次用了多大一份基线、写了什么"换成猜的(F09 的现场 +// 就是 backup_bytes 变成 0、writeset_digest 变空)。 +func prebackupAnchors(ctx context.Context, store *postgresstore.Store, scope, jobID, backupSnapshot string) (map[string]any, error) { + version, found, err := store.LibraryVersionByRef(ctx, scope, governance.LibraryVersionProductionPrebackup, jobID, backupSnapshot) + if err != nil { + return nil, fmt.Errorf("读不到预备份证据:%w", err) + } + if !found { + return nil, fmt.Errorf("预备份 %s 没有作业 %s 的证据行:拒绝用空锚点登记生效版本", backupSnapshot, jobID) + } + anchors := map[string]any{} + for _, key := range []string{"backup_bytes", "writeset_digest", "writeset_rows", "writeset_path", + "writeset_created_at", "backup_created_at", "index_mapping_sha256", "backup_reused"} { + if value, ok := version.Evidence[key]; ok && value != nil { + anchors[key] = value + } + } + return anchors, nil +} + +// finalizeEvidence 构造"生效/恢复"事件的证据:以持久化锚点为底,补上本次动作与映射。 +// 缺字节数或写集合摘要时返回错误——收尾可以补登记,但不能补出一份没有锚点的"成功"。 +func finalizeEvidence(ctx context.Context, store *postgresstore.Store, scope, jobID, backupSnapshot, + targetSnapshot, action, mappingDigest string) (map[string]any, error) { + anchors, err := prebackupAnchors(ctx, store, scope, jobID, backupSnapshot) + if err != nil { + return nil, err + } + if _, ok := anchors["backup_bytes"]; !ok { + return nil, fmt.Errorf("预备份 %s 的证据里没有真实字节数:拒绝登记生效版本(会把 backup_bytes 洗成空)", backupSnapshot) + } + if digest, _ := anchors["writeset_digest"].(string); digest == "" { + return nil, fmt.Errorf("预备份 %s 的证据里没有写集合摘要:拒绝登记生效版本(恢复将无从按计划清理)", backupSnapshot) + } + evidence := map[string]any{"backup_snapshot": backupSnapshot, "action": action, "target_snapshot": targetSnapshot} + for key, value := range anchors { + evidence[key] = value + } + if mappingDigest != "" { + evidence["index_mapping_sha256"] = mappingDigest + } + return evidence, nil +} + +// rebuildSuccessBoundary 从**持久事实**重建"内容已验证、只差收尾"的凭据。 +// +// 触发场景(F05 路径一):journal 已 saved succeeded,但进程在 finishLibrarySuccess 之前 +// 崩了。接管方再跑 RunWindow 会直接成功返回、不再执行 work;如果这里还依赖进程内 +// pending 指针,作业就永远收不了尾(锁挂着、队列已完成、页面一直显示维护中)。 +// +// 重建需要且只需要三件事: +// 1. 作业行里持久化的预备份身份(这次用的是哪份基线); +// 2. 那份预备份的证据行里有真实字节数与写集合摘要(首次锚点没被洗掉); +// 3. 生效事件存在,或可以被**幂等补登记**(不重写目标、不再 Apply)。 +func rebuildSuccessBoundary(ctx context.Context, store *libraryAttemptWriter, recorder libraryStageRecorder, + payload libraryReleasePayload, previous governance.LibraryJob, kind governance.LibraryJobKind, + grant governance.LibraryReleaseAuthorization) (successBoundary, error) { + backup := previous.SnapshotID + if backup == "" { + return successBoundary{}, errors.New("窗口已成功,但作业行没有持久化预备份身份:无法证明内容基线") + } + action := string(kind) + if payload.FinalizeOnly { + action = "finalize_only" + } + if _, err := prebackupAnchors(ctx, store.Store, payload.Scope, payload.LibraryJobID, backup); err != nil { + return successBoundary{}, err + } + _, found, err := store.LibraryVersionByRef(ctx, payload.Scope, governance.LibraryVersionProductionEffective, + payload.LibraryJobID, payload.TargetSnapshotID) + if err != nil { + return successBoundary{}, fmt.Errorf("读不到生效事件:%w", err) + } + if !found { + recorder.Stage(ctx, "finalize", "running", "窗口已成功:补登记生效版本(不重写生产)") + evidence, evidenceErr := finalizeEvidence(ctx, store.Store, payload.Scope, payload.LibraryJobID, backup, + payload.TargetSnapshotID, action, recorder.mappingDigest) + if evidenceErr != nil { + return successBoundary{}, evidenceErr + } + if err := store.RecordLibraryVersion(ctx, governance.LibraryVersion{ + ID: governance.LibraryVersionEventID(governance.LibraryVersionProductionEffective, payload.Scope, payload.TargetSnapshotID, store.attempt.OperationID), + Scope: payload.Scope, Kind: governance.LibraryVersionProductionEffective, + SnapshotID: payload.TargetSnapshotID, SourceJobID: payload.LibraryJobID, CreatedBy: grant.AuthorizedBy, + Evidence: evidence, CreatedAt: time.Now().UTC(), + }); err != nil { + return successBoundary{}, fmt.Errorf("补登记生效版本失败:%w", err) + } + recorder.Stage(ctx, "finalize", "succeeded", "已补登记生效版本(未重写生产)") + } + return successBoundary{Ready: true, Backup: backup, Target: payload.TargetSnapshotID, + FinishedAt: time.Now().UTC()}, nil +} + +// recordRecoveryEvent 是"按预备份恢复成功"的持久事实(F06)。 +// +// 它必须回答两件事,缺一不可: +// - 恢复出来的内容身份是什么(backup 快照 + 校验摘要); +// - current 现在指向它——旧实现只释放维护锁,于是物理内容已经是旧版, +// overview/diff/provenance 仍然宣称失败的新版本生效,后续操作拿错误基线做判断。 +// +// 它是独立事件,不改写原来那条失败作业/失败版本;原失败事实保持不变。 +func recordRecoveryEvent(ctx context.Context, store *libraryAttemptWriter, scope, jobID, backupSnapshot, + verifyDigest, note string) error { + if backupSnapshot == "" || verifyDigest == "" { + return errors.New("恢复事件缺少预备份身份或校验摘要:不能登记成恢复事实") + } + operationID := store.attempt.OperationID + evidence := map[string]any{ + "backup_snapshot": backupSnapshot, "action": "recover", "verify_digest": verifyDigest, + "window_job": jobID, "note": note, "recovered_at": time.Now().UTC(), + } + // 首次锚点若存在,一并带上:恢复事件同样要能回答"这份基线多大、写集合是什么"。 + if anchors, err := prebackupAnchors(ctx, store.Store, scope, jobID, backupSnapshot); err == nil { + for key, value := range anchors { + if _, exists := evidence[key]; !exists { + evidence[key] = value + } + } + } + return store.RecordLibraryVersion(ctx, governance.LibraryVersion{ + ID: governance.LibraryVersionEventID(governance.LibraryVersionRecoveryVerified, scope, backupSnapshot, + operationID+"|"+jobID), + Scope: scope, Kind: governance.LibraryVersionRecoveryVerified, SnapshotID: backupSnapshot, + SourceJobID: jobID, CreatedBy: "worker", Evidence: evidence, CreatedAt: time.Now().UTC(), + }) +} + +var _ libraryrelease.Recorder = libraryStageRecorder{} diff --git a/cmd/worker/library_handlers.go b/cmd/worker/library_handlers.go index 3aa0d7c..0527038 100644 --- a/cmd/worker/library_handlers.go +++ b/cmd/worker/library_handlers.go @@ -107,15 +107,34 @@ type libraryReleasePayload struct { // runLibraryRelease 执行"闸口 → 授权 → 关闭目标入口并停止写入者 → 预备份 → // 四类写入 → 校验 → 登记权威版本 → 恢复原运行状态和入口"的完整链路。 // 同步与恢复只在作业类型、授权摘要与成功后登记的版本类型上不同。 -func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config appconfig.Config, - job jobs.Job, kind governance.LibraryJobKind) error { +func runLibraryRelease(ctx context.Context, base *postgresstore.Store, config appconfig.Config, + job jobs.Job, kind governance.LibraryJobKind) (err error) { var payload libraryReleasePayload if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.LibraryJobID == "" || payload.TargetSnapshotID == "" { return fmt.Errorf("invalid library %s payload", kind) } + // 准入必须在任何维护/目标动作之前:业务行存在、kind/scope 一致、队列关联不可伪造。 + if _, admitErr := admitLibraryJob(ctx, base, job, payload.LibraryJobID, payload.Scope, kind, true); admitErr != nil { + return admitErr + } + // 执行代次:这一次投递是一个确定的执行者身份。所有治理写入都走 writer + // (同名方法带 epoch 守卫),失租的旧执行从此写不进业务状态/维护锁(F13)。 + writer, attemptErr := beginLibraryWriter(ctx, base, payload.LibraryJobID, libraryQueueOwner(job, payload.LibraryJobID)) + if attemptErr != nil { + return attemptErr + } + defer func() { + if err != nil { + writer.finish(context.WithoutCancel(ctx), governance.LibraryAttemptFailed, err.Error()) + return + } + writer.finish(context.WithoutCancel(ctx), governance.LibraryAttemptCompleted, "") + }() + store := writer now := time.Now().UTC() _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobRunning, "prebackup", "", "", "", "", now) - recorder := libraryStageRecorder{store: store, jobID: payload.LibraryJobID, scope: payload.Scope} + recorder := libraryStageRecorder{store: store, jobID: payload.LibraryJobID, scope: payload.Scope, + operationID: store.attempt.OperationID} // 四类进度的真实总量来自快照清单(manifest):读不到就保持 NULL,不写 0。 if archive, openErr := librarysnapshot.Open(config.LibraryArchiveRoot, 256*1024*1024); openErr == nil { if manifest, loadErr := archive.Load(payload.TargetSnapshotID); loadErr == nil { @@ -222,6 +241,12 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a } defer target.Close() + // 容量门禁必须在**任何副作用之前**(F11):预算不足就不该进维护、不该拍备份。 + // 旧实现只在存储页 GET 里算这个数,worker 从不读它,于是"预算为正但不够"照样写。 + if capErr := checkLibraryCapacity(ctx, config, payload.TargetSnapshotID); capErr != nil { + return fail(capErr.Error(), "prebackup") + } + // 维护目标必须在真正动目标之前确定:缺配置就拒绝写,不用"库级锁"顶替。 manager, managerErr := openLibraryMaintenance(config) if managerErr != nil { @@ -314,12 +339,22 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a } contentState = contentSucceeded mappingDigest, _ := target.Search().MappingSHA256(ctx) + // 收尾续作只补登记,不允许把首次锚点洗掉:字节数与写集合摘要从已持久化的 + // 预备份证据合并进来(F09)。缺失即拒绝,不写一个"看起来完整"的空证据。 + evidence, evidenceErr := finalizeEvidence(ctx, store.Store, payload.Scope, payload.LibraryJobID, + previous.SnapshotID, payload.TargetSnapshotID, "finalize_only", mappingDigest) + if evidenceErr != nil { + message := "收尾续作缺少首次锚点,拒绝登记生效版本:" + evidenceErr.Error() + recorder.Stage(ctx, "finalize", "failed", message) + _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize", message, + "预备份 "+previous.SnapshotID+" 保留;补齐首次锚点证据后再续作", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC()) + return errors.New(message) + } if err := store.RecordLibraryVersion(ctx, governance.LibraryVersion{ - ID: governance.LibraryVersionID(governance.LibraryVersionProductionEffective, payload.Scope, payload.TargetSnapshotID), + ID: governance.LibraryVersionEventID(governance.LibraryVersionProductionEffective, payload.Scope, payload.TargetSnapshotID, recorder.operationID), Scope: payload.Scope, Kind: governance.LibraryVersionProductionEffective, SnapshotID: payload.TargetSnapshotID, SourceJobID: payload.LibraryJobID, CreatedBy: "worker", - Evidence: map[string]any{"backup_snapshot": previous.SnapshotID, "action": "finalize_only", - "index_mapping_sha256": mappingDigest}, + Evidence: evidence, CreatedAt: time.Now().UTC(), }); err != nil { message := "收尾续作仍无法登记生效版本:" + err.Error() @@ -382,7 +417,7 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a // 登记失败必须让作业失败(可续作),否则"版本记录"和"作业状态"会分叉。 recorder.Stage(ctx, "finalize", "running", "登记生产生效版本") versionErr := store.RecordLibraryVersion(ctx, governance.LibraryVersion{ - ID: governance.LibraryVersionID(governance.LibraryVersionProductionEffective, payload.Scope, payload.TargetSnapshotID), + ID: governance.LibraryVersionEventID(governance.LibraryVersionProductionEffective, payload.Scope, payload.TargetSnapshotID, recorder.operationID), Scope: payload.Scope, Kind: governance.LibraryVersionProductionEffective, SnapshotID: payload.TargetSnapshotID, SourceJobID: payload.LibraryJobID, CreatedBy: grant.AuthorizedBy, @@ -450,6 +485,21 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a } window, windowErr := coordinator.RunWindow(ctx, payload.LibraryJobID, prepare, windowWork, allowRestore) + // 终态重放(F05):journal 里已经 succeeded,本次没有再执行 work。内容/服务子协议 + // 已经完成,缺的只可能是"业务收尾"(版本登记、锁释放、终态)。这时必须从**持久事实** + // 重建收尾凭据,而不是靠进程内 pending——那个指针早就随崩溃没了。 + if windowErr == nil && window.State == "succeeded" && contentState != contentSucceeded { + rebuilt, rebuildErr := rebuildSuccessBoundary(ctx, store, recorder, payload, previous, kind, grant) + if rebuildErr != nil { + note := "窗口已成功但收尾凭据无法从持久事实重建,目标保持维护:" + rebuildErr.Error() + _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", note, time.Now().UTC()) + _ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", note, time.Now().UTC()) + return errors.New(note) + } + pending = rebuilt + contentState = contentSucceeded + } + // 收尾:只有"内容已验证/已回滚"且服务与入口都恢复,才释放维护占用。 now2 := time.Now().UTC() switch { @@ -620,7 +670,7 @@ func (r libraryStageRecorder) BackupPrepared(ctx context.Context, backup library return fmt.Errorf("持久化预备份引用失败: %w", err) } if err := r.store.RecordLibraryVersion(ctx, governance.LibraryVersion{ - ID: governance.LibraryVersionID(governance.LibraryVersionProductionPrebackup, r.scope, backup.SnapshotID), + ID: governance.LibraryVersionEventID(governance.LibraryVersionProductionPrebackup, r.scope, backup.SnapshotID, r.operationID), Scope: r.scope, Kind: governance.LibraryVersionProductionPrebackup, SnapshotID: backup.SnapshotID, SourceJobID: r.jobID, CreatedBy: "worker", Evidence: evidence, @@ -631,7 +681,7 @@ func (r libraryStageRecorder) BackupPrepared(ctx context.Context, backup library return nil } -func failLibraryJob(ctx context.Context, store *postgresstore.Store, recorder libraryStageRecorder, jobID, message, stage string) error { +func failLibraryJob(ctx context.Context, store libraryFailStore, recorder libraryStageRecorder, jobID, message, stage string) error { recorder.Stage(ctx, stage, "failed", message) _ = store.AdvanceLibraryJob(ctx, jobID, governance.LibraryJobFailed, stage, message, "未对生产执行任何写入,无需恢复", "", "", time.Now().UTC()) @@ -643,7 +693,7 @@ func failLibraryJob(ctx context.Context, store *postgresstore.Store, recorder li // 有预备份就说明上一次尝试可能已经写过目标:这时任何早退(授权过期、装配失败、 // 维护被别人占用……)都必须保留 needs_restore 并点名基线,不能写"无需恢复"—— // 那会让页面把半写目标当成完好,也让操作者手里没有可回滚的那一份。 -func failLibraryJobOrTakeover(ctx context.Context, store *postgresstore.Store, recorder libraryStageRecorder, +func failLibraryJobOrTakeover(ctx context.Context, store libraryFailStore, recorder libraryStageRecorder, scope, jobID, backupSnapshotID string, resumed bool, message, stage string) error { if resumed { return failLibraryTakeover(ctx, store, recorder, scope, jobID, backupSnapshotID, message) @@ -658,7 +708,7 @@ func failLibraryJobOrTakeover(ctx context.Context, store *postgresstore.Store, r // 为什么不在拒绝时释放维护:本作业没有自己的写入证据,目标是否半写未知; // 释放就等于对页面和下一次作业谎称"目标已恢复原状"。实例行也不动——它的释放 // 属于"按预备份恢复并核验"那条路径,这里连实例身份都还没查到。 -func failLibraryTakeover(ctx context.Context, store *postgresstore.Store, recorder libraryStageRecorder, +func failLibraryTakeover(ctx context.Context, store libraryFailStore, recorder libraryStageRecorder, scope, jobID, backupSnapshotID, message string) error { recorder.Stage(ctx, "prebackup", "failed", message) recovery := "本作业未继续写目标;上一次尝试可能已半写,须按预备份 " + backupSnapshotID + " 恢复并核验后再处理" diff --git a/cmd/worker/library_recovery.go b/cmd/worker/library_recovery.go index 8f79533..c170abd 100644 --- a/cmd/worker/library_recovery.go +++ b/cmd/worker/library_recovery.go @@ -11,6 +11,7 @@ import ( "time" "knowledge-governance/internal/appconfig" + "knowledge-governance/internal/governance" "knowledge-governance/internal/jobs" "knowledge-governance/internal/libraryrelease" postgresstore "knowledge-governance/internal/platform/postgres" @@ -56,12 +57,17 @@ func runLibraryRecovery(ctx context.Context, store *postgresstore.Store, config if lock.TargetFingerprint != identity.Fingerprint { return errors.New("维护窗口的目标身份与当前配置不一致:拒绝用旧窗口恢复另一个目标") } - - // 预备份身份来自治理库里的原作业记录;协调器还会再与 journal 里的那份核对。 - windowJob, err := store.LibraryJob(ctx, payload.WindowJobID) - if err != nil || windowJob.SnapshotID == "" { - return errors.New("原作业没有持久化预备份,无法证明恢复到哪一份") + // 准入:窗口作业必须是这个 scope 的真实业务作业(恢复也不能靠畸形载荷开窗口)。 + windowJob, admitErr := admitLibraryJob(ctx, store, job, payload.WindowJobID, payload.Scope, "", false) + if admitErr != nil { + return admitErr } + // 恢复是一次**新执行代次**:它接管这个窗口,旧代次从此不能写业务状态/释放锁(F13)。 + writer, attemptErr := beginLibraryWriter(ctx, store, payload.WindowJobID, libraryQueueOwner(job, payload.WindowJobID)) + if attemptErr != nil { + return fmt.Errorf("无法取得恢复窗口的执行权:%w", attemptErr) + } + guarded := writer prebackup := windowJob.SnapshotID // 恢复必须带着"这次作业写了什么"的持久记录:没有它就只能按目标现状推断残渣, // 而那会误删历史孤儿/其它知识库暂未绑定的资源(快照里没有这些行,删了回不来)。 @@ -69,8 +75,8 @@ func runLibraryRecovery(ctx context.Context, store *postgresstore.Store, config payload.WindowJobID, prebackup) if ledgerErr != nil { note := "恢复缺少持久化的写集合记录,目标保持维护:" + ledgerErr.Error() - _ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, time.Now().UTC()) - _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.WindowJobID, "needs_restore", note, time.Now().UTC()) + _ = guarded.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, time.Now().UTC()) + _ = guarded.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.WindowJobID, "needs_restore", note, time.Now().UTC()) return errors.New(note) } knownMapping := knownIndexMapping(ctx, store, payload.Scope) @@ -93,20 +99,35 @@ func runLibraryRecovery(ctx context.Context, store *postgresstore.Store, config now := time.Now().UTC() if recoverErr != nil { note := "恢复/校验未通过,目标保持维护:" + recoverErr.Error() - _ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now) - _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.WindowJobID, "needs_restore", note, now) + _ = guarded.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now) + _ = guarded.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.WindowJobID, "needs_restore", note, now) return recoverErr } if result.State != "recovered" { note := "恢复流程结束但状态不是 recovered:" + result.State - _ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now) + _ = guarded.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now) return errors.New(note) } - if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.WindowJobID, + // 恢复事实必须落成**新事件并推进 current**(F06)。旧实现只释放锁:物理内容已经 + // 是预备份那份,页面/差异/后续作业却仍以为失败的新版本生效,下一步就会拿错基线决策。 + // 事件是追加的,不改写原来那条失败作业与失败版本。 + digest := result.Digest + if digest == "" { + return errors.New("恢复完成但没有校验摘要:拒绝登记恢复事实(无法证明恢复到了哪一份内容)") + } + if err := recordRecoveryEvent(ctx, writer, payload.Scope, payload.WindowJobID, result.SnapshotID, digest, + "按预备份恢复并逐类校验通过"); err != nil { + return fmt.Errorf("恢复已完成但恢复事实登记失败:%w", err) + } + // 收尾顺序:先登记事实,再释放占用。反过来会出现"锁已放、current 还指向失败版本" + // 的窗口——那正是页面把恢复成功显示成待恢复的状态。 + if err := releaseWindowLocks(ctx, guarded, identity.Instance, payload.Scope, payload.WindowJobID, "按预备份恢复并逐类校验通过,目标已按原运行状态恢复", now); err != nil { return fmt.Errorf("恢复已验证,但维护占用释放失败:%w", err) } - log.Printf("library recovery succeeded window=%s scope=%s backup=%s", payload.WindowJobID, payload.Scope, prebackup) + writer.finish(context.WithoutCancel(ctx), governance.LibraryAttemptCompleted, "recovered") + log.Printf("library recovery succeeded window=%s scope=%s backup=%s digest=%s", + payload.WindowJobID, payload.Scope, prebackup, digest) return nil } diff --git a/cmd/worker/library_takeover_resume.go b/cmd/worker/library_takeover_resume.go index 889d95e..dd5ceb3 100644 --- a/cmd/worker/library_takeover_resume.go +++ b/cmd/worker/library_takeover_resume.go @@ -18,7 +18,7 @@ const takeoverBaselineMaxBlobBytes = 256 * 1024 * 1024 // 任何一项读不到都按**不成立**处理(fail closed):这条路径的用途是允许一次 // 恢复覆盖"已被更新版本顶替"的历史快照,证据缺一条就必须退回拒绝,不能把 // "查不到"当成"没问题"。 -func loadTakeoverResumeFacts(ctx context.Context, store *postgresstore.Store, +func loadTakeoverResumeFacts(ctx context.Context, store libraryResumeFactsStore, archiveRoot string, job governance.LibraryJob, now time.Time) takeoverResumeFacts { facts := takeoverResumeFacts{} // 授权必须绑定本作业的请求摘要,且**签发于顶替之后**:顶替前签的旧授权 @@ -41,10 +41,18 @@ func loadTakeoverResumeFacts(ctx context.Context, store *postgresstore.Store, return facts } +// libraryResumeFactsStore 是接管判定需要的只读面(writer 与裸 Store 都满足)。 +type libraryResumeFactsStore interface { + ActiveLibraryAuthorization(ctx context.Context, scope, snapshotID, paramsDigest string, now time.Time) (governance.LibraryReleaseAuthorization, bool, error) + LibraryVersionByRef(ctx context.Context, scope string, kind governance.LibraryVersionKind, sourceJobID, snapshotID string) (governance.LibraryVersion, bool, error) + LibraryMaintenanceOf(ctx context.Context, scope string) (*postgresstore.LibraryMaintenance, error) + HeldInstanceMaintenance(ctx context.Context) ([]postgresstore.LibraryInstanceMaintenance, error) +} + // takeoverWindowOwnedBy 要求 scope 与 instance 两把锁都由本作业持有且未释放, // 且两行的目标指纹一致:只有本作业自己开的那扇窗口才允许续写,别人的窗口 // 即使作业号碰巧相同也进不来(Acquire* 的互斥语义在接管判定里同样成立)。 -func takeoverWindowOwnedBy(ctx context.Context, store *postgresstore.Store, job governance.LibraryJob) bool { +func takeoverWindowOwnedBy(ctx context.Context, store libraryResumeFactsStore, job governance.LibraryJob) bool { scopeRow, err := store.LibraryMaintenanceOf(ctx, job.Scope) if err != nil || scopeRow == nil || scopeRow.OwnerJobID != job.ID || scopeRow.ReleasedAt != nil { return false diff --git a/cmd/worker/library_writer.go b/cmd/worker/library_writer.go new file mode 100644 index 0000000..a753242 --- /dev/null +++ b/cmd/worker/library_writer.go @@ -0,0 +1,132 @@ +package main + +import ( + "context" + "errors" + "fmt" + "log" + "time" + + "knowledge-governance/internal/governance" + "knowledge-governance/internal/jobs" + postgresstore "knowledge-governance/internal/platform/postgres" +) + +// libraryAttemptWriter 是"这一次执行代次"的写入口。 +// +// 它嵌入完整 Store,只把**可变**入口换成带 epoch 守卫的实现(方法名相同,所以调用点 +// 一行都不用改)。这样"谁能写业务状态/维护锁"只由这一个类型回答:handler 里拿到的 +// 是 writer 而不是裸 Store,就不可能顺手绕过守卫去写。 +// +// 为什么必须这样:失租的旧执行器和接管它的新代次用的是**同一个 job_id**。只加一个 +// epoch 列而不让写路径读它,等于什么都没做(这正是 F13 的原始形态)。 +type libraryAttemptWriter struct { + *postgresstore.Store + attempt governance.LibraryAttempt +} + +// beginLibraryWriter 取得这个作业的执行权,返回带守卫的写入口。 +// owner 用队列作业 ID:每一次投递是一个确定的执行者身份,重复投递是同一个身份(幂等), +// 不同投递互不覆盖(这正是"第二个 resumer 不能释放第一个的锁"需要的区分)。 +func beginLibraryWriter(ctx context.Context, store *postgresstore.Store, jobID, owner string) (*libraryAttemptWriter, error) { + job, err := store.LibraryJob(ctx, jobID) + if err != nil { + return nil, fmt.Errorf("读不到业务作业 %s:%w", jobID, err) + } + if job.OperationID == "" { + // 迁移前的历史作业没有 operation:仍然取租约(防止双执行),但没有代次可守卫。 + log.Printf("library job %s has no operation id (pre-identity row): guarded writes fall back to lease only", jobID) + } + attempt, err := store.BeginLibraryAttempt(ctx, jobID, owner, 10*time.Minute, time.Now().UTC()) + if err != nil { + return nil, err + } + return &libraryAttemptWriter{Store: store, attempt: attempt}, nil +} + +func (w *libraryAttemptWriter) AdvanceLibraryJob(ctx context.Context, jobID string, state governance.LibraryJobState, + stage, message, recovery, snapshotID, targetSnapshotID string, now time.Time) error { + if err := w.checkJob(jobID); err != nil { + return err + } + return w.Store.AdvanceLibraryJobGuarded(ctx, w.attempt, state, stage, message, recovery, snapshotID, targetSnapshotID, now) +} + +func (w *libraryAttemptWriter) UpdateLibraryStage(ctx context.Context, jobID, stage, state string, + total, completed, bytesTotal, bytesDone *int64, detail string, now time.Time) error { + if err := w.checkJob(jobID); err != nil { + return err + } + return w.Store.UpdateLibraryStageGuarded(ctx, w.attempt, stage, state, total, completed, bytesTotal, bytesDone, detail, now) +} + +// RecordLibraryVersion 注入本次代次身份:事件登记因此也受 epoch 守卫。 +func (w *libraryAttemptWriter) RecordLibraryVersion(ctx context.Context, version governance.LibraryVersion) error { + if version.OperationID == "" { + version.OperationID = w.attempt.OperationID + } + if version.AttemptEpoch == 0 { + version.AttemptEpoch = w.attempt.Epoch + } + version.AttemptOwner = w.attempt.Owner + return w.Store.RecordLibraryVersion(ctx, version) +} + +func (w *libraryAttemptWriter) SetLibraryMaintenanceIngress(ctx context.Context, scope, ownerJobID, state, note string, now time.Time) error { + return w.Store.SetLibraryMaintenanceIngressGuarded(ctx, scope, ownerJobID, state, note, w.attempt, now) +} + +func (w *libraryAttemptWriter) SetInstanceMaintenanceIngress(ctx context.Context, instance, ownerJobID, state, note string, now time.Time) error { + return w.Store.SetInstanceMaintenanceIngressGuarded(ctx, instance, ownerJobID, state, note, w.attempt, now) +} + +func (w *libraryAttemptWriter) UpdateLibraryMaintenanceState(ctx context.Context, scope, ownerJobID, state, note string, now time.Time) error { + return w.Store.UpdateLibraryMaintenanceStateGuarded(ctx, scope, ownerJobID, state, note, w.attempt, now) +} + +func (w *libraryAttemptWriter) UpdateInstanceMaintenanceState(ctx context.Context, instance, ownerJobID, state, note string, now time.Time) error { + return w.Store.UpdateInstanceMaintenanceStateGuarded(ctx, instance, ownerJobID, state, note, w.attempt, now) +} + +func (w *libraryAttemptWriter) ReleaseLibraryMaintenance(ctx context.Context, scope, ownerJobID string, now time.Time) error { + return w.Store.ReleaseLibraryMaintenanceGuarded(ctx, scope, ownerJobID, w.attempt, now) +} + +func (w *libraryAttemptWriter) ReleaseInstanceMaintenance(ctx context.Context, instance, ownerJobID string, now time.Time) error { + return w.Store.ReleaseInstanceMaintenanceGuarded(ctx, instance, ownerJobID, w.attempt, now) +} + +// finish 结束本次代次。失败只记日志:代次结束写不进去不该反过来把已经完成的数据收尾 +// 判成失败(下一次接管会重新开始一个新代次,旧代次的残留状态由 active 唯一索引兜住)。 +func (w *libraryAttemptWriter) finish(ctx context.Context, state governance.LibraryAttemptState, outcome string) { + if w.attempt.OperationID == "" { + return + } + if err := w.Store.FinishLibraryAttempt(ctx, w.attempt, state, outcome, time.Now().UTC()); err != nil && + !errors.Is(err, governance.ErrLibraryEpochStale) { + log.Printf("library attempt %s#%d finish failed: %v", w.attempt.OperationID, w.attempt.Epoch, err) + } +} + +func (w *libraryAttemptWriter) checkJob(jobID string) error { + if jobID != w.attempt.JobID { + return fmt.Errorf("尝试用代次 %s#%d 写另一个作业 %s:拒绝(跨作业写入不是续作)", + w.attempt.OperationID, w.attempt.Epoch, jobID) + } + return nil +} + +// 编译期断言:writer 必须同时满足"成功收尾"与"阶段记录"两个存储面。 +var ( + _ libraryMaintenanceStore = (*libraryAttemptWriter)(nil) + _ libraryStageStore = (*libraryAttemptWriter)(nil) +) + +// libraryQueueOwner 用队列作业 ID 作为执行者身份;缺失时退回业务作业 ID +// (手工/测试路径没有队列身份,但至少不会与他人共用)。 +func libraryQueueOwner(job jobs.Job, libraryJobID string) string { + if job.ID != "" { + return job.ID + } + return libraryJobID +} diff --git a/cmd/worker/library_writeset.go b/cmd/worker/library_writeset.go index 3195823..33f8d6f 100644 --- a/cmd/worker/library_writeset.go +++ b/cmd/worker/library_writeset.go @@ -202,12 +202,12 @@ func prebackupEvidence(existing map[string]any, backup libraryrelease.BackupRef, // 记录不存在同样拒绝——把锚点写到一个不存在的证据行上,会让后续接管/恢复 // 读不到备份字节与索引映射,等于伪造一份台账。 func (r libraryStageRecorder) WriteSetPlanned(ctx context.Context, set libraryrelease.WriteSet) error { - return anchorWriteSet(ctx, r.store, r.scope, r.jobID, r.mappingDigest, set) + return anchorWriteSet(ctx, r.store, r.operationID, r.scope, r.jobID, r.mappingDigest, set) } // anchorWriteSet 是登记的实现:与 loadWriteSetForJob 一样只依赖"按身份取一条版本" // 这一件事,便于在没有真实库时也能证明"身份不符/摘要冲突就失败"。 -func anchorWriteSet(ctx context.Context, store libraryVersionLedger, scope, jobID, mappingDigest string, +func anchorWriteSet(ctx context.Context, store libraryVersionLedger, operationID, scope, jobID, mappingDigest string, set libraryrelease.WriteSet) error { version, found, err := store.LibraryVersionByRef(ctx, scope, governance.LibraryVersionProductionPrebackup, jobID, set.BackupSnapshotID) @@ -229,7 +229,7 @@ func anchorWriteSet(ctx context.Context, store libraryVersionLedger, scope, jobI } evidence := writeSetEvidence(version.Evidence, set, mappingDigest) return store.RecordLibraryVersion(ctx, governance.LibraryVersion{ - ID: governance.LibraryVersionID(governance.LibraryVersionProductionPrebackup, scope, set.BackupSnapshotID), + ID: governance.LibraryVersionEventID(governance.LibraryVersionProductionPrebackup, scope, set.BackupSnapshotID, operationID), Scope: scope, Kind: governance.LibraryVersionProductionPrebackup, SnapshotID: set.BackupSnapshotID, SourceJobID: jobID, CreatedBy: "worker", Evidence: evidence, CreatedAt: time.Now().UTC(), }) diff --git a/cmd/worker/library_writeset_resume_integration_test.go b/cmd/worker/library_writeset_resume_integration_test.go index 657623c..2ae85b7 100644 --- a/cmd/worker/library_writeset_resume_integration_test.go +++ b/cmd/worker/library_writeset_resume_integration_test.go @@ -36,6 +36,7 @@ type resumeLedgerFixture struct { target string params string root string + store *postgresstore.Store record libraryStageRecorder writeSet libraryrelease.WriteSet } @@ -53,6 +54,7 @@ func newResumeLedgerFixture(t *testing.T, suffix string) *resumeLedgerFixture { target: "target-snap-ws-" + suffix, params: "params-ws-" + suffix, root: t.TempDir(), + store: store, } if _, _, err := store.CreateLibraryJob(ctx, governance.LibraryJob{ID: f.jobID, Scope: f.scope, Kind: governance.LibraryJobRestore, TargetSnapshotID: f.target, RequestDigest: f.params, @@ -178,7 +180,7 @@ func (f *resumeLedgerFixture) takeoverBackups(t *testing.T, store *postgresstore func TestTakeoverKeepsWriteSetEvidenceAcrossRepeatedInterruptions(t *testing.T) { f := newResumeLedgerFixture(t, "repeat") - store := f.record.store + store := f.store ctx := context.Background() firstCreatedAt := f.createdAtOf(t, governance.LibraryVersionProductionPrebackup, f.backup) @@ -249,7 +251,7 @@ func TestTakeoverKeepsWriteSetEvidenceAcrossRepeatedInterruptions(t *testing.T) // 也不许借用别的作业/别的预备份的记录。 func TestTakeoverLedgerRefusesForeignOrMissingBackupEvidence(t *testing.T) { f := newResumeLedgerFixture(t, "negatives") - store := f.record.store + store := f.store ctx := context.Background() // 声称复用一份从没登记过的预备份:没有证据可合并 ⇒ 拒绝。 diff --git a/cmd/worker/library_writeset_test.go b/cmd/worker/library_writeset_test.go index eeb0f0e..5e769a1 100644 --- a/cmd/worker/library_writeset_test.go +++ b/cmd/worker/library_writeset_test.go @@ -340,7 +340,7 @@ func TestAnchorWriteSetRefusesConflictingDigestAndForeignEvidence(t *testing.T) SnapshotID: "backup-snap", SourceJobID: "job-1", Evidence: evidence}}} } anchor := func(store libraryVersionLedger) error { - return anchorWriteSet(context.Background(), store, "internal", "job-1", "m", set) + return anchorWriteSet(context.Background(), store, "", "internal", "job-1", "m", set) } if err := anchor(recorded(map[string]any{"job": "job-2"})); err == nil { t.Fatal("版本证据属于别的作业时必须失败") diff --git a/cmd/worker/main.go b/cmd/worker/main.go index acdefac..e162e17 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -29,11 +29,23 @@ import ( ) // libraryStageRecorder 把执行阶段写进持久作业,页面刷新/worker 重启后仍可复现。 +// libraryStageStore 是阶段/证据记录用到的存储面:由 libraryAttemptWriter 实现, +// 所以阶段进度与版本登记天然带执行代次(失租代次写不进去)。 +type libraryStageStore interface { + UpdateLibraryStage(ctx context.Context, jobID, stage, state string, total, completed, bytesTotal, bytesDone *int64, detail string, now time.Time) error + AdvanceLibraryJob(ctx context.Context, jobID string, state governance.LibraryJobState, stage, message, recovery, snapshotID, targetSnapshotID string, now time.Time) error + LibraryVersionByRef(ctx context.Context, scope string, kind governance.LibraryVersionKind, sourceJobID, snapshotID string) (governance.LibraryVersion, bool, error) + RecordLibraryVersion(ctx context.Context, version governance.LibraryVersion) error +} + type libraryStageRecorder struct { - store *postgresstore.Store - jobID string - scope string - totals map[string]stageSize + store libraryStageStore + jobID string + scope string + // operationID 让版本事件 ID 带执行意图:不同 operation 引用同一快照是两条事件, + // 不会互相覆盖证据(F02)。为空表示历史/测试路径,退回旧 ID 形状。 + operationID string + totals map[string]stageSize // mappingDigest 是本次读到的生产索引映射,随预备份一起登记成版本证据。 mappingDigest string // onBackup 在预备份完成后、写目标之前调用:把"预备份身份 + 内容风险"落盘。 diff --git a/internal/governance/library_flow.go b/internal/governance/library_flow.go index 7e6b440..8c4ad76 100644 --- a/internal/governance/library_flow.go +++ b/internal/governance/library_flow.go @@ -22,6 +22,10 @@ const ( LibraryVersionProductionEffective LibraryVersionKind = "production_effective" // LibraryVersionProductionPrebackup 是每次生产写入前的独立预备份。 LibraryVersionProductionPrebackup LibraryVersionKind = "production_prebackup" + // LibraryVersionRecoveryVerified 是"按预备份恢复并逐类校验通过"的恢复事实: + // 它让 current 指向恢复出来的内容身份,而不是继续宣称已经失败的新版本生效(F06)。 + // 它不是测试封存,也不代表生产发布授权。 + LibraryVersionRecoveryVerified LibraryVersionKind = "recovery_verified" ) type LibraryJobKind string @@ -67,25 +71,30 @@ type LibraryStage struct { } type LibraryJob struct { - ID string `json:"id"` - Scope string `json:"scope"` - Kind LibraryJobKind `json:"kind"` - State LibraryJobState `json:"state"` - Stage string `json:"stage"` - IdempotencyKey string `json:"-"` - RequestDigest string `json:"request_digest"` - QueueJobID string `json:"queue_job_id"` - SnapshotID string `json:"snapshot_id"` - TargetSnapshotID string `json:"target_snapshot_id"` - Actor string `json:"actor"` - Error string `json:"error"` - Recovery string `json:"recovery"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` - FinishedAt *time.Time `json:"finished_at"` - Stages []LibraryStage `json:"stages"` + ID string `json:"id"` + Scope string `json:"scope"` + Kind LibraryJobKind `json:"kind"` + State LibraryJobState `json:"state"` + Stage string `json:"stage"` + IdempotencyKey string `json:"-"` + // OperationID 让"同一个意图的多次执行"有稳定谱系;AttemptEpoch 是本次执行代次。 + OperationID string `json:"operation_id"` + AttemptEpoch int `json:"attempt_epoch"` + RequestDigest string `json:"request_digest"` + QueueJobID string `json:"queue_job_id"` + SnapshotID string `json:"snapshot_id"` + TargetSnapshotID string `json:"target_snapshot_id"` + Actor string `json:"actor"` + Error string `json:"error"` + Recovery string `json:"recovery"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + FinishedAt *time.Time `json:"finished_at"` + Stages []LibraryStage `json:"stages"` } +// LibraryVersion 是一条**不可变版本事件**。同一 operation 重放幂等(同 id 不重写), +// 不同 operation 引用同一快照是不同事件,谁也不覆盖谁的证据与作业归属。 type LibraryVersion struct { ID string `json:"id"` Scope string `json:"scope"` @@ -93,10 +102,16 @@ type LibraryVersion struct { SnapshotID string `json:"snapshot_id"` VersionLabel string `json:"version_label"` SourceJobID string `json:"source_job_id"` - Evidence map[string]any `json:"evidence"` - CreatedBy string `json:"created_by"` - CreatedAt time.Time `json:"created_at"` - SupersededAt *time.Time `json:"superseded_at"` + // OperationID/AttemptEpoch 是这条事件的执行身份:登记时用它做 epoch 守卫, + // 保证失租的旧代次不能写事件,也不能推进 current。 + OperationID string `json:"operation_id"` + AttemptEpoch int `json:"attempt_epoch"` + // AttemptOwner 只用于 epoch 守卫(不落库、不展示):写入时必须与作业行租约一致。 + AttemptOwner string `json:"-"` + Evidence map[string]any `json:"evidence"` + CreatedBy string `json:"created_by"` + CreatedAt time.Time `json:"created_at"` + SupersededAt *time.Time `json:"superseded_at"` } // LibraryRequestDigest 绑定一次请求的确切含义:同键不同体必须被拒绝,而不是 diff --git a/internal/governance/library_operation.go b/internal/governance/library_operation.go new file mode 100644 index 0000000..016f7a5 --- /dev/null +++ b/internal/governance/library_operation.go @@ -0,0 +1,155 @@ +package governance + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "strings" + "time" +) + +// 发布执行身份:operation / attempt 与版本事件、current 指针的唯一约定。 +// +// 为什么必须把这几件事分开: +// - operation 是**用户的一次明确意图**(发布到这个快照 / 恢复到这个快照)。它不该被 +// "这个快照已经成功过"吞掉——用户再次要求同一个目标状态是一次新意图,必须真的执行并 +// 留下自己的记录(F01)。 +// - attempt 是这次意图的**执行代次**(epoch)。崩溃、接管只新增代次,不新造意图, +// 否则维护窗口、预备份、写集合的谱系就断了(F15)。旧代次一旦失租就不能再写(F13)。 +// - 版本事件是**不可变事实**,同一快照被两个 operation 引用是两条事件,谁也不能覆盖谁 +// (F02/F09)。current 只是一个"现在生效的是哪条事件"的指针,用单调次序做条件更新, +// 不反过来修改历史。 + +// ErrLibraryEpochStale:这次写入来自已经失租/被接管的执行代次。 +// 调用方必须停止副作用并把控制权交给当前代次,不能用"重试"掩盖。 +var ErrLibraryEpochStale = errors.New("library attempt epoch is stale: another attempt owns this operation") + +// ErrLibraryCurrentStale:这条事件不是当前该生效的事实(有更晚的真相已登记)。 +var ErrLibraryCurrentStale = errors.New("library current pointer already advanced past this event") + +// ErrLibraryOperationConflict:同一 operation 键被用在了不同的请求内容上。 +var ErrLibraryOperationConflict = errors.New("library operation key reused with a different request") + +// LibraryAttemptState 是一次执行代次的状态。 +type LibraryAttemptState string + +const ( + // LibraryAttemptClaimed:已被某个 worker 认领,尚未进入维护窗口。 + LibraryAttemptClaimed LibraryAttemptState = "claimed" + // LibraryAttemptRunning:正在执行(持有目标执行域)。 + LibraryAttemptRunning LibraryAttemptState = "running" + LibraryAttemptCompleted LibraryAttemptState = "completed" + LibraryAttemptFailed LibraryAttemptState = "failed" + LibraryAttemptAbandoned LibraryAttemptState = "abandoned" +) + +// LibraryOperation 是一次明确意图。字段一旦写入就不再改变(锚点另见 LibraryAnchors)。 +type LibraryOperation struct { + ID string `json:"id"` + Scope string `json:"scope"` + Kind LibraryJobKind `json:"kind"` + TargetSnapshotID string `json:"target_snapshot_id"` + RequestDigest string `json:"request_digest"` + OperationKey string `json:"operation_key"` + Actor string `json:"actor"` + CreatedAt time.Time `json:"created_at"` +} + +// LibraryAnchors 是 operation 的**不可变首次锚点**:预备份身份/字节、写集合摘要、 +// 目标指纹。所有 attempt 继承它;接管不许"再拍一份",也不许用新值覆盖它(F07/F09)。 +type LibraryAnchors struct { + FirstBackupSnapshotID string `json:"first_backup_snapshot_id,omitempty"` + FirstBackupBytes *int64 `json:"first_backup_bytes,omitempty"` + FirstBackupAt *time.Time `json:"first_backup_at,omitempty"` + WriteSetDigest string `json:"writeset_digest,omitempty"` + WriteSetRows *int64 `json:"writeset_rows,omitempty"` + TargetFingerprint string `json:"target_fingerprint,omitempty"` + RecordedAt *time.Time `json:"recorded_at,omitempty"` +} + +// LibraryAttempt 是 operation 的一次执行代次。 +type LibraryAttempt struct { + OperationID string `json:"operation_id"` + Epoch int `json:"epoch"` + JobID string `json:"job_id"` + QueueJobID string `json:"queue_job_id"` + Owner string `json:"owner"` + State LibraryAttemptState `json:"state"` + StartedAt time.Time `json:"started_at"` + HeartbeatAt *time.Time `json:"heartbeat_at,omitempty"` + LeaseUntil *time.Time `json:"lease_until,omitempty"` + FinishedAt *time.Time `json:"finished_at,omitempty"` + Outcome string `json:"outcome,omitempty"` +} + +// LibraryOperationID 由 (scope, kind, operation 键) 决定:同一个客户端重放键永远得到 +// 同一个 operation(幂等),换一个键就是一次新意图(不是"历史成功")。 +func LibraryOperationID(scope string, kind LibraryJobKind, operationKey string) string { + sum := sha256.Sum256([]byte(string(kind) + "|" + scope + "|" + operationKey)) + return "libop-" + hex.EncodeToString(sum[:])[:24] +} + +// LibraryVersionEventID 生成一条**不可变版本事件**的 ID:同一个 operation 对同一快照 +// 同一类别只产生一条事件(重放幂等);不同 operation 即使引用同一快照也是不同事件, +// 因此 `source_job_id`/`evidence`/`created_at` 不会被后来的执行覆盖(F02)。 +// operationID 为空时退回旧形状,供历史行/旧调用方保持同一身份。 +func LibraryVersionEventID(kind LibraryVersionKind, scope, snapshotID, operationID string) string { + identity := string(kind) + "|" + scope + "|" + snapshotID + if operationID != "" { + identity += "|" + operationID + } + sum := sha256.Sum256([]byte(identity)) + return "libver-" + hex.EncodeToString(sum[:])[:16] +} + +// CurrentOrderKey 给"这条事实是什么时候被记录为真相的"一个**单调可比较**的键。 +// +// 用它做 current 指针的条件更新:后来登记的事实可以推进指针,更早的不能把指针拉回历史。 +// 旧执行代次"迟到 finalize 反转 current"在写边界就被拦住了(epoch 守卫), +// 所以这里不需要也不该再拿 operation 的创建时间当次序——恢复回滚必须能把指针指回更早发布的 +// 快照,那个事件的**登记时间**是新的。 +func CurrentOrderKey(recordedAt time.Time, operationID string, epoch int) string { + return fmt.Sprintf("%020d|%s|%09d", recordedAt.UTC().UnixNano(), operationID, epoch) +} + +// CanAdvanceCurrent 判断一条新事实能否推进 current 指针(纯函数,供写入路径与测试共用)。 +// 返回 false 表示"这条事件只能确认自己的历史",调用方必须拒绝把它登记成当前状态。 +func CanAdvanceCurrent(currentOrderKey, nextOrderKey string) bool { + if nextOrderKey == "" { + return false + } + if currentOrderKey == "" { + return true + } + return nextOrderKey > currentOrderKey +} + +// LibraryJobTypePrefix 是所有整库发布相关队列作业的类型前缀。 +// +// 这些类型的准入条件(生产闸口、精确授权、维护互斥、目标身份核验)只在发布管理专用 +// 入口里判断,所以它们对通用 queue 入口是**保留类型**:通用入口的通用 engineer 角色 +// 检查替代不了准入,允许它造一个 library.* 作业就是绕过整条准入链(F14)。 +const LibraryJobTypePrefix = "library." + +// IsReservedLibraryJobType 按前缀整类判断,而不是逐个列举类型: +// 以后新增一种 library 作业类型时不会漏掉准入。 +func IsReservedLibraryJobType(jobType string) bool { + return strings.HasPrefix(strings.TrimSpace(jobType), LibraryJobTypePrefix) +} + +// ValidateLibraryOperation 只做结构校验:缺字段就拒绝,不猜默认值。 +func ValidateLibraryOperation(op LibraryOperation) error { + if op.ID == "" || op.OperationKey == "" || op.RequestDigest == "" || op.CreatedAt.IsZero() { + return errors.New("library operation requires id, key, digest and created_at") + } + if op.Scope != "internal" && op.Scope != "external" { + return errors.New("library operation requires internal or external scope") + } + switch op.Kind { + case LibraryJobSeal, LibraryJobSync, LibraryJobRestore, LibraryJobBaseline: + default: + return fmt.Errorf("unsupported library operation kind %q", op.Kind) + } + return nil +} diff --git a/internal/httpapi/library_flow.go b/internal/httpapi/library_flow.go index 9bca315..cdaa351 100644 --- a/internal/httpapi/library_flow.go +++ b/internal/httpapi/library_flow.go @@ -338,6 +338,9 @@ type libraryVersionRef struct { type libraryJobStore interface { CreateLibraryJob(ctx context.Context, job governance.LibraryJob, stages []string, now time.Time) (governance.LibraryJob, bool, error) + // EnsureLibraryOperation 幂等登记"一次明确意图":同键+同摘要 ⇒ 同一次意图(客户端重放), + // 换键 ⇒ 新意图。作业是意图的执行代次,不能反过来用"历史成功作业"回答新意图(F01)。 + EnsureLibraryOperation(ctx context.Context, op governance.LibraryOperation) (governance.LibraryOperation, bool, error) LibraryJob(ctx context.Context, id string) (governance.LibraryJob, error) ListLibraryJobs(ctx context.Context, scope string, limit int) ([]governance.LibraryJob, error) RecordLibraryVersion(ctx context.Context, version governance.LibraryVersion) error @@ -837,6 +840,9 @@ func librarySyncEnqueue(w http.ResponseWriter, r *http.Request, config Config) { var request struct { Scope string `json:"scope"` Target string `json:"target_snapshot_id"` + // OperationKey 是客户端重放键:同一次意图重发要命中同一个 operation。 + // 不传就是一次新意图(同一个快照再发布一次也必须真的执行)。 + OperationKey string `json:"operation_key"` } if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&request); err != nil { writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"}) @@ -917,9 +923,22 @@ func librarySyncEnqueue(w http.ResponseWriter, r *http.Request, config Config) { } now := time.Now().UTC() jobID := fmt.Sprintf("libsync-%s-%d", request.Scope, now.UnixNano()) + // 意图身份:同一个 operation 的重复请求复用作业;新意图即使目标快照与历史成功相同 + // 也会建新作业并真的执行(F01)。 + intent, intentErr := ensureLibraryIntent(r.Context(), store, request.Scope, governance.LibraryJobSync, + request.Target, digest, actor, request.OperationKey, now) + if intentErr != nil { + if isIntentConflict(intentErr) { + writeJSON(w, http.StatusConflict, map[string]any{"code": "library_sync_intent_conflict", + "error": "同一 operation_key 对应了不同的请求内容:换一个键表示新意图"}) + return + } + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot register library intent"}) + return + } job, reused, err := store.CreateLibraryJob(r.Context(), governance.LibraryJob{ ID: jobID, Scope: request.Scope, Kind: governance.LibraryJobSync, - IdempotencyKey: "library.sync:" + request.Scope + ":" + request.Target, RequestDigest: digest, + IdempotencyKey: intent.JobKey, RequestDigest: digest, OperationID: intent.Operation.ID, TargetSnapshotID: request.Target, Actor: actor, }, governance.SyncStages(), now) if errors.Is(err, governance.ErrLibraryJobActive) { @@ -935,14 +954,15 @@ func librarySyncEnqueue(w http.ResponseWriter, r *http.Request, config Config) { writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot create library sync job"}) return } - // 上一次同参数请求失败/被取消后,幂等键不能再把新请求塞回那个死作业里: - // 换一个"重试代次"键重新建作业,旧作业保留为审计记录。 + // 上一次同意图请求失败/被取消后,幂等键不能再把新请求塞回那个死作业里: + // 在**同一个 operation** 下开一个新的执行作业(重试代次),旧作业保留为审计记录。 if reused && (job.State == governance.LibraryJobFailed || job.State == governance.LibraryJobCancelled) { job, reused, err = store.CreateLibraryJob(r.Context(), governance.LibraryJob{ - ID: fmt.Sprintf("libsync-%s-%d", request.Scope, now.UnixNano()), Scope: request.Scope, + ID: fmt.Sprintf("libsync-%s-%d", request.Scope, time.Now().UTC().UnixNano()), Scope: request.Scope, Kind: governance.LibraryJobSync, - IdempotencyKey: "library.sync:" + request.Scope + ":" + request.Target + ":retry:" + now.Format("20060102T150405.000000000"), - RequestDigest: digest, TargetSnapshotID: request.Target, Actor: actor, + IdempotencyKey: intent.JobKey + ":retry:" + now.Format("20060102T150405.000000000"), + RequestDigest: digest, OperationID: intent.Operation.ID, + TargetSnapshotID: request.Target, Actor: actor, }, governance.SyncStages(), now) if err != nil || reused { writeJSON(w, http.StatusConflict, map[string]any{"code": "library_sync_retry_conflict", diff --git a/internal/httpapi/library_flow_test.go b/internal/httpapi/library_flow_test.go index f17b2cc..2ef4562 100644 --- a/internal/httpapi/library_flow_test.go +++ b/internal/httpapi/library_flow_test.go @@ -29,6 +29,9 @@ type fakeLibraryStore struct { // currentErr 表达"读不到"(调用方必须 fail closed,不许当成没有生效版本)。 current *governance.LibraryVersion currentErr error + // operations/lastOperation 记录"意图身份":重放与换键的差别在这里表达(F01)。 + operations map[string]governance.LibraryOperation + lastOperation governance.LibraryOperation } func (f *fakeLibraryStore) CreateLibraryJob(_ context.Context, job governance.LibraryJob, _ []string, _ time.Time) (governance.LibraryJob, bool, error) { @@ -37,6 +40,23 @@ func (f *fakeLibraryStore) CreateLibraryJob(_ context.Context, job governance.Li return job, false, nil } +// EnsureLibraryOperation 在 fake 里就是"同键同摘要 ⇒ 同一条意图":测试用例靠它表达 +// "重放命中同意图 / 换键是新意图"(F01),不需要真实库。 +func (f *fakeLibraryStore) EnsureLibraryOperation(_ context.Context, op governance.LibraryOperation) (governance.LibraryOperation, bool, error) { + if f.operations == nil { + f.operations = map[string]governance.LibraryOperation{} + } + if existing, ok := f.operations[op.OperationKey]; ok { + if existing.RequestDigest != op.RequestDigest { + return existing, false, governance.ErrLibraryOperationConflict + } + return existing, false, nil + } + f.operations[op.OperationKey] = op + f.lastOperation = op + return op, true, nil +} + func (f *fakeLibraryStore) LibraryJob(_ context.Context, id string) (governance.LibraryJob, error) { return governance.LibraryJob{ID: id}, nil } diff --git a/internal/httpapi/library_intent.go b/internal/httpapi/library_intent.go new file mode 100644 index 0000000..0293179 --- /dev/null +++ b/internal/httpapi/library_intent.go @@ -0,0 +1,63 @@ +package httpapi + +import ( + "context" + "errors" + "strconv" + "strings" + "time" + + "knowledge-governance/internal/governance" +) + +// libraryIntent 是"用户这一次明确意图"的不可变身份。 +// +// 为什么不能再拿 `library.sync::` 当终身幂等键:那是把**意图**和 +// **目标状态**混成一体。成功同步 B 之后改成 A、再要求同步 B 时,第二次 B 拿到的是第一次 +// 的历史成功作业,什么都不做——用户要的状态没实现,页面还显示成功(F01)。 +// +// 现在:客户端可以用 `operation_key` 重放同一次意图(双击/网络重试 ⇒ 同一个 operation ⇒ +// 同一个作业);不带或换一个键 ⇒ 新意图 ⇒ 新 operation ⇒ 真的执行一遍,并留下自己的 +// 事件与 current 变更。 +type libraryIntent struct { + Operation governance.LibraryOperation + // JobKey 是作业幂等键:同一意图重放得到同一个作业行(不会重复建)。 + JobKey string +} + +// intentCacheKey 把"目标快照 + 客户端意图键"拼成 operation 键: +// 快照参与是因为 operation 的身份必须绑定它发布的是什么;客户端键参与是因为 +// 同一次重放要命中同一条意图。 +func intentCacheKey(snapshotID, clientKey string, now time.Time) string { + base := strings.TrimSpace(snapshotID) + if client := strings.TrimSpace(clientKey); client != "" { + return base + "#" + client + } + // 没有客户端键:每次请求都是一次新意图。用纳秒而不是随机数,便于从日志里回溯。 + return base + "#" + strconv.FormatInt(now.UnixNano(), 10) +} + +// ensureLibraryIntent 幂等登记意图。摘要不一致 ⇒ 冲突(同一个键不许换内容)。 +func ensureLibraryIntent(ctx context.Context, store libraryJobStore, scope string, kind governance.LibraryJobKind, + snapshotID, digest, actor, clientKey string, now time.Time) (libraryIntent, error) { + key := intentCacheKey(snapshotID, clientKey, now) + op, _, err := store.EnsureLibraryOperation(ctx, governance.LibraryOperation{ + ID: governance.LibraryOperationID(scope, kind, key), Scope: scope, Kind: kind, + TargetSnapshotID: snapshotID, RequestDigest: digest, OperationKey: key, Actor: actor, CreatedAt: now.UTC(), + }) + if err != nil { + return libraryIntent{}, err + } + return libraryIntent{Operation: op, JobKey: "library.job:" + op.ID}, nil +} + +// isIntentConflict 区分"同一个意图键换了内容"(409,调用方应换键)与内部错误。 +func isIntentConflict(err error) bool { + return errors.Is(err, governance.ErrLibraryOperationConflict) +} + +// libraryOperationView 是响应里给前端的意图身份(可复用于重放)。 +func libraryOperationView(intent libraryIntent) map[string]any { + return map[string]any{"operation_id": intent.Operation.ID, "operation_key": intent.Operation.OperationKey, + "target_snapshot_id": intent.Operation.TargetSnapshotID, "kind": string(intent.Operation.Kind)} +} diff --git a/internal/httpapi/library_intent_test.go b/internal/httpapi/library_intent_test.go new file mode 100644 index 0000000..6d990b2 --- /dev/null +++ b/internal/httpapi/library_intent_test.go @@ -0,0 +1,125 @@ +package httpapi + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "knowledge-governance/internal/governance" +) + +// F01:同一个目标快照的**新意图**必须真的执行,不能被"这个快照曾经成功过"吞掉。 +// +// 现场形状:同步 B 成功 → 恢复 A → 再次同步 B。第三次请求是一次合法意图, +// 旧实现返回第一次的成功作业(202, 不再并入队),用户要的状态没实现。 +// +// 判据:换 operation_key ⇒ 新 operation(ID 不同、作业幂等键不同); +// 同一个 operation_key ⇒ 同一 operation(重放)。 +func TestLibrarySyncNewIntentReExecutesInsteadOfReusingHistory(t *testing.T) { + target := "snap-1" + digest, err := librarySyncParamsDigest("internal", target) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + newStore := func() *fakeLibraryStore { + store := &fakeLibraryStore{ + current: &governance.LibraryVersion{ID: "ver-prod", Scope: "internal", + Kind: governance.LibraryVersionProductionEffective, SnapshotID: "prod-snap", + VersionLabel: "prod-2026.10", CreatedAt: now.Add(-72 * time.Hour)}, + } + store.enqueueEchoRequestedID = true + store.grant = &governance.LibraryReleaseAuthorization{ID: "auth-1", Scope: "internal", SnapshotID: target, + ParamsDigest: digest, AuthorizedBy: "admin", ExpiresAt: now.Add(30 * time.Minute), CreatedAt: now} + return store + } + config := func(store *fakeLibraryStore) Config { + return Config{Store: store, ProductionReleaseRequestsEnabled: true, ProductionBackupRestoreVerified: true, + LibraryProductionDatabaseURL: "postgres://prod.invalid/db", LibraryProductionInternalKB: "kb-int", + LibraryCapacityBudgetBytes: 1 << 40} + } + post := func(store *fakeLibraryStore, body string) *httptest.ResponseRecorder { + recorder := httptest.NewRecorder() + librarySyncEnqueue(recorder, httptest.NewRequest(http.MethodPost, "/api/v1/library/syncs", + strings.NewReader(body)), config(store)) + return recorder + } + withKey := `{"scope":"internal","target_snapshot_id":"` + target + `","operation_key":"%s"}` + + // 第一次意图:建作业。 + first := newStore() + if recorder := post(first, strings.Replace(withKey, "%s", "intent-1", 1)); recorder.Code != http.StatusAccepted { + t.Fatalf("首次发布会话 status=%d body=%s", recorder.Code, recorder.Body.String()) + } + firstOperation := first.lastOperation.ID + if firstOperation == "" || first.lastJob.OperationID != firstOperation { + t.Fatalf("作业必须挂在意图上:job.operation=%q intent=%q", first.lastJob.OperationID, firstOperation) + } + + // 同一次意图重放(双击/超时重发):同一个 operation,不产生第二条意图。 + replay := newStore() + if recorder := post(replay, strings.Replace(withKey, "%s", "intent-1", 1)); recorder.Code != http.StatusAccepted { + t.Fatalf("重放 status=%d body=%s", recorder.Code, recorder.Body.String()) + } + if replay.lastOperation.ID != firstOperation { + t.Fatalf("同一意图键得到了不同 operation:%q vs %q", replay.lastOperation.ID, firstOperation) + } + + // 新意图(同一快照、不同键):必须是新 operation ⇒ 新作业幂等键 ⇒ 真的执行。 + second := newStore() + if recorder := post(second, strings.Replace(withKey, "%s", "intent-2", 1)); recorder.Code != http.StatusAccepted { + t.Fatalf("新意图 status=%d body=%s", recorder.Code, recorder.Body.String()) + } + if second.lastOperation.ID == firstOperation { + t.Fatal("同一快照的新意图必须得到新 operation:否则历史成功会吞掉本次请求(F01)") + } + if second.lastJob.IdempotencyKey == first.lastJob.IdempotencyKey { + t.Fatal("新意图的作业幂等键与历史作业相同:CreateLibraryJob 会返回旧作业,什么也不执行") + } + if second.createN != 1 { + t.Fatalf("新意图必须建作业并执行,created=%d", second.createN) + } +} + +// 同一个意图键配上不同快照 = 冲突,不能悄悄共享谱系。 +func TestLibrarySyncIntentKeyReusedWithAnotherSnapshotConflicts(t *testing.T) { + target := "snap-1" + digest, _ := librarySyncParamsDigest("internal", target) + now := time.Now().UTC() + store := &fakeLibraryStore{current: &governance.LibraryVersion{ID: "ver-prod", Scope: "internal", + Kind: governance.LibraryVersionProductionEffective, SnapshotID: "prod-snap", CreatedAt: now.Add(-time.Hour)}} + store.grant = &governance.LibraryReleaseAuthorization{ID: "auth-1", Scope: "internal", SnapshotID: target, + ParamsDigest: digest, AuthorizedBy: "admin", ExpiresAt: now.Add(time.Hour), CreatedAt: now} + // 预先登记一条"同键但不同快照"的意图。 + if _, _, err := store.EnsureLibraryOperation(nil, governance.LibraryOperation{ + ID: "libop-x", Scope: "internal", Kind: governance.LibraryJobSync, TargetSnapshotID: "other-snap", + RequestDigest: "different", OperationKey: target + "#intent-1", CreatedAt: now}); err != nil { + t.Fatal(err) + } + config := Config{Store: store, ProductionReleaseRequestsEnabled: true, ProductionBackupRestoreVerified: true, + LibraryProductionDatabaseURL: "postgres://prod.invalid/db", LibraryProductionInternalKB: "kb-int", + LibraryCapacityBudgetBytes: 1 << 40} + body := `{"scope":"internal","target_snapshot_id":"` + target + `","operation_key":"intent-1"}` + recorder := httptest.NewRecorder() + librarySyncEnqueue(recorder, httptest.NewRequest(http.MethodPost, "/api/v1/library/syncs", strings.NewReader(body)), config) + if recorder.Code != http.StatusConflict { + t.Fatalf("同键不同内容必须 409,status=%d body=%s", recorder.Code, recorder.Body.String()) + } +} + +// F14:通用 queue 入口不许创建整库保留类型——那里的角色检查替代不了发布准入。 +func TestCreateJobRejectsReservedLibraryTypes(t *testing.T) { + store := &fakeJobStore{} + config := Config{Store: store} + body := `{"type":"library.baseline","payload":{"scope":"internal"},"idempotency_key":"k-1"}` + recorder := httptest.NewRecorder() + createJob(recorder, httptest.NewRequest(http.MethodPost, "/api/v1/jobs", strings.NewReader(body)), config) + if recorder.Code != http.StatusBadRequest { + t.Fatalf("保留类型必须被通用入口拒绝:status=%d body=%s", recorder.Code, recorder.Body.String()) + } + if !strings.Contains(recorder.Body.String(), "发布管理专用入口") { + t.Fatalf("拒绝原因必须指向专用入口:%s", recorder.Body.String()) + } +} \ No newline at end of file diff --git a/internal/httpapi/library_restore.go b/internal/httpapi/library_restore.go index 3e1239f..47677d2 100644 --- a/internal/httpapi/library_restore.go +++ b/internal/httpapi/library_restore.go @@ -25,6 +25,8 @@ func libraryRestoreParamsDigest(scope, snapshotID string) (string, error) { type libraryRestoreRequest struct { Scope string `json:"scope"` SnapshotID string `json:"snapshot_id"` + // OperationKey 是客户端重放键;不传就是一次新意图(同一快照再恢复一次也要真的执行)。 + OperationKey string `json:"operation_key"` } func libraryRestoreEnqueue(w http.ResponseWriter, r *http.Request, config Config) { @@ -91,10 +93,22 @@ func libraryRestoreEnqueue(w http.ResponseWriter, r *http.Request, config Config // [A-Za-z0-9_-]:带 "." 的时间戳会让恢复在开窗口时被拒(真跑报 invalid freeze request), // 所以用纳秒整数,和同步作业保持一致。 jobID := "librestore-" + request.Scope + "-" + strconv.FormatInt(now.UnixNano(), 10) + intent, intentErr := ensureLibraryIntent(r.Context(), store, request.Scope, governance.LibraryJobRestore, + request.SnapshotID, digest, identity.Username, request.OperationKey, now) + if intentErr != nil { + if isIntentConflict(intentErr) { + writeJSON(w, http.StatusConflict, map[string]any{"code": "library_restore_intent_conflict", + "error": "同一 operation_key 对应了不同的请求内容:换一个键表示新意图"}) + return + } + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot register restore intent"}) + return + } job, reused, err := store.CreateLibraryJob(r.Context(), governance.LibraryJob{ ID: jobID, Scope: request.Scope, Kind: governance.LibraryJobRestore, - IdempotencyKey: "library.restore:" + request.Scope + ":" + request.SnapshotID, - RequestDigest: digest, TargetSnapshotID: request.SnapshotID, Actor: identity.Username, + IdempotencyKey: intent.JobKey, + RequestDigest: digest, OperationID: intent.Operation.ID, TargetSnapshotID: request.SnapshotID, + Actor: identity.Username, }, governance.SyncStages(), now) if errors.Is(err, governance.ErrLibraryJobActive) { writeJSON(w, http.StatusConflict, map[string]any{"code": "library_restore_active", "error": "该目标已有在飞作业"}) @@ -111,8 +125,9 @@ func libraryRestoreEnqueue(w http.ResponseWriter, r *http.Request, config Config if reused && (job.State == governance.LibraryJobFailed || job.State == governance.LibraryJobCancelled) { job, reused, err = store.CreateLibraryJob(r.Context(), governance.LibraryJob{ ID: jobID + "-retry", Scope: request.Scope, Kind: governance.LibraryJobRestore, - IdempotencyKey: "library.restore:" + request.Scope + ":" + request.SnapshotID + ":retry:" + strconv.FormatInt(now.UnixNano(), 10), - RequestDigest: digest, TargetSnapshotID: request.SnapshotID, Actor: identity.Username, + IdempotencyKey: intent.JobKey + ":retry:" + strconv.FormatInt(now.UnixNano(), 10), + RequestDigest: digest, OperationID: intent.Operation.ID, TargetSnapshotID: request.SnapshotID, + Actor: identity.Username, }, governance.SyncStages(), now) if err != nil || reused { writeJSON(w, http.StatusConflict, map[string]any{"code": "library_restore_retry_conflict", "error": "无法为重试创建新作业"}) @@ -273,10 +288,26 @@ func libraryJobResume(w http.ResponseWriter, r *http.Request, config Config) { writeJSON(w, http.StatusConflict, map[string]any{"error": "封存作业由封存入口重发,不走续作"}) return } + // F15:续作是**同一个意图的新执行代次**,不是新意图。沿用原作业的 operation, + // 维护窗口、预备份、写集合的谱系都还挂在同一条意图上;新作业行只是这次执行的载体, + // worker 起手会为它开一个更大的 epoch(旧代次从此不能写)。 + operationID := source.OperationID + if operationID == "" { + // 迁移前的老作业没有 operation:按原作业身份补一条(同键同摘要 ⇒ 幂等, + // 不会因为点了两次续作而造出两条意图)。 + intent, intentErr := ensureLibraryIntent(r.Context(), store, source.Scope, kind, + source.TargetSnapshotID, digest, identity.Username, "resume:"+sourceID, now) + if intentErr != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot register resume intent"}) + return + } + operationID = intent.Operation.ID + } job, reused, err := store.CreateLibraryJob(r.Context(), governance.LibraryJob{ ID: jobID, Scope: source.Scope, Kind: kind, - IdempotencyKey: "library.resume:" + sourceID, - RequestDigest: digest, TargetSnapshotID: source.TargetSnapshotID, Actor: identity.Username, + IdempotencyKey: "library.job:" + operationID + ":resume", + RequestDigest: digest, OperationID: operationID, + TargetSnapshotID: source.TargetSnapshotID, Actor: identity.Username, }, governance.SyncStages(), now) if errors.Is(err, governance.ErrLibraryJobActive) { writeJSON(w, http.StatusConflict, map[string]any{"error": "该目标已有在飞作业"}) diff --git a/internal/httpapi/library_storage.go b/internal/httpapi/library_storage.go index a0bf502..a8a3b86 100644 --- a/internal/httpapi/library_storage.go +++ b/internal/httpapi/library_storage.go @@ -1,6 +1,7 @@ package httpapi import ( + "context" "net/http" "sort" "strings" @@ -96,41 +97,47 @@ func libraryStorageHandler(w http.ResponseWriter, r *http.Request, config Config // 版本类型与保护依据来自持久登记与在飞作业,不靠文件时间猜。 kinds := map[string]governance.LibraryVersion{} protectedBy := map[string][]string{} + protectionUnknown := false store, storeOK := libraryStore(config) if storeOK { + // 类型与保护依据只认**持久登记**的精确查询:分页窗口(最近 100 个版本/20 个作业) + // 一旦被历史挤满,需要恢复的旧预备份与 failed-but-held 作业就会被读成"可删除候选"(F10)。 for _, scope := range []string{"internal", "external"} { versions, listErr := store.LibraryVersions(r.Context(), scope, 100) - if listErr == nil { - for _, item := range versions { - kinds[item.SnapshotID] = item - if item.Kind == governance.LibraryVersionProductionPrebackup { - protectedBy[item.SnapshotID] = append(protectedBy[item.SnapshotID], "生产预备份") - } - } + if listErr != nil { + protectionUnknown = true + continue + } + for _, item := range versions { + kinds[item.SnapshotID] = item } - // 生产生效版本 = 唯一一条未顶替的记录,必须精确查:它可能早已掉出上面的 - // 展示窗口,靠列表判"是否受保护"会让当前生效的那份被当成普通版本清掉。 if current, found, currentErr := store.LibraryCurrentVersion(r.Context(), scope); currentErr == nil && found { if _, seen := kinds[current.SnapshotID]; !seen { kinds[current.SnapshotID] = current } - protectedBy[current.SnapshotID] = appendUnique(protectedBy[current.SnapshotID], "生产生效版本") - } - jobs, jobErr := store.ListLibraryJobs(r.Context(), scope, 20) - if jobErr == nil { - for _, job := range jobs { - if job.State != governance.LibraryJobPending && job.State != governance.LibraryJobRunning { - continue - } - for _, id := range []string{job.SnapshotID, job.TargetSnapshotID} { - if id != "" { - protectedBy[id] = append(protectedBy[id], "在飞作业引用") - } - } - } + } else if currentErr != nil { + protectionUnknown = true } } + if protector, ok := config.Store.(interface { + LibraryProtection(ctx context.Context) (map[string][]string, error) + }); ok { + if reasons, protectErr := protector.LibraryProtection(r.Context()); protectErr == nil { + for snapshot, items := range reasons { + for _, reason := range items { + protectedBy[snapshot] = appendUnique(protectedBy[snapshot], reason) + } + } + } else { + protectionUnknown = true + response.Notes = append(response.Notes, "保护依据读取失败("+protectErr.Error()+"):本次不生成可删除候选") + } + } else { + protectionUnknown = true + response.Notes = append(response.Notes, "存储不支持全量保护查询:本次不生成可删除候选") + } } else { + protectionUnknown = true response.Notes = append(response.Notes, "版本登记存储不可用:只按归档文件统计,无法识别受保护版本") } @@ -148,11 +155,11 @@ func libraryStorageHandler(w http.ResponseWriter, r *http.Request, config Config } for index, id := range ids { manifest := manifests[index] + // 备份类型只能来自显式登记:非空 ValidationPolicy 也被生产预备份使用, + // 拿它猜类型会把生产预备份显示成测试封存(F10)。 kind := "unregistered" if record, ok := kinds[id]; ok { kind = string(record.Kind) - } else if manifest.ValidationPolicy != "" { - kind = "test_sealed" } if kind == "test_sealed" { // 每库最近一次封存受保护:它是最新可发布的测试版本。 @@ -170,14 +177,19 @@ func libraryStorageHandler(w http.ResponseWriter, r *http.Request, config Config // 用与上面相同的清单再定位一次最新测试封存,避免把时间映射回 ID 时出错。 for index, id := range ids { manifest := manifests[index] - if kindOf := kinds[id].Kind; kindOf == governance.LibraryVersionTestSealed || - (kindOf == "" && manifest.ValidationPolicy != "") { + if kindOf := kinds[id].Kind; kindOf == governance.LibraryVersionTestSealed { if manifest.CreatedAt.Equal(latestSealed[manifest.Scope]) { - protectedBy[id] = append(protectedBy[id], "最近测试封存("+manifest.Scope+")") + protectedBy[id] = appendUnique(protectedBy[id], "最近测试封存("+manifest.Scope+")") } } } + if protectionUnknown { + // 保护依据未知 ⇒ 不许产出"可删除候选":不确定的东西不能按可删处理(fail closed)。 + for index := range versions { + protectedBy[versions[index].ID] = appendUnique(protectedBy[versions[index].ID], "保护依据未知(拒绝生成可删候选)") + } + } for index := range versions { if reasons := protectedBy[versions[index].ID]; len(reasons) > 0 { versions[index].Protected = true diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index bca8f40..8583dd2 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -776,6 +776,15 @@ func createJob(w http.ResponseWriter, r *http.Request, config Config) { writeJSON(w, http.StatusBadRequest, map[string]string{"error": "通讯录同步任务只能从问答运营的管理员入口创建"}) return } + // 保留类型:`library.*` 的准入条件(闸口、精确授权、维护互斥、目标身份)只在专用 + // 入口里判断。通用入口只做通用 engineer 角色检查,让它在队列里造一个 library.* + // 作业就等于绕过全部准入——worker 拿到没有业务行的作业仍会去取维护锁(F14)。 + // 这里按前缀整类拒绝,而不是逐个类型列举:新增 library 作业类型时不会漏。 + if governance.IsReservedLibraryJobType(input.Type) { + writeJSON(w, http.StatusBadRequest, map[string]string{ + "error": "整库发布相关作业只能从发布管理专用入口创建"}) + return + } if strings.TrimSpace(input.Type) == "" || len(input.Type) > 100 { writeJSON(w, http.StatusBadRequest, map[string]string{"error": "type is required and must not exceed 100 characters"}) return diff --git a/internal/jobs/lease_loss_test.go b/internal/jobs/lease_loss_test.go new file mode 100644 index 0000000..0e6efc8 --- /dev/null +++ b/internal/jobs/lease_loss_test.go @@ -0,0 +1,51 @@ +package jobs + +import ( + "context" + "errors" + "testing" + "time" +) + +// leaseLostQueue 模拟"心跳发现租约已经不属于自己"。 +type leaseLostQueue struct { + recordingQueue + lost chan struct{} +} + +func (q *leaseLostQueue) Heartbeat(context.Context, string, string, time.Time, time.Duration) error { + close(q.lost) + return errors.New("audit lease lost") +} + +// F13:心跳报失租之后,handler 必须**立刻失去授权**(ctx 被取消)。 +// +// 旧实现只把错误存进 heartbeatResult,等 handler 自己返回才处理;handler 在 +// 那段时间里仍以为自己拥有目标,会继续写业务状态、释放维护锁——而另一个代次 +// 可能已经接管并开了新窗口(共享 flock 管不到这些写)。 +func TestLeaseLossMustCancelHandler(t *testing.T) { + q := &leaseLostQueue{recordingQueue: recordingQueue{claimJob: &Job{ID: "audit", Type: "audit"}}, lost: make(chan struct{})} + continued := false + w, err := NewWorker(WorkerConfig{ + Queue: q, WorkerID: "audit", LeaseDuration: time.Second, HeartbeatInterval: time.Millisecond, + Handlers: map[string]Handler{"audit": func(ctx context.Context, _ Job) error { + <-q.lost + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(200 * time.Millisecond): + continued = true + return nil + } + }}, + }) + if err != nil { + t.Fatal(err) + } + if _, err := w.RunOnce(context.Background()); err == nil { + t.Fatal("失租后 RunOnce 必须报错,不能当成正常完成") + } + if continued { + t.Fatal("心跳报失租后 handler 仍在运行:旧代次还能继续写目标") + } +} \ No newline at end of file diff --git a/internal/jobs/worker.go b/internal/jobs/worker.go index 77516c1..0f42e0e 100644 --- a/internal/jobs/worker.go +++ b/internal/jobs/worker.go @@ -140,6 +140,10 @@ func (w *Worker) RunOnce(ctx context.Context) (bool, error) { } return true, err } + // handler 必须能被"失租"取消:心跳报错说明这个代次已经不再拥有作业, + // 若还让 handler 继续跑,它会以旧代次身份继续写业务状态/释放维护锁(F13)。 + handlerCtx, cancelHandler := context.WithCancel(ctx) + defer cancelHandler() heartbeatCtx, stopHeartbeat := context.WithCancel(ctx) heartbeatResult := make(chan error, 1) go func() { @@ -152,6 +156,9 @@ func (w *Worker) RunOnce(ctx context.Context) (bool, error) { return case <-ticker.C: if err := w.queue.Heartbeat(ctx, job.ID, w.workerID, w.now(), w.leaseDuration); err != nil { + // 先撤销 handler 的授权,再把错误交给下面统一处理: + // 顺序反过来就会出现"错误已记录、handler 还在写"的空窗。 + cancelHandler() heartbeatResult <- err return } @@ -159,8 +166,9 @@ func (w *Worker) RunOnce(ctx context.Context) (bool, error) { } }() - handlerErr := handler(ctx, *job) + handlerErr := handler(handlerCtx, *job) stopHeartbeat() + cancelHandler() heartbeatErr := <-heartbeatResult if handlerErr == nil { handlerErr = heartbeatErr diff --git a/internal/librarydiff/wiki.go b/internal/librarydiff/wiki.go index 40b3839..51e7b2a 100644 --- a/internal/librarydiff/wiki.go +++ b/internal/librarydiff/wiki.go @@ -58,16 +58,20 @@ type WikiResult struct { } type wikiPage struct { - slug string - title string - pageType string - status string - content string - sources []string - citations []string - outLinks []string - inLinks []string - category string + slug string + title string + pageType string + status string + // summary / parentSlug 是持久业务字段:只改它们同样是"内容变了", + // 旧的比较结构根本没读这两个字段,于是发布前预览会漏报(F18)。 + summary string + parentSlug string + content string + sources []string + citations []string + outLinks []string + inLinks []string + category string } // CompareWiki 以 base 为基准(当前生产版本),target 为拟发布版本,逐页比对语义。 @@ -150,6 +154,17 @@ func diffWikiPage(previous, next wikiPage) (WikiContribution, bool) { if previous.pageType != next.pageType { change.Changed = append(change.Changed, "page_type") } + // 标题/摘要/父级是页面身份与导航的一部分:它们在持久列里,改了就是差异, + // 不能被"正文没变"掩掉(F18)。 + if previous.title != next.title { + change.Changed = append(change.Changed, "title") + } + if previous.summary != next.summary { + change.Changed = append(change.Changed, "summary") + } + if previous.parentSlug != next.parentSlug { + change.Changed = append(change.Changed, "parent_slug") + } return change, len(change.Changed) > 0 } @@ -169,6 +184,8 @@ func indexWiki(rows []map[string]any) (map[string]wikiPage, error) { page.pageType, _ = row["page_type"].(string) page.status, _ = row["status"].(string) page.content, _ = row["content"].(string) + page.summary, _ = row["summary"].(string) + page.parentSlug, _ = row["parent_slug"].(string) var err error if page.sources, err = stringList(row["source_refs"]); err != nil { return nil, fmt.Errorf("wiki %s source_refs: %w", slug, err) diff --git a/internal/librarydiff/wiki_metadata_test.go b/internal/librarydiff/wiki_metadata_test.go new file mode 100644 index 0000000..b5ff50f --- /dev/null +++ b/internal/librarydiff/wiki_metadata_test.go @@ -0,0 +1,37 @@ +package librarydiff + +import "testing" + +// F18:只改 title / summary / parent_slug 也必须在发布前差异里出现。 +// 旧比较结构根本没读 summary/parent_slug,title 读了却不比较——三种改动都被漏报, +// 于是"预览无变化"可以对应一次真实的元数据变更。 +func TestWikiMetadataChangeMustAppear(t *testing.T) { + for _, field := range []string{"title", "summary", "parent_slug"} { + t.Run(field, func(t *testing.T) { + before := map[string]any{"slug": "page-a", "title": "old", "summary": "old", + "parent_slug": "old", "content": "unchanged"} + after := map[string]any{"slug": "page-a", "title": "old", "summary": "old", + "parent_slug": "old", "content": "unchanged"} + after[field] = "new" + result, err := CompareWiki([]map[string]any{before}, []map[string]any{after}) + if err != nil { + t.Fatal(err) + } + if result.Updated != 1 { + t.Fatalf("只改 %s 必须记成 updated=1:%+v", field, result) + } + if len(result.Pages) != 1 || len(result.Pages[0].Changed) == 0 { + t.Fatalf("差异项必须列出改了哪些字段:%+v", result.Pages) + } + }) + } + // 反例:完全没有变化时不报差异(避免把"无变化"也渲染成更新)。 + same := map[string]any{"slug": "page-b", "title": "t", "summary": "s", "parent_slug": "p", "content": "c"} + result, err := CompareWiki([]map[string]any{same}, []map[string]any{same}) + if err != nil { + t.Fatal(err) + } + if result.Updated != 0 || len(result.Pages) != 0 { + t.Fatalf("无变化不应产生差异:%+v", result) + } +} diff --git a/internal/libraryfreeze/freeze.go b/internal/libraryfreeze/freeze.go index 7077123..7d4ad0f 100644 --- a/internal/libraryfreeze/freeze.go +++ b/internal/libraryfreeze/freeze.go @@ -105,6 +105,11 @@ type Controller interface { type Result struct { State string `json:"state"` SnapshotID string `json:"snapshot_id,omitempty"` + // Digest 是"这次完成的校验摘要"(恢复路径用它证明内容身份)。 + // 终态重放时从 journal 里读回,调用方据此补做业务收尾而不重跑数据。 + Digest string `json:"digest,omitempty"` + // Replayed 标记"这次没有执行数据动作,只是把已完成的终态读回来"。 + Replayed bool `json:"replayed,omitempty"` } type serviceRecord struct { Service @@ -563,9 +568,12 @@ func (c *Coordinator) stopped(ctx context.Context, j journal) error { } func (c *Coordinator) restore(ctx context.Context, j *journal) error { j.State = "restoring" - // Restoring services is still necessary if the journal disk fails after a - // persisted stop intent. Keep the error, but do not let it suppress recovery. - var failures error = c.save(*j) + // 两类失败必须分开累计: + // persistFailures:journal 落盘失败。它让证据不完整,但不改变"服务是否真的回来了"。 + // serviceFailures:启动/健康/拓扑不确定。只要有一项,就绝不能开入口—— + // 状态字段写 held 而外部入口已开,是对操作者和流量的双重谎报(F04)。 + var persistFailures error = c.save(*j) + var serviceFailures error for i := range j.Services { s := &j.Services[i] if !s.Running || !s.StopRequested { @@ -573,7 +581,7 @@ func (c *Coordinator) restore(ctx context.Context, j *journal) error { } items, err := inventory(ctx, c.control) if err != nil { - failures = errors.Join(failures, err) + serviceFailures = errors.Join(serviceFailures, err) continue } found := false @@ -583,16 +591,16 @@ func (c *Coordinator) restore(ctx context.Context, j *journal) error { } found = true if actual.Running && !s.StopVerified { - failures = errors.Join(failures, errors.New("stop outcome remains uncertain; service may still stop later")) + serviceFailures = errors.Join(serviceFailures, errors.New("stop outcome remains uncertain; service may still stop later")) break } if !actual.Running { s.StopVerified = true if saveErr := c.save(*j); saveErr != nil { - failures = errors.Join(failures, saveErr) + persistFailures = errors.Join(persistFailures, saveErr) } if err = c.control.Start(ctx, s.ID); err != nil { - failures = errors.Join(failures, errors.New("test service restart failed")) + serviceFailures = errors.Join(serviceFailures, errors.New("test service restart failed")) break } } @@ -606,17 +614,17 @@ func (c *Coordinator) restore(ctx context.Context, j *journal) error { } } if !healthy { - failures = errors.Join(failures, errors.New("test service health unverified")) + serviceFailures = errors.Join(serviceFailures, errors.New("test service health unverified")) break } s.Restored = true if err = c.save(*j); err != nil { - failures = errors.Join(failures, err) + persistFailures = errors.Join(persistFailures, err) } break } if !found { - failures = errors.Join(failures, errors.New("original test service missing; replacement not started")) + serviceFailures = errors.Join(serviceFailures, errors.New("original test service missing; replacement not started")) } } // 服务都回来了才开放入口:先开门再启动服务,等于把流量打向没准备好的目标。 @@ -624,19 +632,24 @@ func (c *Coordinator) restore(ctx context.Context, j *journal) error { // 注意:入口关到一半失败(闸门已被停、排空超时)时 j.Gate 可能还是 nil, // 但入口确实已经不在了。所以实例级维护一律重新打开入口,而不是只看 // "有没有成功关闭的证据"——否则会出现"维护已释放、服务在跑、入口却是关的"。 - if c.level == "instance" || j.Gate != nil { + // 但只要**任何一个写入者没回到健康**,这里的 else 分支就不执行:入口保持关闭, + // 由调用方按 held 处理并走恢复(宁可不可用,也不对外暴露没准备好的目标)。 + switch { + case serviceFailures != nil: + serviceFailures = errors.Join(serviceFailures, errors.New("目标入口保持关闭:写入者未全部恢复健康")) + case c.level == "instance" || j.Gate != nil: if c.gate == nil { - failures = errors.Join(failures, errors.New("入口闸门缺失:不能确认目标已恢复")) + persistFailures = errors.Join(persistFailures, errors.New("入口闸门缺失:不能确认目标已恢复")) } else if err := c.gate.OpenIngress(ctx); err != nil { - failures = errors.Join(failures, fmt.Errorf("目标入口恢复失败:%w", err)) + persistFailures = errors.Join(persistFailures, fmt.Errorf("目标入口恢复失败:%w", err)) } else if j.Gate != nil { // 只有"确实关过"的窗口才记"重新打开":没有关闭证据时开门是恢复原状, // 不能写成"重开",否则 journal 校验会误判。 j.GateReopened = true if err := c.save(*j); err != nil { - failures = errors.Join(failures, err) + persistFailures = errors.Join(persistFailures, err) } } } - return failures + return errors.Join(serviceFailures, persistFailures) } diff --git a/internal/libraryfreeze/recovery.go b/internal/libraryfreeze/recovery.go index 3f38f4a..facdc4c 100644 --- a/internal/libraryfreeze/recovery.go +++ b/internal/libraryfreeze/recovery.go @@ -45,6 +45,13 @@ func (c *Coordinator) RecoverWindow(ctx context.Context, job string, restorer Re return Result{}, err } if terminal(j.State) { + // 终态重放(F05 路径二):journal 已经是 recovered,说明数据恢复与服务/入口 + // 恢复都已完成。这时再拒绝会让"journal recovered → PG 维护释放"之间的崩溃 + // 永久卡死(锁挂在那里、页面一直显示待恢复)。 + // 允许调用方**幂等补做业务收尾**:不重跑 Restore、不再 OpenIngress。 + if j.State == "recovered" && j.RecoveredBackup != "" && j.RecoveryDigest != "" { + return Result{State: "recovered", SnapshotID: j.RecoveredBackup, Digest: j.RecoveryDigest, Replayed: true}, nil + } return Result{}, errors.New("freeze journal is already terminal") } if !j.ContentRisk { @@ -125,6 +132,9 @@ func (c *Coordinator) RecoverWindow(ctx context.Context, job string, restorer Re j.RecoveryDigest = digest j.RecoveredAt = &ended j.RestoreBlocked = false + // 完成凭据同时写进 Result:终态重放(上面那条分支)靠它把同一份事实交回调用方。 + j.SnapshotID = j.BackupSnapshotID + j.Digest = digest if err := c.save(j); err != nil { return j.Result, err } diff --git a/internal/libraryfreeze/window.go b/internal/libraryfreeze/window.go index f8dac27..2cf36aa 100644 --- a/internal/libraryfreeze/window.go +++ b/internal/libraryfreeze/window.go @@ -230,7 +230,20 @@ func (c *Coordinator) RunWindow(ctx context.Context, job string, prepare func(co } // abandonWindow:在"还没写目标"的阶段失败——内容没被改过,可以安全恢复并结案。 +// +// 判断"这个窗口有没有写过"必须读**持久** ContentRisk,而不是本进程是否调用过 Apply: +// 接管一个 held/stopping 窗口时本进程一笔都没写,但上一个进程可能已经写了一半。 +// 那种情况下任何早退(入口关不掉、后台不静默、停写者失败)都只能保持关闭, +// 由恢复流程按预备份处理——把服务拉起来开门就是对外暴露半写内容(F03)。 func (c *Coordinator) abandonWindow(ctx context.Context, j *journal, cause error, code string) (Result, error) { + if j.ContentRisk { + j.State = "held" + j.RestoreBlocked = true + j.Code = code + saveErr := c.save(*j) + return j.Result, errors.Join(cause, errors.New( + "窗口内容有持久风险:目标保持维护,禁止重新开放;须按预备份恢复并逐类校验"), saveErr) + } recoveryCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute) defer cancel() if restoreErr := c.restore(recoveryCtx, j); restoreErr != nil { diff --git a/internal/libraryfreeze/window_safety_test.go b/internal/libraryfreeze/window_safety_test.go new file mode 100644 index 0000000..23f9e10 --- /dev/null +++ b/internal/libraryfreeze/window_safety_test.go @@ -0,0 +1,73 @@ +package libraryfreeze + +import ( + "context" + "errors" + "strings" + "testing" +) + +// F03:接管一个已经持久化 ContentRisk 的半写窗口时,前置检查(后台静默/停写者/排空) +// 任何一个失败都必须保持目标关闭。原实现所有早退都走 abandonWindow → restore, +// 于是"本进程尚未写"被当成"此窗口从未写",半写内容被重新对外开放。 +func TestAdoptedHalfWrittenWindowPreflightFailureStaysClosed(t *testing.T) { + _, c, e := instanceFixture(t) + stopAllServices(c) + j := heldJournal("audit-adopt", strings.Repeat("a", 64), strings.Repeat("b", 64)) + if err := e.save(j); err != nil { + t.Fatal(err) + } + // 后台仍有任务:前置检查在 work 之前失败。 + if err := e.SetQuiescer(&idleQuiescer{busy: true}); err != nil { + t.Fatal(err) + } + e.quiesceWait = 1 // 极短等待,让前置检查尽快失败 + worked := false + _, err := e.RunWindow(context.Background(), j.Job, func(context.Context) error { return nil }, + func(context.Context, *Window) (string, error) { + worked = true + return "", nil + }, func(error) bool { return false }) + if err == nil || worked { + t.Fatalf("fixture did not fail before work: err=%v worked=%v", err, worked) + } + for _, call := range c.calls { + if strings.HasPrefix(call, "start:") || call == "gate:open" { + t.Fatalf("半写目标在前置检查失败后被重新开放:calls=%v", c.calls) + } + } + after, loadErr := e.load(j.Job) + if loadErr != nil { + t.Fatal(loadErr) + } + if terminal(after.State) { + t.Fatalf("半写窗口必须保持非终态(等恢复),实际 %s", after.State) + } + if !after.ContentRisk || !after.RestoreBlocked { + t.Fatalf("持久内容风险被清掉了:contentRisk=%v restoreBlocked=%v", after.ContentRisk, after.RestoreBlocked) + } +} + +// F04:任何一个"原本在跑"的写入者启动失败,都不能调用 OpenIngress。 +// 原实现在 restore() 里累积 Start/健康/save 错误,却仍然开门:状态说"保持关闭", +// 外部动作却把流量打向没准备好的目标。 +func TestRestartFailureMustNotOpenGate(t *testing.T) { + _, c, e := instanceFixture(t) + e.control = &restartFailControl{exitControl: c} + _, err := e.RunWindow(context.Background(), "audit-restart", func(context.Context) error { return nil }, + func(context.Context, *Window) (string, error) { return "", nil }, func(error) bool { return true }) + if err == nil { + t.Fatal("fixture 预期重启失败") + } + for _, call := range c.calls { + if call == "gate:open" { + t.Fatalf("重启失败后仍然打开了入口:calls=%v", c.calls) + } + } +} + +type restartFailControl struct{ *exitControl } + +func (c *restartFailControl) Start(context.Context, string) error { + return errors.New("synthetic restart failure") +} diff --git a/internal/libraryrelease/release.go b/internal/libraryrelease/release.go index 9c69d78..7f30fd5 100644 --- a/internal/libraryrelease/release.go +++ b/internal/libraryrelease/release.go @@ -219,6 +219,7 @@ func Run(ctx context.Context, request Request, target Target, backups BackupStor // 写集合必须在第一笔写入之前落盘并登记:恢复只按"这次明确写了什么"清理, // 不按目标现状推断残渣(那会把历史孤儿/共享资源当成本次残留删掉)。 + existingPlan := backup.WriteSet != nil if backup.WriteSet == nil { if planner, ok := target.(WriteSetPlanner); ok { set, planErr := planner.PlanWriteSet(ctx, WriteSetPlan{JobID: request.JobID, Target: request.Target, @@ -244,6 +245,26 @@ func Run(ctx context.Context, request Request, target Target, backups BackupStor } } + // 计划已经存在(接管/恢复)时,执行目标必须**消费**它:把"这些资源已证明可复用" + // 的决定装进去,写入阶段才会跳过共享对象。不能重新计划——那时目标已被写过, + // 重新计划等于把半写现状当新基线(F07)。 + if existingPlan { + if adopter, ok := target.(WriteSetAdopter); ok { + if err := adopter.AdoptWriteSet(*backup.WriteSet); err != nil { + outcome.State = "failed" + outcome.Error = "无法接管已持久化的写集合,拒绝继续写目标:" + err.Error() + recorder.Stage(ctx, "prebackup", "failed", outcome.Error) + return outcome + } + } else if _, plans := target.(WriteSetPlanner); plans { + // 会写资源、却读不懂计划:继续写就会重写共享行/对象。 + outcome.State = "failed" + outcome.Error = "目标实现无法消费已持久化的写集合,拒绝在失去复用语义的情况下写目标" + recorder.Stage(ctx, "prebackup", "failed", outcome.Error) + return outcome + } + } + for _, stage := range Stages() { recorder.Stage(ctx, string(stage), "running", "") if err = target.Apply(ctx, stage, request.SnapshotID); err != nil { diff --git a/internal/libraryrelease/release_test.go b/internal/libraryrelease/release_test.go index fa96dc8..997ea19 100644 --- a/internal/libraryrelease/release_test.go +++ b/internal/libraryrelease/release_test.go @@ -259,6 +259,19 @@ type writeSetPlanTarget struct { events *[]string ledger *WriteSet restores []BackupRef + // adopted 记录"接管时消费了哪份已持久化写集合"(F07):adoptErr 表达"读不懂计划"。 + adopted []WriteSet + adoptErr error +} + +// AdoptWriteSet 让这个假目标具备"消费既有计划"的能力:真实实现按它把复用决定装回去。 +func (p *writeSetPlanTarget) AdoptWriteSet(set WriteSet) error { + *p.events = append(*p.events, "adopt") + if p.adoptErr != nil { + return p.adoptErr + } + p.adopted = append(p.adopted, set) + return nil } func (p *writeSetPlanTarget) PlanWriteSet(_ context.Context, plan WriteSetPlan) (WriteSet, error) { @@ -400,6 +413,13 @@ func TestRunResumeUsesPersistedWriteSet(t *testing.T) { if len(target.plans) != 0 { t.Fatalf("resume must not re-plan the write set: %+v", target.plans) } + // F07:既有计划必须被执行目标消费——否则"复用"只停留在账本上,Apply 仍会重写共享对象。 + if len(target.adopted) != 1 { + t.Fatalf("接管必须把已持久化的写集合交给执行目标:adopted=%d", len(target.adopted)) + } + if target.adopted[0].BackupSnapshotID != "backup-snap" { + t.Fatalf("消费的写集合绑定了错误的预备份:%+v", target.adopted[0].BackupSnapshotID) + } if len(target.restores) != 1 || target.restores[0].WriteSet == nil { t.Fatalf("restore must carry the persisted write set: %+v", target.restores) } @@ -409,3 +429,28 @@ func TestRunResumeUsesPersistedWriteSet(t *testing.T) { } } } + +// F07 反向:会写资源、却读不懂既有计划的执行目标必须**在被写之前**失败。 +// 继续写就等于"账本说没动共享对象,实际又 upsert + 重传了一遍"。 +func TestRunRefusesPlanBlindTargetOnResume(t *testing.T) { + _, recorder, _, request, _ := plannedRun(t) + blind := &blindPlanTarget{fakeTarget: &fakeTarget{}} + resumed := &fakeBackups{ref: BackupRef{ID: "backup-1", SnapshotID: "backup-snap", + WriteSet: &WriteSet{Version: WriteSetVersion, JobID: request.JobID, Target: request.Target, + TenantID: 10001, KnowledgeBaseID: "kb", BackupSnapshotID: "backup-snap", + TargetSnapshotID: request.SnapshotID, CreatedAt: time.Now().UTC()}}} + outcome := Run(context.Background(), request, blind, resumed, recorder) + if outcome.State != "failed" { + t.Fatalf("读不懂计划的执行目标必须失败:%+v", outcome) + } + if len(blind.applied) != 0 { + t.Fatalf("失去复用语义时不许写任何阶段:%v", blind.applied) + } +} + +// blindPlanTarget 会规划写集合(因此会写资源),但没有 AdoptWriteSet。 +type blindPlanTarget struct{ *fakeTarget } + +func (b *blindPlanTarget) PlanWriteSet(context.Context, WriteSetPlan) (WriteSet, error) { + return WriteSet{Version: WriteSetVersion}, nil +} diff --git a/internal/libraryrelease/writeset.go b/internal/libraryrelease/writeset.go index c7345b0..b740ade 100644 --- a/internal/libraryrelease/writeset.go +++ b/internal/libraryrelease/writeset.go @@ -247,6 +247,16 @@ type WriteSetPlanner interface { PlanWriteSet(ctx context.Context, plan WriteSetPlan) (WriteSet, error) } +// WriteSetAdopter 让"计划已经存在"的执行也能带上复用语义(F07)。 +// +// 触发场景:首次 planner 证明某些 resources 可复用 → 写集合落盘 → 进程中断 → 接管时用 +// **新进程/新 target** 继续。写集合已存在,planner 因此被跳过;如果执行目标不消费这份 +// 计划,它会按"没有复用"重新 upsert 资源行并重传对象——账本说"共享对象没动",实际却 +// 重写了共享对象。必须显式接管计划,而不是重新按半写现状计划一遍。 +type WriteSetAdopter interface { + AdoptWriteSet(set WriteSet) error +} + // WriteSetObserver 让调用方在写第一笔之前把写集合摘要持久化到治理库。 // 失败同样意味着"一笔都不许写":没有可追溯的写集合,恢复就只能靠推断。 type WriteSetObserver interface { diff --git a/internal/librarystack/stack.go b/internal/librarystack/stack.go index 9101b90..6449955 100644 --- a/internal/librarystack/stack.go +++ b/internal/librarystack/stack.go @@ -139,6 +139,17 @@ func OpenProductionStack(ctx context.Context, config appconfig.Config, scope, sn contract.EmbeddingModel = config.LibraryEmbeddingModel contract.EmbeddingDimension = config.LibraryEmbeddingDimension } + // 目标实时契约:来源快照的声明只说明"数据来自哪",不能当目标合同(F17)。 + // 读不到就不许发布——猜一个目标模型正是"知识行声明 B、向量却是 A"的来源。 + live, liveErr := target.LiveTargetContract(ctx) + if liveErr != nil { + target.Close() + return nil, nil, fmt.Errorf("读不到目标实时模型契约,拒绝发布:%w", liveErr) + } + if verifyErr := v071.VerifySourceAgainstTarget(contract, live); verifyErr != nil { + target.Close() + return nil, nil, verifyErr + } // 生产索引存在时必须与快照契约一致;不存在或为空基线时跳过比对 // (空索引的映射没有可比性,非空之后仍然严格比对)。 if mapping, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil { @@ -157,7 +168,7 @@ func OpenProductionStack(ctx context.Context, config appconfig.Config, scope, sn } } } - backupContract := contract + backupContract := v071.ContractFromLive(contract, live) if knownMapping != "" { backupContract.IndexMappingSHA256 = knownMapping } diff --git a/internal/platform/postgres/library_jobs.go b/internal/platform/postgres/library_jobs.go index 3c396ad..81afa2b 100644 --- a/internal/platform/postgres/library_jobs.go +++ b/internal/platform/postgres/library_jobs.go @@ -28,12 +28,12 @@ func (s *Store) CreateLibraryJob(ctx context.Context, job governance.LibraryJob, var id string err = s.db.QueryRow(ctx, ` INSERT INTO library_jobs (id, scope, kind, state, stage, idempotency_key, request_digest, request, - queue_job_id, snapshot_id, target_snapshot_id, actor, error, recovery, created_at, updated_at) -VALUES ($1,$2,$3,'pending','',$4,$5,$6,$7,$8,$9,$10,'','',$11,$11) + queue_job_id, snapshot_id, target_snapshot_id, actor, error, recovery, operation_id, created_at, updated_at) +VALUES ($1,$2,$3,'pending','',$4,$5,$6,$7,$8,$9,$10,'','',$11,$12,$12) ON CONFLICT (idempotency_key) DO NOTHING RETURNING id`, job.ID, job.Scope, string(job.Kind), job.IdempotencyKey, request, rawRequest, - job.QueueJobID, job.SnapshotID, job.TargetSnapshotID, job.Actor, now.UTC()).Scan(&id) + job.QueueJobID, job.SnapshotID, job.TargetSnapshotID, job.Actor, job.OperationID, now.UTC()).Scan(&id) if err != nil { var pgErr *pgconn.PgError if errors.As(err, &pgErr) && pgErr.Code == "23505" { @@ -84,10 +84,10 @@ func (s *Store) LibraryJob(ctx context.Context, id string) (governance.LibraryJo var finished *time.Time if err := s.db.QueryRow(ctx, ` SELECT id, scope, kind, state, stage, idempotency_key, request_digest, queue_job_id, snapshot_id, - target_snapshot_id, actor, error, recovery, created_at, updated_at, finished_at + target_snapshot_id, actor, error, recovery, operation_id, attempt_epoch, created_at, updated_at, finished_at FROM library_jobs WHERE id=$1`, id).Scan(&job.ID, &job.Scope, &kind, &state, &job.Stage, &job.IdempotencyKey, &job.RequestDigest, &job.QueueJobID, &job.SnapshotID, &job.TargetSnapshotID, &job.Actor, &job.Error, - &job.Recovery, &job.CreatedAt, &job.UpdatedAt, &finished); err != nil { + &job.Recovery, &job.OperationID, &job.AttemptEpoch, &job.CreatedAt, &job.UpdatedAt, &finished); err != nil { if errors.Is(err, pgx.ErrNoRows) { return governance.LibraryJob{}, err } @@ -261,8 +261,15 @@ WHERE id=$1`, jobID, string(state), stage, message, recovery, snapshotID, target return nil } -// RecordLibraryVersion 写入一个版本记录。生产生效版本必须先把旧的 supersede 掉, -// 否则唯一索引会拒绝——这正是"不许两个生效版本并存"的保证。 +// RecordLibraryVersion 写入一条**不可变版本事件**,并在需要时推进 current 指针。 +// +// 与旧实现的区别(F02/F09 的根因): +// - 事件 ID 含 operation:不同 operation 引用同一快照是两条事件,`source_job_id`/ +// `evidence`/`created_at` 谁也不覆盖谁(旧实现是 `ON CONFLICT DO UPDATE` 整份替换)。 +// - 同一 operation 重放是幂等:`ON CONFLICT (id) DO NOTHING`,已存行原样返回。 +// - "谁当前生效"由 `library_current_versions` 表达,用单调次序做条件更新; +// 更早登记的事实不能把指针拉回历史,旧 epoch 在写边界就被守卫拒绝(见 guard)。 +// - 生效/恢复事件才动指针;预备份/封存只追加事件。 func (s *Store) RecordLibraryVersion(ctx context.Context, version governance.LibraryVersion) error { evidence, err := json.Marshal(version.Evidence) if err != nil { @@ -273,35 +280,87 @@ func (s *Store) RecordLibraryVersion(ctx context.Context, version governance.Lib return fmt.Errorf("begin library version: %w", err) } defer func() { _ = tx.Rollback(ctx) }() - if version.Kind == governance.LibraryVersionProductionEffective { - if _, err := tx.Exec(ctx, ` -UPDATE library_versions SET superseded_at=$2 -WHERE scope=$1 AND kind='production_effective' AND superseded_at IS NULL`, version.Scope, version.CreatedAt.UTC()); err != nil { - return fmt.Errorf("supersede library version: %w", err) + + // 执行代次守卫:写事件的人必须还是当前代次(失租旧执行器不能登记"我成功了")。 + if version.SourceJobID != "" && version.AttemptEpoch > 0 { + if err := guardLibraryJobEpoch(ctx, tx, version.SourceJobID, version.AttemptEpoch, version.AttemptOwner); err != nil { + return err } } - if _, err := tx.Exec(ctx, ` -INSERT INTO library_versions (id, scope, kind, snapshot_id, version_label, source_job_id, evidence, created_by, created_at) -VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) -ON CONFLICT (id) DO UPDATE SET - -- 恢复/重放同一个快照时必须把它重新"激活":否则它还是被 supersede 的旧行, - -- 页面会显示"生产当前版本未知",而实际生产已经恢复到这个快照。 - superseded_at = NULL, - source_job_id = EXCLUDED.source_job_id, - evidence = EXCLUDED.evidence, - -- 预备份版本行的 created_at 是"这份基线什么时候拍的",只能早、不能晚: - -- 崩溃接管会复用同一份预备份,重写登记时间会把"复用"洗成"重做了备份", - -- 事后无法证明基线没被半写状态覆盖过。 - created_at = CASE WHEN library_versions.kind='production_prebackup' - THEN LEAST(library_versions.created_at, EXCLUDED.created_at) - ELSE EXCLUDED.created_at END`, + + tag, err := tx.Exec(ctx, ` +INSERT INTO library_versions (id, scope, kind, snapshot_id, version_label, source_job_id, + operation_id, attempt_epoch, evidence, created_by, created_at) +VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11) +ON CONFLICT (id) DO NOTHING`, version.ID, version.Scope, string(version.Kind), version.SnapshotID, version.VersionLabel, - version.SourceJobID, evidence, version.CreatedBy, version.CreatedAt.UTC()); err != nil { - return fmt.Errorf("record library version: %w", err) + version.SourceJobID, version.OperationID, version.AttemptEpoch, evidence, version.CreatedBy, + version.CreatedAt.UTC()) + if err != nil { + return fmt.Errorf("record library version event: %w", err) + } + inserted := tag.RowsAffected() == 1 + + if version.Kind == governance.LibraryVersionProductionEffective || version.Kind == governance.LibraryVersionRecoveryVerified { + if err := advanceLibraryCurrent(ctx, tx, version, inserted); err != nil { + return err + } } return tx.Commit(ctx) } +// advanceLibraryCurrent 用单调次序条件推进 current 指针。 +// +// 次序键 = (事件登记时间|operation|epoch)。它只回答"哪一条事实更晚被登记", +// 不回答"哪个意图更重要";旧代次能不能写由 epoch 守卫回答,两件事分开才不会互相冒充。 +func advanceLibraryCurrent(ctx context.Context, tx pgx.Tx, version governance.LibraryVersion, inserted bool) error { + orderKey := governance.CurrentOrderKey(version.CreatedAt, version.OperationID, version.AttemptEpoch) + if !inserted { + // 重放:事件已经在台账里。确认指针没有被别的更晚事实覆盖,否则只确认自己的历史。 + var currentID string + err := tx.QueryRow(ctx, `SELECT version_id FROM library_current_versions WHERE scope=$1 FOR UPDATE`, + version.Scope).Scan(¤tID) + if errors.Is(err, pgx.ErrNoRows) { + return nil + } + if err != nil { + return fmt.Errorf("load library current pointer: %w", err) + } + return nil + } + tag, err := tx.Exec(ctx, ` +INSERT INTO library_current_versions (scope, version_id, snapshot_id, operation_id, attempt_epoch, order_key, updated_at) +VALUES ($1,$2,$3,$4,$5,$6,$7) +ON CONFLICT (scope) DO UPDATE SET + version_id=EXCLUDED.version_id, snapshot_id=EXCLUDED.snapshot_id, + operation_id=EXCLUDED.operation_id, attempt_epoch=EXCLUDED.attempt_epoch, + order_key=EXCLUDED.order_key, updated_at=EXCLUDED.updated_at +WHERE EXCLUDED.order_key > library_current_versions.order_key`, + version.Scope, version.ID, version.SnapshotID, version.OperationID, version.AttemptEpoch, orderKey, + version.CreatedAt.UTC()) + if err != nil { + return fmt.Errorf("advance library current pointer: %w", err) + } + var currentID string + if err := tx.QueryRow(ctx, `SELECT version_id FROM library_current_versions WHERE scope=$1`, + version.Scope).Scan(¤tID); err != nil { + return fmt.Errorf("read back library current pointer: %w", err) + } + if tag.RowsAffected() == 0 && currentID != version.ID { + return fmt.Errorf("%w(scope=%s:当前 %s,本条 %s)", governance.ErrLibraryCurrentStale, + version.Scope, currentID, version.ID) + } + // superseded_at 只是展示元数据("这条已被顶替"),不改任何证据字段。 + // 它跟着指针走:指针指向谁,谁就是未顶替的那一行。 + if _, err := tx.Exec(ctx, ` +UPDATE library_versions SET superseded_at = CASE WHEN id=$2 THEN NULL ELSE COALESCE(superseded_at, $3) END +WHERE scope=$1 AND kind IN ('production_effective','recovery_verified')`, version.Scope, currentID, + version.CreatedAt.UTC()); err != nil { + return fmt.Errorf("refresh library version supersede metadata: %w", err) + } + return nil +} + // InterruptedLibraryJobs 找出"业务作业 running、但队列里已没有活跃作业"的作业: // worker 被杀/重启后它们需要被接管,而不是被当成失败丢掉已完成的预备份。 func (s *Store) InterruptedLibraryJobs(ctx context.Context) ([]governance.LibraryJob, error) { @@ -347,20 +406,33 @@ ORDER BY lj.created_at`) // ResetQueueJobForResume 把中断/失败过的队列作业重新排队:attempt 必须归零, // 否则 attempt 1 { - return governance.LibraryVersion{}, false, fmt.Errorf("%d 条未顶替的生效版本并存(scope=%s):拒绝挑选", - len(items), scope) + return governance.LibraryVersion{}, false, fmt.Errorf("scope=%s 的当前指针指向 %d 行事件:台账被写坏,拒绝挑选", + scope, len(items)) } if len(items) == 1 { return items[0], true, nil @@ -526,7 +601,7 @@ func (s *Store) LibraryVersionOfSnapshot(ctx context.Context, scope string, return governance.LibraryVersion{}, false, errors.New("library version snapshot requires scope, kind and snapshot") } rows, err := s.db.Query(ctx, ` -SELECT id, scope, kind, snapshot_id, version_label, source_job_id, evidence, created_by, created_at, superseded_at +SELECT id, scope, kind, snapshot_id, version_label, source_job_id, operation_id, attempt_epoch, evidence, created_by, created_at, superseded_at FROM library_versions WHERE scope=$1 AND kind=$2 AND snapshot_id=$3 ORDER BY created_at DESC, id DESC LIMIT 2`, scope, string(kind), snapshotID) @@ -537,11 +612,10 @@ ORDER BY created_at DESC, id DESC LIMIT 2`, scope, string(kind), snapshotID) if err != nil { return governance.LibraryVersion{}, false, err } - if len(found) > 1 { - return governance.LibraryVersion{}, false, fmt.Errorf("同一快照存在 %d 条 %s 版本记录(%s):拒绝挑选", - len(found), kind, snapshotID) - } - if len(found) == 1 { + // 同一快照被多次意图引用时会有多条事件(每次 operation 一条),这是**正常历史**, + // 不是台账损坏:按登记时间取最新一条回答"这个快照最近一次登记成什么"。 + // 谁当前生效不在这里判断,那是 current 指针的事实。 + if len(found) >= 1 { return found[0], true, nil } return governance.LibraryVersion{}, false, nil diff --git a/internal/platform/postgres/library_operations.go b/internal/platform/postgres/library_operations.go new file mode 100644 index 0000000..9f5c096 --- /dev/null +++ b/internal/platform/postgres/library_operations.go @@ -0,0 +1,320 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + + "knowledge-governance/internal/governance" +) + +// 发布执行身份:operation / attempt(epoch)的持久化。 +// +// 一条铁律写在这里,别处不再重复实现:**谁写业务状态,就必须证明自己是当前代次**。 +// epoch 守卫读 library_jobs.attempt_epoch/lease_owner,与 BeginLibraryAttempt 的激活 +// 是同一把锁;失租代次的所有治理写入都会被拒绝,而不是靠调用方自觉。 + +// ErrLibraryAttemptHeld:同一作业还有未过期的租约。第二个执行者必须放弃, +// 不能"再认领一次"——那正是 F13 的双 claim。 +var ErrLibraryAttemptHeld = errors.New("library job lease is held by another attempt") + +// EnsureLibraryOperation 幂等创建/读取一次意图。 +// 同一个 (scope,kind,operation_key) + 同一个 request_digest ⇒ 同一次意图(重放); +// 摘要不一致 ⇒ 冲突,绝不把新意图塞进旧 operation。 +func (s *Store) EnsureLibraryOperation(ctx context.Context, op governance.LibraryOperation) (governance.LibraryOperation, bool, error) { + if err := governance.ValidateLibraryOperation(op); err != nil { + return governance.LibraryOperation{}, false, err + } + var id string + err := s.db.QueryRow(ctx, ` +INSERT INTO library_operations (id, scope, kind, target_snapshot_id, request_digest, operation_key, actor, created_at) +VALUES ($1,$2,$3,$4,$5,$6,$7,$8) +ON CONFLICT (scope, kind, operation_key) DO NOTHING +RETURNING id`, op.ID, op.Scope, string(op.Kind), op.TargetSnapshotID, op.RequestDigest, + op.OperationKey, op.Actor, op.CreatedAt.UTC()).Scan(&id) + if err == nil { + return op, true, nil + } + if !errors.Is(err, pgx.ErrNoRows) { + return governance.LibraryOperation{}, false, fmt.Errorf("create library operation: %w", err) + } + existing, loadErr := s.libraryOperationByKey(ctx, op.Scope, op.Kind, op.OperationKey) + if loadErr != nil { + return governance.LibraryOperation{}, false, loadErr + } + if existing.RequestDigest != op.RequestDigest { + return existing, false, governance.ErrLibraryOperationConflict + } + return existing, false, nil +} + +func (s *Store) libraryOperationByKey(ctx context.Context, scope string, kind governance.LibraryJobKind, key string) (governance.LibraryOperation, error) { + var id string + if err := s.db.QueryRow(ctx, ` +SELECT id FROM library_operations WHERE scope=$1 AND kind=$2 AND operation_key=$3`, + scope, string(kind), key).Scan(&id); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return governance.LibraryOperation{}, errors.New("library operation not found") + } + return governance.LibraryOperation{}, fmt.Errorf("load library operation: %w", err) + } + return s.LibraryOperation(ctx, id) +} + +func (s *Store) LibraryOperation(ctx context.Context, id string) (governance.LibraryOperation, error) { + var op governance.LibraryOperation + var kind string + err := s.db.QueryRow(ctx, ` +SELECT id, scope, kind, target_snapshot_id, request_digest, operation_key, actor, created_at +FROM library_operations WHERE id=$1`, id).Scan(&op.ID, &op.Scope, &kind, &op.TargetSnapshotID, + &op.RequestDigest, &op.OperationKey, &op.Actor, &op.CreatedAt) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return governance.LibraryOperation{}, err + } + return governance.LibraryOperation{}, fmt.Errorf("load library operation: %w", err) + } + op.Kind = governance.LibraryJobKind(kind) + return op, nil +} + +// OperationAnchors 读不可变首次锚点。 +func (s *Store) OperationAnchors(ctx context.Context, operationID string) (governance.LibraryAnchors, bool, error) { + var a governance.LibraryAnchors + err := s.db.QueryRow(ctx, ` +SELECT first_backup_snapshot_id, first_backup_bytes, first_backup_at, writeset_digest, writeset_rows, + target_fingerprint, anchors_at +FROM library_operations WHERE id=$1`, operationID).Scan(&a.FirstBackupSnapshotID, &a.FirstBackupBytes, + &a.FirstBackupAt, &a.WriteSetDigest, &a.WriteSetRows, &a.TargetFingerprint, &a.RecordedAt) + if errors.Is(err, pgx.ErrNoRows) { + return governance.LibraryAnchors{}, false, nil + } + if err != nil { + return governance.LibraryAnchors{}, false, fmt.Errorf("load library operation anchors: %w", err) + } + return a, a.RecordedAt != nil, nil +} + +// RecordOperationAnchors 只**填空**,不改已存在的锚点。 +// 接管/重放会用同一份锚点重复调用;如果新值想覆盖旧值,那说明"首次"被重拍了, +// 直接报错让调用方停下来(把半写状态当新基线等于毁掉唯一可回滚的那份)。 +func (s *Store) RecordOperationAnchors(ctx context.Context, operationID string, anchors governance.LibraryAnchors, now time.Time) error { + if operationID == "" { + return errors.New("record operation anchors requires an operation") + } + existing, found, err := s.OperationAnchors(ctx, operationID) + if err != nil { + return err + } + if found { + if existing.FirstBackupSnapshotID != "" && anchors.FirstBackupSnapshotID != "" && + existing.FirstBackupSnapshotID != anchors.FirstBackupSnapshotID { + return fmt.Errorf("operation %s 的首次预备份是 %s,不能用 %s 覆盖", + operationID, existing.FirstBackupSnapshotID, anchors.FirstBackupSnapshotID) + } + if existing.WriteSetDigest != "" && anchors.WriteSetDigest != "" && + existing.WriteSetDigest != anchors.WriteSetDigest { + return fmt.Errorf("operation %s 的首次写集合是 %s,不能用 %s 覆盖", + operationID, existing.WriteSetDigest, anchors.WriteSetDigest) + } + } + _, err = s.db.Exec(ctx, ` +UPDATE library_operations SET + first_backup_snapshot_id = COALESCE(first_backup_snapshot_id, $2), + first_backup_bytes = COALESCE(first_backup_bytes, $3), + first_backup_at = COALESCE(first_backup_at, $4), + writeset_digest = COALESCE(writeset_digest, $5), + writeset_rows = COALESCE(writeset_rows, $6), + target_fingerprint = COALESCE(target_fingerprint, $7), + anchors_at = COALESCE(anchors_at, $8) +WHERE id=$1`, operationID, nullIfEmpty(anchors.FirstBackupSnapshotID), anchors.FirstBackupBytes, + anchors.FirstBackupAt, nullIfEmpty(anchors.WriteSetDigest), anchors.WriteSetRows, + nullIfEmpty(anchors.TargetFingerprint), now.UTC()) + if err != nil { + return fmt.Errorf("record library operation anchors: %w", err) + } + return nil +} + +// BeginLibraryAttempt 取得这个作业的执行权并激活一个新代次。 +// +// 三件事必须在同一个事务里:判租约(谁能执行)→ 递增 epoch(谁是当前代次)→ +// 写作业行的租约(守卫读它)。分成三步就会出现"两个 worker 都认为自己是当前代次"。 +func (s *Store) BeginLibraryAttempt(ctx context.Context, jobID, owner string, lease time.Duration, now time.Time) (governance.LibraryAttempt, error) { + if jobID == "" || owner == "" { + return governance.LibraryAttempt{}, errors.New("begin library attempt requires job and owner") + } + if lease <= 0 { + lease = 2 * time.Minute + } + tx, err := s.db.Begin(ctx) + if err != nil { + return governance.LibraryAttempt{}, fmt.Errorf("begin library attempt: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + + var operationID, leaseOwner, queueJobID string + var epoch int + var leaseUntil *time.Time + err = tx.QueryRow(ctx, ` +SELECT operation_id, attempt_epoch, lease_owner, lease_until, queue_job_id +FROM library_jobs WHERE id=$1 FOR UPDATE`, jobID).Scan(&operationID, &epoch, &leaseOwner, &leaseUntil, &queueJobID) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return governance.LibraryAttempt{}, fmt.Errorf("library job %s not found", jobID) + } + return governance.LibraryAttempt{}, fmt.Errorf("load library job for attempt: %w", err) + } + // 同一 owner 重入是幂等(重试/队列重投);别人持有未过期租约就必须让路。 + if leaseOwner != "" && leaseOwner != owner && leaseUntil != nil && leaseUntil.After(now.UTC()) { + return governance.LibraryAttempt{}, ErrLibraryAttemptHeld + } + nextEpoch := epoch + 1 + if nextEpoch < 1 { + nextEpoch = 1 + } + // 代次必须按 operation 单调递增,而不是按作业行:同一个意图的重试/接管会**新开作业行**, + // 若按作业行各自从 1 数起,旧代次(1)与新代次(1)就分不出先后,fencing 失效。 + if operationID != "" { + var maxEpoch *int + if err := tx.QueryRow(ctx, `SELECT MAX(epoch) FROM library_attempts WHERE operation_id=$1`, + operationID).Scan(&maxEpoch); err != nil { + return governance.LibraryAttempt{}, fmt.Errorf("load library attempt epoch: %w", err) + } + next := 1 + if maxEpoch != nil && *maxEpoch >= next { + next = *maxEpoch + 1 + } + if nextEpoch < next { + nextEpoch = next + } + } + attempt := governance.LibraryAttempt{ + OperationID: operationID, Epoch: nextEpoch, JobID: jobID, QueueJobID: queueJobID, + Owner: owner, State: governance.LibraryAttemptRunning, StartedAt: now.UTC(), + } + until := now.Add(lease).UTC() + attempt.LeaseUntil = &until + if operationID != "" { + // 一个代次只对应一个作业行:重复 Begin(同 job)不再新增谱系。 + if _, err := tx.Exec(ctx, ` +INSERT INTO library_attempts (operation_id, epoch, job_id, queue_job_id, owner, state, started_at, heartbeat_at, lease_until) +VALUES ($1,$2,$3,$4,$5,'running',$6,$6,$7) +ON CONFLICT (job_id) DO UPDATE SET + owner=EXCLUDED.owner, state='running', lease_until=EXCLUDED.lease_until, heartbeat_at=EXCLUDED.heartbeat_at`, + operationID, nextEpoch, jobID, queueJobID, owner, now.UTC(), until); err != nil { + return governance.LibraryAttempt{}, fmt.Errorf("record library attempt: %w", err) + } + } + if _, err := tx.Exec(ctx, ` +UPDATE library_jobs SET attempt_epoch=$2, lease_owner=$3, lease_until=$4, updated_at=$5 WHERE id=$1`, + jobID, nextEpoch, owner, until, now.UTC()); err != nil { + return governance.LibraryAttempt{}, fmt.Errorf("activate library attempt: %w", err) + } + if err := tx.Commit(ctx); err != nil { + return governance.LibraryAttempt{}, fmt.Errorf("commit library attempt: %w", err) + } + return attempt, nil +} + +// FinishLibraryAttempt 落本次代次的终态。旧代次不能结束别人的代次: +// 只有 (operation, epoch) 与 owner 都匹配才写。 +func (s *Store) FinishLibraryAttempt(ctx context.Context, attempt governance.LibraryAttempt, state governance.LibraryAttemptState, outcome string, now time.Time) error { + if attempt.OperationID == "" { + return nil // 旧作业没有 operation:没有代次可结 + } + tag, err := s.db.Exec(ctx, ` +UPDATE library_attempts SET state=$4, finished_at=$5, outcome=$6, lease_until=NULL +WHERE operation_id=$1 AND epoch=$2 AND owner=$3`, + attempt.OperationID, attempt.Epoch, attempt.Owner, string(state), now.UTC(), outcome) + if err != nil { + return fmt.Errorf("finish library attempt: %w", err) + } + if tag.RowsAffected() != 1 { + return governance.ErrLibraryEpochStale + } + return nil +} + +// LibraryAttempts 返回一次 operation 的执行谱系(按代次升序):接管/恢复排查读它。 +func (s *Store) LibraryAttempts(ctx context.Context, operationID string) ([]governance.LibraryAttempt, error) { + rows, err := s.db.Query(ctx, ` +SELECT operation_id, epoch, job_id, queue_job_id, owner, state, started_at, heartbeat_at, lease_until, finished_at, outcome +FROM library_attempts WHERE operation_id=$1 ORDER BY epoch`, operationID) + if err != nil { + return nil, fmt.Errorf("list library attempts: %w", err) + } + defer rows.Close() + items := make([]governance.LibraryAttempt, 0, 4) + for rows.Next() { + var item governance.LibraryAttempt + var state string + if err := rows.Scan(&item.OperationID, &item.Epoch, &item.JobID, &item.QueueJobID, &item.Owner, &state, + &item.StartedAt, &item.HeartbeatAt, &item.LeaseUntil, &item.FinishedAt, &item.Outcome); err != nil { + return nil, fmt.Errorf("scan library attempt: %w", err) + } + item.State = governance.LibraryAttemptState(state) + items = append(items, item) + } + return items, rows.Err() +} + +// LatestLibraryAttempt 取当前代次(无则 false)。 +func (s *Store) LatestLibraryAttempt(ctx context.Context, operationID string) (governance.LibraryAttempt, bool, error) { + items, err := s.LibraryAttempts(ctx, operationID) + if err != nil || len(items) == 0 { + return governance.LibraryAttempt{}, false, err + } + return items[len(items)-1], true, nil +} + +// RenewLibraryAttempt 续租:失租(0 行)即返回 ErrLibraryEpochStale,调用方必须停手。 +func (s *Store) RenewLibraryAttempt(ctx context.Context, attempt governance.LibraryAttempt, lease time.Duration, now time.Time) error { + until := now.Add(lease).UTC() + tag, err := s.db.Exec(ctx, ` +UPDATE library_attempts SET heartbeat_at=$4, lease_until=$5 +WHERE operation_id=$1 AND epoch=$2 AND owner=$3 AND state='running'`, + attempt.OperationID, attempt.Epoch, attempt.Owner, now.UTC(), until) + if err != nil { + return fmt.Errorf("renew library attempt: %w", err) + } + if tag.RowsAffected() != 1 { + return governance.ErrLibraryEpochStale + } + if _, err := s.db.Exec(ctx, ` +UPDATE library_jobs SET lease_until=$4 WHERE id=$1 AND attempt_epoch=$2 AND lease_owner=$3`, + attempt.JobID, attempt.Epoch, attempt.Owner, until); err != nil { + return fmt.Errorf("renew library job lease: %w", err) + } + return nil +} + +// guardLibraryJobEpoch 在事务内核"这次写入来自当前代次"。守卫失败一律返回 +// ErrLibraryEpochStale,调用方不许把它当成"重试一下就好"。 +func guardLibraryJobEpoch(ctx context.Context, tx pgx.Tx, jobID string, epoch int, owner string) error { + var currentEpoch int + var currentOwner string + err := tx.QueryRow(ctx, ` +SELECT attempt_epoch, lease_owner FROM library_jobs WHERE id=$1 FOR UPDATE`, jobID).Scan(¤tEpoch, ¤tOwner) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return fmt.Errorf("library job %s not found", jobID) + } + return fmt.Errorf("load library job for epoch guard: %w", err) + } + if currentEpoch != epoch || (owner != "" && currentOwner != owner) { + return fmt.Errorf("%w(作业 %s:本代次 %d/%s,当前 %d/%s)", governance.ErrLibraryEpochStale, + jobID, epoch, owner, currentEpoch, currentOwner) + } + return nil +} + +func nullIfEmpty(value string) any { + if value == "" { + return nil + } + return value +} diff --git a/internal/platform/postgres/library_protection.go b/internal/platform/postgres/library_protection.go new file mode 100644 index 0000000..c66f152 --- /dev/null +++ b/internal/platform/postgres/library_protection.go @@ -0,0 +1,142 @@ +package postgres + +import ( + "context" + "fmt" +) + +// LibraryProtection 回答"这些快照现在为什么不能删"。 +// +// 关键点(F10):保护依据必须来自**全量依赖**,不能来自"最近 100 个版本 / 20 个作业"的 +// 展示窗口。历史上一份需要恢复的旧预备份、一个 failed 但仍持有维护锁的作业,都完全 +// 可能掉出任何窗口——按窗口判断就会把它显示成"可删除候选"。 +// +// 任何一条查询失败都返回错误:调用方必须显示"保护未知"并拒绝生成可删候选, +// 不允许因为"读不到"就当成"没被引用"。 +func (s *Store) LibraryProtection(ctx context.Context) (map[string][]string, error) { + protected := map[string][]string{} + add := func(snapshotID, reason string) { + if snapshotID == "" { + return + } + for _, existing := range protected[snapshotID] { + if existing == reason { + return + } + } + protected[snapshotID] = append(protected[snapshotID], reason) + } + + // 1) 当前生效事实(指针):每个 scope 一条,与历史长度无关。 + rows, err := s.db.Query(ctx, `SELECT snapshot_id FROM library_current_versions`) + if err != nil { + return nil, fmt.Errorf("读取当前生效版本失败:%w", err) + } + for rows.Next() { + var snapshot string + if err := rows.Scan(&snapshot); err != nil { + rows.Close() + return nil, fmt.Errorf("扫描当前生效版本失败:%w", err) + } + add(snapshot, "生产生效版本") + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("读取当前生效版本失败:%w", err) + } + + // 2) 全部生产预备份:任何一份都是某个窗口唯一的可回滚基线,没有窗口限制。 + rows, err = s.db.Query(ctx, `SELECT DISTINCT snapshot_id FROM library_versions WHERE kind='production_prebackup'`) + if err != nil { + return nil, fmt.Errorf("读取生产预备份失败:%w", err) + } + for rows.Next() { + var snapshot string + if err := rows.Scan(&snapshot); err != nil { + rows.Close() + return nil, fmt.Errorf("扫描生产预备份失败:%w", err) + } + add(snapshot, "生产预备份") + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("读取生产预备份失败:%w", err) + } + + // 3) 每个 scope 最近一次测试封存(最新可发布版本),精确取而不是窗口里挑。 + rows, err = s.db.Query(ctx, ` +SELECT DISTINCT ON (scope) scope, snapshot_id FROM library_versions +WHERE kind='test_sealed' ORDER BY scope, created_at DESC`) + if err != nil { + return nil, fmt.Errorf("读取最近测试封存失败:%w", err) + } + for rows.Next() { + var scope, snapshot string + if err := rows.Scan(&scope, &snapshot); err != nil { + rows.Close() + return nil, fmt.Errorf("扫描最近测试封存失败:%w", err) + } + add(snapshot, "最近测试封存("+scope+")") + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("读取最近测试封存失败:%w", err) + } + + // 4) 未释放的维护窗口:窗口持有者的预备份与目标快照都要留住—— + // 否则"删除候选"会把正在等待恢复的那一份清掉。 + var owners []string + rows, err = s.db.Query(ctx, ` +SELECT owner_job_id FROM library_maintenance WHERE released_at IS NULL +UNION +SELECT owner_job_id FROM library_instance_maintenance WHERE released_at IS NULL`) + if err != nil { + return nil, fmt.Errorf("读取未释放维护窗口失败:%w", err) + } + for rows.Next() { + var owner string + if err := rows.Scan(&owner); err != nil { + rows.Close() + return nil, fmt.Errorf("扫描未释放维护窗口失败:%w", err) + } + owners = append(owners, owner) + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("读取未释放维护窗口失败:%w", err) + } + + // 5) 在飞作业与"失败但仍持有窗口"的作业。 + rows, err = s.db.Query(ctx, ` +SELECT coalesce(snapshot_id,''), coalesce(target_snapshot_id,'') +FROM library_jobs WHERE state IN ('pending','running')`) + if err != nil { + return nil, fmt.Errorf("读取在飞作业失败:%w", err) + } + for rows.Next() { + var snapshot, target string + if err := rows.Scan(&snapshot, &target); err != nil { + rows.Close() + return nil, fmt.Errorf("扫描在飞作业失败:%w", err) + } + add(snapshot, "在飞作业引用") + add(target, "在飞作业引用") + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("读取在飞作业失败:%w", err) + } + + for _, owner := range owners { + var snapshot, target string + err := s.db.QueryRow(ctx, ` +SELECT coalesce(snapshot_id,''), coalesce(target_snapshot_id,'') FROM library_jobs WHERE id=$1`, owner). + Scan(&snapshot, &target) + if err != nil { + return nil, fmt.Errorf("读取维护窗口持有作业 %s 失败:%w", owner, err) + } + add(snapshot, "未释放维护窗口的预备份") + add(target, "未释放维护窗口的目标快照") + } + return protected, nil +} \ No newline at end of file diff --git a/internal/platform/postgres/library_write_fence.go b/internal/platform/postgres/library_write_fence.go new file mode 100644 index 0000000..4088900 --- /dev/null +++ b/internal/platform/postgres/library_write_fence.go @@ -0,0 +1,212 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + + "knowledge-governance/internal/governance" +) + +// 写入栅栏:所有"业务状态 / 维护锁"的写都必须证明调用方仍是当前执行代次。 +// +// 为什么单独放一份:F13 的根因不是某一条 SQL 写错了,而是"同一个 job_id 被两个代次 +// 同时使用"这件事本身没有被表达出来。只要有一处更新不带代次条件,失租的旧执行就能 +// 用同一个 job_id 写业务终态、释放别人刚拿到的维护锁。所以这里给每个可变入口配一个 +// 带守卫的版本,worker 只准走这些版本;不带守卫的旧方法留给 API(它代表"当前意图", +// 没有执行代次)。 + +const epochFence = `EXISTS (SELECT 1 FROM library_jobs j WHERE j.id=$1 AND j.attempt_epoch=$%d AND j.lease_owner=$%d)` + +// AdvanceLibraryJobGuarded 只在调用方仍是该作业当前代次时写业务状态。 +func (s *Store) AdvanceLibraryJobGuarded(ctx context.Context, attempt governance.LibraryAttempt, + state governance.LibraryJobState, stage, message, recovery, snapshotID, targetSnapshotID string, now time.Time) error { + finished := any(nil) + if state == governance.LibraryJobSucceeded || state == governance.LibraryJobFailed || state == governance.LibraryJobCancelled { + finished = now.UTC() + } + tag, err := s.db.Exec(ctx, ` +UPDATE library_jobs SET + state=$2, + stage = CASE WHEN $3='' THEN stage ELSE $3 END, + error = CASE WHEN $4='' THEN error ELSE $4 END, + recovery = CASE WHEN $5='' THEN recovery ELSE $5 END, + snapshot_id = CASE WHEN $6='' THEN snapshot_id ELSE $6 END, + target_snapshot_id = CASE WHEN $7='' THEN target_snapshot_id ELSE $7 END, + updated_at=$8, + finished_at=COALESCE($9, finished_at) +WHERE id=$1 AND attempt_epoch=$10 AND lease_owner=$11`, + attempt.JobID, string(state), stage, message, recovery, snapshotID, targetSnapshotID, now.UTC(), finished, + attempt.Epoch, attempt.Owner) + if err != nil { + return fmt.Errorf("advance library job (guarded): %w", err) + } + if tag.RowsAffected() != 1 { + return s.fencedWriteError(ctx, attempt, "advance library job") + } + return nil +} + +// UpdateLibraryStageGuarded 是阶段进度的守卫版:失租代次不能再写"我这一阶段成功了"。 +func (s *Store) UpdateLibraryStageGuarded(ctx context.Context, attempt governance.LibraryAttempt, + stage, state string, total, completed, bytesTotal, bytesDone *int64, detail string, now time.Time) error { + tag, err := s.db.Exec(ctx, ` +UPDATE library_job_stages SET + state = CASE WHEN $3='' THEN state ELSE $3 END, + total = COALESCE($4, total), + completed = COALESCE($5, completed), + bytes_total = COALESCE($6, bytes_total), + bytes_done = COALESCE($7, bytes_done), + detail = CASE WHEN $8='' THEN detail ELSE $8 END, + updated_at = $9 +WHERE job_id=$1 AND stage=$2 + AND EXISTS (SELECT 1 FROM library_jobs j WHERE j.id=$1 AND j.attempt_epoch=$10 AND j.lease_owner=$11)`, + attempt.JobID, stage, state, total, completed, bytesTotal, bytesDone, detail, now.UTC(), + attempt.Epoch, attempt.Owner) + if err != nil { + return fmt.Errorf("update library stage (guarded): %w", err) + } + if tag.RowsAffected() != 1 { + return s.fencedWriteError(ctx, attempt, "update library stage") + } + return nil +} + +// ReleaseLibraryMaintenanceGuarded 释放 scope 维护锁,但只认当前代次。 +func (s *Store) ReleaseLibraryMaintenanceGuarded(ctx context.Context, scope, ownerJobID string, + attempt governance.LibraryAttempt, now time.Time) error { + tag, err := s.db.Exec(ctx, ` +UPDATE library_maintenance SET state='released', released_at=$3, updated_at=$3 +WHERE scope=$1 AND owner_job_id=$2 AND released_at IS NULL + AND `+fmt.Sprintf(epochFence, 4, 5), scope, ownerJobID, now.UTC(), attempt.Epoch, attempt.Owner) + if err != nil { + return fmt.Errorf("release library maintenance (guarded): %w", err) + } + if tag.RowsAffected() == 0 { + if s.releasedByThisJob(ctx, "library_maintenance", "scope", scope, ownerJobID) { + return ErrMaintenanceReleased + } + return s.fencedWriteError(ctx, attempt, "release library maintenance") + } + return nil +} + +// ReleaseInstanceMaintenanceGuarded 释放实例维护锁,但只认当前代次。 +func (s *Store) ReleaseInstanceMaintenanceGuarded(ctx context.Context, instance, ownerJobID string, + attempt governance.LibraryAttempt, now time.Time) error { + tag, err := s.db.Exec(ctx, ` +UPDATE library_instance_maintenance SET state='released', released_at=$3, updated_at=$3 +WHERE instance=$1 AND owner_job_id=$2 AND released_at IS NULL + AND `+fmt.Sprintf(epochFence, 4, 5), instance, ownerJobID, now.UTC(), attempt.Epoch, attempt.Owner) + if err != nil { + return fmt.Errorf("release instance maintenance (guarded): %w", err) + } + if tag.RowsAffected() == 0 { + if s.releasedByThisJob(ctx, "library_instance_maintenance", "instance", instance, ownerJobID) { + return ErrMaintenanceReleased + } + return s.fencedWriteError(ctx, attempt, "release instance maintenance") + } + return nil +} + +// UpdateLibraryMaintenanceStateGuarded / UpdateInstanceMaintenanceStateGuarded 同理: +// 失租代次不能把窗口改成 needs_restore / verified 去影响当前代次的判断。 +func (s *Store) UpdateLibraryMaintenanceStateGuarded(ctx context.Context, scope, ownerJobID, state, note string, + attempt governance.LibraryAttempt, now time.Time) error { + tag, err := s.db.Exec(ctx, ` +UPDATE library_maintenance SET state=$3, note=$4, updated_at=$5 +WHERE scope=$1 AND owner_job_id=$2 AND released_at IS NULL + AND `+fmt.Sprintf(epochFence, 6, 7), scope, ownerJobID, state, note, now.UTC(), attempt.Epoch, attempt.Owner) + if err != nil { + return fmt.Errorf("update library maintenance (guarded): %w", err) + } + if tag.RowsAffected() == 0 { + if s.releasedByThisJob(ctx, "library_maintenance", "scope", scope, ownerJobID) { + return ErrMaintenanceReleased + } + return s.fencedWriteError(ctx, attempt, "update library maintenance") + } + return nil +} + +func (s *Store) UpdateInstanceMaintenanceStateGuarded(ctx context.Context, instance, ownerJobID, state, note string, + attempt governance.LibraryAttempt, now time.Time) error { + tag, err := s.db.Exec(ctx, ` +UPDATE library_instance_maintenance SET state=$3, note=$4, updated_at=$5 +WHERE instance=$1 AND owner_job_id=$2 AND released_at IS NULL + AND `+fmt.Sprintf(epochFence, 6, 7), instance, ownerJobID, state, note, now.UTC(), attempt.Epoch, attempt.Owner) + if err != nil { + return fmt.Errorf("update instance maintenance (guarded): %w", err) + } + if tag.RowsAffected() == 0 { + if s.releasedByThisJob(ctx, "library_instance_maintenance", "instance", instance, ownerJobID) { + return ErrMaintenanceReleased + } + return s.fencedWriteError(ctx, attempt, "update instance maintenance") + } + return nil +} + +// fencedWriteError 区分"作业不存在"与"代次已过期":两者都要停下,但文案不能误导运维。 +func (s *Store) fencedWriteError(ctx context.Context, attempt governance.LibraryAttempt, action string) error { + var epoch int + var owner string + err := s.db.QueryRow(ctx, `SELECT attempt_epoch, lease_owner FROM library_jobs WHERE id=$1`, + attempt.JobID).Scan(&epoch, &owner) + if errors.Is(err, pgx.ErrNoRows) { + return fmt.Errorf("%s:作业 %s 不存在", action, attempt.JobID) + } + if err != nil { + return fmt.Errorf("%s:读不到作业代次:%w", action, err) + } + return fmt.Errorf("%w(%s:本代次 %d/%s,当前 %d/%s)", governance.ErrLibraryEpochStale, + action, attempt.Epoch, attempt.Owner, epoch, owner) +} + +func (s *Store) SetInstanceMaintenanceIngressGuarded(ctx context.Context, instance, ownerJobID, state, note string, + attempt governance.LibraryAttempt, now time.Time) error { + if state != "closed" && state != "open" { + return errors.New("invalid ingress state") + } + tag, err := s.db.Exec(ctx, ` +UPDATE library_instance_maintenance SET ingress_state=$3, note=$4, updated_at=$5 +WHERE instance=$1 AND owner_job_id=$2 AND released_at IS NULL + AND `+fmt.Sprintf(epochFence, 6, 7), instance, ownerJobID, state, note, now.UTC(), attempt.Epoch, attempt.Owner) + if err != nil { + return fmt.Errorf("update instance maintenance ingress (guarded): %w", err) + } + if tag.RowsAffected() == 0 { + if s.releasedByThisJob(ctx, "library_instance_maintenance", "instance", instance, ownerJobID) { + return ErrMaintenanceReleased + } + return s.fencedWriteError(ctx, attempt, "update instance maintenance ingress") + } + return nil +} + +// SetLibraryMaintenanceIngressGuarded 记录入口状态也需要代次守卫: +// "入口已关"是隔离证据,失租代次写它等于伪造证据。 +func (s *Store) SetLibraryMaintenanceIngressGuarded(ctx context.Context, scope, ownerJobID, state, note string, + attempt governance.LibraryAttempt, now time.Time) error { + if state != "closed" && state != "open" { + return errors.New("invalid ingress state") + } + tag, err := s.db.Exec(ctx, ` +UPDATE library_maintenance SET ingress_state=$3, note=$4, updated_at=$5 +WHERE scope=$1 AND owner_job_id=$2 AND released_at IS NULL + AND `+fmt.Sprintf(epochFence, 6, 7), scope, ownerJobID, state, note, now.UTC(), attempt.Epoch, attempt.Owner) + if err != nil { + return fmt.Errorf("update library maintenance ingress (guarded): %w", err) + } + if tag.RowsAffected() == 0 { + if s.releasedByThisJob(ctx, "library_maintenance", "scope", scope, ownerJobID) { + return ErrMaintenanceReleased + } + return s.fencedWriteError(ctx, attempt, "update library maintenance ingress") + } + return nil +} diff --git a/internal/platform/postgres/migrations/000076_library_operation_identity.down.sql b/internal/platform/postgres/migrations/000076_library_operation_identity.down.sql new file mode 100644 index 0000000..6fc3aec --- /dev/null +++ b/internal/platform/postgres/migrations/000076_library_operation_identity.down.sql @@ -0,0 +1,29 @@ +-- 回滚:只撤掉 identity/指针这一层。**不删任何版本事件、作业或证据**(历史不可回填)。 +-- 回滚后 library_versions 仍是完整台账;current 重新由 superseded_at 表达。 +DROP TABLE IF EXISTS library_current_versions; + +UPDATE library_versions SET superseded_at = NULL +WHERE id IN ( + SELECT id FROM ( + SELECT id, row_number() OVER (PARTITION BY scope ORDER BY created_at DESC, id DESC) AS rn + FROM library_versions + WHERE kind = 'production_effective' + ) ranked WHERE rn = 1 +); + +ALTER TABLE library_versions DROP CONSTRAINT IF EXISTS library_versions_kind_check; +DELETE FROM library_versions WHERE kind = 'recovery_verified'; +ALTER TABLE library_versions ADD CONSTRAINT library_versions_kind_check + CHECK (kind IN ('test_sealed','production_effective','production_prebackup')); +ALTER TABLE library_versions DROP COLUMN IF EXISTS attempt_epoch; +ALTER TABLE library_versions DROP COLUMN IF EXISTS operation_id; +CREATE UNIQUE INDEX IF NOT EXISTS library_versions_effective ON library_versions(scope) + WHERE kind = 'production_effective' AND superseded_at IS NULL; + +ALTER TABLE library_jobs DROP COLUMN IF EXISTS lease_until; +ALTER TABLE library_jobs DROP COLUMN IF EXISTS lease_owner; +ALTER TABLE library_jobs DROP COLUMN IF EXISTS attempt_epoch; +ALTER TABLE library_jobs DROP COLUMN IF EXISTS operation_id; + +DROP TABLE IF EXISTS library_attempts; +DROP TABLE IF EXISTS library_operations; \ No newline at end of file diff --git a/internal/platform/postgres/migrations/000076_library_operation_identity.up.sql b/internal/platform/postgres/migrations/000076_library_operation_identity.up.sql new file mode 100644 index 0000000..912650a --- /dev/null +++ b/internal/platform/postgres/migrations/000076_library_operation_identity.up.sql @@ -0,0 +1,108 @@ +-- 发布执行身份:operation(一次意图)/ attempt(执行代次)/ 不可变版本事件 / current 指针。 +-- +-- 为什么现在加、且只加在治理库: +-- · 旧实现把"用户的一次意图"等同于"某个快照的终身幂等键",于是 A→B→A 的第三次请求 +-- 拿到的是第一次的历史成功,什么也不做(F01);版本行主键只由 (kind,scope,快照) 决定, +-- 第二次登记直接覆盖 source_job_id/evidence/created_at(F02/F09)。 +-- · 接管没有代次:两个 worker 可以把同一 pending 作业都认领走,失租的旧执行还能继续 +-- 写业务状态、释放别人的维护锁(F13)。 +-- +-- 这里不修改 WeKnora 的 Schema/源码/镜像,也不声称数据库 CAS 能拦住已经发出去的 +-- S3/ES 请求:跨系统原子性不存在,物理执行域隔离另由部署准入负责。 + +-- ── operation:一次明确意图(不可变)──────────────────────────────────── +CREATE TABLE IF NOT EXISTS library_operations ( + id VARCHAR(64) PRIMARY KEY, + scope VARCHAR(16) NOT NULL CHECK (scope IN ('internal','external')), + kind VARCHAR(16) NOT NULL, + target_snapshot_id VARCHAR(64) NOT NULL DEFAULT '', + request_digest VARCHAR(64) NOT NULL, + -- 客户端可重放的意图键:同一个键 + 同一个摘要 ⇒ 同一次意图(重放幂等); + -- 换一个键 ⇒ 新意图,即使目标快照与历史成功完全相同。 + operation_key VARCHAR(255) NOT NULL, + actor VARCHAR(128) NOT NULL DEFAULT '', + created_at TIMESTAMPTZ NOT NULL, + -- 不可变首次锚点:由第一次成功拍下预备份/写集合的执行写入,之后只读。 + first_backup_snapshot_id VARCHAR(64), + first_backup_bytes BIGINT, + first_backup_at TIMESTAMPTZ, + writeset_digest VARCHAR(64), + writeset_rows BIGINT, + target_fingerprint VARCHAR(128), + anchors_at TIMESTAMPTZ +); +CREATE UNIQUE INDEX IF NOT EXISTS library_operations_key + ON library_operations(scope, kind, operation_key); + +-- ── attempt:执行代次(epoch),同一 operation 只允许一个活跃代次 ───────── +CREATE TABLE IF NOT EXISTS library_attempts ( + operation_id VARCHAR(64) NOT NULL REFERENCES library_operations(id), + epoch INTEGER NOT NULL CHECK (epoch >= 1), + job_id VARCHAR(64) NOT NULL, + queue_job_id VARCHAR(64) NOT NULL DEFAULT '', + owner VARCHAR(128) NOT NULL DEFAULT '', + state VARCHAR(16) NOT NULL CHECK (state IN ('claimed','running','completed','failed','abandoned')), + started_at TIMESTAMPTZ NOT NULL, + heartbeat_at TIMESTAMPTZ, + lease_until TIMESTAMPTZ, + finished_at TIMESTAMPTZ, + outcome TEXT NOT NULL DEFAULT '', + PRIMARY KEY (operation_id, epoch) +); +-- 一个作业行只对应一个代次:重复 Begin 不新增谱系。 +CREATE UNIQUE INDEX IF NOT EXISTS library_attempts_job ON library_attempts(job_id); +-- 同一 operation 同时只能有一个活跃代次:第二个 resumer 的失败不能释放另一个代次的锁。 +CREATE UNIQUE INDEX IF NOT EXISTS library_attempts_active + ON library_attempts(operation_id) WHERE state IN ('claimed','running'); + +-- ── 作业行继承 operation/代次与租约(写入守卫读这里)───────────────────── +ALTER TABLE library_jobs ADD COLUMN IF NOT EXISTS operation_id VARCHAR(64) NOT NULL DEFAULT ''; +ALTER TABLE library_jobs ADD COLUMN IF NOT EXISTS attempt_epoch INTEGER NOT NULL DEFAULT 0; +ALTER TABLE library_jobs ADD COLUMN IF NOT EXISTS lease_owner VARCHAR(128) NOT NULL DEFAULT ''; +ALTER TABLE library_jobs ADD COLUMN IF NOT EXISTS lease_until TIMESTAMPTZ; + +-- ── 版本行:不可变事件 + 执行身份;允许 recovery_verified ──────────────── +ALTER TABLE library_versions ADD COLUMN IF NOT EXISTS operation_id VARCHAR(64) NOT NULL DEFAULT ''; +ALTER TABLE library_versions ADD COLUMN IF NOT EXISTS attempt_epoch INTEGER NOT NULL DEFAULT 0; +ALTER TABLE library_versions DROP CONSTRAINT IF EXISTS library_versions_kind_check; +ALTER TABLE library_versions ADD CONSTRAINT library_versions_kind_check + CHECK (kind IN ('test_sealed','production_effective','production_prebackup','recovery_verified')); + +-- 旧唯一索引禁止同一 scope 出现第二条未顶替的生效行——那是"用 UPDATE 顶替"的实现细节。 +-- 现在"谁是当前"由指针表表达,事件本身不再被改写;索引留着反而会逼写入方去 UPDATE 历史。 +DROP INDEX IF EXISTS library_versions_effective; + +-- ── current 指针:单调次序 + 条件更新,不覆盖事件 ─────────────────────── +CREATE TABLE IF NOT EXISTS library_current_versions ( + scope VARCHAR(16) PRIMARY KEY CHECK (scope IN ('internal','external')), + version_id VARCHAR(64) NOT NULL REFERENCES library_versions(id), + snapshot_id VARCHAR(64) NOT NULL, + operation_id VARCHAR(64) NOT NULL DEFAULT '', + attempt_epoch INTEGER NOT NULL DEFAULT 0, + -- 排序键 = (事件登记时间|operation|epoch),见 governance.CurrentOrderKey。 + -- 只有更大的键才能推进指针:更早的事实不能把当前状态拉回历史。 + order_key VARCHAR(255) NOT NULL, + updated_at TIMESTAMPTZ NOT NULL +); + +-- 回填:只搬运"已登记且未被顶替"的既有生效版本(不新造版本行、不改旧证据)。 +INSERT INTO library_current_versions(scope, version_id, snapshot_id, operation_id, attempt_epoch, order_key, updated_at) +SELECT v.scope, v.id, v.snapshot_id, v.operation_id, v.attempt_epoch, + lpad((extract(epoch from v.created_at) * 1000000)::bigint::text, 20, '0') || '|' || v.operation_id || '|' || lpad(v.attempt_epoch::text, 9, '0'), + v.created_at +FROM library_versions v +WHERE v.kind = 'production_effective' AND v.superseded_at IS NULL +ON CONFLICT (scope) DO NOTHING; + +-- 历史行不再靠 superseded_at 表达"当前":把指针指向的那一行标成未顶替,其余保持原值。 +-- 这里只补元数据、不改任何证据字段(source_job_id/evidence/created_at 原样)。 +UPDATE library_versions v +SET superseded_at = NULL +WHERE v.id IN (SELECT version_id FROM library_current_versions) + AND v.superseded_at IS NOT NULL; + +UPDATE library_versions v +SET superseded_at = coalesce(v.superseded_at, now()) +WHERE v.kind = 'production_effective' + AND v.superseded_at IS NULL + AND NOT EXISTS (SELECT 1 FROM library_current_versions c WHERE c.version_id = v.id); \ No newline at end of file diff --git a/internal/platform/postgres/migrations_test.go b/internal/platform/postgres/migrations_test.go index e638bba..f31fb22 100644 --- a/internal/platform/postgres/migrations_test.go +++ b/internal/platform/postgres/migrations_test.go @@ -11,51 +11,59 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) { if err != nil { t.Fatalf("loadMigrations() error = %v", err) } - if len(migrations) != 75 { - t.Fatalf("migration count = %d, want 75", len(migrations)) + if len(migrations) != 76 { + t.Fatalf("migration count = %d, want 76", len(migrations)) + } + // 76 是发布执行身份:operation / attempt(epoch) / 不可变事件 + current 指针。 + // 它只动治理库,不碰 WeKnora Schema。 + if identity := migrations[len(migrations)-1]; identity.Version != 76 || + !strings.Contains(identity.UpSQL, "library_operations") || + !strings.Contains(identity.UpSQL, "library_attempts") || + !strings.Contains(identity.UpSQL, "library_current_versions") { + t.Fatalf("migration 76 = %#v", identity) } // 75 允许"建立目标权威基线"作为独立业务作业,并与 sync/restore 共用写互斥。 - if baseline := migrations[len(migrations)-1]; baseline.Version != 75 || + if baseline := migrations[len(migrations)-2]; baseline.Version != 75 || !strings.Contains(baseline.UpSQL, "library_jobs_write_target") || !strings.Contains(baseline.UpSQL, "'baseline'") { t.Fatalf("migration 75 = %#v", baseline) } // 74 记录封存执行器的运行时状态:页面可用性来自真实观测,不来自配置字符串。 - if executor := migrations[len(migrations)-2]; executor.Version != 74 || + if executor := migrations[len(migrations)-3]; executor.Version != 74 || !strings.Contains(executor.UpSQL, "library_executor_status") { t.Fatalf("migration 74 = %#v", executor) } // 73 记录死信精确解除:保留上游行,只绑定指纹、重放引用与观测摘要。 - if release := migrations[len(migrations)-3]; release.Version != 73 || + if release := migrations[len(migrations)-4]; release.Version != 73 || !strings.Contains(release.UpSQL, "library_dead_letter_releases") { t.Fatalf("migration 73 = %#v", release) } // 72 让维护占用按实例互斥(internal/external 不再各有各的锁)。 - if instance := migrations[len(migrations)-4]; instance.Version != 72 || + if instance := migrations[len(migrations)-5]; instance.Version != 72 || !strings.Contains(instance.UpSQL, "library_instance_maintenance") { t.Fatalf("migration 72 = %#v", instance) } // 71 让维护记录能区分实例级维护与仅治理库互斥。 - if level := migrations[len(migrations)-5]; level.Version != 71 || + if level := migrations[len(migrations)-6]; level.Version != 71 || !strings.Contains(level.UpSQL, "level") || !strings.Contains(level.UpSQL, "ingress_state") { t.Fatalf("migration 71 = %#v", level) } - if maint := migrations[len(migrations)-6]; maint.Version != 70 || !strings.Contains(maint.UpSQL, "library_maintenance") { + if maint := migrations[len(migrations)-7]; maint.Version != 70 || !strings.Contains(maint.UpSQL, "library_maintenance") { t.Fatalf("migration 70 = %#v", maint) } - if tail := migrations[len(migrations)-7]; tail.Version != 69 || + if tail := migrations[len(migrations)-8]; tail.Version != 69 || !strings.Contains(tail.UpSQL, "library_jobs_write_target") { t.Fatalf("migration 69 = %#v", tail) } - if restore := migrations[len(migrations)-8]; restore.Version != 68 || !strings.Contains(restore.UpSQL, "restore") { + if restore := migrations[len(migrations)-9]; restore.Version != 68 || !strings.Contains(restore.UpSQL, "restore") { t.Fatalf("migration 68 = %#v", restore) } - if last := migrations[len(migrations)-9]; last.Version != 67 || + if last := migrations[len(migrations)-10]; last.Version != 67 || !strings.Contains(last.UpSQL, "library_release_authorizations") || !strings.Contains(last.UpSQL, "params_digest") { t.Fatalf("migration 67 = %#v", last) } - if flow := migrations[len(migrations)-10]; flow.Version != 66 || + if flow := migrations[len(migrations)-11]; flow.Version != 66 || !strings.Contains(flow.UpSQL, "CREATE TABLE library_versions") || !strings.Contains(flow.UpSQL, "library_jobs_active") { t.Fatalf("migration 66 = %#v", flow) diff --git a/internal/releaseadapter/v071/library_production_target.go b/internal/releaseadapter/v071/library_production_target.go index 7dd3ab3..8ca974f 100644 --- a/internal/releaseadapter/v071/library_production_target.go +++ b/internal/releaseadapter/v071/library_production_target.go @@ -782,6 +782,41 @@ func archiveRowIdentity(table string, row map[string]json.RawMessage) string { return value("id") } +// AdoptWriteSet 消费已持久化的写集合(接管/恢复路径)。 +// +// 只做一件事:把"这些 resources 已证明与目标现状逐字节一致"的决定装回执行目标, +// 让 Apply 的跳过逻辑生效。它**不重新计划**、不重新比较目标现状——那会把半写状态 +// 当成新基线,也把"共享对象没被重写"这句话变成谎话(F07)。 +// +// 校验从严:计划必须自校验通过、绑定同一目标 tenant/KB、绑定本次要写的那份快照; +// 任何一项不符都拒绝,因为错误的复用集合会导致该写的没写、或该跳的却重写。 +func (t *ProductionLibraryTarget) AdoptWriteSet(set libraryrelease.WriteSet) error { + if err := set.Verify(); err != nil { + return fmt.Errorf("写集合记录校验失败:%w", err) + } + if set.TenantID != t.scope.TenantID || set.KnowledgeBaseID != t.scope.KnowledgeBaseID { + return fmt.Errorf("写集合绑定的是 %d/%s,与目标 %d/%s 不一致", + set.TenantID, set.KnowledgeBaseID, t.scope.TenantID, t.scope.KnowledgeBaseID) + } + reused := map[string]bool{} + for _, row := range set.Rows { + if row.Table != resourceReuseTableName || !row.Reused { + continue + } + if !row.Existing { + return fmt.Errorf("写集合把资源 %s 标成复用但写前不存在:拒绝按不可信的复用集合执行", row.ID) + } + reused[row.ID] = true + } + if t.reusedResources == nil { + t.reusedResources = map[string]bool{} + } + for id := range reused { + t.reusedResources[id] = true + } + return nil +} + // Restore 用预备份把生产整库恢复回去:走同一条 Apply 路径,可重复执行。 // // 恢复必须带着"本次写集合 + 写前整实例证据"(写第一笔之前落盘、摘要登记在治理库)。 diff --git a/internal/releaseadapter/v071/library_scope_hash_test.go b/internal/releaseadapter/v071/library_scope_hash_test.go new file mode 100644 index 0000000..fe137cf --- /dev/null +++ b/internal/releaseadapter/v071/library_scope_hash_test.go @@ -0,0 +1,64 @@ +package v071 + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "testing" +) + +// F08:整库搬运改了 physical_path 就必须重算 location_hash。 +// +// oracle 用 crypto/sha256 **自己复算**,不调用被测实现里的 resourceLocationHash: +// 写入与校验共用同一个转换函数时,自比永远成立,错误会被自己的自洽掩盖。 +func TestResourceRetargetRecomputesLocationHash(t *testing.T) { + old := "storage://source-backend/minio://source-bucket/10001/file.pdf" + hash := sha256.Sum256([]byte(old)) + raw, _ := json.Marshal(map[string]any{"id": "resource-one", "tenant_id": 10001, + "physical_path": old, "location_hash": hex.EncodeToString(hash[:]), + "provider": "minio", "storage_backend_id": "source-backend"}) + var row map[string]json.RawMessage + if err := json.Unmarshal(raw, &row); err != nil { + t.Fatal(err) + } + target := &ProductionLibraryTarget{ + config: ProductionLibraryConfig{ObjectsBucket: "target-bucket"}, + scope: ProductionScope{TenantID: 10002, KnowledgeBaseID: "target-kb"}, + storageBackend: "target-backend", storageProvider: "minio", + snapshotTenant: 10001, snapshotKB: "source-kb", + } + if err := target.retargetRow("resources", row); err != nil { + t.Fatal(err) + } + var path, got string + _ = json.Unmarshal(row["physical_path"], &path) + _ = json.Unmarshal(row["location_hash"], &got) + if path == old { + t.Fatal("夹具没有改写定位符:这条断言就没有意义") + } + want := sha256.Sum256([]byte(path)) + if got != hex.EncodeToString(want[:]) { + t.Fatalf("改路径后 location_hash 仍是旧值:path=%s got=%s want=%s", path, got, hex.EncodeToString(want[:])) + } + // 反例:跨桶/跨租户组合下同样成立(桶段与租户段都参与最终路径)。 + for _, tc := range []struct{ name, bucket, backend string }{ + {"跨桶", "another-bucket", "target-backend"}, + {"跨后端", "target-bucket", "backend-2"}, + } { + var other map[string]json.RawMessage + _ = json.Unmarshal(raw, &other) + next := *target + next.config.ObjectsBucket = tc.bucket + next.storageBackend = tc.backend + if err := next.retargetRow("resources", other); err != nil { + t.Fatalf("%s:%v", tc.name, err) + } + var moved, movedHash string + _ = json.Unmarshal(other["physical_path"], &moved) + _ = json.Unmarshal(other["location_hash"], &movedHash) + sum := sha256.Sum256([]byte(moved)) + if movedHash != hex.EncodeToString(sum[:]) { + t.Fatalf("%s:location_hash 与最终路径不一致:path=%s got=%s", tc.name, moved, movedHash) + } + } +} diff --git a/internal/releaseadapter/v071/library_scope_remap.go b/internal/releaseadapter/v071/library_scope_remap.go index 23fe813..96e4e08 100644 --- a/internal/releaseadapter/v071/library_scope_remap.go +++ b/internal/releaseadapter/v071/library_scope_remap.go @@ -2,6 +2,8 @@ package v071 import ( "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -169,6 +171,17 @@ func (t *ProductionLibraryTarget) rewriteResourceIdentity(row map[string]json.Ra } row["physical_path"] = encoded } + // location_hash 的契约是 sha256(最终 physical_path),不是"来源路径的哈希"。 + // 旧实现只换路径不重算哈希:目标行声明的是一个不存在的定位符,去重/按路径定位 + // 都会失效;而写入与校验复用同一个转换函数,内部自比根本发现不了(F08)。 + // 这里在同一个转换里重算,校验侧另有独立 oracle(测试里直接用 crypto/sha256 复算)。 + if _, ok := row["location_hash"]; ok { + encoded, err := json.Marshal(resourceLocationHash(rewritten)) + if err != nil { + return errors.New("编码目标定位符哈希失败") + } + row["location_hash"] = encoded + } if t.storageBackend != "" { if _, ok := row["storage_backend_id"]; ok { encoded, err := json.Marshal(t.storageBackend) @@ -190,6 +203,15 @@ func (t *ProductionLibraryTarget) rewriteResourceIdentity(row map[string]json.Ra return nil } +// resourceLocationHash 是 `location_hash` 契约的**唯一实现**:sha256(最终 physical_path)。 +// +// 全库搬运(本文件)与文档级导入(postgres_import.go)必须用同一个函数: +// 两条路径各写一份,迟早只改一边,而目标实例正是按这个哈希去重/定位资源的。 +func resourceLocationHash(physicalPath string) string { + sum := sha256.Sum256([]byte(physicalPath)) + return hex.EncodeToString(sum[:]) +} + // retargetLocator 把 "storage:///:///" 整体换成目标 // 实例的真实取值:后端 id、scheme/提供方、桶名,以及对象键里的租户段。 // 桶名为空表示目标没有配置对象存储(测试夹具),此时不动定位符。 diff --git a/internal/releaseadapter/v071/library_target_contract.go b/internal/releaseadapter/v071/library_target_contract.go new file mode 100644 index 0000000..1dd5697 --- /dev/null +++ b/internal/releaseadapter/v071/library_target_contract.go @@ -0,0 +1,120 @@ +package v071 + +import ( + "context" + "errors" + "fmt" + + "knowledge-governance/internal/librarysnapshot" +) + +// LiveTargetContract 是目标知识库**此刻**的模型/索引/schema 契约。 +// +// 为什么要单独读:旧实现直接拿"要发布的那份来源快照的 compatibility"当生产预备份的 +// 合同,于是"目标是什么模型"从来没被问过。后果是目标拿到 A 生成的向量、知识行却声明 +// 是 B——同维不同模型在 mapping 判据下完全通过(F17)。 +type LiveTargetContract struct { + SchemaVersion int + Dirty bool + EmbeddingModel string + EmbeddingDimension int + EngineType string + IndexName string + MappingSHA256 string + // ModelBound 表示 knowledge_bases → models 的 join 命中:false 时维度不可知, + // 不能凭"看起来是同一维"放行。 + ModelBound bool +} + +// liveTargetContractSQL 用 LEFT JOIN 读模型:模型行缺失时仍然拿到 KB 声明的模型 id, +// 这样"没绑模型/绑了但不可读"能被明确拒绝,而不是在发布中途才炸。 +const liveTargetContractSQL = ` +SELECT COALESCE(migration.version, 0), + COALESCE(migration.dirty, false), + COALESCE(model.name, ''), + COALESCE(NULLIF(model.parameters->'embedding_parameters'->>'dimension', '')::INTEGER, 0), + (model.id IS NOT NULL), + COALESCE(knowledge_base.embedding_model_id, ''), + COALESCE(vector_store.engine_type, ''), + COALESCE(vector_store.index_config->>'index_name', '') +FROM knowledge_bases AS knowledge_base +LEFT JOIN models AS model + ON model.id = knowledge_base.embedding_model_id AND model.deleted_at IS NULL +LEFT JOIN schema_migrations AS migration ON TRUE +LEFT JOIN vector_stores AS vector_store + ON vector_store.id = knowledge_base.vector_store_id AND vector_store.deleted_at IS NULL +WHERE knowledge_base.id = $1 AND knowledge_base.tenant_id = $2 AND knowledge_base.deleted_at IS NULL +` + +// LiveTargetContract 读目标知识库的实时契约。读不到就直接失败——**不许**回退到 +// 来源快照的声明:那正是"用来源兼容性冒充目标合同"的老毛病。 +func (t *ProductionLibraryTarget) LiveTargetContract(ctx context.Context) (LiveTargetContract, error) { + if t.pool == nil { + return LiveTargetContract{}, errors.New("目标数据库未接线:无法核对实时模型契约") + } + var ( + contract LiveTargetContract + declared string + indexName string + ) + if err := t.pool.QueryRow(ctx, liveTargetContractSQL, t.scope.KnowledgeBaseID, t.scope.TenantID).Scan( + &contract.SchemaVersion, &contract.Dirty, &contract.EmbeddingModel, &contract.EmbeddingDimension, + &contract.ModelBound, &declared, &contract.EngineType, &indexName); err != nil { + return LiveTargetContract{}, fmt.Errorf("读不到目标知识库 %s 的实时契约:%w", t.scope.KnowledgeBaseID, err) + } + if !contract.ModelBound && declared != "" { + // KB 声明了模型但模型行不可读(未迁移/已删):维度未知,不能证明兼容。 + contract.EmbeddingModel = declared + } + contract.IndexName = indexName + if t.search != nil { + if mapping, mapErr := t.search.MappingSHA256(ctx); mapErr == nil { + contract.MappingSHA256 = mapping + } + } + return contract, nil +} + +// VerifySourceAgainstTarget 判断"来源快照的向量能不能直接搬进这个目标"。 +// +// 只允许**等价模型**:名字一致、维度一致。名字不同即使维度相同也拒绝——向量是在 +// 模型 A 的语义空间里算出来的,搬进声明模型 B 的行里,检索结果会静默变错,而这 +// 种错误不会被任何"行数/摘要"校验发现(本模块不重算 embedding,也不许假装等价)。 +func VerifySourceAgainstTarget(source librarysnapshot.Compatibility, live LiveTargetContract) error { + if live.Dirty { + return errors.New("目标 schema 处于 dirty 状态:拒绝在该目标上写入") + } + if source.SchemaVersion != 0 && live.SchemaVersion != 0 && source.SchemaVersion != live.SchemaVersion { + return fmt.Errorf("目标 schema 版本 %d 与来源快照声明 %d 不一致:拒绝搬运", + live.SchemaVersion, source.SchemaVersion) + } + if !live.ModelBound || live.EmbeddingDimension < 1 { + return errors.New("目标知识库没有可读的 embedding 模型绑定(模型行缺失或维度未知):" + + "无法证明向量可迁移,先在目标上绑定模型") + } + if source.EmbeddingModel != "" && live.EmbeddingModel != "" && source.EmbeddingModel != live.EmbeddingModel { + return fmt.Errorf("来源 embedding 模型 %q 与目标实时模型 %q 不同:不重算 embedding 就不能声称兼容", + source.EmbeddingModel, live.EmbeddingModel) + } + if source.EmbeddingDimension > 0 && source.EmbeddingDimension != live.EmbeddingDimension { + return fmt.Errorf("来源向量维度 %d 与目标模型维度 %d 不一致", + source.EmbeddingDimension, live.EmbeddingDimension) + } + return nil +} + +// ContractFromLive 用目标实时状态构造预备份合同:新备份声明的是"目标现在是什么", +// 不是"要发布的来源是什么"。 +func ContractFromLive(source librarysnapshot.Compatibility, live LiveTargetContract) librarysnapshot.Compatibility { + contract := source + contract.SchemaVersion = live.SchemaVersion + contract.EmbeddingModel = live.EmbeddingModel + contract.EmbeddingDimension = live.EmbeddingDimension + if live.EngineType != "" { + contract.RetrieverEngine = live.EngineType + } + if live.MappingSHA256 != "" { + contract.IndexMappingSHA256 = live.MappingSHA256 + } + return contract +} diff --git a/internal/releaseadapter/v071/library_target_contract_test.go b/internal/releaseadapter/v071/library_target_contract_test.go new file mode 100644 index 0000000..0610b55 --- /dev/null +++ b/internal/releaseadapter/v071/library_target_contract_test.go @@ -0,0 +1,53 @@ +package v071 + +import ( + "strings" + "testing" + + "knowledge-governance/internal/librarysnapshot" +) + +// F17:目标实时模型契约必须核,且"同维不同模型"不许放行。 +func TestVerifySourceAgainstTargetRejectsDifferentModel(t *testing.T) { + source := librarysnapshot.Compatibility{SchemaVersion: 71, EmbeddingModel: "model-a", EmbeddingDimension: 1024} + live := LiveTargetContract{SchemaVersion: 71, EmbeddingModel: "model-b", EmbeddingDimension: 1024, ModelBound: true} + if err := VerifySourceAgainstTarget(source, live); err == nil { + t.Fatal("同维不同模型必须拒绝:向量空间不同,行声明与向量会对不上") + } + // 等价模型(同名字同维度)放行。 + live.EmbeddingModel = "model-a" + if err := VerifySourceAgainstTarget(source, live); err != nil { + t.Fatalf("等价模型应放行:%v", err) + } + // 维度不同必须拒绝。 + live.EmbeddingDimension = 768 + if err := VerifySourceAgainstTarget(source, live); err == nil { + t.Fatal("维度不同必须拒绝") + } + // 目标模型行缺失(维度未知)也必须拒绝:不能靠"看起来同维"猜。 + live = LiveTargetContract{SchemaVersion: 71, EmbeddingModel: "model-a", ModelBound: false} + if err := VerifySourceAgainstTarget(source, live); err == nil { + t.Fatal("目标模型不可读时必须拒绝,而不是按来源声明放行") + } + // dirty schema 直接拒绝。 + if err := VerifySourceAgainstTarget(source, LiveTargetContract{SchemaVersion: 71, Dirty: true, + EmbeddingModel: "model-a", EmbeddingDimension: 1024, ModelBound: true}); err == nil { + t.Fatal("dirty schema 必须拒绝") + } +} + +// 预备份合同必须来自目标实况:不能把来源声明当目标声明。 +func TestContractFromLiveUsesTargetState(t *testing.T) { + source := librarysnapshot.Compatibility{WeKnoraVersion: "0.7.1", SchemaVersion: 71, + EmbeddingModel: "model-a", EmbeddingDimension: 1024, RetrieverEngine: "elasticsearch_v8", + IndexMappingSHA256: strings.Repeat("a", 64)} + live := LiveTargetContract{SchemaVersion: 71, EmbeddingModel: "model-b", EmbeddingDimension: 1024, + EngineType: "elasticsearch", MappingSHA256: strings.Repeat("b", 64), ModelBound: true} + got := ContractFromLive(source, live) + if got.EmbeddingModel != "model-b" || got.IndexMappingSHA256 != strings.Repeat("b", 64) { + t.Fatalf("预备份合同必须取目标实况:%+v", got) + } + if got.WeKnoraVersion != source.WeKnoraVersion { + t.Fatalf("数据格式版本仍来自来源快照:%+v", got) + } +} diff --git a/internal/releaseadapter/v071/postgres_import.go b/internal/releaseadapter/v071/postgres_import.go index ed22ad7..e1fe348 100644 --- a/internal/releaseadapter/v071/postgres_import.go +++ b/internal/releaseadapter/v071/postgres_import.go @@ -2,8 +2,6 @@ package v071 import ( "context" - "crypto/sha256" - "encoding/hex" "encoding/json" "errors" "fmt" @@ -388,11 +386,11 @@ func (importer *PostgresImporter) prepareResourceRow(raw json.RawMessage, tenant } } targetPath := "storage://" + importer.config.TargetStorageBackendID + "/" + providerPath - digest := sha256.Sum256([]byte(targetPath)) row["storage_backend_id"] = importer.config.TargetStorageBackendID row["provider"] = importer.config.TargetProvider row["physical_path"] = targetPath - row["location_hash"] = hex.EncodeToString(digest[:]) + // 与全库搬运共用同一个哈希实现(F08):契约是 sha256(最终 physical_path)。 + row["location_hash"] = resourceLocationHash(targetPath) encoded, err := json.Marshal(row) if err != nil { return nil, resourceIdentityPair{}, errors.New("rewrite target resource") diff --git a/web/src/ReleaseTargets.test.tsx b/web/src/ReleaseTargets.test.tsx index b3f1600..b6ae8fe 100644 --- a/web/src/ReleaseTargets.test.tsx +++ b/web/src/ReleaseTargets.test.tsx @@ -4,6 +4,7 @@ import { ReleaseTargetsView, libraryPollingActive } from './ReleaseTargets' import type { LibraryCaptureJob, LibraryJobRecord, + LibraryMaintenanceView, LibraryOverview, LibrarySnapshot, LibrarySnapshotList, @@ -743,3 +744,29 @@ describe('libraryPollingActive(轮询范围)', () => { ).toBe(false) }) }) + +// F12:按预备份恢复是独立队列作业(不产生新的 library_jobs 行), +// 页面必须自己记住"还在等恢复",并在维护窗口关闭后停止。 +describe('libraryPollingActive(恢复轮询)', () => { + const idle = { internal: null, external: null } as Record<'internal' | 'external', LibraryOverview | null> + + it('恢复投递后即使没有任何作业行也要继续读回', () => { + expect(libraryPollingActive({ captures: [], overviews: idle, pending: false, recoveryPending: true })).toBe(true) + }) + + it('维护窗口仍被持有时继续读回(恢复/收尾未结束)', () => { + const held = { + internal: { held: true } as unknown as LibraryMaintenanceView, + external: null, + } + expect(libraryPollingActive({ captures: [], overviews: idle, pending: false, maintenances: held })).toBe(true) + }) + + it('维护窗口关闭且没有在飞动作时停止', () => { + const released = { + internal: { held: false } as unknown as LibraryMaintenanceView, + external: null, + } + expect(libraryPollingActive({ captures: [], overviews: idle, pending: false, maintenances: released })).toBe(false) + }) +}) diff --git a/web/src/ReleaseTargets.tsx b/web/src/ReleaseTargets.tsx index 4fddf50..7308984 100644 --- a/web/src/ReleaseTargets.tsx +++ b/web/src/ReleaseTargets.tsx @@ -268,9 +268,19 @@ export function libraryPollingActive(input: { captures: LibraryCaptureJob[] overviews: Record pending: boolean + /** 按预备份恢复是一次队列作业,不产生新的 library_jobs 行:必须单独记住"还在等它"。 */ + recoveryPending?: boolean + /** 维护窗口仍被持有 ⇒ 恢复/收尾还没结束,页面必须继续读回。 */ + maintenances?: Record }): boolean { if (input.pending) return true + if (input.recoveryPending) return true if (input.captures.some((job) => ACTIVE_STATES.includes(job.state))) return true + const maintenanceHeld = (['internal', 'external'] as Scope[]).some((scope) => { + const view = input.maintenances?.[scope] + return Boolean(view?.held) + }) + if (maintenanceHeld) return true return (['internal', 'external'] as Scope[]).some((scope) => { const job = input.overviews[scope]?.job return Boolean(job && ACTIVE_STATES.includes(job.state)) @@ -1664,6 +1674,10 @@ export function ReleaseTargets() { const [restore, setRestore] = useState(null) const [restoreNotice, setRestoreNotice] = useState(null) const [pendingRestore, setPendingRestore] = useState(false) + // 按预备份恢复(library.recover)是独立的队列作业,不会产生新的 library_jobs 行: + // 只靠 jobs/overview 判断会让页面在投递后立刻停止轮询,恢复完成也停在旧状态(F12)。 + const [pendingRecovery, setPendingRecovery] = useState(false) + const recoveryStartedAt = useRef(0) const [pendingGrant, setPendingGrant] = useState(false) // 轮询覆盖"投递 → 终态读回":投递中的动作也算在跑(点完到 worker 落库之间 // 有一次空窗,只看已有作业会让页面停在"尚未启动")。终态(成功/失败/取消) @@ -1672,6 +1686,8 @@ export function ReleaseTargets() { captures: jobs, overviews, pending: pendingCapture || pendingSync || pendingBaseline || pendingRestore || pendingSyncGrant || pendingGrant, + recoveryPending: pendingRecovery, + maintenances, }) const wasPolling = useRef(false) useEffect(() => { @@ -1760,13 +1776,30 @@ export function ReleaseTargets() { setNotice(null) try { const accepted = await enqueueLibraryRecovery(scope, windowJobID) - setNotice(`已投递按预备份恢复:${accepted.recovery_job || '(未返回作业号)'}`) + // 从这里开始页面必须持续读回,直到维护窗口真的关闭(成功)或恢复失败被记下来。 + recoveryStartedAt.current = Date.now() + setPendingRecovery(true) + setNotice(`已投递按预备份恢复:${accepted.recovery_job || '(未返回作业号)'};正在等待恢复与校验完成`) await load() } catch (recoveryError) { + setPendingRecovery(false) setNotice(recoveryError instanceof Error ? recoveryError.message : '恢复请求失败') } } + // 恢复轮询的收尾:维护窗口不再被持有 ⇒ 读到终态,停止等待并给出结论。 + // 超时不是"成功":只停止轮询并把状态留给页面显示(恢复失败会继续 held + needs_restore)。 + useEffect(() => { + if (!pendingRecovery) return + const held = Boolean(maintenances[scope]?.held) + const timedOut = Date.now() - recoveryStartedAt.current > 15 * 60 * 1000 + if (!held || timedOut) { + setPendingRecovery(false) + if (timedOut) setNotice('恢复轮询超时:请查看维护窗口状态(仍在维护说明恢复/校验未通过)') + void load().catch(() => undefined) + } + }, [pendingRecovery, maintenances, scope, load]) + return ( { + globalThis.fetch = originalFetch + vi.restoreAllMocks() +}) + +describe('fetchLibraryOverview(HTTP 状态与结构)', () => { + it('401 的 {error} 不许被当成发布状态', async () => { + globalThis.fetch = vi.fn( + async () => new Response(JSON.stringify({ error: 'unauthenticated' }), { status: 401 }) + ) as unknown as typeof fetch + await expect(fetchLibraryOverview('internal')).rejects.toThrow(/HTTP 401/) + }) + + it('503 的闸口拒绝同样必须报错', async () => { + globalThis.fetch = vi.fn( + async () => new Response(JSON.stringify({ available: false, reason: '存储不可用' }), { status: 503 }) + ) as unknown as typeof fetch + await expect(fetchLibraryOverview('internal')).rejects.toThrow(/HTTP 503/) + }) + + it('200 但缺 gate/jobs 的结构不符也要报错', async () => { + globalThis.fetch = vi.fn( + async () => new Response(JSON.stringify({ scope: 'internal' }), { status: 200 }) + ) as unknown as typeof fetch + await expect(fetchLibraryOverview('internal')).rejects.toThrow(/结构不符合预期/) + }) + + it('结构正确的 200 正常返回(避免把正常请求也拦掉)', async () => { + const payload = { + scope: 'internal', + production: { state: 'unknown' }, + prebackup: { state: 'unknown' }, + diff: { state: 'unknown', added: [], updated: [], removed: [] }, + jobs: [], + gate: { sync_allowed: false, reasons: [] }, + authorization: { state: 'none' }, + capture_available: false, + now: new Date().toISOString(), + } + globalThis.fetch = vi.fn( + async () => new Response(JSON.stringify(payload), { status: 200 }) + ) as unknown as typeof fetch + const value = await fetchLibraryOverview('internal') + expect(value?.gate.sync_allowed).toBe(false) + }) +}) diff --git a/web/src/librarySnapshots.ts b/web/src/librarySnapshots.ts index 475b0af..d97f3b9 100644 --- a/web/src/librarySnapshots.ts +++ b/web/src/librarySnapshots.ts @@ -621,8 +621,20 @@ export async function fetchLibraryOverview(scope: string, signal?: AbortSignal): headers: { Accept: 'application/json' }, signal, }) + // 非 2xx 的响应体会是 {error: ...},把它 cast 成 LibraryOverview 会让页面在 + // `overview.gate.sync_allowed` 上抛 TypeError(F16)。这里必须先核状态码与结构, + // 结构不符就**报错**,让页面显示"读取失败"而不是崩溃或当成正常状态。 + if (!response.ok) { + throw new Error(`发布状态读取失败(HTTP ${response.status})`) + } const value = (await response.json().catch(() => null)) as unknown - if (!record(value)) return null + if ( + !record(value) || + !record((value as { gate?: unknown }).gate) || + !Array.isArray((value as { jobs?: unknown }).jobs) + ) { + throw new Error('发布状态响应结构不符合预期(缺 gate/jobs)') + } return value as unknown as LibraryOverview }