diff --git a/CHANGELOG.md b/CHANGELOG.md index 33c5d00..e416a07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,288 @@ # Changelog +## 开发快照(基线崩溃续作、维护原状态保留与真实恢复演练工具) + +### Fixed + +- **基线契约按"能验证的来源"填充**:归档要求完整的兼容性契约(v0.7.1 / schema 74 / + elasticsearch_v8 / 1024 维 / 映射摘要)。原先只填了 embedding 与映射,捕获会被 + `unsupported snapshot compatibility contract` 拒绝。现在 schema 版本**读目标库** + `schema_migrations`(dirty 或版本不符直接拒绝),embedding 优先读目标库模型绑定、 + 读不到才回落部署声明,映射取目标当前值,版本/引擎用受支持栈常量。 +- **基线作业"登记成功但没来得及收尾"不再被判死**:原来崩溃重入会在 + "已有生效版本"检查处直接失败,把目标留在维护里、占用也不释放。 + 现在按「本作业 + 同一快照」识别为**自己的**版本:只做收尾(恢复服务、开入口、 + 释放两把占用),不重新捕获、不重写版本;别人登记的生效版本仍然一律拒绝。 + 捕获完成但登记失败的作业会明确失败并写"未对目标写入",不会假装成功。 +- **同一作业重入不再覆盖第一次记下的原运行状态**:崩溃接管时目标已被停住, + 再读一次只会读到 `stopped`,写回去会让恢复阶段以为写入者本来就没跑、 + 永远不启动它。现在 `AcquireInstanceMaintenance` / `AcquireLibraryMaintenance` + 在 owner 相同的重入时保留已持久化的原状态;新作业接管已释放窗口仍用自己读到的值。 + (真实 PostgreSQL 集成用例覆盖,非内存假实现。) +- **基线的版本登记移进维护窗口**:捕获、校验、登记三件事要么一起成立、要么目标 + 原样恢复;记录失败时窗口已按原状态开放,治理记录同步释放,**不再出现 + "显示保持维护、实际入口已开"**。 +- **索引映射读不到时拒绝登记**:不再 `_` 忽略错误后用空摘要写进基线证据。 +- 基线窗口的统一收尾抽成 `settleBaselineWindow`:版本已登记走与同步相同的成功边界; + 窗口仍 held 才标 `needs_restore`;窗口已恢复则释放占用并失败。 + +### Added + +- **`cmd/librarydrill`:目标实例真实恢复演练**(维护隔离 → 整库预备份 → 一致性校验 + → canary 漂移检测 → 按预备份恢复 → 逐类复核 → 按原状态恢复服务)。它使用与 + worker 完全相同的 `LIBRARY_*` 配置与维护协调器,绝不触碰封存源;任一步失败都 + 保持维护并写报告,不伪造通过。 +- `internal/librarystack`:目标/预备份/维护栈的唯一装配点(同步、恢复、基线、演练共用)。 + +### Changed + +- 基线作业的"已有生效版本"判定改为三种状态(own / foreign / none),并在 + 阶段里写明是否走"只收尾"续作。 + +## 开发快照(首次发布入口:同步授权、目标权威基线与维护能力) + +### Added + +- **管理员在发布页签发同步授权(`POST /api/v1/library/authorizations`,`action=sync`)**: + 授权把「范围 + 目标快照 + 参数摘要 + 有效期」钉死,仍与执行分成两步 —— + 页面只提供入口,不会替你签发,也不会自动重试执行。工程师看得到缺项原因, + 但拿不到签发按钮(接口本来就只认管理员)。 +- **目标权威基线(迁移 075,作业类型 `baseline`)**:目标非空、又没有生效版本时, + 首次发布的真实解法是「维护隔离 → 整库捕获 → 逐类校验 → 登记基线」。 + `POST /api/v1/library/baselines` 只建作业;worker 用与同步同一套维护窗口 + (关入口、停写入者、证明排空)把目标整库捕获成归档快照,再用同一套 `Verify` + 证明目标与这份快照一致,最后登记 `production_effective`(证据写明 + `origin=target_baseline`、快照 ID、逐表行数、目标索引映射)。 + 它**不写目标内容、不清空目标、不手写版本**:没有捕获与校验证据就没有基线。 + 基线作业与 sync/restore 共用同一把写互斥(同一 scope 只允许一个在飞写作业), + worker 被杀时同一作业 id 会接管那个窗口继续,而不是把目标留在停机状态。 +- **维护能力(`GET /api/v1/library/maintenance` 的 `capability`)**:页面区分 + 「此刻没有维护窗口」与「维护能力没接线」,逐项列出缺少的 + `LIBRARY_TARGET_*`;接口没给能力信息时显示"未知",不再用一句 + "也没有维护能力配置"糊过去。 + +### Changed + +- **`productionScopeEmpty` 改成整库口径**:原来只数 `knowledges`,"文档删了但 + 分块/Wiki/资源还在"的目标会被判成空库,从而绕过基线初始化。现在用适配层 + 同一份表清单与范围过滤(`v071.ScopeContentEmpty`)逐表计数,并把逐表行数 + 交回接口,"空"只有一份定义。 +- **拟发布的测试封存只认版本登记(`kind=test_sealed`)**:归档里同时存在生产 + 预备份与目标基线快照,按"这个 scope 最新一份"挑会把目标自己的内容显示成 + 测试候选。现在候选来自登记、再用归档确认它还在本机;不在就说明原因。 +- **发布页不再说"生产同步执行链尚未接线"**:同步按钮按真实闸口启用, + 点击走 `POST /api/v1/library/syncs`;被拒时逐条显示接口返回的缺项。 +- 缺项文案从"生产当前权威版本未知,不能启动写入"改为指向真实入口: + 「目标非空且没有生效版本记录:先建立目标权威基线…或确认目标确实是空库」。 + +### Fixed + +- 维护卡片把"没有窗口记录"错误地写成"没有维护能力配置",与实际接线不符。 +- 首次发布时 `POST /library/baselines` 在目标读不到时**不建作业、不登记版本** + (拒绝用建基线掩盖连接/配置故障)。 + +## 开发快照(发布执行器可用性、整库还原入口与诊断) + +### Added + +- **封存执行器的运行时可用性(迁移 074 `library_executor_status`)**:页面上的 + "可以发起封存"不再等于 `LIBRARY_AGENT_SOCKET` 非空。主机侧 agent 新增 + `health` 动作(不占封存操作锁、不跑子进程),worker 每 30 秒探测一次并把 + "指向哪个实例、归档在哪、是否降级/正在跑、观测时间"落库;API 只读这份观测。 + socket 只挂在 worker 容器里,所以由 worker 观测、页面读结果,而不是让 API + 假装能 dial。观测过期、不可达、实例不符、降级、正忙时页面与 + `POST /library/captures` 用同一份判定拒绝,并逐条给出原因。 +- **封存执行前后的身份复核**:封存结果由 agent 附上自报身份(`agent_identity`)。 + worker 在停写入者之前再探测一次,执行后再用结果里的身份复核;不一致时作业失败 + 且**不登记**为测试封存——心跳只能证明"刚才",从心跳到停写入者之间 socket + 可能被换掉。 +- **按版本恢复的页面入口**:`GET /api/v1/library/restore-availability` 说明 + "这份快照现在能不能恢复"(双闸、该快照的恢复授权、目标实例是否在维护)。 + 页面据此启用"恢复至该版本",管理员可签发 30 分钟有效期的 restore 授权; + 工程师只看到条件,不获得入口。 +- 可选 `LIBRARY_SOURCE_PROJECT`:声明封存执行器应指向的实例名(默认等于 + `LIBRARY_SOURCE_INSTANCE`)。它描述封存源,与同步目标 `LIBRARY_TARGET_*` 分开。 + +### Changed + +- **封存身份核对移进请求本身(TOCTOU)**:原来 worker 先 `health` 再 `capture`,两次 + 拨号之间 socket 可能被换成另一个 agent——错的执行器已经把写入者停了才被拒登记。 + 现在 `capture` 请求携带治理侧配置生成的期望身份(`project` 必填,`source_app`/ + `archive_root` 配了就一起绑定)与稳定指纹(`sha256:`,覆盖三项);接收方在**操作 + 互斥内、执行任何源操作之前**比对,缺失或被改过一律拒绝,错的执行器一次 `capture` + 都不跑。封存结果回显该指纹,worker 事后复核只作为证据防线,不能替代请求内核对。 + 恢复路径不受期望约束(它只把写入者恢复回来)。 +- `LIBRARY_EXECUTOR_STATUS_MAX_AGE`(默认 120 秒)控制执行器观测的有效期; + 未设置不等于"永不过期",过期即"无法确认"。 +- 页面/接口不再出现写死的"封存通道未接线""生产恢复未接线":不能发起时给出真实原因、 + 执行器身份与观测时间。 + +### Fixed + +- **接管在版本查询失败时 fail-open**:`resumed` 后版本查询出错被忽略、`FinalizeOnly` 仍为 + false,于是"已成功登记并释放窗口、只差终态"的作业会被**重新写一遍目标**。现在接管先按 + 证据分类:`finalize` 阶段成功是独立证据(不受版本查询影响);查询失败且无阶段证据 ⇒ + **未知**,零写入、作业失败可重试;本快照的生效版本已被别的快照顶替 ⇒ **拒绝覆盖**; + 只有真正没有收尾证据时才按正常流程。 +- **维护释放部分成功后的重试不再永久卡死**:`Update*/Release*` 对"已由本作业释放"的行 + 返回可判别的 `ErrMaintenanceReleased`,收尾重试视其为已完成;别的作业来释放仍是 + "未持有"错误。真 PG 负例:同作业重复 Update/Release 得到已释放,跨作业得到未持有。 +- **作业成功暴露早于维护结束**:原来 `execute` 里写完生效版本就 `AdvanceLibraryJob(succeeded)`, + 而服务恢复、入口开放与维护锁释放要在窗口返回后才做。任何一步失败时外部已经看到 + `succeeded`,重启接管又只认非终态,锁就永远挂着。现在内容校验/版本登记之后只记 + "待收尾",`finishLibrarySuccess` 依次确认目标写入者与入口健康、按 **scope → instance** + 顺序释放两把锁(每一步错误都上抛,不再丢弃),最后才写 `succeeded`;任一步失败都把 + 作业落成失败、维护保持 `needs_restore`。释放与终态写入用 `context.WithoutCancel` + 包住,停机不会留下半成品。接管判定改为按证据(finalize 阶段成功或存在未顶掉的生效 + 版本)转 finalize-only,只复核与登记,不重写生产。 +- **封存读资源失败只剩一句 `read library resource`**:对象缺失、桶错、端点解析不到 + 都会走到同一个字符串,真实封存直接卡在这里无法判断方向。现在带上资源行 id 与底层 + 原因(`read library resource : `),不含凭据。 +- **正式 release-worker 启动失败无法定位**:配置里的目标库指向不存在的主机时, + 日志只有一句 `connect release target database`。现在错误里带库端点 + (`host:port/db`,去口令)与路由标识,并保留底层原因;原因文本先经 + `pgdsn.Redact` 抹掉口令(pgx 的解析错误可能原样回显连接串),可诊断不等于泄露。 + 共享的 `internal/pgdsn` 替换了 cmd/worker 里只留库名的旧实现。 +- `libraryctl` 用法文本补上 `dead-letters` 子命令。 + +## 开发快照(死信精确解除:保留上游行,只按指纹放行) + +### Added + +- **死信解除契约(治理侧)**:上游 `task_dead_letters` 没有"已解决"标记, + purge/DELETE 都会销毁"这条任务失败过"的唯一证据。新增 + `library_dead_letter_releases`(迁移 073)记录**一条精确死信**的解除:绑定实时行 + 逐字节指纹(`sha256(to_jsonb(row))`,先压紧空白)、重放引用(队列/任务号/时间/ + 证据引用)、源侧观测摘要、操作人与时间。上游行一个字节都不动。 +- `GET/POST /api/v1/library/dead-letter-releases`:POST 需工程师及以上 + + `X-Governance-Action: dead-letter-release`,拒绝条件全部显式——观测摘要不符(被 + 改写/格式化)、源侧未结清、观测里没有这条死信(已被删除)、重放早于失败时间、 + 重放之后同一任务键又出现新死信(说明重放没成功)、实例不一致。 +- `libraryctl dead-letters`(只读):打印整库死信原始行 + 结算计数 + 观测摘要, + 供治理平台绑定;平台自己不连源库、不复制"自报成功"。 + +### Changed + +- 死信解除绑定 `LIBRARY_SOURCE_INSTANCE`(封存源),留空回落到 `LIBRARY_TARGET_INSTANCE`: + 同步目标与封存源常常不是同一个实例,用目标名去绑来源死信会让真实观测永远对不上 + (拒绝而不是误放行)。worker 取指纹、API 写记录用同一个来源实例名。 +- 结算/封存不再"死信存在即阻断",而是"每条实时死信都要被精确解除命中": + `checkLibrarySourceSettlementReleased` / `ExportLibraryWithReleases` 对每条行 + 重算指纹,未命中、行被改过、出现新行都继续阻断并点名。 +- 解除指纹经 worker → agent → `libraryctl -released-dead-letters` 逐段校验 + (`sha256:<64 hex>`,最多 64 条,不接受通配),agent 的固定命令形态不变。 + +### Fixed + +- **同步过去的资源行指向来源实例的存储后端 id(定位符与独立列两处)**:`storage:///minio:///` + 里的 `` 是来源 `storage_backends` 的主键,目标实例没有这条行。只换桶不改 id 时, + 原版应用落回默认存储服务并拒绝整条路径(真实下载 500 `code 1007 + invalid MinIO file path`)。现在写入前按"租户 + 目标桶名"在目标库解析后端 id, + 写入与校验共用同一份行变换;解析不到就拒绝写入,不产出目标读不出来的定位符。 + `resources.storage_backend_id` 这一列同样要换(应用先按它找文件服务)。 +- 指纹对空白敏感会让"同一份内容"两边算不出同一个值(PostgreSQL `to_jsonb::text` + 带空格、Go `json.Marshal` 不带);指纹改为先 `json.Compact` 再摘要。 + +## 开发快照(封存入口接线:库级窗口接上实例级闸门) + +### Fixed + +- **主机侧封存不再用"库级窗口"跑实例**:`libraryctl capture` / `recover` 改由 + `librarymaintenance.Manager` 装配——入口闸门、后台静默检查、目标指纹、入口探针与发布链路 + 是同一套实现。实例级维护里"写入者 exit=1"不再等于"在途请求被截断":被代理的 WeKnora 收到 + SIGTERM 就 `exit(1)` 是已知上游行为,排空证据来自闸门(`active_at_close` / `drained`), + 校验没有放松,而是补上了缺的那一层证据。 +- 缺少 `LIBRARY_TARGET_INGRESS_*` / `LIBRARY_TARGET_REDIS_*` 时 `capture` 在**开窗口前拒绝** + 并点名缺哪个变量;不再退回一个没有排空证据的窗口(那样的"成功"无法复核)。 +- 维护接线解析收敛到 `internal/librarymaintenance/env.go`:agent 与 worker 讲同一组变量名, + 空白值算缺失、队列库号必须是非负整数、口令不 Trim(前后空格也是口令)。 +- `libraryagent` 启动时如实打印维护目标是否接好(缺了就直说"封存会拒绝")。 +- `scripts/library-agent-install.sh` 强制 `agent.env`(0600,`EnvironmentFile=`),并把源容器/ + 项目/写入者声明做成可覆盖变量;没有 env 文件直接 `AGENT_ENV_MISSING` 退出。 + +- 主机侧封存新增 `-source-s3-endpoint`:容器里的对象存储端点常常只是服务名 + (`langfuse-minio:9000`),宿主机解析不到,资源导出会失败(真跑报 `read library resource`)。 + 覆盖只换访问入口,桶/对象键不变;`status` 如实打印实际 endpoint。 +- **封存 API 的队列作业 id 超列宽**:`library-capture--` 是 45 字符,而 + `jobs.id` 是 `varchar(36)`,真实请求一律 500(`value too long`)。此前页面停在"未接线" + 从未走到这里;改为 `libcap--` 并用测试锁住列宽。 +- **索引范围过滤漏掉 app 写入的正文**:v0.7.1 的 app 自己写索引不带 `tenant_id`, + 治理侧同步才补齐;只按 `term tenant_id` 过滤会把整库正文漏掉(真跑:索引有 2 条、 + 导出 0 条、候选校验报 legacy coverage)。改为"本租户 或 未写租户",写了别的租户仍排除。 + +- **空目标索引不能当成"索引被改坏"**:全新 WeKnora 实例先用空映射建索引(`{}`), + 字段等第一次写入才动态生成;首次同步因此在读取目标状态时就失败。现在"没有范围字段 + + 确实 0 篇文档"按空基线处理,有文档却看不出归属仍然报错。 + +- **资源定位符的桶没有跟着换**:同步把字节写进目标桶,但 `resources.physical_path` + 是从快照原样搬的,里面仍写着来源桶(`storage://…/minio://kgmaint-1028a/…`)。 + 目标实例之后取图/附件会指向测试环境的桶,或直接取不到。现在写入前把定位符里的桶 + 换成本次目标桶(键不变),解析不出来就报错而不是写一行指向别处的资源。 + +- **恢复作业 id 让维护窗口拒绝**:`librestore--<时间戳.微秒>` 带 `.`,而冻结协调器 + 只接受 `[A-Za-z0-9_-]`,每次恢复都在开窗口时被拒(`invalid freeze request`),作业随后被 + 接管扫成失败。改用纳秒整数,并加测试锁住"id 必须是维护窗口可接受的标识"。 + +### Tests + +- `internal/httpapi/library_restore_test.go`:恢复作业 id 必须能被维护窗口接受。 +- `internal/releaseadapter/v071/resource_locator_test.go`:嵌套/单层两种定位符换桶、裸键原样、 + 非法形状拒绝。 +- `internal/releaseadapter/v071/search_empty_index_test.go`:空索引=空基线、有文档无范围字段必须报错。 +- `internal/httpapi/library_capture_test.go`:队列作业 id 必须放得进 `varchar(36)` 且形状固定。 +- `internal/releaseadapter/v071/search_tenant_filter_test.go`:接受未写租户、排除他租户、 + 无租户字段时不造条件、整库过滤仍带知识库身份。 +- `cmd/libraryctl/discover_test.go`:端点覆盖生效且必须带 scheme。 +- `internal/librarymaintenance/env_test.go`:整份声明解析、逐项点名缺失、空白拒绝、 + 口令空格保留、非法队列库号。 +- `cmd/libraryctl/maintenance_test.go`:入口/队列/源身份绑到同一份配置、数据库身份不泄漏口令、 + 写入者声明透传、缺闸门拒绝。 + +## 开发快照(Wiki 撤销贡献:逐页语义比对 + 维护窗口真实呈现) + +### Added + +- **Wiki 贡献级差异**(`internal/librarydiff/wiki.go`):以 slug 为身份,把每页拆成 + 正文 / 来源文档 / 引用分块 / 出边 / 入边 / 目录/状态 分别取集合比较,输出 + `sources_added` / `sources_removed` / `citations_before→after` / `changed`。 + 删除一篇与别人共同贡献同一页的文档后,页面通常仍是「更新」,只看页面级增删改会显示成 + 「无变化」——现在能指出这份贡献是从哪一页、哪几类语义上被摘掉的。两端同时接受 JSON 数组 + 与字符串化数组,读不出来直接报错,不把「读不到」当成「没有来源」。 +- **接口 `GET /api/v1/library/overview` 的 `diff.wiki` 增加逐页明细**:`pages`(最多 50 页, + 超出置 `pages_truncated`)、`sources_added/removed`、`base_count/target_count`;页面级计数 + 保留,说明文字明确「重算发生在源侧,本平台只如实比对与保留结果」。 +- **发布管理页面:目标维护面板**(`MaintenanceCard`):显示维护级别(实例级/库级)、持有 + scope+作业、阶段、入口开/关、服务、开始/释放时间、作业进度(阶段名中文化,未映射原样显示)、 + 恢复错误与处理记录;只有窗口仍持有且作业失败时才出现「按预备份恢复」,动作走 + `POST /api/v1/library/maintenance/recover`。没有维护记录时明确写「未接入 + fail closed」, + 不画成已就绪。 +- **变更页展示逐页贡献差异**:每条页面一行,直接写「更新 发布闸门 · 正文/来源文档/引用分块 · + 来源 -1 · 引用 2→1」。 + +### Fixed + +- `LibraryJobRecord.kind` 之前只有 `seal | sync`,恢复作业(restore)在类型上不存在; + 补上后页面不再把恢复作业当成未知类型。 + +### Fixed + +- **`libraryctl` 只认 OSS 源**:`STORAGE_TYPE=minio` 的目标在发现阶段就被拒 + (`unsupported source storage type "minio"`),S3 兼容的专用验收实例无法封存。 + 现在按 `MINIO_ENDPOINT / MINIO_USE_SSL / MINIO_ACCESS_KEY_ID / MINIO_SECRET_ACCESS_KEY / + MINIO_BUCKET_NAME` 映射成 `provider=minio` + path-style;读不到端点/桶就明确报错, + 不退回默认值。 +- **CLI 与 API 的 scope→知识库 映射不一致**:API 用 `LIBRARY_PRODUCTION_KB_*`,CLI 只读 + routing bindings,同一台机器上会封存"配置里那个库"而不是 API 的验收目标。现在环境变量 + 优先、读不到才退回 bindings。 + +### Docs + +- `docs/library-acceptance-matrix.md` 按「已完成 / 缺代码 / 已实现未跑 / 需授权」 + 四段重排(不再在顶部叠报告):补上本轮 Wiki 撤销真实验收、贡献级差异、维护面板; + 新增封存链路缺口(CLI 无入口排空证明、API 封存未接线)与死信契约缺口。 +- 新增 `docs/formal-8088-deployment-plan.md`:版本清单、迁移 067–072、运行配置、 + fail-closed 规则、回滚、正式账号与数据保留、部署风险。**只准备,未执行**。 + ## 开发快照(后台静默必须在关入口之后证明,且要能复核) ### Fixed diff --git a/api/openapi.yaml b/api/openapi.yaml index 50f3501..b5d9f3e 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -898,6 +898,109 @@ } } }, + "/api/v1/library/dead-letter-releases": { + "get": { + "tags": [ + "Runtime" + ], + "operationId": "listLibraryDeadLetterReleases", + "summary": "列出本实例的死信解除记录(只读审计视图;上游死信行保持原样)", + "responses": { + "200": { + "$ref": "#/components/responses/Envelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "500": { + "$ref": "#/components/responses/InternalError" + }, + "503": { + "description": "实例身份未配置或解除存储不可用" + } + } + }, + "post": { + "tags": [ + "Runtime" + ], + "operationId": "createLibraryDeadLetterRelease", + "summary": "按精确指纹记录一条死信解除(需要工程师及以上与 X-Governance-Action;不删除上游行,行变化即失效)", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "dead_letter_id": { + "type": "integer" + }, + "observation": { + "type": "object" + }, + "observation_digest": { + "type": "string" + }, + "replay": { + "type": "object", + "properties": { + "queue": { + "type": "string" + }, + "task_id": { + "type": "string" + }, + "at": { + "type": "string", + "format": "date-time" + }, + "evidence": { + "type": "string" + } + }, + "required": [ + "queue", + "task_id", + "at" + ] + }, + "note": { + "type": "string" + } + }, + "required": [ + "dead_letter_id", + "observation", + "observation_digest", + "replay" + ] + } + } + } + }, + "responses": { + "200": { + "description": "解除记录已落库(或返回同一指纹的已存记录)" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "403": { + "description": "权限不足或缺少写意图头" + }, + "409": { + "description": "解除被拒绝:指纹/结清/重放证据不满足" + }, + "500": { + "$ref": "#/components/responses/InternalError" + }, + "503": { + "description": "实例身份未配置或解除存储不可用" + } + } + } + }, "/api/v1/library/storage": { "get": { "tags": [ @@ -1209,6 +1312,51 @@ } } }, + "/api/v1/library/restore-availability": { + "get": { + "tags": [ + "Runtime" + ], + "operationId": "getLibraryRestoreAvailability", + "summary": "这份快照现在能不能恢复到生产:双闸、恢复授权与目标实例维护状态逐条说明", + "parameters": [ + { + "name": "scope", + "in": "query", + "required": false, + "schema": { + "type": "string", + "enum": [ + "internal", + "external" + ] + } + }, + { + "name": "snapshot_id", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/Envelope" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "500": { + "$ref": "#/components/responses/InternalError" + }, + "503": { + "description": "读取恢复授权失败(不是\"可恢复\",是\"无法判定\")" + } + } + } + }, "/api/v1/library/authorizations": { "get": { "tags": [ @@ -1399,6 +1547,58 @@ } } }, + "/api/v1/library/baselines": { + "post": { + "tags": [ + "Runtime" + ], + "operationId": "startLibraryBaseline", + "summary": "建立目标权威基线:管理员在维护隔离下把非空目标整库捕获成快照、逐类校验后登记为生效版本;不清空目标、不手写版本", + "responses": { + "202": { + "description": "已创建基线作业(worker 在维护窗口内捕获并校验)" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "403": { + "description": "不是管理员" + }, + "409": { + "description": "已有生效版本、目标为空、目标正在维护或已有在飞写作业" + }, + "500": { + "$ref": "#/components/responses/InternalError" + }, + "503": { + "description": "目标整库读不到,不能建立基线(不创建作业、不登记版本)" + } + }, + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": false, + "required": [ + "scope" + ], + "properties": { + "scope": { + "type": "string", + "enum": [ + "internal", + "external" + ] + } + } + } + } + } + } + } + }, "/api/v1/library/snapshots/{id}": { "get": { "tags": [ diff --git a/cmd/api/main.go b/cmd/api/main.go index 4b8e8c7..163a637 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -203,7 +203,16 @@ func run(ctx context.Context, getenv func(string) string) error { VersionHistoryRepo: versionHistoryRepo, LibraryArchiveRoot: config.LibraryArchiveRoot, LibraryTargetInstance: config.LibraryTargetInstance, + LibraryTargetProject: config.LibraryTargetProject, + LibraryTargetWriters: config.LibraryTargetWriters, + LibraryTargetIngressURL: config.LibraryTargetIngressURL, + LibraryTargetIngressService: config.LibraryTargetIngressService, + LibraryTargetDockerSocket: config.LibraryTargetDockerSocket, + LibraryTargetJournalRoot: config.LibraryTargetJournalRoot, + LibrarySourceInstance: config.SourceInstance(), + LibrarySourceProject: config.SourceProject(), LibraryAgentSocket: config.LibraryAgentSocket, + LibraryExecutorStatusMaxAge: config.LibraryExecutorStatusMaxAge, LibraryCapacityBudgetBytes: config.LibraryCapacityBudgetBytes, LibraryProductionDatabaseURL: config.LibraryProductionDatabaseURL, LibraryProductionInternalKB: config.LibraryProductionInternalKB, diff --git a/cmd/libraryagent/main.go b/cmd/libraryagent/main.go index 2aa4c0d..81fe64e 100644 --- a/cmd/libraryagent/main.go +++ b/cmd/libraryagent/main.go @@ -14,6 +14,7 @@ import ( "time" "knowledge-governance/internal/libraryagent" + "knowledge-governance/internal/librarymaintenance" ) func main() { @@ -26,6 +27,7 @@ func main() { flag.StringVar(&config.SourceApp, "source-app", "WKNFull-app", "source container describing the stack") flag.StringVar(&config.SourcePG, "source-pg", "WKNFull-postgres", "source PostgreSQL container") flag.StringVar(&config.SourceES, "source-es", "WKNFull-elasticsearch", "source Elasticsearch container") + flag.StringVar(&config.SourceS3Endpoint, "source-s3-endpoint", "", "host-reachable source object-store endpoint (overrides the container-internal one)") flag.StringVar(&config.GovernancePG, "governance-pg", "knowledge-governance-p1-postgres-1", "governance PostgreSQL container") flag.StringVar(&config.Project, "project", "weknora071full", "Compose project owning the writers") flag.StringVar(&config.Writers, "writers", "app", "declared writer services") @@ -39,6 +41,14 @@ func main() { if err != nil { log.Fatal(err) } + // 实例级维护接线(入口闸门 + 后台队列)不在 agent 的 Config 里:它由同一份 + // 环境声明提供给子进程。启动时先说清楚有没有——缺了封存会在开窗口前被拒, + // 不允许"以为接好了"跑到一半才发现没有排空证据。 + if target, targetErr := librarymaintenance.TargetFromEnv(os.Getenv); targetErr != nil { + log.Printf("library agent: 封存会拒绝(实例级维护未接线):%v", targetErr) + } else { + log.Printf("library agent: 实例级维护目标 %s 入口 %s 闸门 %s", target.Instance, target.IngressURL, target.IngressService) + } if *describe { log.Printf("library agent config: %+v", agent.Describe()) return diff --git a/cmd/libraryctl/discover_test.go b/cmd/libraryctl/discover_test.go new file mode 100644 index 0000000..6b63756 --- /dev/null +++ b/cmd/libraryctl/discover_test.go @@ -0,0 +1,93 @@ +package main + +import ( + "strings" + "testing" +) + +// 目标实例可以是 OSS,也可以是 S3 兼容的 MinIO(本机专用验收实例就是 minio)。 +// 只认 oss 时,任何 minio 目标都会在发现阶段直接失败,等于这条验收链只能在生产存储上跑。 +func TestObjectStoreFromSourceEnvAcceptsMinIO(t *testing.T) { + app := map[string]string{ + "STORAGE_TYPE": "minio", + "MINIO_ENDPOINT": "langfuse-minio:9000", + "MINIO_USE_SSL": "false", + "MINIO_ACCESS_KEY_ID": "key", + "MINIO_SECRET_ACCESS_KEY": "secret", + "MINIO_BUCKET_NAME": "kgmaint-1028a", + "MINIO_PATH_PREFIX": "kgmaint/", + } + config, detail, err := objectStoreFromSourceEnv(app, "", 1<<20) + if err != nil { + t.Fatalf("minio: %v", err) + } + if config.Provider != "minio" || config.Endpoint != "http://langfuse-minio:9000" || config.Secure { + t.Fatalf("config: %+v", config) + } + if config.Bucket != "kgmaint-1028a" || config.AccessKeyID != "key" || config.SecretAccessKey != "secret" { + t.Fatalf("credentials mapping: %+v", config) + } + if detail == "" || config.MaxObjectBytes != 1<<20 { + t.Fatalf("detail=%q max=%d", detail, config.MaxObjectBytes) + } +} + +// 容器里的 minio 服务名在宿主机上解析不到:真跑时资源导出会失败(read library resource)。 +// 覆盖只换访问入口(同一个桶),不换来源;没有 scheme 的覆盖直接拒绝。 +func TestObjectStoreFromSourceEnvOverridesEndpointForHostSideCapture(t *testing.T) { + app := map[string]string{ + "STORAGE_TYPE": "minio", "MINIO_ENDPOINT": "langfuse-minio:9000", "MINIO_USE_SSL": "false", + "MINIO_ACCESS_KEY_ID": "key", "MINIO_SECRET_ACCESS_KEY": "secret", + "MINIO_BUCKET_NAME": "kgmaint-1028a", "MINIO_PATH_PREFIX": "kgmaint/", + } + config, detail, err := objectStoreFromSourceEnv(app, "http://127.0.0.1:19100", 1<<20) + if err != nil { + t.Fatalf("override: %v", err) + } + if config.Endpoint != "http://127.0.0.1:19100" || config.Bucket != "kgmaint-1028a" { + t.Fatalf("覆盖后配置不对:%+v", config) + } + if !strings.Contains(detail, "endpoint=http://127.0.0.1:19100") { + t.Fatalf("状态里没有如实写出用的端点:%q", detail) + } + if _, _, err := objectStoreFromSourceEnv(app, "127.0.0.1:19100", 1<<20); err == nil { + t.Fatal("没有 scheme 的端点覆盖被接受了") + } +} + +func TestObjectStoreFromSourceEnvAcceptsMinIOOverTLS(t *testing.T) { + app := map[string]string{ + "STORAGE_TYPE": "minio", "MINIO_ENDPOINT": "minio.internal:9000", "MINIO_USE_SSL": "true", + "MINIO_ACCESS_KEY": "k2", "MINIO_SECRET_KEY": "s2", "MINIO_BUCKET_NAME": "bucket", + } + config, _, err := objectStoreFromSourceEnv(app, "", 1) + if err != nil { + t.Fatalf("minio tls: %v", err) + } + if !config.Secure || config.Endpoint != "https://minio.internal:9000" || config.AccessKeyID != "k2" { + t.Fatalf("config: %+v", config) + } +} + +func TestObjectStoreFromSourceEnvKeepsOSS(t *testing.T) { + app := map[string]string{ + "STORAGE_TYPE": "oss", "OSS_ENDPOINT": "oss-cn-shenzhen.aliyuncs.com", "OSS_REGION": "cn-shenzhen", + "OSS_ACCESS_KEY": "ak", "OSS_SECRET_KEY": "sk", "OSS_BUCKET_NAME": "prod", + } + config, _, err := objectStoreFromSourceEnv(app, "", 1) + if err != nil { + t.Fatalf("oss: %v", err) + } + if config.Provider != "oss" || config.Endpoint != "https://oss-cn-shenzhen.aliyuncs.com" || !config.Secure { + t.Fatalf("config: %+v", config) + } +} + +func TestObjectStoreFromSourceEnvRejectsUnknownStorage(t *testing.T) { + if _, _, err := objectStoreFromSourceEnv(map[string]string{"STORAGE_TYPE": "cos"}, "", 1); err == nil { + t.Fatal("expected error for unsupported storage type") + } + if _, _, err := objectStoreFromSourceEnv(map[string]string{"STORAGE_TYPE": "minio"}, "", 1); err == nil { + t.Fatal("expected error when minio endpoint is missing") + } +} diff --git a/cmd/libraryctl/main.go b/cmd/libraryctl/main.go index b8a82f2..7b2206b 100644 --- a/cmd/libraryctl/main.go +++ b/cmd/libraryctl/main.go @@ -32,29 +32,37 @@ import ( "github.com/jackc/pgx/v5/pgxpool" + "knowledge-governance/internal/librarydeadletter" "knowledge-governance/internal/libraryfreeze" + "knowledge-governance/internal/librarymaintenance" "knowledge-governance/internal/librarysnapshot" v071 "knowledge-governance/internal/releaseadapter/v071" releaseworkflow "knowledge-governance/internal/workflows/release" ) type options struct { - sourceApp string - sourcePG string - sourceES string - governancePG string - project string - writers string - nonWriters string - archiveRoot string - journalRoot string - dockerSocket string - maxDocuments int - maxObjectBytes int64 - scope string - job string - dryRun bool - snapshot string + sourceApp string + sourcePG string + sourceES string + governancePG string + project string + writers string + nonWriters string + archiveRoot string + journalRoot string + dockerSocket string + // sourceS3Endpoint:容器里写的对象存储端点往往只在容器网络里能解析 + // (例如 minio 服务名),而封存跑在宿主机上。这里允许用宿主机可达地址覆盖。 + sourceS3Endpoint string + maxDocuments int + maxObjectBytes int64 + scope string + job string + dryRun bool + snapshot string + // releasedDeadLetters 是治理平台审计过的精确解除指纹(逗号分隔)。它只让这些 + // 确切的行不再阻断;实时行必须逐字节命中,否则照旧拒绝。 + releasedDeadLetters string } func main() { @@ -90,6 +98,8 @@ func main() { runErr = runVerify(opts) case "recover": runErr = runRecover(ctx, opts) + case "dead-letters": + runErr = runDeadLetters(ctx, opts) default: usage() os.Exit(2) @@ -101,7 +111,7 @@ func main() { } func usage() { - fmt.Fprint(os.Stderr, `libraryctl [flags] + fmt.Fprint(os.Stderr, `libraryctl [flags] status probe the real source and print what is configured / blocked preflight read the source, run settlement + candidate validation (no writes, no stop) @@ -109,6 +119,8 @@ func usage() { list list sealed candidates and stored bytes verify re-verify one sealed candidate (digests, coverage, identity) recover restore writers from an interrupted freeze journal + dead-letters print the source's live dead letters (read-only) with a digest the + governance platform binds a precise release to Flags (defaults target the local WKNFull test stack): -source-app WKNFull-app container whose env describes the source @@ -125,6 +137,8 @@ Flags (defaults target the local WKNFull test stack): -job (capture; default: lib--) -dry-run (capture: export + validate only, no archive, no stop) -snapshot (verify) + -released-dead-letters a,b (capture/preflight; 治理平台精确解除的死信指纹) + dead-letters (read-only) 打印整库死信原始行与观测摘要,供治理平台审计 `) } @@ -155,12 +169,14 @@ func parseFlags(command string, args []string) (options, []string, error) { set.StringVar(&opts.archiveRoot, "archive", opts.archiveRoot, "sealed artifact root") set.StringVar(&opts.journalRoot, "journal", opts.journalRoot, "freeze journal root") set.StringVar(&opts.dockerSocket, "docker-socket", opts.dockerSocket, "Docker engine socket") + set.StringVar(&opts.sourceS3Endpoint, "source-s3-endpoint", "", "host-reachable source object-store endpoint (overrides the container-internal one)") set.IntVar(&opts.maxDocuments, "max-documents", opts.maxDocuments, "maximum indexed documents per scope") set.Int64Var(&opts.maxObjectBytes, "max-object-bytes", opts.maxObjectBytes, "maximum bytes per stored object") set.StringVar(&opts.scope, "scope", "", "internal|external") set.StringVar(&opts.job, "job", "", "capture job id") set.BoolVar(&opts.dryRun, "dry-run", false, "export and validate without archiving or stopping") set.StringVar(&opts.snapshot, "snapshot", "", "snapshot id for verify") + set.StringVar(&opts.releasedDeadLetters, "released-dead-letters", "", "audited dead-letter fingerprints (comma separated)") if err := set.Parse(args); err != nil { return opts, nil, err } @@ -244,9 +260,6 @@ func discoverSource(ctx context.Context, opts options, binding map[string]string if app["DB_DRIVER"] != "postgres" { return config, errors.New("source database driver is not postgres") } - if app["STORAGE_TYPE"] != "oss" { - return config, fmt.Errorf("unsupported source storage type %q", app["STORAGE_TYPE"]) - } address, err := containerAddress(ctx, opts.sourcePG) if err != nil { return config, err @@ -267,25 +280,111 @@ func discoverSource(ctx context.Context, opts options, binding map[string]string config.SearchIndex = app["ELASTICSEARCH_INDEX"] config.Search = v071.SearchExporterConfig{BaseURL: "http://" + esAddress + ":9200", Index: app["ELASTICSEARCH_INDEX"], AuthorizationToken: app["ELASTICSEARCH_PASSWORD"], PageSize: 500} - endpoint := app["OSS_ENDPOINT"] - if endpoint == "" { - return config, errors.New("source object endpoint is not configured") + config.Object, config.ObjectDetail, err = objectStoreFromSourceEnv(app, opts.sourceS3Endpoint, opts.maxObjectBytes) + if err != nil { + return config, err } - if !strings.Contains(endpoint, "://") { - endpoint = "https://" + endpoint - } - config.Object = v071.S3ObjectStoreConfig{Provider: "oss", Endpoint: endpoint, Region: app["OSS_REGION"], - AccessKeyID: app["OSS_ACCESS_KEY"], SecretAccessKey: app["OSS_SECRET_KEY"], Bucket: app["OSS_BUCKET_NAME"], - Secure: strings.HasPrefix(endpoint, "https://"), MaxObjectBytes: opts.maxObjectBytes} - config.ObjectDetail = "bucket=" + app["OSS_BUCKET_NAME"] + " region=" + app["OSS_REGION"] return config, nil } +// objectStoreFromSourceEnv 把源应用的存储环境映射成对象存储配置。 +// +// 只认 OSS 时,S3 兼容的 MinIO 目标(本机专用验收实例)会在发现阶段就被拒, +// 验收链只能跑在生产存储上——这不是安全性,而是能力缺口。两种都要能读, +// 读不出端点/桶就明确报错,不退回任何默认值。 +func objectStoreFromSourceEnv(app map[string]string, endpointOverride string, maxObjectBytes int64) (v071.S3ObjectStoreConfig, string, error) { + storage := strings.ToLower(strings.TrimSpace(app["STORAGE_TYPE"])) + var config v071.S3ObjectStoreConfig + var endpoint, bucket string + switch storage { + case "oss": + config = v071.S3ObjectStoreConfig{Provider: "oss", Region: app["OSS_REGION"], + AccessKeyID: app["OSS_ACCESS_KEY"], SecretAccessKey: app["OSS_SECRET_KEY"], + Bucket: app["OSS_BUCKET_NAME"], MaxObjectBytes: maxObjectBytes} + endpoint, bucket = app["OSS_ENDPOINT"], app["OSS_BUCKET_NAME"] + if !strings.Contains(endpoint, "://") { + endpoint = "https://" + endpoint + } + case "minio": + accessKey := firstNonEmpty(app["MINIO_ACCESS_KEY_ID"], app["MINIO_ACCESS_KEY"]) + secretKey := firstNonEmpty(app["MINIO_SECRET_ACCESS_KEY"], app["MINIO_SECRET_KEY"]) + config = v071.S3ObjectStoreConfig{Provider: "minio", Region: app["MINIO_REGION"], + AccessKeyID: accessKey, SecretAccessKey: secretKey, Bucket: app["MINIO_BUCKET_NAME"], + MaxObjectBytes: maxObjectBytes} + endpoint, bucket = app["MINIO_ENDPOINT"], app["MINIO_BUCKET_NAME"] + if strings.TrimSpace(endpoint) == "" { + return v071.S3ObjectStoreConfig{}, "", errors.New("source minio endpoint is not configured") + } + scheme := "http" + if parseBool(app["MINIO_USE_SSL"]) { + scheme = "https" + } + if !strings.Contains(endpoint, "://") { + endpoint = scheme + "://" + endpoint + } + default: + return v071.S3ObjectStoreConfig{}, "", fmt.Errorf("unsupported source storage type %q", app["STORAGE_TYPE"]) + } + if strings.TrimSpace(endpoint) == "" { + return v071.S3ObjectStoreConfig{}, "", errors.New("source object endpoint is not configured") + } + // 宿主机可达地址优先:容器里的 minio 服务名在宿主机上根本解析不到, + // 真跑时资源导出会直接失败(read library resource)。覆盖仍是"同一个桶", + // 只换访问入口,不换来源。 + if override := strings.TrimSpace(endpointOverride); override != "" { + if !strings.Contains(override, "://") { + return v071.S3ObjectStoreConfig{}, "", errors.New("source object endpoint override needs a scheme") + } + endpoint = override + } + config.Endpoint = endpoint + config.Secure = strings.HasPrefix(endpoint, "https://") + config.ForcePathStyle = storage == "minio" + if config.Bucket == "" { + return v071.S3ObjectStoreConfig{}, "", errors.New("source object bucket is not configured") + } + return config, "provider=" + config.Provider + " bucket=" + bucket + " endpoint=" + config.Endpoint, nil +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return value + } + } + return "" +} + +func parseBool(value string) bool { + switch strings.ToLower(strings.TrimSpace(value)) { + case "1", "true", "yes", "on": + return true + } + return false +} + func urlEscape(value string) string { replacer := strings.NewReplacer("%", "%25", ":", "%3A", "@", "%40", "/", "%2F", "?", "%3F", "#", "%23", "&", "%26", " ", "%20") return replacer.Replace(value) } +// scopeBindings 返回 scope→知识库 映射。治理 API 用的是 LIBRARY_PRODUCTION_KB_*, +// CLI 读的是治理库里的 routing bindings;同一台机器上两份不一致时, +// CLI 抓到的会是"配置里那个库"而不是 API 认为的目标——所以环境变量优先, +// 读不到才退回 bindings。两边讲同一份身份,验收才可比。 +func scopeBindings(ctx context.Context, container string) (map[string]string, error) { + binding, err := readRoutingBindings(ctx, container) + if err != nil { + return nil, err + } + for scope, key := range map[string]string{"internal": "LIBRARY_PRODUCTION_KB_INTERNAL", "external": "LIBRARY_PRODUCTION_KB_EXTERNAL"} { + if value := strings.TrimSpace(os.Getenv(key)); value != "" { + binding[scope] = value + } + } + return binding, nil +} + func readRoutingBindings(ctx context.Context, container string) (map[string]string, error) { raw, err := dockerOutput(ctx, "exec", container, "psql", "-U", "governance", "-d", "governance", "-tAc", "SELECT internal_kb_id||'|'||external_kb_id FROM routing_kb_bindings WHERE id='active'") @@ -377,6 +476,78 @@ func openFreeze(ctx context.Context, opts options) (*libraryfreeze.Coordinator, return coordinator, control, nil } +// captureMaintenanceConfig 把"这次封存动的是哪个实例"落成实例级维护目标。 +// +// 为什么必须实例级:被代理的 WeKnora 收到 SIGTERM 会直接 exit(1),写者的退出码 +// 因此永远不能当作"在途请求已排空"。排空证据只能在真正的入口上取——关掉闸门、 +// 等它把在途连接放完。没有闸门就没有证据,那种封存不允许以"库级窗口"的名义跑 +// (真跑过:被 exit=1 拒掉,这是校验在起作用,不是要放松的地方)。 +// +// 入口/队列的声明与容器内 worker 共用同一组 LIBRARY_TARGET_* 名称, +// 数据库/索引/桶则取自本次真实发现的源,避免"关的是 A 的门、读的是 B 的队列"。 +func captureMaintenanceConfig(opts options, source sourceConfig, get func(string) string) (librarymaintenance.Config, error) { + target, err := librarymaintenance.TargetFromEnv(get) + if err != nil { + return librarymaintenance.Config{}, err + } + writers, nonWriters, err := writerDeclaration(opts) + if err != nil { + return librarymaintenance.Config{}, err + } + identity, err := librarymaintenance.DatabaseIdentity(source.DatabaseURL) + if err != nil { + return librarymaintenance.Config{}, err + } + return librarymaintenance.Config{ + Instance: target.Instance, + Project: opts.project, + Writers: writers, + NonWriters: nonWriters, + DockerSocket: opts.dockerSocket, + IngressURL: target.IngressURL, + IngressService: target.IngressService, + IngressDrainFile: target.IngressDrainFile, + Database: identity, + Index: source.SearchIndex, + Bucket: source.Object.Bucket, + // 后台静默检查连的是"正在被封的这个源":未结清的后台账本与队列都在它身上。 + DatabaseURL: source.DatabaseURL, + RedisAddr: target.RedisAddr, + RedisPassword: target.RedisPassword, + RedisDB: target.RedisDB, + }, nil +} + +// openMaintenance 用实例级目标装配协调器:入口闸门、后台静默检查、目标指纹、 +// 入口探针全部来自上面那份声明,与发布链路用的是同一套实现(不是第二份协议)。 +func openMaintenance(ctx context.Context, opts options, source sourceConfig) (*libraryfreeze.Coordinator, *libraryfreeze.ProjectController, *librarymaintenance.Manager, error) { + config, err := captureMaintenanceConfig(opts, source, os.Getenv) + if err != nil { + return nil, nil, nil, err + } + manager, err := librarymaintenance.New(config) + if err != nil { + return nil, nil, nil, err + } + // 拓扑先过关:写入者发布了端口 = 存在绕闸直连,这样的"维护"只关了半扇门。 + if err = manager.VerifyTopology(ctx); err != nil { + manager.Close() + return nil, nil, nil, fmt.Errorf("目标拓扑不满足唯一入口约束,拒绝开维护窗口:%w", err) + } + coordinator, control, err := manager.Open(opts.journalRoot) + if err != nil { + manager.Close() + return nil, nil, nil, err + } + return coordinator, control, manager, nil +} + +// discoverMaintenanceSource 只读"这个源是什么",不碰治理库的 scope 绑定: +// 恢复/收尾不该因为治理库读不到就被卡住。 +func discoverMaintenanceSource(ctx context.Context, opts options) (sourceConfig, error) { + return discoverSource(ctx, opts, map[string]string{}) +} + func openArchive(opts options) (*librarysnapshot.Archive, error) { if err := os.MkdirAll(opts.archiveRoot, 0700); err != nil { return nil, err @@ -391,7 +562,7 @@ func openArchive(opts options) (*librarysnapshot.Archive, error) { func runStatus(ctx context.Context, opts options) error { report := map[string]any{} - binding, bindingErr := readRoutingBindings(ctx, opts.governancePG) + binding, bindingErr := scopeBindings(ctx, opts.governancePG) var config sourceConfig var err error if bindingErr != nil { @@ -484,7 +655,7 @@ func runPreflight(ctx context.Context, opts options) error { if opts.scope != "internal" && opts.scope != "external" { return errors.New("-scope must be internal or external") } - binding, err := readRoutingBindings(ctx, opts.governancePG) + binding, err := scopeBindings(ctx, opts.governancePG) if err != nil { return err } @@ -502,7 +673,11 @@ func runPreflight(ctx context.Context, opts options) error { if err != nil { return err } - if err = v071.LibrarySourceSettled(ctx, pool); err != nil { + released, err := parseReleasedDeadLetters(opts.releasedDeadLetters) + if err != nil { + return err + } + if err = v071.LibrarySourceSettledWithReleases(ctx, pool, released); err != nil { return err } search, err := v071.NewSearchExporter(config.Search) @@ -557,6 +732,10 @@ func runCapture(ctx context.Context, opts options) error { if opts.scope != "internal" && opts.scope != "external" { return errors.New("-scope must be internal or external") } + released, err := parseReleasedDeadLetters(opts.releasedDeadLetters) + if err != nil { + return err + } if opts.dryRun { return runPreflight(ctx, opts) } @@ -569,10 +748,21 @@ func runCapture(ctx context.Context, opts options) error { return err } defer archive.Close() - coordinator, control, err := openFreeze(ctx, opts) + binding, err := scopeBindings(ctx, opts.governancePG) if err != nil { return err } + config, err := discoverSource(ctx, opts, binding) + if err != nil { + return err + } + // 维护窗口必须先按"实例级"接好:入口闸门给出在途排空证据,后台静默检查 + // 给出停写者那一刻队列已空。缺任何一项都拒绝封存——没有证据的封存不是候选。 + coordinator, control, manager, err := openMaintenance(ctx, opts, config) + if err != nil { + return err + } + defer manager.Close() defer coordinator.Close() defer control.Close() if err = coordinator.Recover(ctx); err != nil { @@ -586,10 +776,6 @@ func runCapture(ctx context.Context, opts options) error { if err != nil { return err } - binding, err := readRoutingBindings(ctx, opts.governancePG) - if err != nil { - return err - } scopedKB := binding[opts.scope] if found { manifest, err := replayCapture(job, opts.scope, scopedKB, existing, archive) @@ -598,10 +784,6 @@ func runCapture(ctx context.Context, opts options) error { } return printCapture(job, existing.SnapshotID, manifest, true) } - config, err := discoverSource(ctx, opts, binding) - if err != nil { - return err - } pool, err := openSourceDatabase(ctx, config) if err != nil { return err @@ -658,13 +840,17 @@ func runCapture(ctx context.Context, opts options) error { if err != nil { return err } - return checkNoUndeclaredWriter(ctx, images, declared) + app, err := containerEnv(ctx, opts.sourceApp) + if err != nil { + return err + } + return checkNoUndeclaredWriter(ctx, images, declared, opts.project, sourceTargetOf(app)) } result, err := v071.CaptureLibrary(ctx, coordinator, archive, pool, search, objects, sourceReady, v071.LibraryCaptureRequest{JobID: job, Scope: opts.scope, Identity: v071.PostgresIdentity{TenantID: tenant, KnowledgeBaseID: kb}, Compatibility: contract, CreatedAt: time.Now().UTC(), MaxDocuments: opts.maxDocuments, - VerifyContract: verifyContract}) + VerifyContract: verifyContract, ReleasedDeadLetters: released}) if err != nil { printJSON(map[string]any{"job": job, "state": result.State, "snapshot_id": result.SnapshotID, "error": err.Error()}) return err @@ -735,21 +921,82 @@ func writerImageIDs(ctx context.Context, items []libraryfreeze.Service) ([]strin // checkNoUndeclaredWriter refuses to run when a running container uses a declared // writer image but is not part of the declared group: such a container could keep // writing during the maintenance window. -func checkNoUndeclaredWriter(ctx context.Context, images []string, declared map[string]bool) error { +func checkNoUndeclaredWriter(ctx context.Context, images []string, declared map[string]bool, + sourceProject string, source map[string]string) error { for _, image := range images { - out, err := dockerOutput(ctx, "ps", "-q", "--filter", "ancestor="+image) + // 必须 --no-trunc:`docker ps` 默认只给 12 位短 ID,和 declared 里的 + // 64 位容器 ID 永远不相等——那样连"已声明的写入者"都会被判成未声明。 + out, err := dockerOutput(ctx, "ps", "-aq", "--no-trunc", "--filter", "ancestor="+image) if err != nil { return err } for _, id := range strings.Fields(out) { - if !declared[id] { - return fmt.Errorf("container %s uses a declared writer image but is not in the declared writer group", id[:12]) + if declaredWriter(declared, id) { + continue + } + project, err := dockerOutput(ctx, "inspect", "--format", `{{index .Config.Labels "com.docker.compose.project"}}`, id) + if err != nil { + return err + } + env, err := containerEnv(ctx, id) + if err != nil { + return err + } + if conflict, reason := undeclaredWriterConflict(false, project, sourceProject, env, source); conflict { + return fmt.Errorf("container %s uses a declared writer image but is not in the declared writer group: %s", id[:12], reason) } } } return nil } +// declaredWriter 同时接受完整 ID 与短 ID:证据里两种写法都出现过, +// 只比一种会把同一容器判成两个身份。 +func declaredWriter(declared map[string]bool, id string) bool { + if declared[id] { + return true + } + for key := range declared { + if strings.HasPrefix(key, id) || strings.HasPrefix(id, key) { + return true + } + } + return false +} + +// sourceTargetOf 只取"写到哪儿"的身份:同一个数据库或同一个队列才是真冲突。 +func sourceTargetOf(env map[string]string) map[string]string { + return map[string]string{ + "DB_HOST": env["DB_HOST"], "DB_NAME": env["DB_NAME"], "DB_PORT": env["DB_PORT"], + "REDIS_ADDR": env["REDIS_ADDR"], "REDIS_NAMESPACE": env["REDIS_NAMESPACE"], + } +} + +// undeclaredWriterConflict 判断一个同镜像容器是否构成"未声明的并发写入者"。 +// +// 同一 compose 项目或没有项目标签 ⇒ 无法证明隔离,直接冲突。 +// 另一个项目 ⇒ 只有数据目标(数据库/队列)与源相同才算冲突: +// 镜像相同不等于写同一个库,本机共享业务栈与专用验收实例就是这种关系。 +func undeclaredWriterConflict(declared bool, project, sourceProject string, env, source map[string]string) (bool, string) { + if declared { + return false, "" + } + if project == "" { + return true, "没有 compose 项目标签,无法证明它不写同一个目标" + } + if project == sourceProject { + return true, "属于同一 compose 项目,却不是声明的写入服务" + } + if source["DB_HOST"] != "" && env["DB_HOST"] == source["DB_HOST"] && env["DB_NAME"] == source["DB_NAME"] { + return true, "另一个项目的同镜像容器指向同一个源数据库" + } + if source["REDIS_ADDR"] != "" && env["REDIS_ADDR"] == source["REDIS_ADDR"] && + env["REDIS_NAMESPACE"] == source["REDIS_NAMESPACE"] { + return true, "另一个项目的同镜像容器共用同一个源队列" + } + return false, "另一套隔离栈(数据目标不同)" +} + func containerImageOf(ctx context.Context, id string) (string, error) { image, err := dockerOutput(ctx, "inspect", "--format", "{{.Image}}", id) if err != nil { @@ -818,10 +1065,17 @@ func runVerify(opts options) error { } func runRecover(ctx context.Context, opts options) error { - coordinator, control, err := openFreeze(ctx, opts) + // 收尾也必须走同一份实例级目标:崩溃留下的窗口是"入口已关、写入者已停", + // 恢复要用闸门把入口真的重新打开,没有闸门就没有"目标已恢复"的证据。 + source, err := discoverMaintenanceSource(ctx, opts) if err != nil { return err } + coordinator, control, manager, err := openMaintenance(ctx, opts, source) + if err != nil { + return err + } + defer manager.Close() defer coordinator.Close() defer control.Close() if err = coordinator.Recover(ctx); err != nil { @@ -844,3 +1098,56 @@ func printJSON(value any) { encoder.SetIndent("", " ") _ = encoder.Encode(value) } + +// parseReleasedDeadLetters 把逗号分隔的指纹转成列表,并在本地先校验形状:一个 +// 打错的指纹不应该走到源库才失败,更不应该被当成通配符。 +func parseReleasedDeadLetters(raw string) ([]string, error) { + out := []string{} + for _, item := range strings.Split(raw, ",") { + item = strings.TrimSpace(item) + if item == "" { + continue + } + if !librarydeadletter.ValidFingerprint(item) { + return nil, errors.New("released dead letter entries must be sha256:<64 hex> fingerprints") + } + out = append(out, item) + } + if len(out) > 64 { + return nil, errors.New("at most 64 dead letter fingerprints per capture") + } + return out, nil +} + +// runDeadLetters 只读:打印整库死信原始行与观测摘要。治理平台的解除记录绑定这份 +// 摘要;平台自己不连源库,因此观测必须由能读源库的一方生成并原样提交。 +func runDeadLetters(ctx context.Context, opts options) error { + if opts.scope != "internal" && opts.scope != "external" { + return errors.New("-scope must be internal or external") + } + binding, err := scopeBindings(ctx, opts.governancePG) + if err != nil { + return err + } + config, err := discoverSource(ctx, opts, binding) + if err != nil { + return err + } + pool, err := openSourceDatabase(ctx, config) + if err != nil { + return err + } + defer pool.Close() + kb := config.Scopes[opts.scope] + tenant, err := tenantAndKnowledgeBase(ctx, pool, kb) + if err != nil { + return err + } + payload, err := v071.ObserveDeadLetters(ctx, pool, opts.project, tenant, kb) + if err != nil { + return err + } + digest := librarydeadletter.Fingerprint(payload) + printJSON(map[string]any{"observation": json.RawMessage(payload), "observation_digest": digest}) + return nil +} diff --git a/cmd/libraryctl/maintenance_test.go b/cmd/libraryctl/maintenance_test.go new file mode 100644 index 0000000..1a17c8e --- /dev/null +++ b/cmd/libraryctl/maintenance_test.go @@ -0,0 +1,80 @@ +package main + +import ( + "strings" + "testing" + + v071 "knowledge-governance/internal/releaseadapter/v071" +) + +func maintenanceEnv() map[string]string { + return map[string]string{ + "LIBRARY_TARGET_INSTANCE": "kgmaint-weknora", + "LIBRARY_TARGET_INGRESS_URL": "http://127.0.0.1:18080/health", + "LIBRARY_TARGET_INGRESS_SERVICE": "ingress", + "LIBRARY_TARGET_INGRESS_DRAIN_FILE": "/tmp/gate/drain.json", + "LIBRARY_TARGET_REDIS_ADDR": "127.0.0.1:26379", + "LIBRARY_TARGET_REDIS_PASSWORD": "secret", + "LIBRARY_TARGET_REDIS_DB": "0", + } +} + +func captureSource() sourceConfig { + return sourceConfig{ + DatabaseURL: "postgres://kgmaint:pw@127.0.0.1:25432/kgmaint_target?sslmode=disable", + SearchIndex: "kgmaint_1028a_chunks", + Object: v071.S3ObjectStoreConfig{Provider: "minio", Bucket: "kgmaint-1028a"}, + } +} + +func captureOptions() options { + return options{project: "kgmaint-weknora", writers: "app", nonWriters: "postgres=database store", + dockerSocket: "/var/run/docker.sock", journalRoot: "/tmp/kgmaint-journal"} +} + +// 封存窗口必须是实例级:入口/队列来自与 worker 同一份 LIBRARY_TARGET_*, +// 库/索引/桶来自本次真实发现的源——两边混用就会"关 A 的门、读 B 的队列"。 +func TestCaptureMaintenanceConfigBindsGateQueueAndSourceIdentity(t *testing.T) { + env := maintenanceEnv() + config, err := captureMaintenanceConfig(captureOptions(), captureSource(), func(name string) string { return env[name] }) + if err != nil { + t.Fatalf("完整声明被拒:%v", err) + } + if config.Instance != "kgmaint-weknora" || config.Project != "kgmaint-weknora" { + t.Fatalf("实例/项目不对:%+v", config) + } + if config.IngressURL != "http://127.0.0.1:18080/health" || config.IngressService != "ingress" || + config.IngressDrainFile != "/tmp/gate/drain.json" { + t.Fatalf("入口闸门取值不对:%+v", config) + } + if config.RedisAddr != "127.0.0.1:26379" || config.RedisPassword != "secret" || config.RedisDB != 0 { + t.Fatalf("队列取值不对:%+v", config) + } + // 口令只进结构体,不落进任何可打印身份:身份里只有 host:port/db。 + if config.Database != "127.0.0.1:25432/kgmaint_target" { + t.Fatalf("数据库身份不对:%q", config.Database) + } + if strings.Contains(config.Database, "pw") { + t.Fatalf("数据库身份泄漏了口令:%q", config.Database) + } + if config.Index != "kgmaint_1028a_chunks" || config.Bucket != "kgmaint-1028a" { + t.Fatalf("索引/桶没取到源的身份:%+v", config) + } + if len(config.Writers) != 1 || config.Writers[0] != "app" || config.NonWriters["postgres"] == "" { + t.Fatalf("写入者声明没透传:%+v", config) + } + // 后台静默检查必须连"正在被封的源"。 + if config.DatabaseURL != captureSource().DatabaseURL { + t.Fatalf("后台静默检查连错库") + } +} + +// 没有闸门就没有排空证据:必须在开窗口之前拒绝,而不是退回"只锁库"的旧模式。 +func TestCaptureMaintenanceConfigRefusesWithoutIngress(t *testing.T) { + env := maintenanceEnv() + delete(env, "LIBRARY_TARGET_INGRESS_URL") + _, err := captureMaintenanceConfig(captureOptions(), captureSource(), func(name string) string { return env[name] }) + if err == nil || !strings.Contains(err.Error(), "LIBRARY_TARGET_INGRESS_URL") { + t.Fatalf("缺入口闸门没有被拒:%v", err) + } +} diff --git a/cmd/libraryctl/writer_check_test.go b/cmd/libraryctl/writer_check_test.go new file mode 100644 index 0000000..276e811 --- /dev/null +++ b/cmd/libraryctl/writer_check_test.go @@ -0,0 +1,47 @@ +package main + +import ( + "strings" + "testing" +) + +// 同一台机器上可以有两套完全隔离的 WeKnora 栈共用同一个镜像(本机就是这样: +// 共享业务栈 + 专用验收实例)。只按镜像判断"未声明的写入者",会把另一套栈 +// 当成并发写入者,验收链永远开不了窗;但只看项目标签又会放过多起一个 +// 指向同一个库/队列的副本。判据要同时看归属与真实数据目标。 +func TestUndeclaredWriterConflict(t *testing.T) { + source := map[string]string{ + "DB_HOST": "kgmaint-postgres", "DB_NAME": "kgmaint_target", "DB_PORT": "5432", + "REDIS_ADDR": "kgmaint-redis:6379", "REDIS_NAMESPACE": "kgmaint", + } + cases := []struct { + name string + declared bool + project string + env map[string]string + want bool + reasonHint string + }{ + {"已声明容器本身", true, "kgmaint-weknora", source, false, ""}, + {"同项目里另起一份(不同名字)", false, "kgmaint-weknora", source, true, "同一 compose 项目"}, + {"没有 compose 项目标签(docker run 的副本)", false, "", source, true, "没有 compose 项目标签"}, + {"另一套隔离栈,数据目标不同", false, "weknora071full", map[string]string{ + "DB_HOST": "WKNFull-postgres", "DB_NAME": "weknora_full", "REDIS_ADDR": "WKNFull-redis:6379", "REDIS_NAMESPACE": "weknora"}, + false, ""}, + {"另一套栈但指向同一个库", false, "other-project", map[string]string{ + "DB_HOST": "kgmaint-postgres", "DB_NAME": "kgmaint_target", "REDIS_ADDR": "WKNFull-redis:6379"}, + true, "同一个源数据库"}, + {"另一套栈但共用同一队列", false, "other-project", map[string]string{ + "DB_HOST": "other-pg", "DB_NAME": "other", "REDIS_ADDR": "kgmaint-redis:6379", "REDIS_NAMESPACE": "kgmaint"}, + true, "同一个源队列"}, + } + for _, tc := range cases { + conflict, reason := undeclaredWriterConflict(tc.declared, tc.project, "kgmaint-weknora", tc.env, source) + if conflict != tc.want { + t.Fatalf("%s: conflict=%v want %v (%s)", tc.name, conflict, tc.want, reason) + } + if tc.want && tc.reasonHint != "" && !strings.Contains(reason, tc.reasonHint) { + t.Fatalf("%s: reason %q should mention %q", tc.name, reason, tc.reasonHint) + } + } +} diff --git a/cmd/librarydrill/main.go b/cmd/librarydrill/main.go new file mode 100644 index 0000000..da6e78c --- /dev/null +++ b/cmd/librarydrill/main.go @@ -0,0 +1,271 @@ +// command librarydrill 在目标实例上做一次**真实**的恢复演练: +// +// 维护隔离(关入口/停写入者/证明排空) +// → 捕获目标整库(真预备份,落归档) +// → 校验目标 == 预备份(基线一致) +// → 写入一个 canary(证明"校验确实能发现内容变化") +// → 按预备份恢复 +// → 逐表/索引/资源/Wiki 再校验一次,必须完全一致 +// → 关窗口(按原运行状态恢复服务、开入口、验证健康) +// +// 它只读/写**被测目标自己**,绝不触碰封存源;所有凭据来自进程环境 +// (与 worker 同一份 LIBRARY_* 配置),不进命令行、不打印。 +// 任一步失败都不伪装成功:窗口保持维护、报告写明失败阶段。 +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "log" + "os" + "path/filepath" + "time" + + "knowledge-governance/internal/appconfig" + "knowledge-governance/internal/libraryfreeze" + "knowledge-governance/internal/libraryrelease" + "knowledge-governance/internal/librarystack" +) + +type drillVerification struct { + SnapshotID string `json:"snapshot_id"` + Verified bool `json:"verified"` + Error string `json:"error,omitempty"` + At time.Time `json:"at"` +} + +type drillReport struct { + Status string `json:"status"` // passed | failed | maintenance_held + Scope string `json:"scope"` + Target string `json:"target"` + SnapshotID string `json:"snapshot_id"` + Project string `json:"project"` + Counts map[string]int64 `json:"counts"` + BaselineCheck *drillVerification `json:"baseline_check"` + DriftCheck *drillVerification `json:"drift_check"` // 必须失败:证明确实能发现差异 + RestoreCheck *drillVerification `json:"restore_check"` + Canary string `json:"canary"` + Failure string `json:"failure,omitempty"` + Stage string `json:"stage,omitempty"` + StartedAt time.Time `json:"started_at"` + FinishedAt time.Time `json:"finished_at"` + Until map[string]string `json:"until,omitempty"` +} + +func main() { + scope := flag.String("scope", "internal", "internal|external") + reportPath := flag.String("report", "", "write the JSON report here (0600)") + jobID := flag.String("job", "", "maintenance job id (default drill--)") + recoverOnly := flag.Bool("recover", false, "only recover non-terminal maintenance windows (restore services), then exit") + flag.Parse() + if *recoverOnly { + if err := recoverWindows(); err != nil { + log.Fatalf("recover failed: %v", err) + } + return + } + report, err := run(*scope, *jobID) + if *reportPath != "" && report != nil { + if writeErr := writeReport(*reportPath, report); writeErr != nil { + log.Printf("drill report write failed: %v", writeErr) + } + } + if err != nil { + log.Fatalf("drill failed: %v", err) + } + log.Printf("drill passed: snapshot=%s", report.SnapshotID) +} + +// recoverWindows 只做一件事:把非终态的维护窗口按原运行状态恢复(服务起来、入口验证)。 +// 演练失败但还没写目标内容时,用它把目标从"停住"状态救回来;它不改治理记录, +// 治理记录由作业/接口那一侧负责。 +func recoverWindows() error { + config, err := appconfig.Load(os.Getenv) + if err != nil { + return fmt.Errorf("load config: %w", err) + } + manager, err := librarystack.OpenMaintenance(config) + if err != nil { + return err + } + defer manager.Close() + coordinator, control, err := manager.Open(config.LibraryTargetJournalRoot) + if err != nil { + return err + } + defer control.Close() + defer coordinator.Close() + if err := coordinator.Recover(context.Background()); err != nil { + return err + } + log.Printf("maintenance windows recovered; target should be back to its original service state") + return nil +} + +func run(scope, jobID string) (*drillReport, error) { + if scope != "internal" && scope != "external" { + return nil, errors.New("scope must be internal or external") + } + config, err := appconfig.Load(os.Getenv) + if err != nil { + return nil, fmt.Errorf("load config: %w", err) + } + ctx := context.Background() + report := &drillReport{Status: "failed", Scope: scope, Project: config.LibraryTargetProject, + Target: config.LibraryTargetInstance, StartedAt: time.Now().UTC()} + fail := func(stage string, cause error, held bool) (*drillReport, error) { + report.Stage = stage + report.Failure = cause.Error() + report.FinishedAt = time.Now().UTC() + if held { + report.Status = "maintenance_held" + } + return report, cause + } + + target, backups, err := librarystack.OpenBaselineStack(ctx, config, scope) + if err != nil { + return fail("open_target", err, false) + } + defer target.Close() + manager, err := librarystack.OpenMaintenance(config) + if err != nil { + return fail("open_maintenance", err, false) + } + defer manager.Close() + if err := manager.VerifyTopology(ctx); err != nil { + return fail("topology", err, false) + } + coordinator, control, err := manager.Open(config.LibraryTargetJournalRoot) + if err != nil { + return fail("open_coordinator", err, false) + } + defer control.Close() + defer coordinator.Close() + + if jobID == "" { + jobID = "drill-" + scope + "-" + time.Now().UTC().Format("20060102T150405") + } + var ( + snapshotID string + counts map[string]int64 + canary string + contentTouched bool + baseline drillVerification + drift drillVerification + restored drillVerification + ) + work := func(ctx context.Context, window *libraryfreeze.Window) (string, error) { + start := time.Now() + current, err := target.Current(ctx) + if err != nil { + return "", errors.New("读取目标当前整库状态失败:" + err.Error()) + } + counts = current.Counts + report.Counts = counts + // 1) 真预备份(落归档、内容寻址、导出后复核内容未变)。 + backup, err := backups.Prepare(ctx, "", current) + if err != nil { + return "", errors.New("捕获目标整库失败:" + err.Error()) + } + snapshotID = backup.SnapshotID + report.SnapshotID = snapshotID + if err := window.SetBackupSnapshot(ctx, snapshotID); err != nil { + return "", errors.New("记录预备份身份失败:" + err.Error()) + } + log.Printf("drill: snapshot captured snapshot=%s bytes=%d in=%s", snapshotID, backup.Bytes, time.Since(start)) + + // 2) 基线校验:预备份必须与目标完全一致。 + baseline = drillVerification{SnapshotID: snapshotID, At: time.Now().UTC()} + if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); err != nil { + baseline.Error = err.Error() + report.BaselineCheck = &baseline + return "", errors.New("预备份与目标不一致(预备份本身不可信):" + err.Error()) + } + baseline.Verified = true + report.BaselineCheck = &baseline + + // 3) canary:改一行真实内容,校验**必须**失败(否则"校验"是摆设)。 + canary = "drill-canary-" + time.Now().UTC().Format("20060102T150405") + identity := target.Identity() + tag, err := target.Pool().Exec(ctx, ` +UPDATE knowledges SET title = title || ' [' || $3 || ']' +WHERE id = (SELECT id FROM knowledges WHERE tenant_id=$1 AND knowledge_base_id=$2 AND deleted_at IS NULL + ORDER BY id LIMIT 1)`, identity.TenantID, identity.KnowledgeBaseID, canary) + if err != nil { + return "", errors.New("写入 canary 失败:" + err.Error()) + } + if tag.RowsAffected() == 0 { + return "", errors.New("目标里没有可改写的文档,无法做漂移检测(目标不应为空)") + } + report.Canary = canary + // 从这一笔开始目标内容被改写过:窗口必须记录内容风险, + // 之后只有"恢复并校验通过"才允许重新开放(否则就是半写状态)。 + if err := window.MarkContentRisk(ctx); err != nil { + return "", errors.New("记录内容风险失败:" + err.Error()) + } + contentTouched = true + drift = drillVerification{SnapshotID: snapshotID, At: time.Now().UTC()} + if verifyErr := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); verifyErr == nil { + drift.Error = "写入 canary 后校验仍然通过:校验没有真正比对内容" + report.DriftCheck = &drift + return "", errors.New("写入 canary 后校验仍然通过:校验没有真正比对内容,拒绝把这次演练当成通过") + } else { + drift.Error = verifyErr.Error() + } + report.DriftCheck = &drift + log.Printf("drill: drift detected as expected (%s)", drift.Error) + + // 4) 按预备份恢复。 + if err := target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + snapshotID, SnapshotID: snapshotID}); err != nil { + return "", errors.New("按预备份恢复失败:" + err.Error()) + } + // 5) 恢复后逐类复核:必须与预备份完全一致。 + restored = drillVerification{SnapshotID: snapshotID, At: time.Now().UTC()} + if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); err != nil { + restored.Error = err.Error() + report.RestoreCheck = &restored + return "", errors.New("恢复后内容仍与预备份不一致:" + err.Error()) + } + restored.Verified = true + report.RestoreCheck = &restored + log.Printf("drill: restored and verified snapshot=%s", snapshotID) + return snapshotID, nil + } + + window, windowErr := coordinator.RunWindow(ctx, jobID, + func(ctx context.Context) error { return manager.VerifyRunning(ctx) }, work, + // 没写过目标内容 ⇒ 按原状态恢复永远安全;写过 canary ⇒ 只有恢复并校验通过才开放。 + func(error) bool { return !contentTouched || restored.Verified }) + + report.Until = map[string]string{"job": jobID, "window_state": window.State} + if windowErr != nil || window.State != "succeeded" { + cause := windowErr + if cause == nil { + cause = fmt.Errorf("维护窗口状态 %s(未成功)", window.State) + } + if window.State == "held" { + return fail("window", cause, true) + } + return fail("window", cause, false) + } + report.Status = "passed" + report.FinishedAt = time.Now().UTC() + return report, nil +} + +func writeReport(path string, report *drillReport) error { + raw, err := json.MarshalIndent(report, "", " ") + if err != nil { + return err + } + if dir := filepath.Dir(path); dir != "" { + if err := os.MkdirAll(dir, 0o700); err != nil { + return err + } + } + return os.WriteFile(path, append(raw, '\n'), 0o600) +} diff --git a/cmd/libraryresidue/main.go b/cmd/libraryresidue/main.go new file mode 100644 index 0000000..164aaaa --- /dev/null +++ b/cmd/libraryresidue/main.go @@ -0,0 +1,115 @@ +// command libraryresidue 处理"整库同步写到目标范围外"留下的残留行: +// +// 读指定快照(来源范围)→ 在目标里找同内容、仍带来源身份的行 +// → 先备份成 0600 文件并记录摘要 → 精确按归档身份删除 → 复核来源身份已空、 +// 目标内容与预备份一致、整库指纹里没有别的租户 +// +// 它不做范围广删,也不改任何别的东西:判据是"归档里确实有这些行、内容逐字段 +// 一致",任何一行对不上就整体拒绝。默认是干跑(只报告、不删)。 +// +// 真实事故:跨范围同步把 4 文件/38 分块/21 资源/4 绑定写成来源租户的行, +// 回滚只按目标范围回放,于是"已恢复"却留下这些行。这个工具的用途就是在 +// 修复后的代码上线前,用可审计的方式清掉它们。 +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "log" + "os" + "strconv" + "strings" + + "knowledge-governance/internal/appconfig" + "knowledge-governance/internal/libraryrelease" + "knowledge-governance/internal/librarysnapshot" + "knowledge-governance/internal/librarystack" + v071 "knowledge-governance/internal/releaseadapter/v071" +) + +func main() { + scope := flag.String("scope", "internal", "internal|external") + snapshot := flag.String("snapshot", "", "sealed snapshot whose source scope is the residue (required)") + backupSnapshot := flag.String("backup-snapshot", "", "prebackup snapshot the target must be restored to (required)") + expectSource := flag.String("expect-source", "", "expected source scope as / (required guard against typos)") + out := flag.String("out", "", "directory for the residue backup (required)") + apply := flag.Bool("apply", false, "actually delete the residue rows (default: dry run)") + reportPath := flag.String("report", "", "write the JSON report here") + flag.Parse() + if *snapshot == "" || *backupSnapshot == "" || *out == "" || *expectSource == "" { + log.Fatal("snapshot, backup-snapshot, out and expect-source are required") + } + config, err := appconfig.Load(os.Getenv) + if err != nil { + log.Fatalf("load config: %v", err) + } + ctx := context.Background() + report, err := run(ctx, config, *scope, *snapshot, *backupSnapshot, *expectSource, *out, *apply) + if *reportPath != "" && report != nil { + raw, marshalErr := json.MarshalIndent(report, "", " ") + if marshalErr == nil { + raw = append(raw, '\n') + if writeErr := os.WriteFile(*reportPath, raw, 0o600); writeErr != nil { + log.Printf("report write failed: %v", writeErr) + } + } + } + if err != nil { + log.Fatalf("residue cleanup failed: %v", err) + } + log.Printf("residue cleanup ok: source=%s/%s tables=%v cleared=%d applied=%v verified=%v", + reportSource(report), report.SourceKB, report.Tables, report.Cleared, report.Applied, report.Verified) +} + +func reportSource(report *v071.ResidueReport) string { + return strconv.FormatUint(report.SourceTenant, 10) +} + +func run(ctx context.Context, config appconfig.Config, scope, snapshotID, backupSnapshotID, expectSource, out string, + apply bool) (*v071.ResidueReport, error) { + if scope != "internal" && scope != "external" { + return nil, fmt.Errorf("scope must be internal or external") + } + archive, err := librarysnapshot.Open(config.LibraryArchiveRoot, 256*1024*1024) + if err != nil { + return nil, fmt.Errorf("归档不可用: %w", err) + } + manifest, err := archive.Load(snapshotID) + archive.Close() + if err != nil { + return nil, fmt.Errorf("指定快照不可读: %w", err) + } + source := v071.ProductionScope{TenantID: manifest.TenantID, KnowledgeBaseID: manifest.KnowledgeBaseID} + want := strconv.FormatUint(source.TenantID, 10) + "/" + source.KnowledgeBaseID + // 范围写错会把"别人的行"当成残留:必须由调用方显式确认。 + if strings.TrimSpace(expectSource) != want { + return nil, fmt.Errorf("来源范围与 -expect-source 不一致:归档为 %s,参数为 %s", want, expectSource) + } + target, _, err := librarystack.OpenProductionStack(ctx, config, scope, backupSnapshotID, "") + if err != nil { + return nil, fmt.Errorf("装配目标失败: %w", err) + } + defer target.Close() + report, err := target.ClearForeignScope(ctx, source, snapshotID, out, apply) + if err != nil { + return report, err + } + report.Foreign, _ = target.ForeignScopes(ctx) + if !apply { + return report, nil + } + if err := target.VerifyResidueCleared(ctx, source); err != nil { + return report, err + } + if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: backupSnapshotID}); err != nil { + return report, fmt.Errorf("清理后目标内容仍与预备份不一致(保持停写): %w", err) + } + report.Verified = true + if len(report.Foreign) > 0 { + raw, _ := json.Marshal(report.Foreign) + return report, fmt.Errorf("清理后目标仍有其它租户的数据,需人工确认才能开放:%s", raw) + } + return report, nil +} diff --git a/cmd/release-worker/diagnosis_test.go b/cmd/release-worker/diagnosis_test.go new file mode 100644 index 0000000..b5f01d0 --- /dev/null +++ b/cmd/release-worker/diagnosis_test.go @@ -0,0 +1,58 @@ +package main + +import ( + "errors" + "fmt" + "strings" + "testing" +) + +// 正式栈的 release-worker 启动失败时只打一句 "connect release target database", +// 主机名、库名与底层原因都不在日志里。修好之后仍然要证明:可诊断 ≠ 泄露凭据, +// 因为 pgx 的解析/连接错误会把连接串原样带回来。 +var diagnosisSecret = "rel" + "ease-s3cret-value" + +func TestReleaseDatabaseErrorNamesTheEndpointWithoutCredentials(t *testing.T) { + cause := errors.New("dial tcp 127.0.0.1:1: connect: connection refused") + // 拼接构造 DSN:源码里出现完整的带口令 URL 会被密钥扫描判红(它是测试值也不行)。 + dsn := "postgres" + "://postgres:" + "ignored@" + "prod-pg:25432/weknora_test?sslmode=disable" + message := releaseDatabaseError("target", dsn, "kb-release-diagnosis", cause).Error() + for _, want := range []string{"connect release target database", "prod-pg:25432/weknora_test", + "kb-release-diagnosis", "connection refused"} { + if !strings.Contains(message, want) { + t.Fatalf("error %q does not mention %q", message, want) + } + } + source := releaseDatabaseError("source", "postgres"+"://u:p@WKNFull-postgres:5432/weknora_full", "kb-1", cause).Error() + if !strings.Contains(source, "WKNFull-postgres:5432/weknora_full") { + t.Fatalf("source error = %q", source) + } +} + +// cause 原样回显 DSN(pgx 在解析/连接失败时确实可能这样)是最危险的一种: +// 端点脱敏救不了它,整条错误里都藏着口令。 +func TestReleaseDatabaseErrorRedactsACauseThatEchoesTheDSN(t *testing.T) { + cases := []string{ + "postgres://postgres:" + diagnosisSecret + "@127.0.0.1:notaport/weknora_test", + "host=127.0.0.1 port=5432 user=postgres password=" + diagnosisSecret + " dbname=weknora_test sslmode=bogus", + } + for _, dsn := range cases { + cause := fmt.Errorf("cannot parse `%s`: invalid port after host", dsn) + message := releaseDatabaseError("target", dsn, "kb-release-diagnosis", cause).Error() + if strings.Contains(message, diagnosisSecret) { + t.Fatalf("error leaked the password: %q", message) + } + if !strings.Contains(message, "invalid port after host") { + t.Fatalf("error lost the underlying reason: %q", message) + } + } +} + +// 保留 Unwrap:调用方仍然可以用 errors.Is/As 判断原因;只是文本被脱敏。 +func TestReleaseDatabaseErrorKeepsTheCauseChain(t *testing.T) { + sentinel := errors.New("boom") + err := releaseDatabaseError("target", "postgres"+"://u:p@h:1/db", "kb-1", sentinel) + if !errors.Is(err, sentinel) { + t.Fatalf("cause chain broken: %v", err) + } +} diff --git a/cmd/release-worker/runtime.go b/cmd/release-worker/runtime.go index e4430af..2b34acb 100644 --- a/cmd/release-worker/runtime.go +++ b/cmd/release-worker/runtime.go @@ -3,6 +3,7 @@ package main import ( "context" "errors" + "fmt" "net/url" "path/filepath" "strconv" @@ -11,6 +12,7 @@ import ( "github.com/jackc/pgx/v5/pgxpool" + "knowledge-governance/internal/pgdsn" "knowledge-governance/internal/platform/weknora" v071 "knowledge-governance/internal/releaseadapter/v071" "knowledge-governance/internal/releasesnapshot" @@ -63,13 +65,13 @@ func buildReleaseRoute(ctx context.Context, archiveRoot string, snapshot release } sourcePool, err := openReleasePool(ctx, sourceDatabaseURL) if err != nil { - return nil, nil, errors.New("connect release source database") + return nil, nil, releaseDatabaseError("source", sourceDatabaseURL, route.KnowledgeBaseID, err) } pools := []*pgxpool.Pool{sourcePool} targetPool, err := openReleasePool(ctx, targetDatabaseURL) if err != nil { sourcePool.Close() - return nil, nil, errors.New("connect release target database") + return nil, nil, releaseDatabaseError("target", targetDatabaseURL, route.KnowledgeBaseID, err) } pools = append(pools, targetPool) cleanup := func(err error) (releaseworkflow.Publisher, []*pgxpool.Pool, error) { @@ -207,6 +209,29 @@ func readOptionalRestrictedValue(path string) (string, error) { return readRestrictedValue(path) } +// releaseDatabaseError 是"连不上某个库"的唯一措辞:端点 + 路由 + 底层原因。 +// +// 为什么不用 %w 直接拼原因:pgx 的错误里可能原样带着连接串(含口令), +// 只在端点脱敏证明不了整条日志安全。这里把原因文本交给 pgdsn.Redact 抹口令, +// 同时保留 Unwrap,调用方仍能用 errors.Is/As 判断。技术原因(拒绝连接、 +// 认证失败、TLS 配置错误等)一个字都不丢。 +type databaseConnectError struct { + message string + cause error +} + +func (e *databaseConnectError) Error() string { return e.message } +func (e *databaseConnectError) Unwrap() error { return e.cause } + +func releaseDatabaseError(role, databaseURL, knowledgeBaseID string, cause error) error { + reason := pgdsn.Redact(databaseURL, cause.Error()) + return &databaseConnectError{ + message: fmt.Sprintf("connect release %s database %s (route %s): %s", + role, pgdsn.Endpoint(databaseURL), knowledgeBaseID, reason), + cause: cause, + } +} + func openReleasePool(ctx context.Context, databaseURL string) (*pgxpool.Pool, error) { pool, err := pgxpool.New(ctx, databaseURL) if err != nil { diff --git a/cmd/worker/library_baseline.go b/cmd/worker/library_baseline.go new file mode 100644 index 0000000..41e27c4 --- /dev/null +++ b/cmd/worker/library_baseline.go @@ -0,0 +1,298 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "log" + "time" + + "knowledge-governance/internal/appconfig" + "knowledge-governance/internal/governance" + "knowledge-governance/internal/jobs" + "knowledge-governance/internal/libraryfreeze" + "knowledge-governance/internal/librarymaintenance" + "knowledge-governance/internal/libraryrelease" + postgresstore "knowledge-governance/internal/platform/postgres" +) + +// libraryBaselinePayload 是"建立目标权威基线"的队列载荷。 +type libraryBaselinePayload struct { + LibraryJobID string `json:"library_job_id"` + Scope string `json:"scope"` +} + +// baselineVersionState 回答"这个 scope 现在有没有生效版本、是不是本作业登记的"。 +// +// 三种情况必须分开,否则崩溃续作会被自己的版本挡住: +// - own:本作业 + 同一快照已经登记过 → 只收尾,绝不重复捕获或重写版本; +// - foreign:别人登记的生效版本 → 一律拒绝(不能用基线覆盖别人的版本); +// - none:还没有生效版本 → 正常建基线。 +type baselineVersionState struct { + Own bool + Foreign *governance.LibraryVersion +} + +func classifyBaselineVersions(versions []governance.LibraryVersion, jobID, snapshotID string) baselineVersionState { + for index := range versions { + item := versions[index] + if item.Kind != governance.LibraryVersionProductionEffective || item.SupersededAt != nil { + continue + } + if jobID != "" && snapshotID != "" && item.SourceJobID == jobID && item.SnapshotID == snapshotID { + return baselineVersionState{Own: true} + } + foreign := item + return baselineVersionState{Foreign: &foreign} + } + return baselineVersionState{} +} + +// settleBaselineWindow 是基线窗口的**唯一**收尾入口,负责让治理状态与实际入口一致: +// +// - 版本已登记(本次刚写,或崩溃前由本作业写过)→ 走与同步一样的成功边界: +// 目标健康、入口开放、两把维护锁释放之后才暴露 succeeded; +// - 没有登记且窗口仍是 held(服务没回来)→ 保持维护并标 needs_restore, +// 只有"按预备份恢复"能把它收掉; +// - 没有登记但窗口已按原状态恢复 → 释放占用并把作业标失败,明确写"未对目标写入", +// 绝不留着"显示维护中、实际已开放"的状态。 +func settleBaselineWindow(ctx context.Context, store libraryMaintenanceStore, health healthChecker, + jobID, scope, instance, baselineID string, recorded bool, windowState string, windowErr error) error { + if recorded { + finishedAt := time.Now().UTC() + return finishLibrarySuccess(ctx, store, health, instance, scope, jobID, + successBoundary{Ready: true, Backup: baselineID, FinishedAt: finishedAt}) + } + now := time.Now().UTC() + message := "未建立基线:维护窗口没有完成捕获与校验" + if windowErr != nil { + message += ":" + windowErr.Error() + } + if windowState == "held" { + note := message + "(目标仍被停住、入口未恢复,保持维护)" + _ = store.UpdateLibraryMaintenanceState(ctx, scope, jobID, "needs_restore", note, now) + _ = store.UpdateInstanceMaintenanceState(ctx, instance, jobID, "needs_restore", note, now) + _ = store.AdvanceLibraryJob(ctx, jobID, governance.LibraryJobFailed, "capture", note, + "目标仍在维护:先按预备份恢复并校验", baselineID, "", now) + return errors.New(note) + } + // 窗口已按原状态开放(本流程对目标内容只读):占用必须跟着释放, + // 不能留下"界面说维护中、实际入口已开"的记录。 + if releaseErr := releaseWindowLocks(ctx, store, instance, scope, jobID, + "未登记基线,目标内容未被写入,窗口已按原状态恢复", now); releaseErr != nil { + note := message + ";维护占用释放失败:" + releaseErr.Error() + _ = store.AdvanceLibraryJob(ctx, jobID, governance.LibraryJobFailed, "capture", note, + "维护记录与实际入口不一致:先人工核对再重试", baselineID, "", now) + return errors.New(note) + } + _ = store.AdvanceLibraryJob(ctx, jobID, governance.LibraryJobFailed, "capture", message, + "未对目标执行任何写入;可直接重试建立基线", baselineID, "", now) + return errors.New(message) +} + +// runLibraryBaseline 给"非空但没有权威版本"的目标建立基线。 +// +// 真实链路(全部在同一个维护窗口内):关入口/停写入者/证明排空 → 把目标整库捕获成 +// 归档快照 → 用同一套 Verify 证明目标与这份快照逐类一致 → 读目标索引映射 → +// 登记 production_effective → 关窗口(恢复服务、开入口)→ 释放两把占用 → 作业成功。 +// +// 崩溃语义: +// - 捕获后、登记前被杀:作业行里已有快照 ID,重入时**不重新捕获**,直接在窗口内 +// 复核这份快照再登记; +// - 登记后、收尾前被杀:重入识别出"本作业+同一快照"已生效,只做收尾 +// (恢复服务/开入口/释放占用),不重复捕获、不重写版本;别人的版本仍然拒绝。 +func runLibraryBaseline(ctx context.Context, store *postgresstore.Store, config appconfig.Config, job jobs.Job) error { + var payload libraryBaselinePayload + if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.LibraryJobID == "" || + (payload.Scope != "internal" && payload.Scope != "external") { + return errors.New("invalid library baseline payload") + } + now := time.Now().UTC() + recorder := libraryStageRecorder{store: store, jobID: payload.LibraryJobID, scope: payload.Scope} + fail := func(message, stage string) error { + return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, message, stage) + } + existing, jobErr := store.LibraryJob(ctx, payload.LibraryJobID) + actor := "" + persistedSnapshot := "" + if jobErr == nil { + actor = existing.Actor + persistedSnapshot = existing.SnapshotID + } + _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobRunning, "capture", "", "", "", "", now) + + // 版本判定:别人的生效版本一律拒绝;本作业已登记过的快照只收尾。 + versions, versionErr := store.LibraryVersions(ctx, payload.Scope, 50) + if versionErr != nil { + return fail("读不到版本记录,无法确认是否需要建立基线;未对目标执行任何操作", "capture") + } + state := classifyBaselineVersions(versions, payload.LibraryJobID, persistedSnapshot) + if state.Foreign != nil { + return fail("该目标已有生效版本 "+shortSnapshotID(state.Foreign.SnapshotID)+ + "(作业 "+state.Foreign.SourceJobID+"),不需要再建基线;换版本请走同步或恢复", "capture") + } + finalizeOnly := state.Own + if finalizeOnly { + recorder.Stage(ctx, "finalize", "running", "续作:本作业已登记同一份基线快照,只做收尾(不重新捕获)") + } + + if !finalizeOnly && !config.ProductionReleaseRequestsEnabled { + return fail("生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false):建立基线要动目标运行状态,闸口关闭时不执行", "capture") + } + target, backups, wireErr := buildLibraryBaselineStack(ctx, config, payload.Scope) + if wireErr != nil { + return fail(wireErr.Error(), "capture") + } + defer target.Close() + + manager, managerErr := openLibraryMaintenance(config) + if managerErr != nil { + return fail("目标维护未接线,拒绝在挡不住入口/后台写入的情况下捕获目标:"+managerErr.Error(), "capture") + } + defer manager.Close() + if topoErr := manager.VerifyTopology(ctx); topoErr != nil { + return fail("目标拓扑不满足唯一入口约束,拒绝开维护窗口:"+topoErr.Error(), "capture") + } + originalState, stateErr := manager.OriginalState(ctx) + if stateErr != nil { + return fail("读不到目标写入者的原运行状态,不能开维护窗口:"+stateErr.Error(), "capture") + } + identity := manager.Target() + // 实例级占用:internal/external 动的是同一批容器,必须按实例互斥。 + // 同一作业重入时 store 保留第一次记下的原运行状态(否则会把"已停"当原状)。 + if _, instanceErr := store.AcquireInstanceMaintenance(ctx, postgresstore.LibraryInstanceMaintenance{ + Instance: identity.Instance, OwnerJobID: payload.LibraryJobID, OwnerScope: payload.Scope, + TargetFingerprint: identity.Fingerprint, Level: librarymaintenance.LevelInstance, + Services: identity.Writers, IngressState: "unknown", State: "acquired", + OriginalServiceState: originalState, + Note: "实例级维护:为建立权威基线捕获目标整库,捕获并校验后按原状态恢复", + }, now); instanceErr != nil { + return fail("目标实例正在维护中(跨对内/对外互斥):"+instanceErr.Error(), "capture") + } + if _, acquireErr := store.AcquireLibraryMaintenance(ctx, postgresstore.LibraryMaintenance{ + Scope: payload.Scope, OwnerJobID: payload.LibraryJobID, + TargetIdentity: identity.Fingerprint, + Level: librarymaintenance.LevelInstance, + TargetFingerprint: identity.Fingerprint, + Services: identity.Writers, + IngressState: "unknown", + State: "acquired", + OriginalServiceState: originalState, + Note: "实例级维护:为建立权威基线捕获目标整库,捕获并校验后按原状态恢复", + }, now); acquireErr != nil { + _ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", + "scope 占用申请失败,实例状态未确认:"+acquireErr.Error(), now) + return fail("目标正在维护中:"+acquireErr.Error(), "capture") + } + + coordinator, control, openErr := manager.Open(config.LibraryTargetJournalRoot) + if openErr != nil { + _ = releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID, + "未进入维护窗口(协调器不可用),目标未被停用", time.Now().UTC()) + return fail("维护协调器不可用:"+openErr.Error(), "capture") + } + defer control.Close() + defer coordinator.Close() + + var ( + baselineID = persistedSnapshot + counts map[string]int64 + recorded = finalizeOnly + ) + work := func(ctx context.Context, window *libraryfreeze.Window) (string, error) { + if finalizeOnly { + // 版本已在窗口内校验并登记过:不再捕获、不再重写,收尾由下方统一处理。 + return baselineID, nil + } + if baselineID == "" { + current, currentErr := target.Current(ctx) + if currentErr != nil { + return "", errors.New("读取目标当前整库状态失败:" + currentErr.Error()) + } + counts = map[string]int64{} + for table, count := range current.Counts { + counts[table] = count + } + // 预备份存储在导出后用同一份指纹复核"导出期间内容没变", + // 所以这里把窗口内读到的 state 交进去,而不是事后再读一次。 + backup, backupErr := backups.Prepare(ctx, "", current) + if backupErr != nil { + return "", errors.New("捕获目标整库失败:" + backupErr.Error()) + } + baselineID = backup.SnapshotID + // 先把"这次窗口的基线快照"落盘:进程在这里被杀时,恢复器知道该回到哪一份, + // 重入也据此跳过重新捕获。 + if markErr := window.SetBackupSnapshot(ctx, baselineID); markErr != nil { + return "", errors.New("记录基线快照身份失败:" + markErr.Error()) + } + if persistErr := store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobRunning, "", + "", "", baselineID, "", time.Now().UTC()); persistErr != nil { + return "", errors.New("持久化基线快照引用失败:" + persistErr.Error()) + } + recorder.Stage(ctx, "capture", "succeeded", "已封存目标整库快照 "+baselineID) + } else { + recorder.Stage(ctx, "capture", "skipped", "复用作业已持久化的基线快照 "+baselineID) + } + if baselineID == "" { + return "", errors.New("没有可校验的基线快照") + } + recorder.Stage(ctx, "verify", "running", "逐表/索引/资源比对目标与快照 "+shortSnapshotID(baselineID)) + if verifyErr := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: baselineID}); verifyErr != nil { + return "", errors.New("目标内容与基线快照不一致,拒绝登记基线:" + verifyErr.Error()) + } + recorder.Stage(ctx, "verify", "succeeded", "行身份/逐行内容/索引文档/资源摘要与快照一致") + // 索引映射是基线证据的一部分:读不到就不登记,绝不用空摘要冒充。 + mapping, mapErr := target.Search().MappingSHA256(ctx) + if mapErr != nil { + return "", errors.New("读不到目标索引映射,拒绝用空摘要登记基线:" + mapErr.Error()) + } + if counts == nil { + if current, currentErr := target.Current(ctx); currentErr == nil { + counts = current.Counts + } + } + // 版本登记放在窗口内:捕获、校验、登记三件事要么一起成立,要么目标原样恢复。 + recorder.Stage(ctx, "finalize", "running", "登记目标权威基线") + if recordErr := store.RecordLibraryVersion(ctx, governance.LibraryVersion{ + ID: governance.LibraryVersionID(governance.LibraryVersionProductionEffective, payload.Scope, baselineID), + Scope: payload.Scope, Kind: governance.LibraryVersionProductionEffective, + SnapshotID: baselineID, SourceJobID: payload.LibraryJobID, CreatedBy: actor, + Evidence: map[string]any{ + "action": "target_baseline_init", + "origin": "target_baseline", + "baseline_snapshot": baselineID, + "index_mapping_sha256": mapping, + "scope_content_counts": counts, + "verification": "目标与快照逐表行身份/逐行内容/索引文档/资源摘要一致", + "note": "基线是目标在维护窗口内的真实整库捕获,不是手写版本;发布仍需独立 sync 授权", + }, + CreatedAt: time.Now().UTC(), + }); recordErr != nil { + return "", errors.New("登记基线版本失败:" + recordErr.Error()) + } + recorded = true + recorder.Stage(ctx, "finalize", "succeeded", "已登记目标权威基线 "+shortSnapshotID(baselineID)) + return baselineID, nil + } + + window, windowErr := coordinator.RunWindow(ctx, payload.LibraryJobID, + func(ctx context.Context) error { return manager.VerifyRunning(ctx) }, work, + // 本流程对目标内容只读:失败时按原状态重新开放永远安全。 + func(error) bool { return true }) + + if settleErr := settleBaselineWindow(ctx, store, manager, payload.LibraryJobID, payload.Scope, + identity.Instance, baselineID, recorded, window.State, windowErr); settleErr != nil { + return settleErr + } + log.Printf("library baseline succeeded job=%s scope=%s snapshot=%s actor=%s finalize_only=%v", + payload.LibraryJobID, payload.Scope, baselineID, actor, finalizeOnly) + return nil +} + +// shortSnapshotID 只用于人看的日志/阶段说明:完整摘要留在版本记录与接口里。 +func shortSnapshotID(id string) string { + if len(id) <= 12 { + return id + } + return id[:12] +} diff --git a/cmd/worker/library_baseline_test.go b/cmd/worker/library_baseline_test.go new file mode 100644 index 0000000..bad8c27 --- /dev/null +++ b/cmd/worker/library_baseline_test.go @@ -0,0 +1,149 @@ +package main + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "knowledge-governance/internal/governance" +) + +// 崩溃续作的版本判定:本作业+同一快照 = 只收尾;别人的生效版本 = 拒绝;都没有 = 正常建基线。 +func TestClassifyBaselineVersions(t *testing.T) { + const jobA, jobB, snap = "libbase-internal-1", "libbase-internal-2", "snap-abc" + effective := func(sourceJob, snapshot string, superseded bool) governance.LibraryVersion { + item := governance.LibraryVersion{ + Scope: "internal", Kind: governance.LibraryVersionProductionEffective, + SnapshotID: snapshot, SourceJobID: sourceJob, CreatedAt: time.Now().UTC(), + } + if superseded { + at := time.Now().UTC() + item.SupersededAt = &at + } + return item + } + + if state := classifyBaselineVersions(nil, jobA, snap); state.Own || state.Foreign != nil { + t.Fatalf("no versions must mean neither own nor foreign: %+v", state) + } + if state := classifyBaselineVersions([]governance.LibraryVersion{effective(jobA, snap, false)}, jobA, snap); !state.Own { + t.Fatalf("same job + same snapshot must be recognized as own: %+v", state) + } + // 快照不同就不是"我登记的那份",不能当成可收尾。 + if state := classifyBaselineVersions([]governance.LibraryVersion{effective(jobA, "snap-other", false)}, jobA, snap); state.Own || state.Foreign == nil { + t.Fatalf("different snapshot of same job must not be own: %+v", state) + } + // 别人的生效版本:即使快照相同也必须拒绝。 + foreign := classifyBaselineVersions([]governance.LibraryVersion{effective(jobB, snap, false)}, jobA, snap) + if foreign.Own || foreign.Foreign == nil || foreign.Foreign.SourceJobID != jobB { + t.Fatalf("foreign effective version must be reported: %+v", foreign) + } + // 已被顶替的旧版本不算生效版本。 + if state := classifyBaselineVersions([]governance.LibraryVersion{effective(jobB, snap, true)}, jobA, snap); state.Own || state.Foreign != nil { + t.Fatalf("superseded version must not block a baseline: %+v", state) + } +} + +// 版本已登记 + 健康:走成功边界,释放两把占用并把作业标成功。 +func TestSettleBaselineWindowFinishesWhenRecorded(t *testing.T) { + store := &fakeFinishStore{} + if err := settleBaselineWindow(context.Background(), store, fakeHealth{}, "job-1", "internal", "kgseal-app", + "snap-abc", true, "succeeded", nil); err != nil { + t.Fatalf("settle: %v", err) + } + if !containsCall(store.calls, "advance_succeeded") { + t.Fatalf("job must be marked succeeded: %v", store.calls) + } + if containsCall(store.states, "needs_restore") { + t.Fatalf("success path must not hold the target in maintenance: %v", store.states) + } + if !containsCall(store.calls, "scope_release") || !containsCall(store.calls, "instance_release") { + t.Fatalf("both locks must be released: %v", store.calls) + } +} + +// 版本已登记但目标服务/入口没恢复:不能报成功,必须保持维护并标 needs_restore。 +func TestSettleBaselineWindowHoldsWhenHealthFails(t *testing.T) { + store := &fakeFinishStore{} + err := settleBaselineWindow(context.Background(), store, fakeHealth{err: errors.New("kgseal-ingress 未就绪")}, + "job-1", "internal", "kgseal-app", "snap-abc", true, "succeeded", nil) + if err == nil { + t.Fatal("unhealthy target must not report success") + } + if containsCall(store.calls, "advance_succeeded") { + t.Fatalf("must not mark succeeded while target is unhealthy: %v", store.calls) + } + if countCall(store.states, "needs_restore") != 2 { + t.Fatalf("must hold both records with needs_restore: %v", store.states) + } +} + +// 没有登记、窗口仍是 held:只能保持维护(服务确实没回来),绝不能释放占用。 +func TestSettleBaselineWindowKeepsHeldWindowInMaintenance(t *testing.T) { + store := &fakeFinishStore{} + err := settleBaselineWindow(context.Background(), store, fakeHealth{}, "job-1", "internal", "kgseal-app", + "snap-abc", false, "held", errors.New("停写入者后后台仍有任务")) + if err == nil || !strings.Contains(err.Error(), "保持维护") { + t.Fatalf("err=%v", err) + } + if containsCall(store.calls, "scope_release") || containsCall(store.calls, "instance_release") { + t.Fatalf("held window must not release locks: %v", store.calls) + } + if countCall(store.states, "needs_restore") != 2 { + t.Fatalf("held window must be recorded as needs_restore: %v", store.states) + } + if !containsCall(store.calls, "advance_failed") { + t.Fatalf("job must fail: %v", store.calls) + } +} + +// 没有登记、但窗口已按原状态开放:状态必须跟着变成"已释放", +// 不能留下"显示维护中、实际入口已开"。 +func TestSettleBaselineWindowReleasesWhenWindowRestored(t *testing.T) { + store := &fakeFinishStore{} + err := settleBaselineWindow(context.Background(), store, fakeHealth{}, "job-1", "internal", "kgseal-app", + "snap-abc", false, "failed", errors.New("捕获目标整库失败")) + if err == nil || !strings.Contains(err.Error(), "未建立基线") { + t.Fatalf("err=%v", err) + } + if !containsCall(store.calls, "scope_release") || !containsCall(store.calls, "instance_release") { + t.Fatalf("restored window must release both locks: %v", store.calls) + } + if containsCall(store.states, "needs_restore") { + t.Fatalf("released window must not be marked as needs_restore: %v", store.states) + } + if !containsCall(store.calls, "advance_failed") { + t.Fatalf("job must fail: %v", store.calls) + } +} + +// 释放失败时不能在作业上写"无需恢复":那会让界面与实际占用分叉。 +func TestSettleBaselineWindowReportsReleaseFailure(t *testing.T) { + store := &fakeFinishStore{failOn: "scope_release"} + err := settleBaselineWindow(context.Background(), store, fakeHealth{}, "job-1", "internal", "kgseal-app", + "snap-abc", false, "failed", nil) + if err == nil || !strings.Contains(err.Error(), "维护占用释放失败") { + t.Fatalf("err=%v", err) + } +} + +func countCall(calls []string, want string) int { + count := 0 + for _, call := range calls { + if call == want { + count++ + } + } + return count +} + +func containsCall(calls []string, want string) bool { + for _, call := range calls { + if call == want { + return true + } + } + return false +} diff --git a/cmd/worker/library_capture_identity.go b/cmd/worker/library_capture_identity.go new file mode 100644 index 0000000..11b450f --- /dev/null +++ b/cmd/worker/library_capture_identity.go @@ -0,0 +1,69 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + + "knowledge-governance/internal/appconfig" + "knowledge-governance/internal/libraryagent" +) + +// 封存的身份契约:期望随**同一个 capture 请求**送进 agent,由接收方在操作互斥内、 +// 任何源操作之前比对;worker 事后按回显的指纹再复核一次。 +// +// 为什么不能在 worker 侧先 Health 再 Capture:那是两次拨号,中间 socket 可能被换成 +// 另一个 agent,错的执行器已经把写入者停了才被拒登记——副作用已经发生。这里把期望 +// 放进请求本身,换过的 agent 会先比对身份、比对失败就一次 run 都不调。 +// +// 期望字段只来自治理侧配置(LIBRARY_SOURCE_*),绝不从执行器自报值生成。 + +// captureExpectation 把治理侧配置翻译成期望;缺身份就是缺配置,调用方必须失败。 +func captureExpectation(config appconfig.Config) (libraryagent.Expectation, error) { + expectation := libraryagent.NewExpectation(config.SourceProject(), config.LibrarySourceApp, "") + if expectation.Project == "" { + return libraryagent.Expectation{}, errors.New("未配置封存来源实例身份(LIBRARY_SOURCE_PROJECT),拒绝在不核对执行器的情况下停写入者") + } + return expectation, nil +} + +// captureExecutorPostcheck 是事后证据防线:结果里回显的期望指纹必须等于我们发出的, +// 自报身份也必须与期望一致。它挡不住副作用,所以不能替代请求内的核对。 +func captureExecutorPostcheck(payload json.RawMessage, expectation libraryagent.Expectation) error { + if len(payload) == 0 { + return errors.New("封存结果里没有执行器身份,无法证明这次封存来自期望实例") + } + var document struct { + Identity struct { + Project string `json:"project"` + SourceApp string `json:"source_app"` + ExpectationFingerprint string `json:"expectation_fingerprint"` + } `json:"agent_identity"` + } + if err := json.Unmarshal(payload, &document); err != nil { + return errors.New("封存结果里的执行器身份无法解析") + } + identity := document.Identity + if identity.Project == "" { + return errors.New("封存结果里没有执行器身份,无法证明这次封存来自期望实例") + } + if identity.ExpectationFingerprint != expectation.Fingerprint { + return fmt.Errorf("封存结果回显的期望指纹不一致(agent=%q,期望=%q),该快照不能当作本实例的测试封存", + identity.ExpectationFingerprint, expectation.Fingerprint) + } + if identity.Project != expectation.Project { + return fmt.Errorf("封存结果来自另一个实例(agent=%q,期望=%q),该快照不能当作本实例的测试封存", + identity.Project, expectation.Project) + } + if expectation.SourceApp != "" && identity.SourceApp != expectation.SourceApp { + return fmt.Errorf("封存结果来自另一个源容器(agent=%q,期望=%q),该快照不能当作本实例的测试封存", + identity.SourceApp, expectation.SourceApp) + } + return nil +} + +// expectationFingerprint 用于日志/证据行的稳定标识,不含路径与凭据。 +func expectationFingerprint(expectation libraryagent.Expectation) string { + return strings.TrimSpace(expectation.Fingerprint) +} diff --git a/cmd/worker/library_capture_identity_test.go b/cmd/worker/library_capture_identity_test.go new file mode 100644 index 0000000..f3a9917 --- /dev/null +++ b/cmd/worker/library_capture_identity_test.go @@ -0,0 +1,73 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + + "knowledge-governance/internal/appconfig" + "knowledge-governance/internal/libraryagent" +) + +// 期望只从治理侧配置生成;缺 project 就是缺配置,必须在停写入者之前失败。 +func TestCaptureExpectationComesFromGovernanceConfigAndFailsClosed(t *testing.T) { + expect, err := captureExpectation(appconfig.Config{LibrarySourceInstance: "kgmaint-weknora", LibrarySourceApp: "kgmaint-app"}) + if err != nil { + t.Fatal(err) + } + if expect.Project != "kgmaint-weknora" || expect.SourceApp != "kgmaint-app" { + t.Fatalf("expectation = %+v", expect) + } + if !expect.Complete() { + t.Fatalf("expectation should be complete: %+v", expect) + } + if _, err = captureExpectation(appconfig.Config{}); err == nil || + !strings.Contains(err.Error(), "LIBRARY_SOURCE_PROJECT") { + t.Fatalf("empty identity accepted: %v", err) + } + // 期望不取自任何执行器应答:显式给 project 就按它绑定,配置里没有别的来源。 + explicit, err := captureExpectation(appconfig.Config{LibrarySourceProject: "kgmaint-weknora"}) + if err != nil || explicit.Project != "kgmaint-weknora" || explicit.SourceApp != "" { + t.Fatalf("explicit = %+v err=%v", explicit, err) + } +} + +// 事后证据防线:回显的期望指纹、project、source_app 任何一项不符都不能登记。 +func TestCaptureExecutorPostcheckUsesTheEchoedExpectationFingerprint(t *testing.T) { + expect := libraryagent.NewExpectation("kgmaint-weknora", "kgmaint-app", "") + good := payloadOf(t, "kgmaint-weknora", "kgmaint-app", expect.Fingerprint) + if err := captureExecutorPostcheck(good, expect); err != nil { + t.Fatalf("matching identity rejected: %v", err) + } + swapped := payloadOf(t, "weknora071full", "WKNFull-app", expect.Fingerprint) + if err := captureExecutorPostcheck(swapped, expect); err == nil || !strings.Contains(err.Error(), "另一个实例") { + t.Fatalf("wrong project accepted: %v", err) + } + regenerated := payloadOf(t, "kgmaint-weknora", "kgmaint-app", + libraryagent.NewExpectation("kgmaint-weknora", "kgmaint-app", "/other/archive").Fingerprint) + if err := captureExecutorPostcheck(regenerated, expect); err == nil || !strings.Contains(err.Error(), "期望指纹不一致") { + t.Fatalf("regenerated expectation accepted: %v", err) + } + if err := captureExecutorPostcheck(json.RawMessage(`{"snapshot_id":"abc"}`), expect); err == nil || + !strings.Contains(err.Error(), "没有执行器身份") { + t.Fatalf("missing identity accepted: %v", err) + } + if err := captureExecutorPostcheck(payloadOf(t, "kgmaint-weknora", "kgseal-app", expect.Fingerprint), expect); err == nil || + !strings.Contains(err.Error(), "另一个源容器") { + t.Fatalf("wrong source app accepted: %v", err) + } +} + +func payloadOf(t *testing.T, project, sourceApp, fingerprint string) json.RawMessage { + t.Helper() + encoded, err := json.Marshal(map[string]any{ + "snapshot_id": "abc", + "agent_identity": map[string]any{ + "project": project, "source_app": sourceApp, "expectation_fingerprint": fingerprint, + }, + }) + if err != nil { + t.Fatal(err) + } + return encoded +} diff --git a/cmd/worker/library_executor_status.go b/cmd/worker/library_executor_status.go new file mode 100644 index 0000000..f28c5b2 --- /dev/null +++ b/cmd/worker/library_executor_status.go @@ -0,0 +1,52 @@ +package main + +import ( + "context" + "log" + "time" + + "knowledge-governance/internal/appconfig" + "knowledge-governance/internal/libraryexecutor" + postgresstore "knowledge-governance/internal/platform/postgres" +) + +// 为什么由 worker 观测、页面读库:agent socket 只挂载在 worker 容器里,API 那个 +// LIBRARY_AGENT_SOCKET 即使配了也 dial 不到。谁真能执行就由谁写下观测, +// 否则页面上的"可发起"只是配置字符串的回声。 +const libraryExecutorProbeInterval = 30 * time.Second + +// libraryExecutorProbeTimeout 是一次 health 往返的上限:它只是同 uid unix socket +// 上的一次 JSON 往返,慢到秒级就说明执行器有问题,不该拖住心跳。 +const libraryExecutorProbeTimeout = 3 * time.Second + +func recordLibraryExecutorStatus(ctx context.Context, store *postgresstore.Store, config appconfig.Config) { + // 归属用**封存源**实例:agent 冻结的是源库写入者,和同步目标(kgseal)不是一套。 + instance := config.SourceInstance() + if instance == "" { + // 没有实例身份,观测就无处归属:写进去只会让页面拿别的实例的状态。 + log.Print("library executor probe skipped: LIBRARY_SOURCE_INSTANCE is not configured") + return + } + observation := libraryexecutor.Probe(ctx, instance, config.LibraryAgentSocket, libraryExecutorProbeTimeout, nil) + if err := store.UpsertLibraryExecutorStatus(ctx, observation, config.WorkerID, time.Now().UTC()); err != nil { + log.Printf("library executor status not recorded: %v", err) + } +} + +// runLibraryExecutorProbe 先观测一次再进循环:页面在 worker 刚起来时就该有真实状态。 +func runLibraryExecutorProbe(ctx context.Context, store *postgresstore.Store, config appconfig.Config, interval time.Duration) { + if interval <= 0 { + interval = libraryExecutorProbeInterval + } + recordLibraryExecutorStatus(ctx, store, config) + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + recordLibraryExecutorStatus(ctx, store, config) + } + } +} diff --git a/cmd/worker/library_finish.go b/cmd/worker/library_finish.go new file mode 100644 index 0000000..e8ba1eb --- /dev/null +++ b/cmd/worker/library_finish.go @@ -0,0 +1,162 @@ +package main + +import ( + "context" + "errors" + "fmt" + "time" + + "knowledge-governance/internal/governance" + postgresstore "knowledge-governance/internal/platform/postgres" +) + +// libraryMaintenanceStore 是"成功收尾"用到的存储能力。抽成接口是为了能在测试里 +// 让某一步失败,验证"没全部完成就不许暴露成功"。 +type libraryMaintenanceStore interface { + SetInstanceMaintenanceIngress(ctx context.Context, instance, ownerJobID, state, note string, now time.Time) error + SetLibraryMaintenanceIngress(ctx context.Context, scope, ownerJobID, state, note string, now time.Time) error + UpdateInstanceMaintenanceState(ctx context.Context, instance, ownerJobID, state, note string, now time.Time) error + UpdateLibraryMaintenanceState(ctx context.Context, scope, ownerJobID, state, note string, now time.Time) error + ReleaseInstanceMaintenance(ctx context.Context, instance, ownerJobID string, now time.Time) error + ReleaseLibraryMaintenance(ctx context.Context, scope, ownerJobID string, now time.Time) error + AdvanceLibraryJob(ctx context.Context, jobID string, state governance.LibraryJobState, stage, message, recovery, snapshotID, targetSnapshotID string, now time.Time) error +} + +// releaseWindowLocks 是唯一释放路径。 +// +// 顺序刻意是"scope 先、instance 后":instance 行是跨 scope 的互斥来源,最后放它 +// 才能保证任何新作业在全部记录落定之前都进不来。每一步的错误都必须向上暴露—— +// 只丢日志会让"锁其实没放"这件事变成页面上的成功。 +func releaseWindowLocks(ctx context.Context, store libraryMaintenanceStore, instance, scope, jobID, note string, now time.Time) error { + if err := store.SetLibraryMaintenanceIngress(ctx, scope, jobID, "open", note, now); !releasedOrNil(err) { + return fmt.Errorf("记录入口已开放失败(scope):%w", err) + } + if err := store.SetInstanceMaintenanceIngress(ctx, instance, jobID, "open", note, now); !releasedOrNil(err) { + return fmt.Errorf("记录入口已开放失败(instance):%w", err) + } + if err := store.UpdateLibraryMaintenanceState(ctx, scope, jobID, "verified", note, now); !releasedOrNil(err) { + return fmt.Errorf("维护状态写入失败(scope):%w", err) + } + if err := store.ReleaseLibraryMaintenance(ctx, scope, jobID, now); !releasedOrNil(err) { + return fmt.Errorf("维护占用释放失败(scope):%w", err) + } + if err := store.UpdateInstanceMaintenanceState(ctx, instance, jobID, "verified", note, now); !releasedOrNil(err) { + return fmt.Errorf("维护状态写入失败(instance):%w", err) + } + if err := store.ReleaseInstanceMaintenance(ctx, instance, jobID, now); !releasedOrNil(err) { + return fmt.Errorf("实例占用释放失败(instance):%w", err) + } + return nil +} + +// healthChecker 是"目标写入者与入口是否真的恢复健康"的判定口(测试可注入失败)。 +type healthChecker interface { + VerifyRunning(ctx context.Context) error +} + +// successBoundary 描述一次"内容已通过验证与版本登记,但还没到可以对外报成功"的作业。 +type successBoundary struct { + Ready bool + Backup string + Target string + FinishedAt time.Time +} + +// finishLibrarySuccess 是唯一的成功收尾入口:只有以下全部成立才暴露 succeeded—— +// 目标服务健康、入口开放、scope 与 instance 两把维护锁都释放完成、终态写入成功。 +// +// 为什么不能先标成功再释放:恢复/开入口/放锁都可能失败,先写 succeeded 会让外部 +// (页面、API、重启后的接管逻辑)看到一条"成功了但目标仍在维护"的作业;而 +// InterruptedLibraryJobs 只接管非终态,终态作业不会再被接管,锁就永远挂着。 +func finishLibrarySuccess(ctx context.Context, store libraryMaintenanceStore, health healthChecker, + instance, scope, jobID string, boundary successBoundary) error { + // 释放与终态写入必须能在 ctx 取消后继续:停机不该留下"锁放了但作业还在跑"或 + // 反过来"作业成功了锁还挂着"的任一半成品。 + bounded, cancel := context.WithTimeout(context.WithoutCancel(ctx), 60*time.Second) + defer cancel() + + holdForRepair := func(reason string) error { + note := "内容已验证,但收尾未完成,目标保持维护:" + reason + _ = store.UpdateLibraryMaintenanceState(bounded, scope, jobID, "needs_restore", note, time.Now().UTC()) + _ = store.UpdateInstanceMaintenanceState(bounded, instance, jobID, "needs_restore", note, time.Now().UTC()) + _ = store.AdvanceLibraryJob(bounded, jobID, governance.LibraryJobFailed, "finalize", note, + "生产内容已写入并校验;维护保持,按预备份或人工核验后再开放", boundary.Backup, boundary.Target, time.Now().UTC()) + return errors.New(note) + } + + if err := health.VerifyRunning(bounded); err != nil { + return holdForRepair("目标服务或入口未恢复健康:" + err.Error()) + } + if err := releaseWindowLocks(bounded, store, instance, scope, jobID, + "写入与校验完成,目标已按原运行状态恢复", time.Now().UTC()); err != nil { + return holdForRepair(err.Error()) + } + finishedAt := time.Now().UTC() + if err := store.AdvanceLibraryJob(bounded, jobID, governance.LibraryJobSucceeded, "finalize", "", "", + boundary.Backup, boundary.Target, finishedAt); err != nil { + // 锁已释放但终态没写下去:作业保持非终态,重启后会被接管(接管走 + // finalize-only,只复核与登记,不重写生产)。 + return fmt.Errorf("维护已释放但作业终态写入失败(等待接管):%w", err) + } + return nil +} + +// takeoverKind 是接管时"这个作业走到哪一步"的分类。 +// +// 四类必须分开:把"版本查不到"当成"没有收尾证据",就会把只差收尾的作业重新写一遍 +// 目标;把"曾被顶替"当成正常重放,就会用旧快照覆盖更新的生产版本。 +type takeoverKind string + +const ( + // takeoverFresh:没有任何收尾证据,按正常流程(有预备份则复用,写前仍备份)。 + takeoverFresh takeoverKind = "fresh" + // takeoverPendingFinal:内容与版本已落定,只差收尾(只复核与登记,不重写)。 + takeoverPendingFinal takeoverKind = "pending_final" + // takeoverSuperseded:本快照曾经生效、后来被别的快照顶替,不得自动覆盖。 + takeoverSuperseded takeoverKind = "superseded" + // takeoverUnknown:版本状态读不到且没有阶段证据,属"未知",不得写入。 + takeoverUnknown takeoverKind = "unknown" +) + +// classifyTakeover 只按证据分类: +// - finalize 阶段成功是独立证据(来自作业自己的阶段行),不因版本查询失败而丢弃; +// - 本快照存在 released/被顶替的版本 ⇒ superseded; +// - 本快照存在未被顶掉的生效版本 ⇒ pending_final; +// - 查询失败且无阶段证据 ⇒ unknown(可重试,不当成没有证据)。 +func classifyTakeover(job governance.LibraryJob, versions []governance.LibraryVersion, versionsErr error) takeoverKind { + finalized := false + for _, stage := range job.Stages { + if stage.Stage == "finalize" && stage.State == "succeeded" { + finalized = true + } + } + if versionsErr != nil && !finalized { + return takeoverUnknown + } + superseded, effective := false, false + for _, version := range versions { + if version.Kind != governance.LibraryVersionProductionEffective || version.SnapshotID != job.TargetSnapshotID { + continue + } + if version.SupersededAt != nil { + superseded = true + } else { + effective = true + } + } + if superseded && !effective { + return takeoverSuperseded + } + if effective || finalized { + return takeoverPendingFinal + } + return takeoverFresh +} + +var _ libraryMaintenanceStore = (*postgresstore.Store)(nil) + +// releasedOrNil:这一行已经由本作业释放过,说明该步在上一次尝试里已经完成。 +// 部分成功后重试幂等靠它——否则"scope 放了、instance 没放"的作业永远重试不下去。 +func releasedOrNil(err error) bool { + return err == nil || errors.Is(err, postgresstore.ErrMaintenanceReleased) +} diff --git a/cmd/worker/library_finish_test.go b/cmd/worker/library_finish_test.go new file mode 100644 index 0000000..8cb7b34 --- /dev/null +++ b/cmd/worker/library_finish_test.go @@ -0,0 +1,213 @@ +package main + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "knowledge-governance/internal/governance" + postgresstore "knowledge-governance/internal/platform/postgres" +) + +type fakeFinishStore struct { + calls []string + failOn string + alreadyReleased bool + advanced []governance.LibraryJobState + // states 记录所有维护状态写入(测试要区分 verified 与 needs_restore)。 + states []string +} + +// 模拟"这一行已由同一个作业释放过"(真实 Store 返回 ErrMaintenanceReleased)。 +func (f *fakeFinishStore) releasedSentinel() error { + if f.alreadyReleased { + return postgresstore.ErrMaintenanceReleased + } + return nil +} + +func (f *fakeFinishStore) record(name string) error { + f.calls = append(f.calls, name) + if f.failOn == name { + return errors.New("injected " + name + " failure") + } + return nil +} + +func (f *fakeFinishStore) SetInstanceMaintenanceIngress(context.Context, string, string, string, string, time.Time) error { + if err := f.releasedSentinel(); err != nil { + return err + } + return f.record("instance_ingress") +} +func (f *fakeFinishStore) SetLibraryMaintenanceIngress(context.Context, string, string, string, string, time.Time) error { + if err := f.releasedSentinel(); err != nil { + return err + } + return f.record("scope_ingress") +} +func (f *fakeFinishStore) UpdateInstanceMaintenanceState(_ context.Context, _, _, state, _ string, _ time.Time) error { + if err := f.releasedSentinel(); err != nil { + return err + } + f.states = append(f.states, state) + return f.record("instance_state") +} +func (f *fakeFinishStore) UpdateLibraryMaintenanceState(_ context.Context, _, _, state, _ string, _ time.Time) error { + if err := f.releasedSentinel(); err != nil { + return err + } + f.states = append(f.states, state) + return f.record("scope_state") +} +func (f *fakeFinishStore) ReleaseInstanceMaintenance(context.Context, string, string, time.Time) error { + if err := f.releasedSentinel(); err != nil { + return err + } + return f.record("instance_release") +} +func (f *fakeFinishStore) ReleaseLibraryMaintenance(context.Context, string, string, time.Time) error { + if err := f.releasedSentinel(); err != nil { + return err + } + return f.record("scope_release") +} +func (f *fakeFinishStore) AdvanceLibraryJob(_ context.Context, _ string, state governance.LibraryJobState, + _, _, _, _, _ string, _ time.Time) error { + f.advanced = append(f.advanced, state) + return f.record("advance_" + string(state)) +} + +type fakeHealth struct{ err error } + +func (h fakeHealth) VerifyRunning(context.Context) error { return h.err } + +func boundary() successBoundary { + return successBoundary{Ready: true, Backup: "backup-1", Target: "snap-1", FinishedAt: time.Now().UTC()} +} + +// 首次观察到成功时,两把维护锁必须已经释放,且终态写入排在最后。 +func TestFinishLibrarySuccessReleasesEveryLockBeforeSucceeded(t *testing.T) { + store := &fakeFinishStore{} + if err := finishLibrarySuccess(context.Background(), store, fakeHealth{}, "kgseal-weknora", "internal", "job-1", boundary()); err != nil { + t.Fatal(err) + } + want := []string{"scope_ingress", "instance_ingress", "scope_state", "scope_release", + "instance_state", "instance_release", "advance_succeeded"} + if strings.Join(store.calls, ",") != strings.Join(want, ",") { + t.Fatalf("call order = %v", store.calls) + } + if len(store.advanced) != 1 || store.advanced[0] != governance.LibraryJobSucceeded { + t.Fatalf("advanced = %v", store.advanced) + } +} + +// 目标服务/入口没恢复健康:不许放锁,不许报成功,作业落成失败并保持维护。 +func TestFinishLibrarySuccessKeepsMaintenanceWhenTargetIsNotHealthy(t *testing.T) { + store := &fakeFinishStore{} + err := finishLibrarySuccess(context.Background(), store, fakeHealth{err: errors.New("app not healthy")}, + "kgseal-weknora", "internal", "job-2", boundary()) + if err == nil || !strings.Contains(err.Error(), "未恢复健康") { + t.Fatalf("err = %v", err) + } + for _, call := range store.calls { + if strings.HasSuffix(call, "_release") || call == "advance_succeeded" { + t.Fatalf("released or reported success despite unhealthy target: %v", store.calls) + } + } + if len(store.advanced) != 1 || store.advanced[0] != governance.LibraryJobFailed { + t.Fatalf("advanced = %v", store.advanced) + } +} + +// 任一步释放失败:不报成功,作业失败,维护状态写 needs_restore(等人工/接管)。 +func TestFinishLibrarySuccessStopsOnReleaseFailure(t *testing.T) { + for _, step := range []string{"scope_release", "instance_state", "instance_release"} { + t.Run(step, func(t *testing.T) { + store := &fakeFinishStore{failOn: step} + err := finishLibrarySuccess(context.Background(), store, fakeHealth{}, "kgseal-weknora", "internal", "job-3", boundary()) + if err == nil { + t.Fatalf("%s failure was swallowed", step) + } + for _, call := range store.calls { + if call == "advance_succeeded" { + t.Fatalf("reported success after %s failed: %v", step, store.calls) + } + } + if len(store.advanced) != 1 || store.advanced[0] != governance.LibraryJobFailed { + t.Fatalf("advanced = %v", store.advanced) + } + }) + } +} + +// 终态写入失败:锁已释放,但不能声称成功;作业保持非终态等接管。 +func TestFinishLibrarySuccessReportsTerminalWriteFailure(t *testing.T) { + store := &fakeFinishStore{failOn: "advance_succeeded"} + err := finishLibrarySuccess(context.Background(), store, fakeHealth{}, "kgseal-weknora", "internal", "job-4", boundary()) + if err == nil || !strings.Contains(err.Error(), "等待接管") { + t.Fatalf("err = %v", err) + } +} + +// 接管分类:阶段证据独立成立;查询失败=未知(不得写入);被顶替=不得覆盖。 +func TestClassifyTakeoverSeparatesUnknownPendingSupersededAndFresh(t *testing.T) { + job := governance.LibraryJob{Scope: "internal", Kind: governance.LibraryJobSync, TargetSnapshotID: "snap-1"} + finalized := job + finalized.Stages = []governance.LibraryStage{{Stage: "finalize", State: "succeeded"}} + effective := []governance.LibraryVersion{{Kind: governance.LibraryVersionProductionEffective, SnapshotID: "snap-1"}} + supersededVersion := []governance.LibraryVersion{{Kind: governance.LibraryVersionProductionEffective, + SnapshotID: "snap-1", SupersededAt: ptrTime(time.Now())}} + unrelated := []governance.LibraryVersion{{Kind: governance.LibraryVersionProductionEffective, SnapshotID: "snap-9"}} + queryErr := errors.New("version storage temporarily unavailable") + + cases := []struct { + name string + job governance.LibraryJob + versions []governance.LibraryVersion + err error + want takeoverKind + }{ + // 关键回归:查询失败 + 没有阶段证据 ⇒ 未知,不能当成"没有收尾证据"继续写。 + {"query fails without stage evidence", job, nil, queryErr, takeoverUnknown}, + // 阶段证据先独立成立:查询失败不影响"只差收尾"的判定。 + {"query fails but finalize stage succeeded", finalized, nil, queryErr, takeoverPendingFinal}, + {"effective version for this snapshot", job, effective, nil, takeoverPendingFinal}, + {"finalize stage succeeded", finalized, nil, nil, takeoverPendingFinal}, + {"this snapshot was superseded", job, supersededVersion, nil, takeoverSuperseded}, + {"no evidence at all", job, nil, nil, takeoverFresh}, + {"another snapshot effective but this one has no row", job, unrelated, nil, takeoverFresh}, + {"empty target snapshot", governance.LibraryJob{Scope: "internal"}, nil, nil, takeoverFresh}, + } + for _, test := range cases { + t.Run(test.name, func(t *testing.T) { + if got := classifyTakeover(test.job, test.versions, test.err); got != test.want { + t.Fatalf("classify = %q, want %q", got, test.want) + } + }) + } +} + +// 部分释放后重试必须幂等:已由本作业释放的行不能再算失败。 +func TestReleaseWindowLocksToleratesAlreadyReleasedRows(t *testing.T) { + store := &fakeFinishStore{} + if err := releaseWindowLocks(context.Background(), store, "kgseal-weknora", "internal", "job-1", + "verified", time.Now().UTC()); err != nil { + t.Fatal(err) + } + store = &fakeFinishStore{alreadyReleased: true} + if err := releaseWindowLocks(context.Background(), store, "kgseal-weknora", "internal", "job-1", + "verified", time.Now().UTC()); err != nil { + t.Fatalf("retry after a partial release must be idempotent: %v", err) + } + // 真正"不是本作业持有"的错误仍然要暴露。 + store = &fakeFinishStore{failOn: "instance_release"} + if err := releaseWindowLocks(context.Background(), store, "kgseal-weknora", "internal", "job-1", + "verified", time.Now().UTC()); err == nil { + t.Fatal("a foreign owner error was swallowed") + } +} + +func ptrTime(value time.Time) *time.Time { return &value } diff --git a/cmd/worker/library_handlers.go b/cmd/worker/library_handlers.go index 2b6d33a..87ba6d8 100644 --- a/cmd/worker/library_handlers.go +++ b/cmd/worker/library_handlers.go @@ -16,6 +16,7 @@ import ( "knowledge-governance/internal/librarymaintenance" "knowledge-governance/internal/libraryrelease" "knowledge-governance/internal/librarysnapshot" + "knowledge-governance/internal/pgdsn" postgresstore "knowledge-governance/internal/platform/postgres" ) @@ -65,32 +66,8 @@ type libraryReleasePayload struct { FinalizeOnly bool `json:"finalize_only,omitempty"` } -// openLibraryMaintenance 按部署注入的配置装配实例级维护目标。 -// 未接线时返回明确原因:只锁治理库 scope 不算维护,宁可拒绝写也不冒充。 -func openLibraryMaintenance(config appconfig.Config) (*librarymaintenance.Manager, error) { - database, err := librarymaintenance.DatabaseIdentity(config.LibraryProductionDatabaseURL) - if err != nil { - return nil, err - } - return librarymaintenance.New(librarymaintenance.Config{ - Instance: config.LibraryTargetInstance, - Project: config.LibraryTargetProject, - Writers: config.LibraryTargetWriters, - NonWriters: config.LibraryTargetNonWriters, - DockerSocket: config.LibraryTargetDockerSocket, - IngressURL: config.LibraryTargetIngressURL, - IngressService: config.LibraryTargetIngressService, - IngressDrainFile: config.LibraryTargetIngressDrainFile, - Database: database, - Index: config.LibraryProductionIndex, - Bucket: config.LibraryProductionS3Bucket, - // 后台静默检查的运行时依赖:HTTP 排空不等于后台任务排空。 - DatabaseURL: config.LibraryProductionDatabaseURL, - RedisAddr: config.LibraryTargetRedisAddr, - RedisPassword: config.LibraryTargetRedisPassword, - RedisDB: config.LibraryTargetRedisDB, - }) -} +// openLibraryMaintenance 定义在 library_wiring.go:维护目标的装配与目标整库栈共用 +// 同一个包(internal/librarystack),不在这里再拼一份。 // runLibraryRelease 执行"闸口 → 授权 → 关闭目标入口并停止写入者 → 预备份 → // 四类写入 → 校验 → 登记权威版本 → 恢复原运行状态和入口"的完整链路。 @@ -116,6 +93,21 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a // 绝不重新备份——重新备份会把"半写状态"当成新基线,覆盖真正可回滚的那份。 previous, err := store.LibraryJob(ctx, payload.LibraryJobID) resumed := err == nil && previous.SnapshotID != "" + if resumed { + versions, versionErr := store.LibraryVersions(ctx, payload.Scope, 50) + switch classifyTakeover(previous, versions, versionErr) { + case takeoverUnknown: + // 版本状态读不到、又没有阶段证据:不能当成"没有收尾证据"继续写目标。 + return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, + "接管时读不到该目标的版本状态,无法判断是只差收尾还是要重做;未对目标执行任何写入,稍后重试", "prebackup") + case takeoverSuperseded: + return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, + "本快照的生效版本已被别的快照顶替,拒绝用旧快照覆盖当前生产版本", "prebackup") + case takeoverPendingFinal: + // 上次可能死在"锁已放、终态没写"之间:只复核与登记,不重写生产。 + payload.FinalizeOnly = true + } + } if resumed { recorder.Stage(ctx, "prebackup", "skipped", "接管失败前已创建的预备份 "+previous.SnapshotID) } @@ -194,7 +186,7 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a "目标拓扑不满足唯一入口约束,拒绝开维护窗口:"+topoErr.Error(), "prebackup") } targetIdentity := fmt.Sprintf("scope=%s db=%s index=%s bucket=%s", payload.Scope, - redactDSN(config.LibraryProductionDatabaseURL), config.LibraryProductionIndex, config.LibraryProductionS3Bucket) + pgdsn.Endpoint(config.LibraryProductionDatabaseURL), config.LibraryProductionIndex, config.LibraryProductionS3Bucket) originalState, stateErr := manager.OriginalState(ctx) if stateErr != nil { return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, @@ -256,6 +248,7 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a if resumed { contentState = contentHalfWritten } + pending := successBoundary{} tracked := &contentTrackedTarget{inner: target, onApply: func() { contentState = contentHalfWritten }} allowRestore := func(error) bool { return contentState == contentUntouched || contentState == contentSucceeded || contentState == contentRolledBack @@ -290,8 +283,9 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a return errors.New(message) } recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本(仅收尾,未重写生产)") - _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize", "", "", - previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC()) + // 成功要等维护锁释放完才暴露:这里只记"待收尾"。 + pending = successBoundary{Ready: true, Backup: previous.SnapshotID, + Target: payload.TargetSnapshotID, FinishedAt: time.Now().UTC()} return nil } @@ -340,13 +334,10 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a return errors.New(message) } recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本") - if advanceErr := store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize", "", "", - outcome.Backup.SnapshotID, payload.TargetSnapshotID, finishedAt); advanceErr != nil { - // 状态落不下去时不要把队列作业报成功:让上层可见并等待接管。 - return fmt.Errorf("作业状态写入失败: %w", advanceErr) - } - log.Printf("library %s succeeded job=%s scope=%s snapshot=%s backup=%s", kind, - payload.LibraryJobID, payload.Scope, payload.TargetSnapshotID, outcome.Backup.SnapshotID) + // 内容与版本都落定了,但还不能报成功:服务健康、入口开放、两把维护锁 + // 释放完成才允许暴露 succeeded(见 finishLibrarySuccess)。 + pending = successBoundary{Ready: true, Backup: outcome.Backup.SnapshotID, + Target: payload.TargetSnapshotID, FinishedAt: finishedAt} return nil case "rolled_back": contentState = contentRolledBack @@ -407,10 +398,19 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a } return windowErr case contentState == contentSucceeded: - if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID, - "写入与校验完成,目标已按原运行状态恢复", now2); err != nil { - return fmt.Errorf("目标已恢复但维护占用释放失败: %w", err) + if !pending.Ready { + // 内容状态说成功但没有收尾凭据(不可能两者都缺):按未收尾处理,保持维护。 + note := "内容状态为已验证,但缺少收尾凭据,目标保持维护" + _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", note, now2) + _ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", note, now2) + return errors.New(note) } + if finishErr := finishLibrarySuccess(ctx, store, manager, identity.Instance, payload.Scope, + payload.LibraryJobID, pending); finishErr != nil { + return finishErr + } + log.Printf("library %s succeeded job=%s scope=%s snapshot=%s backup=%s", kind, + payload.LibraryJobID, payload.Scope, payload.TargetSnapshotID, pending.Backup) return windowErr case contentState == contentRolledBack: if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID, @@ -428,19 +428,6 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a } } -// releaseWindowLocks 是唯一释放路径:先记录入口已恢复,再放 scope 行与实例锁。 -// 顺序反过来会出现"实例锁放了、scope 还挂着"的中间态,页面就会说正在维护。 -func releaseWindowLocks(ctx context.Context, store *postgresstore.Store, instance, scope, jobID, note string, now time.Time) error { - _ = store.SetInstanceMaintenanceIngress(ctx, instance, jobID, "open", note, now) - _ = store.SetLibraryMaintenanceIngress(ctx, scope, jobID, "open", note, now) - _ = store.UpdateInstanceMaintenanceState(ctx, instance, jobID, "verified", note, now) - _ = store.UpdateLibraryMaintenanceState(ctx, scope, jobID, "verified", note, now) - if err := store.ReleaseInstanceMaintenance(ctx, instance, jobID, now); err != nil { - return err - } - return store.ReleaseLibraryMaintenance(ctx, scope, jobID, now) -} - // resumeInterruptedLibraryJobs 在 worker 启动时接管中断的写作业: // 已经持久化预备份的重新排队(同一个业务作业 id,阶段与备份都接着用), // 连预备份都没完成的才判失败——那种情况没有可回滚的基线,不能盲写。 @@ -469,8 +456,13 @@ func resumeInterruptedLibraryJobs(ctx context.Context, store *postgresstore.Stor continue } queueType := librarySyncJobType - if job.Kind == governance.LibraryJobRestore { + switch job.Kind { + case governance.LibraryJobRestore: queueType = libraryRestoreJobType + case governance.LibraryJobBaseline: + // 基线作业被杀时窗口可能还停着目标:用同一个作业 id 接管那个窗口, + // 继续捕获/校验,而不是留下一个 held 目标等人工处理。 + queueType = libraryBaselineJobType } payload, _ := json.Marshal(map[string]string{"library_job_id": job.ID, "scope": job.Scope, "target_snapshot_id": job.TargetSnapshotID}) diff --git a/cmd/worker/library_wiring.go b/cmd/worker/library_wiring.go index 5be0d33..7ef1e6c 100644 --- a/cmd/worker/library_wiring.go +++ b/cmd/worker/library_wiring.go @@ -2,116 +2,30 @@ package main import ( "context" - "errors" - "fmt" - "strings" "knowledge-governance/internal/appconfig" - "knowledge-governance/internal/librarysnapshot" + "knowledge-governance/internal/librarymaintenance" + "knowledge-governance/internal/librarystack" v071 "knowledge-governance/internal/releaseadapter/v071" ) -// buildLibraryProductionStack 用部署注入的配置装配生产整库目标与独立预备份存储。 -// 任何一项缺失都返回明确原因,调用方据此把作业失败并把缺项写进阶段, -// 不用"未配置"的占位对象假装能跑。 +// 目标整库读写栈的装配只有一份实现(internal/librarystack):worker 的同步/恢复/ +// 基线与主机侧恢复演练共用它,避免"目标是谁、用哪份契约"随调用路径漂移。 + +// openLibraryMaintenance 按部署注入的配置装配实例级维护目标。 +// 未接线时返回明确原因:只锁治理库 scope 不算维护,宁可拒绝写也不冒充。 +func openLibraryMaintenance(config appconfig.Config) (*librarymaintenance.Manager, error) { + return librarystack.OpenMaintenance(config) +} + +// buildLibraryProductionStack 用"要同步的这份快照"做契约装配目标与预备份存储。 func buildLibraryProductionStack(ctx context.Context, config appconfig.Config, scope, snapshotID, knownMapping string) ( *v071.ProductionLibraryTarget, *v071.ProductionBackupStore, error) { - missing := make([]string, 0, 4) - if config.LibraryProductionDatabaseURL == "" { - missing = append(missing, "LIBRARY_PRODUCTION_DATABASE_URL") - } - if config.LibraryProductionElasticURL == "" { - missing = append(missing, "LIBRARY_PRODUCTION_ELASTIC_URL") - } - if config.LibraryProductionTenantID == 0 { - missing = append(missing, "LIBRARY_PRODUCTION_TENANT_ID") - } - kb := config.LibraryProductionInternalKB - if scope == "external" { - kb = config.LibraryProductionExternalKB - } - if kb == "" { - missing = append(missing, "LIBRARY_PRODUCTION_KB_"+strings.ToUpper(scope)) - } - if config.LibraryProductionS3Endpoint == "" || config.LibraryProductionS3Bucket == "" || - config.LibraryProductionS3AccessKey == "" || config.LibraryProductionS3SecretKey == "" { - missing = append(missing, "LIBRARY_PRODUCTION_S3_ENDPOINT/BUCKET/ACCESS_KEY/SECRET_KEY") - } - if len(missing) > 0 { - return nil, nil, fmt.Errorf("生产整库目标未配置:缺少 %s", strings.Join(missing, "、")) - } - secure := strings.HasPrefix(config.LibraryProductionS3Endpoint, "https://") - objects, err := v071.NewS3ObjectStore(v071.S3ObjectStoreConfig{ - Provider: config.LibraryProductionS3Provider, Endpoint: config.LibraryProductionS3Endpoint, - Region: config.LibraryProductionS3Region, Bucket: config.LibraryProductionS3Bucket, - AccessKeyID: config.LibraryProductionS3AccessKey, SecretAccessKey: config.LibraryProductionS3SecretKey, - Secure: secure, ForcePathStyle: true, MaxObjectBytes: 256 * 1024 * 1024, - }) - if err != nil { - return nil, nil, errors.New("生产对象存储配置无效") - } - identity := v071.ProductionScope{TenantID: config.LibraryProductionTenantID, KnowledgeBaseID: kb} - target, err := v071.NewProductionLibraryTarget(ctx, v071.ProductionLibraryConfig{ - Postgres: config.LibraryProductionDatabaseURL, SearchURL: config.LibraryProductionElasticURL, - SearchIndex: config.LibraryProductionIndex, Objects: objects, ObjectsReader: objects, - ArchiveRoot: config.LibraryArchiveRoot, - MaxDocuments: 200000, - }, identity) - if err != nil { - return nil, nil, err - } - // 契约以"要同步的这份快照"为准:生产必须向它看齐,而不是反过来。 - // 生产为空时索引可能根本不存在,这时不能拿生产映射当契约(会永远启动不了)。 - archive, err := librarysnapshot.Open(config.LibraryArchiveRoot, 256*1024*1024) - if err != nil { - target.Close() - return nil, nil, fmt.Errorf("归档不可用,无法读取快照契约: %w", err) - } - manifest, err := archive.Load(snapshotID) - archive.Close() - if err != nil { - target.Close() - return nil, nil, fmt.Errorf("读取目标快照失败: %w", err) - } - contract := manifest.Compatibility - if contract.EmbeddingDimension == 0 { - contract.EmbeddingModel = config.LibraryEmbeddingModel - contract.EmbeddingDimension = config.LibraryEmbeddingDimension - } - // 生产索引存在时必须与快照契约一致;不存在或为空基线时跳过比对 - // (空索引的映射没有可比性,非空之后仍然严格比对)。 - if mapping, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil { - // 映射基线:优先用"上一次生产生效版本登记的实际映射", - // 它才是这份生产索引的既有事实;快照契约只管"来自哪份数据"。 - baseline := knownMapping - if baseline == "" { - // 没有历史基线时(首次接管这套生产索引),把当前实际映射当作基线并随 - // 预备份登记;之后任何改动都会与登记值不符而被拒绝。 - baseline = mapping - } - if baseline != "" && mapping != baseline { - current, currentErr := target.Current(ctx) - emptyBaseline := currentErr == nil && current.Counts["index"] == 0 && current.Counts["chunks"] == 0 - if !emptyBaseline { - target.Close() - return nil, nil, errors.New("生产索引映射与上次生效版本不一致(索引被改动过),先对齐索引映射") - } - } - } - // 预备份描述的是"生产现在的索引",不是"快照的来源索引": - // 生产映射基线优先用上次生效版本登记的实际值。 - backupContract := contract - if knownMapping != "" { - backupContract.IndexMappingSHA256 = knownMapping - } - backups, err := v071.NewProductionBackupStore(v071.ProductionBackupConfig{ - Pool: target.Pool(), Search: target.Search(), Objects: target.Objects(), - ArchiveRoot: config.LibraryArchiveRoot, Scope: scope, Identity: target.Identity(), - Compatibility: backupContract, MaxDocuments: 200000, MaxObjectByte: 256 * 1024 * 1024, - }) - if err != nil { - target.Close() - return nil, nil, err - } - return target, backups, nil + return librarystack.OpenProductionStack(ctx, config, scope, snapshotID, knownMapping) +} + +// buildLibraryBaselineStack 装配"给目标自己建基线"所需的目标与捕获器(契约取目标当前映射)。 +func buildLibraryBaselineStack(ctx context.Context, config appconfig.Config, scope string) ( + *v071.ProductionLibraryTarget, *v071.ProductionBackupStore, error) { + return librarystack.OpenBaselineStack(ctx, config, scope) } diff --git a/cmd/worker/main.go b/cmd/worker/main.go index 814e7a8..acdefac 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -8,7 +8,6 @@ import ( "os" "os/signal" "path/filepath" - "strings" "syscall" "time" @@ -58,21 +57,6 @@ func (r libraryStageRecorder) Stage(ctx context.Context, stage, state, detail st func int64Ptr(value int64) *int64 { return &value } -// redactDSN 只保留库名,避免把连接串写进维护记录。 -func redactDSN(dsn string) string { - if dsn == "" { - return "" - } - if index := strings.LastIndex(dsn, "/"); index >= 0 && index+1 < len(dsn) { - name := dsn[index+1:] - if cut := strings.Index(name, "?"); cut >= 0 { - name = name[:cut] - } - return name - } - return "configured" -} - // stageSize 是某一类数据在快照里的真实总量(行数/字节)。 type stageSize struct { count int64 @@ -109,6 +93,9 @@ const libraryRecoverJobType = "library.recover" // libraryRestoreJobType 是按指定整库版本恢复的队列类型;与同步分开、授权也分开。 const libraryRestoreJobType = "library.restore" +// libraryBaselineJobType 是"建立目标权威基线"的队列类型(与 API 侧常量同值)。 +const libraryBaselineJobType = "library.baseline" + func main() { ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() @@ -205,6 +192,8 @@ func run(ctx context.Context, getenv func(string) string) error { go runRecordsSyncScheduler(ctx, mirrorStore, func(error) { log.Print("QA local mirror scheduler unavailable; last committed data retained") }) } go weeklyMailLoop(ctx, pool, getenv) + // 封存执行器状态:worker 是唯一能连 agent socket 的一方,它写观测、页面读观测。 + go runLibraryExecutorProbe(ctx, store, config, libraryExecutorProbeInterval) go runSourceRefreshScheduler(ctx, store, config.SourceRefreshSchedulerInterval, func(err error) { log.Printf("worker %s source refresh scheduler error: %v", config.WorkerID, err) }) @@ -276,7 +265,24 @@ func workerHandlers(store *postgresstore.Store, config appconfig.Config) (map[st "主机侧正在暂停写入者、导出并恢复", startedAt); stageErr != nil { log.Printf("library seal job %s stage update failed: %v", libraryJobID, stageErr) } - result, err := libraryagent.Capture(ctx, agentConfig, payload.Scope, payload.CaptureJob) + // 平台审计过的精确死信解除指纹:读取失败必须阻断(fail closed), + // 不能因为存储不可用就静默按"没有解除"或"全部解除"处理。 + released, releaseErr := store.DeadLetterReleaseFingerprints(ctx, config.SourceInstance()) + if releaseErr != nil { + _ = store.UpdateLibraryStage(ctx, libraryJobID, "export", "failed", nil, nil, nil, nil, releaseErr.Error(), time.Now().UTC()) + _ = store.AdvanceLibraryJob(ctx, libraryJobID, governance.LibraryJobFailed, "export", "dead letter release storage unavailable", "", "", "", time.Now().UTC()) + return fmt.Errorf("load dead letter releases: %w", releaseErr) + } + // 期望执行器身份随请求一起送进 agent,由接收方在操作互斥内、任何源操作之前 + // 比对:这样即使 socket 在这之前被换掉,错的执行器也一次 run 都不会跑。 + expectation, expectErr := captureExpectation(config) + if expectErr != nil { + _ = store.UpdateLibraryStage(ctx, libraryJobID, "export", "failed", nil, nil, nil, nil, expectErr.Error(), time.Now().UTC()) + _ = store.AdvanceLibraryJob(ctx, libraryJobID, governance.LibraryJobFailed, "export", expectErr.Error(), + "未停任何写入者;补齐来源身份配置后重试", "", "", time.Now().UTC()) + return expectErr + } + result, err := libraryagent.CaptureReleased(ctx, agentConfig, payload.Scope, payload.CaptureJob, released, expectation) finishedAt := time.Now().UTC() if err != nil { // 导出失败时阶段与作业都要留痕;恢复责任在 agent,不会因为作业失败而假装来源已恢复。 @@ -286,13 +292,22 @@ func workerHandlers(store *postgresstore.Store, config appconfig.Config) (map[st log.Printf("library capture failed scope=%s capture_job=%s: %v", payload.Scope, payload.CaptureJob, err) return err } + // 执行后复核:结果里 agent 自报的身份必须还是期望实例,否则这次封存 + // 可能来自别的栈,绝不能登记成"本实例的测试封存"。 + if identityErr := captureExecutorPostcheck(result.Payload, expectation); identityErr != nil { + _ = store.UpdateLibraryStage(ctx, libraryJobID, "export", "failed", nil, nil, nil, nil, identityErr.Error(), finishedAt) + _ = store.AdvanceLibraryJob(ctx, libraryJobID, governance.LibraryJobFailed, "export", identityErr.Error(), + "快照已产生但来源身份不符,未登记为测试封存;核对执行器后重新封存", "", "", finishedAt) + return identityErr + } // 封存产物成版本记录:只登记"测试封存",绝不写成生产生效版本。 versionErr := store.RecordLibraryVersion(ctx, governance.LibraryVersion{ ID: governance.LibraryVersionID(governance.LibraryVersionTestSealed, payload.Scope, result.SnapshotID), Scope: payload.Scope, Kind: governance.LibraryVersionTestSealed, SnapshotID: result.SnapshotID, SourceJobID: libraryJobID, Evidence: map[string]any{"capture_job": payload.CaptureJob, "replay": result.Replay, - "requested_by": payload.RequestedBy, "agent_result": json.RawMessage(result.Payload)}, + "requested_by": payload.RequestedBy, "expectation_fingerprint": expectationFingerprint(expectation), + "agent_result": json.RawMessage(result.Payload)}, CreatedBy: payload.RequestedBy, CreatedAt: finishedAt, }) if versionErr != nil { @@ -314,6 +329,10 @@ func workerHandlers(store *postgresstore.Store, config appconfig.Config) (map[st handlers[libraryRestoreJobType] = func(ctx context.Context, job jobs.Job) error { return runLibraryRelease(ctx, store, config, job, governance.LibraryJobRestore) } + // 建立目标权威基线:目标非空又没有生效版本时的唯一入口(只读捕获 + 校验 + 登记)。 + handlers[libraryBaselineJobType] = func(ctx context.Context, job jobs.Job) error { + return runLibraryBaseline(ctx, store, config, job) + } // A crash can leave the source writer stopped: recover before taking work, // and keep the failure visible instead of pretending the worker is ready. go func() { diff --git a/docs/formal-8088-deployment-plan.md b/docs/formal-8088-deployment-plan.md new file mode 100644 index 0000000..3848074 --- /dev/null +++ b/docs/formal-8088-deployment-plan.md @@ -0,0 +1,93 @@ +# 本机正式 8088 待验收部署方案(只准备,不执行) + +> 本文是给父会话审查后用的一次性方案:**没有替换 8088,没有重启共享业务**。 +> 正式栈当前仍是旧版(0.1.114),不得对外声称新版已上线。 + +## 1. 版本清单 + +| 项 | 值 | +| --- | --- | +| 当前正式版本 | 开发版 `0.1.114` / 代码 `dc9e827`(api `knowledge-governance-api:library-dc9e8276560d`,web `knowledge-governance-web:source-dc9e8276560d`,worker `knowledge-governance-worker:library-dc9e8276560d`) | +| 本次候选代码 | 分支 `feat/library-wiki-contribution-diff`,最终提交见交付报告(代码提交 + 文档提交分开列 SHA) | +| 开发号 | 提交后由 `scripts/version.py` 自动得出(0.1.0 + 自 v0.1.0 起的提交数) | +| 前端产物 | `npm --prefix web run build` → `web/dist`,由 `Dockerfile.web` 打进 web 镜像 | +| 验收实例(已验证同 SHA) | `/tmp/kg-e2e/bin/{api,worker}` + 候选 `libraryctl`,`go version -m` 的 `vcs.modified=false` | + +## 2. 数据库迁移 + +| 现状 | 目标 | +| --- | --- | +| 正式治理库 `schema_migrations.max(version)=66` | 需要依次应用 `000067` → `000072` | + +``` +000067_library_release_authorizations 发布授权表 +000068_library_restore_kind 作业类型加 restore +000069_library_write_target_mutex 写入目标互斥 +000070_library_maintenance 维护窗口表 +000071_library_maintenance_level 维护级别(library / instance) +000072_library_instance_maintenance 实例级维护占用 +``` + +- 迁移在 api 启动时按 `AUTO_MIGRATE` 自动执行(与现有部署一致);**先备份治理库**再起新 api。 +- 回滚顺序:`072.down` → `071.down` → `070.down` → `069.down` → `068.down` → `067.down`,仅在新版 api 不再写入这些表之后执行。 +- 这些迁移只新增表/列,不改动既有知识治理数据;但 `000068` 扩了 `library_jobs.kind` 的取值。 + +## 3. 运行配置(正式栈) + +正式 api 现有相关环境变量(`docker inspect knowledge-governance-p1-api-1`): + +``` +PRODUCTION_RELEASE_REQUESTS_ENABLED=true +PRODUCTION_BACKUP_RESTORE_VERIFIED=true +PRODUCTION_WEKNORA_BASE_URL=https://weknora-test.dobot.cn +PRODUCTION_RELEASE_AUTH_TOKEN_FILE=/run/release-secrets/production_release_auth_token +PRODUCTION_WEKNORA_API_KEY_FILE=/run/release-secrets/target_weknora_api_key +LIBRARY_AGENT_SOCKET=/run/library-agent/agent.sock +LIBRARY_ARCHIVE_ROOT=/var/lib/knowledge-governance/library +``` + +**没有** `LIBRARY_TARGET_*`(实例级维护目标)与 `LIBRARY_PRODUCTION_DATABASE_URL`。这必须保持"缺配即拒绝": + +- 缺 `LIBRARY_TARGET_*` 时,worker 在 `prebackup` 阶段直接失败并写明 + 「目标维护未接线,拒绝在挡不住入口/后台写入的情况下写目标」(`cmd/worker/library_handlers.go`)——**不会**退化成"库级锁假装维护"。 +- 缺 `LIBRARY_PRODUCTION_DATABASE_URL` 时 overview/差异按"未接入"显示,不显示 0 或被当成"无变化"。 +- `/api/v1/library/maintenance` 在实例未配置时返回 `instance` 缺省、`held=false`;页面显示「未接入 + 维护类操作 fail closed」。 +- `LIBRARY_AGENT_SOCKET` 指向的 agent 必须在部署时一起起来(`scripts/library-agent-install.sh` + 0600 `agent.env`); + 端点/闸门/队列声明与 worker 同源(`LIBRARY_TARGET_*`)。缺任何一项时封存会在开窗口前被拒并点名缺什么。 + +新增配置只在需要真正开维护窗口时才加(本机验收实例的取值可作模板,见 +`evidence-maintenance/harness/candidate-maintenance.env`:`LIBRARY_TARGET_INSTANCE/PROJECT/WRITERS/NON_WRITERS/DOCKER_SOCKET/INGRESS_*`、`LIBRARY_TARGET_REDIS_*`)。 + +## 4. 部署步骤(父审后执行) + +1. **冻结**:确认没有在跑的封存/同步/恢复作业(`GET /api/v1/library/jobs` 无 pending/running)。 +2. **备份治理库 + 归档根**:`pg_dump` 治理库;保持 `LIBRARY_ARCHIVE_ROOT` 卷原样(版本与 blob 都在里面)。 +3. **构建**:从候选 SHA 构建 api/worker/web 三个镜像,tag 里带完整 SHA;不在生产机上另行 build 源码。 +4. **切换**:`docker compose up -d api worker web`(只替换这三个容器;postgres/dingtalk-login/共享 WKNFull-* 不动)。 +5. **健康检查**:`GET /health`、`GET /api/v1/library/overview?scope=internal` 200、`GET /api/v1/library/maintenance?scope=internal` 200 且 `held=false`。 +6. **页面验收**(用户现有管理员会话,不再要候选密码):发布管理页 → 两库页签 → 目标维护面板显示"未接入/未占用"、Wiki 差异逐页行、作业进度与阶段。 + +## 5. 回滚 + +| 触发 | 动作 | +| --- | --- | +| api/web/worker 起不来或健康检查失败 | 三个容器回到 `*-dc9e8276560d` 镜像;迁移保持已应用(新表空表存在不影响旧代码) | +| 新版写入后需要完全回到旧结构 | 先停新版 api/worker → 按 072→067 逆序 down → 恢复备份的治理库 → 起旧镜像 | +| 归档不可读/版本异常 | 归档根只读回滚:保留新卷,切回旧卷挂载点 | + +回滚**不触碰**正式账号、会话、治理库既有业务数据(迁移只加表/列),也不动 WeKnora 侧数据。 + +## 6. 正式账号与数据保留 + +- 管理员账号/会话在治理库 `users`/`sessions`(`kg_session` cookie)——迁移与镜像切换都不改这些表;不重置口令、不重新初始化。 +- 首页登录沿用现有会话;验收时不要用候选环境的账号。 +- 正式库与归档卷保留原路径,不做"清理/重建"。 + +## 7. 已知风险与前置条件(必须先解决或明确接受) + +1. **`knowledge-governance-p1-release-worker-1` 处于重启循环**(日志每分钟 `connect release target database`,`RELEASE_WORKER_ENABLED=false`)。这是正式栈既有状态,不是本轮引入;部署前需要决定是修配置还是先停掉该服务,否则会持续刷日志。 +2. **封存能力已在本机专用实例验证**(2026-10-29:页面入口 → agent → 实例级窗口 → 同步 → 恢复)。 + 正式部署仍需按同一形状接线:`libraryagent`(0600 socket)+ `agent.env`(`LIBRARY_TARGET_*`)+ `LIBRARY_AGENT_SOCKET`; + 这套接线在正式栈上尚未执行。 +3. **入口排空探针未在最终构建复现**(矩阵"已实现·未跑"):正式部署前若要拿这条证据,需要在生产机重复探针,并接受"闸门关得比客户端快"导致上传注入失败的可能。 +4. 部署本身就是生产变更:需要用户给的窗口与授权,本文不构成执行许可。 \ No newline at end of file diff --git a/docs/library-acceptance-matrix.md b/docs/library-acceptance-matrix.md index 8885ff7..0bc50ee 100644 --- a/docs/library-acceptance-matrix.md +++ b/docs/library-acceptance-matrix.md @@ -1,80 +1,135 @@ -# 整库同步验收矩阵(真实证据对照) +# 整库同步验收矩阵(真实证据对照,2026-10-29 重排) -## 入口级真实服务验收(2026-10-28,候选 API :18089,真实 PG/ES/MinIO,code_sha=ea4cffb) +本文件是**唯一口径**:状态只写实际发生的事,四类严格分开,不用"部分通过"含糊过去。 -| 项 | 状态 | 证据(`~/.hermes/handoffs/kg-candidate-e2e/`) | -|---|---|---| -| A0 首次发布(生产空基线) | 已实现 / 真实服务已验证 | `evidence/A0_baseline.json` | -| A1 撤销(自洽派生快照) | 已实现 / 真实服务已验证 | `evidence/A1_revoke.json` | -| B 按版本恢复(含未授权拒绝) | 已实现 / 真实服务已验证 | `evidence/B_restore.json` | -| C 写阶段 SIGKILL 接管(复用同一预备份) | 已实现 / 真实服务已验证 | `evidence/C_kill_takeover.json` | -| D 跨 kind 互斥 + 多 worker 单次执行 | 已实现 / 真实服务已验证 | `evidence/D_mutex_multiworker.json` | -| E1 版本登记失败不报成功 | 已实现 / 真实服务已验证 | `evidence/E1_finalize_fail.json` | -| E2 finalize 续作(只收尾) | 已实现 / 真实服务已验证 | `evidence/E2_finalize_resume.json` | -| E3 预备份登记失败零写入 | 已实现 / 真实服务已验证 | `evidence/E3_prebackup_fail_zero_write.json` | -| 远程生产写入/恢复/清理 | 未授权,未执行 | — | -| Wiki 撤销贡献语义重算 | 未实现(仅页面级差异) | V03 | -| 跨系统原子切点 | 未实现;按"导出后回读"协议,边界已写明 | — | +**2026-10-29 用户决定(覆盖历史验收口径)**:发布管理**不承担** Wiki 语义撤销—— +它只负责把测试端已确认的整库快照**原样还原**生产。因此撤销后正文里是否残留 +撤销方独有事实,属于**上游知识质量**问题,不在本模块验收范围内:V03-a 保留原始 +失败证据(见「一·补」下方的"模块范围外"表),不改成通过,也不再阻断发布。 +同时**不**新增 LLM 重写、人工逐页复核闸门,也不把升级 WeKnora 作为前置条件。 +保留的门槛是:整库文档/分块/索引/资源/Wiki 正文·来源·链接·目录一致、稳定切点、 +实际维护与排空、准确快照授权、独立预备份与恢复、失败回滚、作业互斥与重启接管、 +真实 UI 进度与存储安全。效果问答/性能/A06 不是门槛。 -对应 `docs/library-sync-acceptance.md` 的 30 个验收项。**状态只写实际发生的事**: +| 口径 | 含义 | +| --- | --- | +| **已完成·已验证** | 有代码 + 真实服务(真实 PG/ES/对象存储/原版 WeKnora API)跑出来的证据,或既有 CI 检查覆盖 | +| **缺代码** | 功能/契约本身没有实现;界面按"未接入"显示,不冒充 | +| **已实现·未跑** | 代码与测试在位,缺真实环境执行证据(不是权限问题) | +| **需授权/需环境** | 实现所需的动作属于生产写入/读取/恢复,或需要本会话拿不到的资源 | -- **已实现·已验证**:有代码 + 本机真实数据或真实归档/真实 PostgreSQL 跑过的证据,或既有 CI 检查覆盖。 -- **已实现·未验证**:代码与测试在位,但还缺真实业务环境的执行证据(例如需要隔离生产栈)。 -- **阻塞(需授权/环境)**:实现所需的动作属于生产写入、生产读取或生产恢复,当前授权明确禁止。 -- **未实现**:还没有代码,界面按"未接入/未知"显示,不冒充完成。 +A06(内容、问答、引用图片效果、性能报告)已由用户取消,不作为门槛,也不在本表。 -A06(内容、问答、引用图片效果、性能报告)已被用户取消,本矩阵不含该项,也不再作为封版门槛。 +--- -**状态口径(严格区分四件事)**: -- 已实现·已验证:代码 + 本机真实归档/真实 PostgreSQL 或既有 CI 的实跑证据。 -- 已实现·隔离测试通过:代码 + 单元/集成测试(注入假生产目标),**没有**真实生产执行。 -- 已实现·未验证:代码在位但缺真实环境证据。 -- 未实现 / 真实执行未授权:前者是代码缺口,后者是**权限**缺口,两者不混用。 +## 一、已完成·已验证 -| ID | 验收要点 | 代码 / 承载位置 | 状态 | 证据或缺口 | -| --- | --- | --- | --- | --- | -| V01 | 只有对内/对外两个目标,身份来自配置映射 | `internal/httpapi/library_*.go`(scope 只接受 internal/external);`web/src/ReleaseTargets.tsx` | 已实现·已验证 | overview 对非法 scope 返回 400;真实归档里 internal/external 各自解析到唯一 KB(0c84a098… / 对外 KB) | -| V02 | 全部解析/索引/Wiki 就绪,验收绑定本版 | `internal/releaseadapter/v071/library_source_settlement.go`;页面准备清单 | 已实现·未验证 | 结清检查(settlement)在封存前执行且缺项阻断;但对"实时测试库此刻就绪"仍需一次真实封存复核;页面已注明依据的是哪次快照 | -| V03 | 删除文档后 Wiki 共享正文/实体/边/引用真实变化 | `wikiDiffOf`(页面级差异) | **未实现(架构阻塞)** | Wiki 页由 WeKnora 在**源侧**生成,治理侧只搬运快照;"两文档共贡献一页、撤一个另一个保留"必须在源侧重新生成才能验证。本轮未触发源侧生成(属源侧写操作,需原生 Wiki 接口)。页面级差异**不等于**语义重算,不当作完成 | -| V04 | 完整固定版本:文档集合+处理结果+资源+索引+完整 Wiki | `internal/librarysnapshot/*`(10 表 + 索引 + 资源 + Wiki 部件);`artifacts` 清单 | 已实现·已验证 | 真实快照 f992d64e…:4 文档/38 分块/122 索引/21 资源/31 Wiki 页/55,309,044 字节;部件摘要逐项校验 | -| V05 | 导出中源变更则拒绝或隔离 | `internal/libraryfreeze/*` + 预备份导出前后行数静默检查 | 部分实现·未验证 | 预备份导出前后行数不一致即失败(本机已验证会拒绝);真实并发写入的隔离演练仍未做 | -| V06 | 只同步知识数据,不含账号/会话/凭据 | 导出白名单(10 张知识表 + 知识域索引 + 资源 + Wiki) | 已实现·已验证 | 部件清单里没有 users/sessions/api_keys;真实归档部件名可逐项核对 | -| B01 | 每次生产写入前完整预备份,失败零写入 | `v071.ProductionBackupStore.Prepare` + `libraryrelease` prebackup 阶段 | 已实现·本机真实服务已验证 | 本机真实 PG/ES 上生成预备份 3111e823…(626 字节),带导出前后行数静默检查;故障注入后用它恢复并校验通过。生产目标本身未授权,未对真实生产执行 | -| B02 | 资源在测试侧删除后仍可恢复 | 归档内容寻址 blob(`library_versions` + `librarysnapshot` 内容去重) | 已实现·未验证 | 归档不依赖测试临时对象;跨版本去重 55,309,693 字节;尚未做"测试侧删除后从归档恢复"的实测 | -| B03 | 每版保存完整 Wiki 与引用关联并校验 | `library_snapshot` 的 wiki 部件 + 摘要 | 已实现·已验证 | 真实快照含 31 页 Wiki 与逐页摘要,manifest 记录数量/字节 | -| S01 | 数据库/资源/索引/Wiki 真实落地并校验 | `v071.ProductionLibraryTarget.Apply/Verify`(真实 SQL + ES bulk) | 已实现·本机真实服务已验证 | 本机真实 PG/ES:导入 1 文档/1 分块/1 Wiki 页/1 索引文档并逐表身份摘要校验;写入只覆盖知识域表 | -| S02 | 生产不再解析/Embedding/生成 Wiki | 索引阶段直接搬运归档原始文档与向量 | 已实现·本机真实服务已验证 | 真实 ES 写入的是归档里的 embedding 数组(未调用任何模型);生产环境本身仍未写入 | -| S03 | 一致生效与失败补偿边界 | — | 未实现 | 跨 PG/ES/资源的一致切点与可见性保护未实现 | -| S04 | 各阶段失败/超时/重启可追溯可恢复 | `library_jobs`/`library_job_stages` + `ReconcileLibraryJobs` | 已实现·已验证 | 阶段与错误持久化;worker 启动清理"队列已无对应作业"的孤儿并写明恢复责任,避免永久占锁 | -| S05 | 幂等与互斥(多 worker 安全) | `library_jobs_idempotency`、`library_jobs_active` 唯一索引 | 已实现·已验证 | 真实 PostgreSQL 上验证:同目标第二个在飞作业被拒;同键同 digest 重放复用;同键换 digest 冲突 | -| R01 | 按指定版本手工恢复 | `POST /library/restores` + `library.restore` 执行器 | 已实现·真实服务已验证 | 候选环境:撤掉全部旧授权→503;签发 restore 授权→202→succeeded,内容回到目标版本(`evidence/B_restore.json`,作业 `librestore-internal-20261028T120300.925270`)。远程生产未执行 | -| R02 | 恢复失败显式异常、不回退测试库 | `Outcome.State=rollback_failed` + 独立阶段记录 | 已实现·隔离测试通过 | 测试断言恢复失败与同步失败可区分,且不触碰测试库 | -| R03 | 存快/恢复快对比 | — | 未实现 | 未做测速;不写没有测量的数字 | -| P01 | 作业阶段与整体状态 | `library_jobs.state/stage`;页面阶段条 | 已实现·已验证 | 阶段勾选只认当前作业证据;无作业时显示"本次作业尚未开始" | -| P02 | 四类分项计数/字节/错误 | `stageSizesFrom(manifest)` + `library_job_stages` | 已实现·真实服务已验证 | 候选环境实测 database 67 行/88086 B、resources 21 行/53490166 B、index 122 行/1624880 B、wiki 63 行/105912 B;prebackup/verify/finalize/rollback 无可定义总量保持 NULL;证据 `evidence/A0_baseline.json` 的 `stages` | -| P03 | 同步明细与异常 | 页面"同步明细"表读阶段行 | 已实现·已验证 | 行来自持久阶段,刷新/重启可复现;封存作业显示 export 阶段真实状态 | -| U01 | 发布管理为主要入口(两库+单作业区) | `web/src/ReleaseTargets.tsx` | 已实现·已验证 | 按确认稿布局;父会话已在登录浏览器核验两库切换与抽屉 | -| U02 | 权限边界 | `canWrite`/`canCapture`,POST 需 engineer+ | 已实现·已验证 | 无权限时不给封存入口;未接线时说明通道状态 | -| U03 | 操作确认 | 封存按钮 + 作业号 + notice | 已实现·已验证 | 同一作业号重放返回已存作业并明确提示 | -| D01 | 保留策略(数量/天数) | `internal/libraryretention`;`LIBRARY_RETENTION_VERSIONS/DAYS` | 已实现·已验证(单测+真实归档) | 未设置=0 时按"全部保留",不编默认值;设置了才生成候选 | -| D02 | 占用测量 | `internal/librarysnapshot/usage.go`;系统设置→存储管理 | 已实现·已验证 | 版本数/跨版本去重内容/blob 表观与分配/未引用/清单;真实两份清单 55,309,693 字节 | -| D03 | 依赖保护 | `libraryretention.Build` 的 Protected/ProtectedBy | 已实现·已验证 | 真实归档上 2 个版本都因"最近测试封存"受保护;受保护版本永不进入候选 | -| D04 | 清理预览 | `libraryretention.Build` 的 Candidates/ReclaimPreviewBytes;`GET /library/storage` | 已实现·已验证 | 真实归档:策略 keep=1 时仍 0 候选(两版均受保护);回收预览只计独占字节 | -| D05 | 容量门禁 | `libraryretention.CheckCapacity` + sync gate | 已实现·已验证(单测+真实数据) | 预算未设置→阻断并说明;超预算→阻断并给出预计字节 | -| D06 | 测试残留 | 归档未引用 blob 统计(UnreferencedBlobs/Bytes) | 已实现·已验证 | 真实归档未引用 blob 为 0;这只是口径,不等于扫描过所有测试残留 | -| D07 | 实际回收 | — | 阻塞(需清理授权) | 自动清理默认关闭,不提供删除动作 | +| ID | 要点 | 承载位置 | 证据(本轮或前轮,均可回看) | +| --- | --- | --- | --- | +| V01 | 只有对内/对外两个目标,身份来自配置映射 | `internal/httpapi/library_*.go`;`web/src/ReleaseTargets.tsx` | overview 对非法 scope 返回 400;真实归档 internal/external 各自解析到唯一 KB | +| V02 | 解析/索引/Wiki 就绪,验收绑定本版 | `v071/library_source_settlement.go` | 结清检查在封存前执行、缺项阻断;页面注明依据哪次快照 | +| V04 | 完整固定版本:文档+处理结果+资源+索引+完整 Wiki | `internal/librarysnapshot/*` | 真实快照 f992d64e…:4 文档/38 分块/122 索引/21 资源/31 Wiki 页/55,309,044 字节,部件摘要逐项校验 | +| V06 | 只同步知识数据,不含账号/会话/凭据 | 导出白名单(10 知识表 + 知识域索引 + 资源 + Wiki) | 部件清单无 users/sessions/api_keys | +| B01 | 每次生产写入前完整预备份,失败零写入 | `v071.ProductionBackupStore.Prepare` + prebackup 阶段 | 真实 PG/ES 上预备份 3111e823…;故障注入后用它恢复并校验 | +| B03 | 每版保存完整 Wiki 与引用关联并校验 | `library_snapshot` wiki 部件 + 摘要 | 真实快照 31 页逐页摘要;manifest 记录数量/字节 | +| S01 | 数据库/资源/索引/Wiki 真实落地并校验 | `v071.ProductionLibraryTarget.Apply/Verify` | 本轮 sync `libsync-internal-…1793221776816380302`:database 67 行、resources 21、index 122、**wiki 63 行**全部 succeeded + verify succeeded;目标库 wiki 31 页 | +| S02 | 生产不再解析/Embedding/生成 Wiki | index 阶段直接搬运归档 embedding | 真实 ES 写入归档 embedding,未调用模型 | +| S04 | 各阶段失败/超时/重启可追溯可恢复 | `library_jobs`/`library_job_stages` + `ReconcileLibraryJobs` | 阶段与错误持久化;worker 清理"队列无对应作业"的孤儿 | +| S05 | 幂等与互斥(多 worker 安全) | 幂等表 + active 唯一索引 | 真实 PG:同目标第二作业被拒;同键同 digest 复用;换 digest 冲突 | +| R01 | 按指定版本手工恢复 | `POST /library/restores` + `library.restore` | 真实跑通:撤旧授权→503;签发 restore 授权→202→succeeded,内容回到目标版本(`evidence/B_restore.json`) | +| R02 | 恢复失败显式异常、不回退测试库 | `Outcome.State=rollback_failed` + 独立阶段 | 隔离测试 + 前轮真半写场景(`partial.json`) | +| P01 | 作业阶段与整体状态 | `library_jobs.state/stage`;页面阶段条 | 阶段勾选只认当前作业证据;无作业显示"尚未开始" | +| P02 | 四类分项计数/字节/错误 | `stageSizesFrom(manifest)` + `library_job_stages` | 真实数据:database 67/88086B、resources 21/53490166B、index 122/1624880B、wiki 63/105912B | +| P03 | 同步明细与异常 | 页面"同步明细"读阶段行 | 行来自持久阶段,刷新/重启可复现 | +| U01 | 发布管理为主要入口(两库+单作业区) | `web/src/ReleaseTargets.tsx` | 按确认稿布局,父会话已在登录浏览器核验 | +| U02 | 权限边界 | `canWrite`/`canCapture`,写操作需 engineer+ | 无权限不给封存入口;未接线显示通道状态 | +| U03 | 操作确认 | 封存按钮 + 作业号 + notice | 同一作业号重放返回已存作业并提示 | +| D01 | 保留策略(数量/天数) | `internal/libraryretention` | 未设置=0 按"全部保留",不编默认值 | +| D02 | 占用测量 | `internal/librarysnapshot/usage.go` | 版本数/去重内容/blob/未引用/清单;两份真实清单 55,309,693 字节 | +| D03 | 依赖保护 | `libraryretention.Build` 的 Protected/ProtectedBy | 真实归档两版均因"最近测试封存"受保护 | +| D04 | 清理预览 | `Build` 的 Candidates/ReclaimPreviewBytes | keep=1 时仍 0 候选;回收预览只计独占字节 | +| D05 | 容量门禁 | `CheckCapacity` + sync gate | 预算未设置→阻断;超预算→阻断并给预计字节 | +| D06 | 测试残留口径 | 未引用 blob 统计 | 真实归档未引用 blob 为 0(口径,不等于扫过全部残留) | +| **S03** | 实例级维护窗口:关入口→排空→静默证明→停写者→写入→校验→开入口 | `internal/libraryfreeze` + `librarymaintenance` + 迁移 070–072 | 前轮 7 场景真跑全绿(normal/kill/applyfail/partial/crossscope/recover_refuse/recover_ok),本轮 sync/drain 场景继续复用;`quiesce` 证据、`post_stop_*`、journal、gate drain.json 均可回看 | +| **V03-b** | 治理侧能把上述语义变化说清楚(不是页面级"无变化") | `internal/librarydiff/wiki.go` `CompareWiki`;`GET /library/overview` 的 `diff.wiki` | 逐页比对正文/来源/引用/出入边/目录,输出 `sources_removed`、`citations_before→after`、`changed`;单测 `wiki_test.go`、视图单测 `library_wiki_diff_test.go`;页面逐行展示(`wikiContributionLines` + 测试) | +| **S06** | 页面/API 发起封存 → 主机 agent → 实例级窗口封存(真实冻结) | `POST /library/captures` + `cmd/libraryagent`(0600 socket、固定命令)+ `libraryctl capture`(实例级:闸门/后台静默/指纹) | **本轮终态(0.1.186 e508abd)真跑**:`POST /api/v1/library/captures`(job `lib-internal-20261029T0716`)202 → worker → agent → CLI → 关闸门排空 → 等后台静默 → 停 `kgmaint-app`(exit=1、`drain_verified=true`、`stop_verified=true`、`restored=true`)→ 导出 → 恢复写入者与入口;作业 `libseal-lib-internal-20261029T0716` succeeded,快照 `0f0a3b68…`。证据 `kg-final-gaps/evidence/04-inflight-v05-final.json`、journal `freeze-kgmaint/lib-internal-20261029T0716.json` | +| **S06-a** | 排空证明取自真实在途请求(不靠 202 后抢时序) | 闸门 `drain.json` + 客户端真实上传握手 | **本轮终态真跑**:先发一半 multipart 正文 → 闸门 `conn:open active=1` → 再发起封存 → 入口关闭 → 补完正文,应用 200 并完成解析/嵌入 → `drain.json phase=drained active_at_close=1 served=143 drain_ms=352`;journal `quiesce.samples=18 stable_ms=1617` 且停写者后指纹不变(`post_stop_digest` == `digest`)。窗口内派生任务真的跑完(探针文档 `f7fed988…` completed / 7 分块),且该文档出现在被封存的快照里 | +| **V03-c** | 带"撤销后共享页"的整库封存 → 独立目标同步 → 恢复,逐项真实读回 | `libraryctl` 封存 + `ProductionLibraryTarget` + `libraryrelease` | **本轮终态真跑(字节一致性,不是撤销语义)**:KB `e64e82b5`(9 Wiki 页、1 文档、2 分块、1 资源)→ 快照 `0f0a3b68…` 与源逐页一致(`05-snapshot-compare.json` ok=True)→ 同步到独立实例 kgseal `libsync-internal-1793229249632461601` succeeded → 目标逐页/索引/资源字节一致(`07-target-compare.json` ok=True,含目标桶对象 sha256)→ `librestore-internal-1793229274077866298` succeeded → 目标仍逐项一致。**注意**:该共享页正文仍带撤销方残留(见 V03-a 不通过),本行只证明"搬运/恢复逐字节一致" | +| **S06-b** | 主机侧封存/恢复必须与 worker 讲同一份维护声明 | `internal/librarymaintenance/env.go`(`LIBRARY_TARGET_*` 唯一解析处) | 缺项时 `capture` 在开窗口前拒绝并点名;agent 启动打印维护目标;单测 5 项(逐项缺失/空白/口令空格/队列库号) | +| **Neo4j 启用场景** | 启用图谱的库必须被真实拒绝 | `v071/library_export.go` 读 `indexing_strategy.graph_enabled` | **本轮真跑**:在 kgmaint 建 `graph_enabled=true` 的真实 KB `8de41ca7…`,用真实 CLI 预检 → 退出码 1、`LIBRARYCTL_FAILED: Neo4j graph snapshot is not supported`(`evidence/10-neo4j-refuse.txt`)。不是 skip | +| **执行器可用性(运行时观测)** | 页面的"可发起封存"来自 worker 的真实探测,不是配置字符串 | `internal/libraryexecutor` + agent `health` + 迁移 074 `library_executor_status` + worker 30s 心跳 | **本轮真跑(候选栈,代码 SHA 4c79052 / 0.1.192 的工作树构建)**:`GET /library/overview` 返回 `capture_available=true` 且 `executor={instance:kgmaint-weknora, project:kgmaint-weknora, source_app:kgmaint-app, observed_at:…}`;停候选 agent 后同一接口转 `false`、原因 `主机侧封存执行器不可达:library agent is unreachable`,此时 `POST /library/captures` 返回 **503 `library_capture_unavailable`**;重启 agent 后自动恢复 `true`。证据 `$TMPDIR/kg-candidate-verify/evidence/{01,03,04,05}-*.json` | +| **按版本恢复入口** | 页面不再把"恢复至该版本"写死 disabled;条件来自双闸/授权/维护 | `GET /api/v1/library/restore-availability` + `web/src/ReleaseTargets.tsx` | 接口真实返回 200(候选栈,缺授权时逐条列出原因);页面按 `allowed` 启用、管理员才见"签发恢复授权(30 分钟)",授权与执行是两步。证据 `02-restore-availability.json` + 前端用例 `libraryRestore.test.ts` / `ReleaseTargets.test.tsx` | +| **U04** | 维护级别/进度/恢复错误真实呈现 | `MaintenanceCard`(`web/src/ReleaseTargets.tsx`)、`/api/v1/library/maintenance` | 显示实例级/库级、持有 scope+作业、阶段、入口开/关、服务、开始/释放、进度(阶段名中文化)、恢复错误与处理记录;仅"窗口仍持有且作业失败"时给"按预备份恢复";无记录时"未接入 + fail closed"。组件测试 6 项 | -## 差异比对(本轮新增) +| **V05** | 导出中源变更必须被入口/写入者阻断,切点一致 | 闸门 `drain.json` + 真实并发客户端 | **本轮终态真跑**:入口关闭的同一刻(固定协调点,非 sleep)对专用源发 8 写 + 4 读 → 8 写全部失败(7 `URLError`/1 `RemoteDisconnected`)、4 读 `RemoteDisconnected`;写入者被停下的那一刻(轮询容器状态捕获 `running→exited→running`)再发 4 写 + 2 读 → 全部 `URLError`。事后源库 `v05` 文档 0 / 分块 0,快照与源逐项一致(ok=True),在途文档 `f7fed988…` completed/7 分块且出现在快照里。证据 `04-inflight-v05-final.json` | +| **B02** | 源测试桶删对象后,归档自包含且目标应用能读回同样字节 | 归档 `resource` 部件 + 目标应用下载接口 | **本轮终态真跑**:源对象 `10001/d4fbe69d…/ab2b628f….md`(507B,sha256 `25a55cd8…`)先备份再 `mc rm`(stat 已 404)→ 同步仍 succeeded(字节只可能来自归档)→ 目标桶 `kgseal-1029a` 同键 507B、sha256 一致 → **目标应用自己的** `GET /api/v1/knowledge/{id}/download` 恢复前后都 200 / 507B / sha256 `25a55cd8…`。修复前该接口真实 500(`code 1007 invalid MinIO file path`),见 0.1.184/0.1.185 两条修复。证据 `11-b02-delete-and-sync.json`、`12-b02-restore-download.json`、`07-target-compare.json` | +| **死信精确解除(治理侧)** | 保留上游死信行,只按精确指纹解除 | 迁移 073 + `internal/librarydeadletter` + `POST/GET /library/dead-letter-releases` + `libraryctl dead-letters` + `-released-dead-letters` | **已实现并有真实验证的部分**:`libraryctl dead-letters` 在真实源上读出 0 条死信与观测摘要;治理 API `GET` 返回 `instance=kgmaint-weknora`、`releases=[]`;`POST` 无写意图头 403;带真实观测但引用不存在的死信 → 409「观测里没有这条死信」。单测覆盖:摘要被改写、源侧未结清、重放早于失败、重放后又出现同键新死信、实例不一致、通配指纹。**未演练**:真实死信→重放→解除→放行(本轮没有死信,也没有源侧管理员凭据) | -`internal/librarydiff` 以**稳定文档 ID** 为身份、以排除 `updated_at`/`updated_by` 后的整行摘要为内容, -比较两份归档快照的 `knowledges` 行数组,输出新增/更新/撤销。真实归档上验证: -对内快照 4 篇对空集为"新增 4",反向为"撤销 4",缺失快照返回原因而不是空差异。 -生产权威版本建立后即可直接给出生产→测试差异;在生产版本未知时页面保持"未接入"。 -Wiki 页面级差异单独返回(`diff.wiki`),明确标注"不代表实体/边/引用重算"。 +## 一·补 失败 / 不通过(不得标通过) -## 未通过项如何阻断 +| ID | 要点 | 承载位置 | 本轮真实结果 | +| --- | --- | --- | --- | +| **(无)** | 本模块没有"不通过"项:撤销语义已明确移出范围(见上表下方"模块范围外"),不在此改写为通过 | — | — | -- `POST /api/v1/library/syncs`:闸口关闭、缺恢复演练证据、生产端点未配置、生产权威版本未知、 - 目标快照无预备份——任一成立即 503 并逐条列原因,**不创建作业**。 -- 页面"启动同步"按同一 gate 禁用,title 与卡片列出全部缺项;不存在"点了才失败"的假按钮。 -- 封存缺项(结清检查不通过、契约不符、导出校验失败)一律阻断封版,不改状态蒙混。 \ No newline at end of file +### 模块范围外(用户 2026-10-29 决定;证据保留,不改判) + +| ID | 要点 | 本轮真实结果(原样保留) | +| --- | --- | --- | +| **V03-a** | 撤销一份共享贡献后,清除撤销方独有正文/引用/实体边 | **上游不通过,模块范围外**:真实 `DELETE /api/v1/knowledge/{id}` 后,共享页 `entity/fa-bu-zha-men` 的来源/引用/边/目录都按上游 retract 更新(source_refs 只剩保留文档、引用 2→1、9 页/18 边无死链),**但正文仍保留撤销方独有的两句事实 `ZEBRA-A-7731`/`ZEBRA-A-9910`**(version=3、3733 字节、sha256 `5fd27348…`)。这是上游 retract 走 LLM 重写的非确定性结果,治理侧只按字节搬运,不能也不应改写它。证据 `kg-final-gaps/evidence/14-wiki-residue-evidence.json` + `01-wiki-source-after-r3.json`,不涂改 | + +## 二、缺代码 + +| ID | 缺口 | 现状与影响 | +| --- | --- | --- | +| R03 | 存快/恢复快对比 | A06 性能报告已由用户取消,不作为门槛。**已记实测耗时**(小测试库,不是压测):封存 15.8s(含停写~5s)、同步 13.6s(预备份 13.4s 占主要)、恢复 13.8s;证据 `13-r03-durations.json`。没有做高负载压测,也不写没有测量的数字 | +| 死信解除·真实重放演练 | 没有可解除的真实死信,也没有源侧系统管理员凭据 | 契约与执行侧已实现(见下节新增行):上游保留原行、治理侧审计+指纹绑定、未结清/变化/新死信仍阻断。**本轮没有真实死信**(`dead_letters=0`),无法演练"真实重放成功→解除→放行"整条链路;上游只有 `系统管理 runtime run_now`(需要源侧管理员权限)与整队列 purge(销毁证据,不采用)。治理平台故意不持有源侧管理员凭据(放大授权),因此这一步**阻塞在真实死信 + 重放执行方**,不伪造成功 | + +## 三、已实现·未跑 + +| ID | 要点 | 为什么未跑 | +| --- | --- | --- | +| S03-b | 死信/孤儿任务阻断的负向演练(本轮只验证了真实死信阻断) | 真实阻断已发生并记录;系统化的负向矩阵未跑 | +| 排空时序(历史结论) | 旧做法"202 之后抢时序上传"在 1faf513 上重跑 6 次全部 `Broken pipe`(`drain-task-final-1faf513*.json` 保留,不记通过) | 已被 S06-a 的确定性做法取代:先证明请求已被闸门接收,再启动维护。旧证据保留,不覆盖 | +| 封存执行期身份复核(真实封存) | 期望身份随 capture 请求送进 agent,由它在操作互斥内、任何源操作之前比对(`swap_test.go` 用两个真实 agent 的 socket 替换做负例,断言错误 agent 的 capture 调用 0 次);真实封存尚未在最终构建上跑,本轮**没有**在候选栈发起真实封存(会产生新快照与停写入者窗口),因此"换 socket 被挡住"只有单测证据 | +| 按版本恢复的真实执行(新构建) | `restore-availability` 真实返回、恢复投递路径有单测;真实恢复作业仍在旧构建上验过(R01),本轮未重跑,不把旧证据算作新构建证据 | +| U05 | 正式 8088 页面验收 | 正式栈仍是 0.1.114,本轮未替换;维护面板/逐页 Wiki 差异只在测试与候选构建里验证过 | + +## 四、需授权 / 需环境 + +| ID | 事项 | 缺什么 | +| --- | --- | --- | +| D07 | 实际回收(真删对象/版本) | 自动清理默认关闭;删除属于真实清理,需单独授权 | +| 远程生产 | 生产写入/恢复/清理 | 用户授权范围只到本机专用验收实例;远程生产一律未执行 | +| 生产目标维护 | 对共享业务栈开维护窗口 | 本轮明确不停 `WKNFull-app`;维护只在 kgmaint 专用实例上跑 | +| 生产封存/同步真实链路 | 需要在生产机部署 `libraryagent` 并接线 | 需要用户批准的部署窗口与凭据录入 | + +### 四·补 运维风险(本轮核对,未擅自处理) + +正式 8088 栈(`knowledge-governance-p1-api-1` / `-worker-1`)**当前引用同一个 agent socket**: +api env `LIBRARY_AGENT_SOCKET=/run/library-agent/agent.sock`,worker 挂载 +`/home/zheromiku/.local/share/knowledge-governance/agent -> /run/library-agent`。而主机侧 user 单元 +`~/.config/systemd/user/knowledge-governance-library-agent.service` 本轮已把目标改成 kgmaint 验收实例 +(`-source-app kgmaint-app`、`-project kgmaint-weknora`、`-archive /tmp/kg-e2e/archive`, +原 WKNFull 版备份为 `*.wkfull.bak`)。后果:**从正式 8088 发起封存会冻结测试实例,并把快照写进 +正式 API 看不到的归档目录**(正式 api 的 `LIBRARY_ARCHIVE_ROOT=/var/lib/knowledge-governance/library`)。 +本轮没有启动任何共享维护,也没有改回该单元——需要用户决定:正式上线前把单元恢复为 WKNFull 版 +(备份已在位),或给候选构建单独的 socket 路径。证据 `00-identity-final.json` 的 `formal_stack` 段。 + +另外:`knowledge-governance-p1-release-worker-1` 自 2026-10-10 起就在崩溃重启(`exit=1`, +`RestartCount` 已达 1500+,日志只有 `connect release target database`)。**本轮只读定位到根因**: +`config/release.sim.json` 的 `target.database_url_file` 指向 `prod-pg:25432`,而该主机名在本机与 +容器网络里都不存在(`getent hosts prod-pg` 无结果、容器内 NXDOMAIN);`source` 指向的 +`WKNFull-postgres` 可解析,所以是**目标库配置遗留**,不是代码缺陷。错误文本吞掉了主机与原因, +本轮已改为带端点 + 路由 + 脱敏原因(可在下次重启时直接看到),但**修正正式配置或停用该 worker +属于正式栈变更,需要部署窗口授权**,本轮没有执行。 + +--- + +## 五、本轮新增结论(写在这里而不是报告顶部) + +1. **Wiki 撤销语义本轮不通过,且不是治理侧能自己补的**:上游 retract 走 LLM 重写,重跑一次留下两句撤销方独有事实(`ZEBRA-A-7731`/`ZEBRA-A-9910`),另有 r1 轮次把两句清掉的记录——同一输入两次结果不同。原版 v0.7.1 的 Wiki API 只有页 CRUD、`rebuild-links`(只重算 `[[slug]]` 边,metadata-only)、`auto-fix`(只修 lint 可修项),**没有"按剩余来源重算页面正文"的安全 API**(`internal/router/router.go:2354` 起的路由表 + `service/wiki_page.go:759`)。因此:①不得把 V03 标通过;②不得用关键词硬删掩盖;③可选方案见交付报告(外部最小治理:保留旧版本+关联撤销作业+受影响页重建或阻断发布+验证与审计,需用户拍板产品语义)。 +2. **治理侧不能只报页面级差异**:撤销共享贡献时页面仍是"更新",页面级计数会显示成"无变化"。已改为逐页语义比对,能指出"来源 -1、引用 2→1、正文变了、出边少了一条"。 +3. **封存链路本轮真的打通了,而且不是放松校验**:CLI 改为实例级窗口(闸门 + 后台静默 + 目标指纹),"写者 exit=1"由闸门的 `drained/active_at_close` 证据解释,而不是被忽略;排空证明改成"先在途、再关入口"的确定性顺序,不再靠 202 之后抢时序。 +4. **接线后暴露的四个真缺陷(都已修,都有用例)**:`jobs.id` 列宽 36 让封存作业根本投不进去;v0.7.1 的 app 写索引不带 `tenant_id` 导致导出 0 条;全新实例的空索引被当成"索引被改坏";恢复作业 id 带 `.` 被维护窗口拒。另有资源定位符仍写来源桶——同步过去的目标取不到图/附件。 +5. **死信解除已从"缺代码"变成"有代码、真实验证了一半"**:迁移 073 记录精确解除(保留上游行、绑定实时行指纹、重放引用、源侧观测摘要、操作人);封存时每条实时死信都必须被精确解除命中,行变化/新行/未结清照旧阻断;`libraryctl dead-letters` 生成只读观测,`libraryctl -released-dead-letters` 逐字节复核。**没有演练的部分**是"真实死信→重放→解除→放行":本轮源侧 `dead_letters=0`,且重放需要源侧系统管理员权限(`/system/admin/runtime/queues/{queue}/tasks/{id}/actions/run_now`),治理平台故意不持有该凭据。不伪造成功,也不 SQL 删行冒充。 +6. **同步过去的资源"能被应用读出来"是这轮才真正验的**:资源行里既有 `physical_path` 也有独立的 `storage_backend_id` 列,两者都指向来源实例的 `storage_backends` 主键;只换桶或不换列,目标应用都 500(`code 1007 invalid MinIO file path`)。已在同一份行变换里同时改后端 id 与桶,解析不到就拒绝写入。 \ No newline at end of file diff --git a/docs/library-capture.md b/docs/library-capture.md index 6a0dfa3..5972a0d 100644 --- a/docs/library-capture.md +++ b/docs/library-capture.md @@ -45,6 +45,33 @@ 同一个作业号重跑是**幂等重放**:直接返回已封存的同一份候选(`replay=true`、`source_read=false`、`writers_stopped=false`),不读来源、不停写入者;同一作业号换了 scope 或绑定库则明确拒绝。 +## 实例级维护接线(封存窗口的唯一入口证据) + +`capture` 与 `recover` 现在都按**实例级维护**开窗口,接线来自与 worker 同一组环境变量 +(名字只在 `internal/librarymaintenance/env.go` 出现一次): + +``` +LIBRARY_TARGET_INSTANCE 实例名(写进维护记录与目标指纹) +LIBRARY_TARGET_INGRESS_URL 入口地址(恢复后要能 200) +LIBRARY_TARGET_INGRESS_SERVICE Compose 里的入口闸门服务名 +LIBRARY_TARGET_INGRESS_DRAIN_FILE 闸门写排空证据的宿主机文件 +LIBRARY_TARGET_REDIS_ADDR/_PASSWORD/_DB 后台队列(asynq)地址 +``` + +另外 `-source-s3-endpoint ` 用来覆盖"源对象存储的访问入口":容器里写的 +`MINIO_ENDPOINT` 往往是服务名(`langfuse-minio:9000`),封存跑在宿主机上时解析不到, +资源导出会失败(`read library resource`)。覆盖只换入口,桶与对象键不变;`status` +会如实打印实际使用的 endpoint。 + +为什么不再保留"库级窗口":被代理的 WeKnora 收到 SIGTERM 会直接 `exit(1)`,写者退出码 +永远不能当作"在途请求已排空"。排空证据只能在真正的入口上取——关闸门、等它把在途连接 +放完(`drain.json` 里的 `active_at_close` / `drained`)。因此缺少上面任何一项时 `capture` +会**在开窗口前拒绝**(报出缺少的变量名),而不是退回一个没有排空证据的窗口。 + +开窗口的顺序仍是:关入口并等在途排空 → 证明后台静默(稳定窗口 + 任务集合指纹)→ 停写入者 +→ 停后复核队列/指纹与入口 → 导出 → 恢复写入者与入口。停写者后仍会**再查一次**源是否结清, +所以窗口内派生的真实任务必须真的跑完,否则封存被拒(不是等一会儿就算了)。 + ## 从页面发起封存(主机侧 agent) 页面点「发起对内/对外封存」只是**投递一个作业**;执行发生在这台主机上的 agent: @@ -52,6 +79,10 @@ - `scripts/library-agent-install.sh` 构建并安装 `libraryagent`(systemd **user** 服务), 监听 `~/.local/share/knowledge-governance/agent/agent.sock`,目录 0700、socket 0600, 并额外用 `SO_PEERCRED` 要求对端与 agent 同 uid。 +- 维护接线由 `~/.local/share/knowledge-governance/agent.env`(0600,`EnvironmentFile=`)提供; + 没有这个文件安装脚本直接拒绝(`AGENT_ENV_MISSING`),免得装出一个"没有闸门"的 agent。 + 源容器/项目/写入者声明可用 `AGENT_SOURCE_APP`、`AGENT_SOURCE_PG`、`AGENT_SOURCE_ES`、 + `AGENT_PROJECT`、`AGENT_WRITERS`、`AGENT_NON_WRITERS` 覆盖,默认是本机 `weknora071full`。 - agent 自己固定 `libraryctl` 的路径、归档与日志根、Compose 项目与写入者声明; 请求只能选 `scope`(internal/external)和作业号,**不能**指定路径、项目、容器或命令。 - 只有 worker 通过挂载的 socket 目录投递作业(`LIBRARY_AGENT_SOCKET`)。容器里 @@ -76,6 +107,19 @@ 两次封存都记录了 `stop_verified=true`、`restored=true`,`WKNFull-app` 恢复后 healthy;来源未做任何写操作。 +## 2026-10-29 实测:页面入口 → agent → 实例级窗口(真实撤销后的库) + +- 请求:`POST /api/v1/library/captures {scope:internal, job:lib-internal-20261029T0614}` → 202, + worker 投递 agent,CLI 以**实例级窗口**执行。 +- 排空证据(确定性顺序,不抢时序):先发一半 multipart 正文 → 闸门日志 `conn:open active=1` + → 再发起封存 → 入口拒绝新连接 → 补完正文,应用返回 200 并完成解析/嵌入 → + `drain.json`:`phase=drained active_at_close=1 served=126 drain_ms=302`。 +- 冻结:`kgmaint-app` 收到 SIGTERM 后 `exit=1`(上游行为),因入口已排空,`drain_verified=true`; + 后台静默 `quiesce.samples=18 stable_ms=1617`,停写者后指纹不变;随后恢复写入者与入口。 +- 产物:快照 `1196c518…`(KB `e64e82b5…`,tenant 10001):1 文档 / 2 分块 / 9 Wiki 页 / 1 资源, + 与源库逐页比对(正文 md5、来源、引用、目录、18 条边)全部一致。 +- 同一份快照随后同步到**独立实例**(自己的 PG/索引/桶)并成功恢复,逐项读回一致。 + ## 它证明了什么 / 没有证明什么 已证明(A04 数据完整性口径):这一份整库内容在**写入者停住**之后被完整导出,PostgreSQL 与 Elasticsearch 双向覆盖、生成问题、资源与摘要逐项核验通过,产物可寻址、可复读、可校验。这正是封版所需的完整性口径。 diff --git a/docs/release-ledger.md b/docs/release-ledger.md index 1cfadc9..10e2367 100644 --- a/docs/release-ledger.md +++ b/docs/release-ledger.md @@ -61,6 +61,61 @@ run164 因 qawemail 依赖外网 DNS 失败,已修,不涂改。 旧记录 0.1.119 起为本台账自身补记:它只改文档,CI 结论以对应 run 为准(交付报告列出 run id 与结论), 不因"文档提交"免检。 +| 0.1.162 | 1f482e4b | Wiki 贡献级差异 + 维护面板(前端) | 203 | **success** | 否 | +| 0.1.163 | 23cc6c0a | libraryctl 支持 MinIO 源 | 204 | **success** | 否 | +| 0.1.164 | 906c2829 | 捕获/预检共用 scope 映射 | 205 | **success** | 否 | +| 0.1.165 | fd71ea34 | 同镜像不等于同一写入者 | 206 | completed/cancelled(被后续提交取消,不计) | 否 | +| 0.1.166 | 1faf5137 | 用完整容器 ID 判定写入者归属(**验收代码 SHA**) | 207 | **success** | 否 | +| 0.1.167 | f82eb073 | 矩阵四段重排 + 8088 待验收部署方案 | 208 | **success** | 否 | + +| 0.1.169 | ee7688f2 | 矩阵改正 Wiki 检查计数(上轮交付写作 0.1.168,按 `git rev-list --count v0.1.0..HEAD` 实为 0.1.169,此处以台账为准) | 210 | **success** | 否 | +| 0.1.170 | 11c52ad8 | 封存窗口接上实例级闸门/后台静默(**本轮第一个代码提交**) | 211 | failure(密钥扫描把"环境变量名字面量"误判为密钥;该提交之后才在下一提交改名修复) | 否 | +| 0.1.171 | 37a8370f | 索引范围过滤接受未写 `tenant_id` 的文档 | 212 | **success** | 否 | +| 0.1.172 | f2d6f3f6 | 封存队列作业 id 按 `jobs.id` 列宽拼短 | 213 | **success** | 否 | +| 0.1.173 | 4d6ecda3 | 源对象存储端点可按宿主机地址覆盖 | 214 | **success** | 否 | +| 0.1.174 | faaf0037 | agent 固定命令形态纳入对象存储端点 | 215 | failure(漏跑 gofmt,下一提交修) | 否 | +| 0.1.175 | 84e8a0df | gofmt | 216 | **success** | 否 | +| 0.1.176 | 7d4ffdbf | 全新目标实例的空索引按空基线 | 217 | **success** | 否 | +| 0.1.177 | cb009080 | 同步资源行定位符换成目标桶 | 218 | completed/cancelled(被后续提交取消,不计) | 否 | +| 0.1.178 | 2e39e495 | 移出一次性核对小工具 | 219 | **success** | 否 | +| 0.1.179 | b061a968 | 资源换桶变换在校验时同用(**封存闭环验收代码 SHA**) | 220 | **success** | 否 | +| 0.1.180 | a6406209 | 恢复作业 id 改成维护窗口可接受的形状 | 221 | **success** | 否 | +| 0.1.181 | bc49b85 | 文档:封存闭环进"已完成"、补死信最小外部契约 | 222 | **success** | 否 | +| 0.1.182 | 958292d | 文档:按实际运行的二进制记封存闭环时序 | 223 | **success** | 否 | +| 0.1.183 | c2402947 | 死信按精确指纹解除(保留上游行、审计、指纹绑定) | 224 | **success** | 否 | +| 0.1.184 | 1aecd9d6 | 资源定位符同时改到目标存储后端 | 225 | **success** | 否 | +| 0.1.185 | c542a133 | 资源行 storage_backend_id 列也改到目标实例 | 226 | **success** | 否 | +| 0.1.186 | e508abd9 | 死信解除绑定封存源实例(LIBRARY_SOURCE_INSTANCE) | 227 | **success** | 否 | + +| 0.1.188 | 47eb23c | 封存执行器可用性来自真实观测(agent health、迁移 074、worker 心跳、API 统一判定、执行期身份复核) | 229 | **success**(run 229 / run_number 206,head 4c79052) | 是(仅候选隔离栈 `/tmp/kg-e2e/bin`,正式 8088 未动) | +| 0.1.189 | c406961 | 页面恢复入口按真实条件启用 + 签发 30 分钟 restore 授权 | 229 | **success**(同上) | 是(仅候选 web 未重建;API/worker 已换) | +| 0.1.190 | cfad003 | release-worker 启动失败带端点/路由/脱敏原因;共享 `internal/pgdsn` | 229 | **success**(同上) | 是(候选栈未跑 release-worker) | +| 0.1.191 | 18276d0 | 文档:验收口径按用户决定收窄(V03-a 移出范围)、矩阵补执行器证据 | 229 | **success**(同上) | 否(只改文档) | +| 0.1.192 | 4c79052 | 测试 DSN 拼接构造(密钥扫描误报修复,语义不变) | 229 | **success**(head 4c79052,必需 context `build / build (push)` = success) | 否(只改测试) | + +| 0.1.195 | 6ed0635 | 期望执行器身份随 capture 请求送达并在执行前核对(TOCTOU) | 233 | **success**(head c14dd01 之前的 run 232/233 连续绿;最终以 8d2ed8e/run234 为准) | 是(候选栈) | +| 0.1.196 | c14dd01 | 封存读资源失败带资源行与底层原因 | 233 | **success**(run 233 / run_number 210,head c14dd01,必需 context success) | 是(候选栈,clean 构建 `vcs.modified=false`) | +| 0.1.197 | 8d2ed8e | 维护释放与服务恢复完成后才暴露作业成功(R4) | 234 | **success**(run 234 / run_number 211,head 8d2ed8e,必需 context success) | 是(候选栈,clean 构建 `vcs.modified=false`;真实封存→同步→恢复与"成功后立即恢复 202"均在该构建上跑过) | + +| 0.1.198 | 691e4e0 | 台账补记 0.1.195–0.1.197 | 235 | **success**(run 235,head 691e4e0) | 否(文档) | +| 0.1.199 | 24d2546 | 接管分类(未知/待收尾/被顶替)+ 维护释放幂等 | 236 | **success**(run 236,head 24d2546,必需 context success) | 是(候选栈,`-ldflags` 注入身份;`build-info` 读回 0.1.199 / 24d2546…,进程 `vcs.modified=false`) | +这批提交的候选栈验证:候选 api/worker/libraryagent/libraryctl 由该工作树重新构建(`vcs.modified=true`, +不是发布产物),二进制 sha256 记在 `$TMPDIR/kg-candidate-verify/evidence/01-binaries.json`; +旧二进制整目录备份在该目录下的 `bin-prev-*`,可一条命令回退。真实观测证据见 +`$TMPDIR/kg-candidate-verify/evidence/{00..10}*.json`。 + +本轮(父审后)真实验收运行的最终代码 SHA 是 **0.1.186(e508abd9)**:封存→同步→恢复、 +B02 删源对象后目标应用读回、V05 并发阻断都在这一份构建上跑完(PID/exe/revision 绑定见 +`kg-final-gaps/evidence/00-identity-final.json`)。0.1.181–0.1.183 是同一轮内的连续修复, +不作为最终验收 SHA。CI run 204(958292d,本轮起点)success。 + +封存闭环的运行时序(按实际运行的二进制记):封存 API→agent→冻结出快照在 **0.1.175(84e8a0d)**; +首次同步成功在 **0.1.179(b061a968)**;恢复与最终逐项读回在 **0.1.180(a6406209)**。 +最终 HEAD 的代码提交即 0.1.180(其后 bc49b85 只改文档,CI run 222 success)。 +本轮真实验收证据:`/home/zheromiku/.hermes/handoffs/kg-candidate-e2e/evidence-wiki-final/`, +交付报告:`/home/zheromiku/.hermes/handoffs/release-wiki-final-acceptance-result.md`。 +正式 8088 仍是 0.1.114(dc9e827),未部署;部署方案见 `docs/formal-8088-deployment-plan.md`。 + ## 说明 - 0.1.108–0.1.115 的 CI 事实是 failure/未取得通过证据,**不因后续修复改写成绿**; diff --git a/internal/appconfig/config.go b/internal/appconfig/config.go index 097703d..48d0f44 100644 --- a/internal/appconfig/config.go +++ b/internal/appconfig/config.go @@ -96,7 +96,16 @@ type Config struct { LibraryRetentionDays int // 目标维护(实例级):同步/恢复期间关闭目标入口、停下写入者、排空在途请求。 // 任一项留空即"未接线",生产写入会被拒绝——只锁治理库 scope 不算维护。 - LibraryTargetInstance string + LibraryTargetInstance string + // LibrarySourceInstance 是死信解除记录绑定的"来源实例"名字(封存源,不是同步目标)。 + // 留空时回落到 LibraryTargetInstance,保持单实例部署的旧行为。 + LibrarySourceInstance string + // LibrarySourceProject 是封存执行器应该指向的实例名(agent -project)。 + // 留空时等于来源实例名:环境里两者本来就同名(kgmaint-weknora)。 + LibrarySourceProject string + // LibrarySourceApp 是封存源容器名(agent -source-app)。project 可能同名而源接线 + // 不同,所以配了就一起绑定进"期望执行器身份"。 + LibrarySourceApp string LibraryTargetProject string LibraryTargetWriters []string LibraryTargetNonWriters map[string]string @@ -112,10 +121,13 @@ type Config struct { LibraryTargetJournalRoot string // LibraryAgentSocket 是主机侧封存 agent 的 unix socket(只由 worker 连接)。 // 留空即"未接线":页面不显示可用的封存入口,接口返回原因而不是假装已排队。 - LibraryAgentSocket string - VersionHistoryLinkOrigin string - VersionHistoryHTTPTimeout time.Duration - VersionHistoryMaxPages int + LibraryAgentSocket string + // LibraryExecutorStatusMaxAge 是执行器状态的有效期:超过它就是"无法确认", + // 不是"仍然可用"。worker 每 30 秒观测一次,默认 120 秒足够容错。 + LibraryExecutorStatusMaxAge time.Duration + VersionHistoryLinkOrigin string + VersionHistoryHTTPTimeout time.Duration + VersionHistoryMaxPages int } func Load(getenv func(string) string) (Config, error) { @@ -199,6 +211,9 @@ func Load(getenv func(string) string) (Config, error) { LibraryProductionS3AccessKey: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_ACCESS_KEY")), LibraryProductionS3SecretKey: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_SECRET_KEY")), LibraryTargetInstance: strings.TrimSpace(getenv("LIBRARY_TARGET_INSTANCE")), + LibrarySourceInstance: strings.TrimSpace(getenv("LIBRARY_SOURCE_INSTANCE")), + LibrarySourceProject: strings.TrimSpace(getenv("LIBRARY_SOURCE_PROJECT")), + LibrarySourceApp: strings.TrimSpace(getenv("LIBRARY_SOURCE_APP")), LibraryTargetProject: strings.TrimSpace(getenv("LIBRARY_TARGET_PROJECT")), LibraryTargetWriters: splitList(getenv("LIBRARY_TARGET_WRITERS")), LibraryTargetNonWriters: splitDeclarations(getenv("LIBRARY_TARGET_NON_WRITERS")), @@ -215,6 +230,7 @@ func Load(getenv func(string) string) (Config, error) { LibraryCapacityBudgetBytes: capacityBudget, LibraryRetentionVersions: retentionVersions, LibraryRetentionDays: retentionDays, + LibraryExecutorStatusMaxAge: 2 * time.Minute, VersionHistoryLinkOrigin: strings.TrimSpace(getenv("VERSION_HISTORY_LINK_ORIGIN")), VersionHistoryHTTPTimeout: 15 * time.Second, VersionHistoryMaxPages: valueOrDefaultInt(getenv("VERSION_HISTORY_MAX_PAGES"), 200), @@ -237,6 +253,9 @@ func Load(getenv func(string) string) (Config, error) { if config.SourceRefreshSchedulerInterval < 5*time.Second { return Config{}, errors.New("SOURCE_REFRESH_SCHEDULER_INTERVAL must be at least 5s") } + if config.LibraryExecutorStatusMaxAge, err = durationOrDefault(getenv("LIBRARY_EXECUTOR_STATUS_MAX_AGE"), config.LibraryExecutorStatusMaxAge); err != nil { + return Config{}, fmt.Errorf("LIBRARY_EXECUTOR_STATUS_MAX_AGE: %w", err) + } if raw := getenv("RELEASE_WORKER_ENABLED"); raw != "" { config.ReleaseWorkerEnabled, err = strconv.ParseBool(raw) if err != nil { @@ -562,3 +581,22 @@ func positiveInt64OrDefault(value string, fallback int64) (int64, error) { } return parsed, nil } + +// SourceInstance 是死信解除记录绑定的实例名:封存源与同步目标常常不是同一个实例, +// 用目标名去绑来源死信会让真实观测永远对不上(拒绝而不是误放行)。 +func (config Config) SourceInstance() string { + if config.LibrarySourceInstance != "" { + return config.LibrarySourceInstance + } + return config.LibraryTargetInstance +} + +// SourceProject 是封存执行器自报 project 的期望值。它描述的是**封存源** +// (agent 冻结哪一套写入者),不是同步目标:拿目标实例去比会把正确的执行器判成 +// 不匹配(本机候选栈就是这种形状:源 kgmaint、目标 kgseal)。 +func (config Config) SourceProject() string { + if config.LibrarySourceProject != "" { + return config.LibrarySourceProject + } + return config.SourceInstance() +} diff --git a/internal/governance/library_flow.go b/internal/governance/library_flow.go index 34f10bd..7e6b440 100644 --- a/internal/governance/library_flow.go +++ b/internal/governance/library_flow.go @@ -31,6 +31,10 @@ const ( LibraryJobSync LibraryJobKind = "sync" // LibraryJobRestore 是按指定整库版本恢复:与同步分开,授权与阶段也独立。 LibraryJobRestore LibraryJobKind = "restore" + // LibraryJobBaseline 建立首次发布目标的权威基线:在维护隔离下把目标整库 + // 读出来封存成快照并逐类校验,再登记成 production_effective。 + // 它不写目标内容、不清空目标、不猜版本——基线只能来自真实捕获与校验。 + LibraryJobBaseline LibraryJobKind = "baseline" ) type LibraryJobState string @@ -134,6 +138,12 @@ func SealStages() []string { return []string{"export"} } +// BaselineStages 是"建立目标权威基线"的阶段:捕获目标整库 → 与快照逐类校验 +// → 登记生效版本。没有 rollback 阶段:基线过程不写目标内容。 +func BaselineStages() []string { + return []string{"capture", "verify", "finalize"} +} + // LibraryReleaseAuthorization 是生产发布授权:绑定目标、快照、参数摘要与有效期。 type LibraryReleaseAuthorization struct { ID string `json:"id"` @@ -150,3 +160,34 @@ type LibraryReleaseAuthorization struct { func (a LibraryReleaseAuthorization) ValidAt(now time.Time) bool { return a.RevokedAt == nil && now.Before(a.ExpiresAt) } + +// LibraryDeadLetterRelease 是治理平台对一条上游死信的精确解除记录。上游行保持原样, +// 这条记录只说明"这一行已被审计、重放并结清";指纹一变就不再命中。 +type LibraryDeadLetterRelease struct { + ID string `json:"id"` + Instance string `json:"instance"` + TenantID uint64 `json:"tenant_id"` + KnowledgeBaseID string `json:"kb_id"` + DeadLetterID int64 `json:"dead_letter_id"` + Fingerprint string `json:"fingerprint"` + TaskType string `json:"task_type"` + Scope string `json:"scope"` + ScopeID string `json:"scope_id"` + RelatedID string `json:"related_id"` + ReplayQueue string `json:"replay_queue"` + ReplayTaskID string `json:"replay_task_id"` + ReplayedAt time.Time `json:"replayed_at"` + ReplayEvidence string `json:"replay_evidence"` + ObservationDigest string `json:"observation_digest"` + Observation json.RawMessage `json:"observation"` + Note string `json:"note"` + Actor string `json:"actor"` + CreatedAt time.Time `json:"created_at"` +} + +// LibraryDeadLetterReleaseID 生成解除记录 ID。同一实例同一指纹永远得到同一个 ID, +// 重复提交返回已存记录而不是造第二条审计。 +func LibraryDeadLetterReleaseID(instance, fingerprint string) string { + sum := sha256.Sum256([]byte(instance + "|" + fingerprint)) + return "dldr-" + hex.EncodeToString(sum[:])[:24] +} diff --git a/internal/httpapi/library_baseline.go b/internal/httpapi/library_baseline.go new file mode 100644 index 0000000..01acebe --- /dev/null +++ b/internal/httpapi/library_baseline.go @@ -0,0 +1,156 @@ +package httpapi + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "time" + + "knowledge-governance/internal/governance" + "knowledge-governance/internal/jobs" + postgresstore "knowledge-governance/internal/platform/postgres" +) + +// 建立目标权威基线:首次发布时目标已经有内容,却没有生效版本记录。 +// +// 唯一诚实的解法是"维护隔离 + 真实整库捕获 + 逐类校验 + 登记版本": +// - 不手写 production_effective,不伪造验收; +// - 不清空目标换库绕过(那会把"非空但未知"变成"空"); +// - 页面只投递作业,停写入者、捕获、校验都在 worker 里执行。 +// +// 这个入口只建作业,不签发发布授权:基线不等于发布,同步仍然要单独的 sync 授权。 +const libraryBaselineJobType = "library.baseline" + +type libraryBaselineRequest struct { + Scope string `json:"scope"` +} + +func libraryBaselineEnqueue(w http.ResponseWriter, r *http.Request, config Config) { + if r.Method != http.MethodPost { + writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"}) + return + } + identity, ok := sessionFrom(r.Context()) + if !ok || roleRank(identity.Role) < roleRank(roleAdmin) { + writeJSON(w, http.StatusForbidden, map[string]any{"error": "建立目标权威基线仅管理员可发起"}) + return + } + var request libraryBaselineRequest + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&request); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"}) + return + } + if request.Scope != "internal" && request.Scope != "external" { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "scope must be internal or external"}) + return + } + store, ok := libraryStore(config) + if !ok { + writeJSON(w, http.StatusServiceUnavailable, map[string]any{"code": "library_baseline_unavailable", + "error": "版本登记存储不可用"}) + return + } + versions, err := store.LibraryVersions(r.Context(), request.Scope, 50) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot read library versions"}) + return + } + for _, item := range versions { + if item.Kind == governance.LibraryVersionProductionEffective && item.SupersededAt == nil { + writeJSON(w, http.StatusConflict, map[string]any{"code": "library_baseline_exists", + "error": "该目标已有生效版本,不需要再建基线;换版本请走同步或恢复", + "snapshot_id": item.SnapshotID}) + return + } + } + // 目标必须真的读得到:读不到就不是"非空无权威版本",而是另一个问题, + // 不允许用建基线去掩盖连接/配置故障。 + probe := config.LibraryScopeProbe + if probe == nil { + probe = func(ctx context.Context, scope string) (bool, map[string]int64, error) { + return productionScopeState(ctx, config, scope) + } + } + empty, counts, probeErr := probe(r.Context(), request.Scope) + if probeErr != nil { + writeJSON(w, http.StatusServiceUnavailable, map[string]any{"code": "library_baseline_unprobeable", + "error": "无法读取目标整库状态,不能建立基线:" + probeErr.Error()}) + return + } + if empty { + writeJSON(w, http.StatusConflict, map[string]any{"code": "library_baseline_target_empty", + "error": "目标整库为空,不需要基线:首次同步会在写入前生成完整预备份并登记生效版本", + "counts": counts}) + return + } + // 维护互斥与同步一致:基线要在目标上开维护窗口,目标正在维护时直接拒绝。 + if body, held := instanceMaintenanceGuard(r.Context(), config, request.Scope); held { + writeJSON(w, http.StatusConflict, body) + return + } + if maintenanceStore, ok := config.Store.(interface { + LibraryMaintenanceOf(ctx context.Context, scope string) (*postgresstore.LibraryMaintenance, error) + }); ok { + if held, err := maintenanceStore.LibraryMaintenanceOf(r.Context(), request.Scope); err == nil && held != nil && held.ReleasedAt == nil { + writeJSON(w, http.StatusConflict, map[string]any{"code": "library_target_in_maintenance", + "error": "目标正在维护中,验证完成前不接受新的写入请求", + "owner_job": held.OwnerJobID, "state": held.State}) + return + } + } + digest, err := governance.LibraryRequestDigest(request.Scope, governance.LibraryJobBaseline, + map[string]any{"scope": request.Scope}) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot digest request"}) + return + } + now := time.Now().UTC() + job, reused, err := store.CreateLibraryJob(r.Context(), governance.LibraryJob{ + ID: fmt.Sprintf("libbase-%s-%d", request.Scope, now.UnixNano()), Scope: request.Scope, + Kind: governance.LibraryJobBaseline, + IdempotencyKey: "library.baseline:" + request.Scope + ":" + now.Format("20060102T150405.000000000"), + RequestDigest: digest, Actor: identity.Username, + }, governance.BaselineStages(), now) + if errors.Is(err, governance.ErrLibraryJobActive) { + writeJSON(w, http.StatusConflict, map[string]any{"code": "library_write_active", + "error": "该目标已有在飞的写作业(同步/恢复/基线互斥),等它结束再发起"}) + return + } + if errors.Is(err, governance.ErrLibraryJobIdempotencyConflict) { + writeJSON(w, http.StatusConflict, map[string]any{"code": "library_baseline_conflict", + "error": "同一幂等键对应了不同的请求"}) + return + } + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot create library baseline job"}) + return + } + if reused { + writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": true}) + return + } + payload, err := json.Marshal(map[string]string{"library_job_id": job.ID, "scope": request.Scope}) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot build baseline payload"}) + return + } + queueJob, err := config.Store.Enqueue(r.Context(), jobs.EnqueueRequest{ + ID: governance.QueueJobIDFor(job.ID), Type: libraryBaselineJobType, Payload: payload, + AvailableAt: now, MaxAttempts: 1, IdempotencyKey: "library.baseline.queue:" + job.ID, + }, now) + if err != nil || queueJob == nil { + // 作业先落库、队列投递失败就把作业取消:不留下"看起来在跑"的空作业, + // 否则它会一直占着该 scope 的写互斥。 + _ = store.CancelLibraryJob(r.Context(), job.ID, "队列投递失败,未开始建立基线", now) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot enqueue baseline job"}) + return + } + if attachErr := store.AttachLibraryQueueJob(r.Context(), job.ID, queueJob.ID, now); attachErr != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot attach baseline job"}) + return + } + writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": false, + "target_counts": counts}) +} diff --git a/internal/httpapi/library_baseline_mutex_test.go b/internal/httpapi/library_baseline_mutex_test.go new file mode 100644 index 0000000..842afc2 --- /dev/null +++ b/internal/httpapi/library_baseline_mutex_test.go @@ -0,0 +1,97 @@ +package httpapi + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" + "time" + + "knowledge-governance/internal/governance" + "knowledge-governance/internal/jobs" +) + +// nonEmptyTarget 模拟"目标整库非空且可读"的探测结果(真实实现直连目标库逐表计数)。 +func nonEmptyTarget(context.Context, string) (bool, map[string]int64, error) { + return false, map[string]int64{"knowledges": 1, "chunks": 2}, nil +} + +// scopedProbe:internal 可读非空,external 没有目标 KB(与当前部署一致)。 +func scopedProbe(_ context.Context, scope string) (bool, map[string]int64, error) { + if scope == "external" { + return false, nil, errors.New("该范围的目标知识库未配置") + } + return nonEmptyTarget(context.Background(), scope) +} + +// baselineMutexStore 在 fake 之外**真的**实现"同一 scope 只允许一个在飞写作业"这条不变量, +// 用来验证接口的重试/双击行为(真实索引由 integration test +// TestLibraryWriteMutexCoversBaselineJobs 在 PostgreSQL 上证明)。 +type baselineMutexStore struct { + fakeLibraryStore + activeByScope map[string]string + enqueues int +} + +func (s *baselineMutexStore) CreateLibraryJob(ctx context.Context, job governance.LibraryJob, stages []string, now time.Time) (governance.LibraryJob, bool, error) { + if s.activeByScope == nil { + s.activeByScope = map[string]string{} + } + if owner := s.activeByScope[job.Scope]; owner != "" { + return governance.LibraryJob{}, false, governance.ErrLibraryJobActive + } + s.activeByScope[job.Scope] = job.ID + return s.fakeLibraryStore.CreateLibraryJob(ctx, job, stages, now) +} + +func (s *baselineMutexStore) Enqueue(ctx context.Context, request jobs.EnqueueRequest, now time.Time) (*jobs.Job, error) { + s.enqueues++ + return s.fakeLibraryStore.fakeJobStore.Enqueue(ctx, request, now) +} + +// 双击/重试:第一次建作业成功,第二次被互斥拒绝,且**不会**再投递一个队列作业。 +func TestLibraryBaselineDoubleSubmitCreatesOneJob(t *testing.T) { + store := &baselineMutexStore{fakeLibraryStore: fakeLibraryStore{fakeJobStore: fakeJobStore{enqueueEchoRequestedID: true}}} + config := Config{Store: store, LibraryScopeProbe: nonEmptyTarget} + first := httptest.NewRecorder() + libraryBaselineEnqueue(first, baselineRequest(`{"scope":"internal"}`, roleAdmin, "admin"), config) + if first.Code != http.StatusAccepted { + t.Fatalf("first status=%d body=%s", first.Code, first.Body.String()) + } + if store.enqueues != 1 { + t.Fatalf("first submit enqueues=%d, want 1", store.enqueues) + } + second := httptest.NewRecorder() + libraryBaselineEnqueue(second, baselineRequest(`{"scope":"internal"}`, roleAdmin, "admin"), config) + if second.Code != http.StatusConflict { + t.Fatalf("second status=%d body=%s", second.Code, second.Body.String()) + } + if body := decodeBody(t, second); body["code"] != "library_write_active" { + t.Fatalf("second body=%v", body) + } + if store.enqueues != 1 { + t.Fatalf("rejected submit must not enqueue again: enqueues=%d", store.enqueues) + } +} + +// 不同 scope 不会被同一把锁挡住(互斥粒度是 scope/实例,不是全局)。 +func TestLibraryBaselineDifferentScopesAreNotBlocked(t *testing.T) { + store := &baselineMutexStore{fakeLibraryStore: fakeLibraryStore{fakeJobStore: fakeJobStore{enqueueEchoRequestedID: true}}} + config := Config{Store: store, LibraryScopeProbe: scopedProbe} + internal := httptest.NewRecorder() + libraryBaselineEnqueue(internal, baselineRequest(`{"scope":"internal"}`, roleAdmin, "admin"), config) + if internal.Code != http.StatusAccepted { + t.Fatalf("internal status=%d body=%s", internal.Code, internal.Body.String()) + } + // external 在探针阶段就会被拒(该范围没有目标 KB),但绝不能是因为互斥。 + external := httptest.NewRecorder() + libraryBaselineEnqueue(external, baselineRequest(`{"scope":"external"}`, roleAdmin, "admin"), config) + if external.Code == http.StatusAccepted { + t.Fatalf("external without a target KB must not be accepted: %s", external.Body.String()) + } + body := decodeBody(t, external) + if body["code"] == "library_write_active" { + t.Fatalf("external must not be blocked by the internal scope's mutex: %v", body) + } +} diff --git a/internal/httpapi/library_baseline_test.go b/internal/httpapi/library_baseline_test.go new file mode 100644 index 0000000..ce83293 --- /dev/null +++ b/internal/httpapi/library_baseline_test.go @@ -0,0 +1,162 @@ +package httpapi + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "knowledge-governance/internal/governance" +) + +// 建立目标权威基线的入口契约: +// - 只有管理员能发起(它是生产侧维护动作); +// - 已有生效版本时拒绝(基线只解决首次发布); +// - 目标读不到时拒绝,且**不建作业、不登记版本**(不能用基线掩盖连接问题)。 + +func baselineRequest(body, role, username string) *http.Request { + request := httptest.NewRequest(http.MethodPost, "/api/v1/library/baselines", strings.NewReader(body)) + request.Header.Set("Content-Type", "application/json") + ctx := context.WithValue(request.Context(), sessionKey{}, sessionIdentity{Username: username, Role: role}) + return request.WithContext(ctx) +} + +func decodeBody(t *testing.T, recorder *httptest.ResponseRecorder) map[string]any { + t.Helper() + var payload map[string]any + if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil { + t.Fatalf("response is not JSON: %s", recorder.Body.String()) + } + return payload +} + +func TestLibraryBaselineRequiresAdmin(t *testing.T) { + store := &fakeLibraryStore{} + recorder := httptest.NewRecorder() + libraryBaselineEnqueue(recorder, baselineRequest(`{"scope":"internal"}`, roleEngineer, "eng"), Config{Store: store}) + if recorder.Code != http.StatusForbidden { + t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String()) + } + if store.createN != 0 { + t.Fatal("non-admin must not create a baseline job") + } +} + +func TestLibraryBaselineRefusesWhenEffectiveVersionExists(t *testing.T) { + store := &fakeLibraryStore{versions: []governance.LibraryVersion{{ + ID: "libver-1", Scope: "internal", Kind: governance.LibraryVersionProductionEffective, + SnapshotID: "snap-effective", CreatedAt: time.Now().UTC(), + }}} + recorder := httptest.NewRecorder() + libraryBaselineEnqueue(recorder, baselineRequest(`{"scope":"internal"}`, roleAdmin, "admin"), Config{Store: store}) + if recorder.Code != http.StatusConflict { + t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String()) + } + if body := decodeBody(t, recorder); body["code"] != "library_baseline_exists" || body["snapshot_id"] != "snap-effective" { + t.Fatalf("body=%v", body) + } + if store.createN != 0 { + t.Fatal("existing effective version must not create another baseline job") + } +} + +// 目标读不到(这里用"没配置目标库"复现):必须 503 并说明原因, +// 既不能建作业,也不能顺手写一条版本记录来"解决"未知状态。 +func TestLibraryBaselineRefusesWhenTargetUnreadable(t *testing.T) { + store := &fakeLibraryStore{} + recorder := httptest.NewRecorder() + libraryBaselineEnqueue(recorder, baselineRequest(`{"scope":"external"}`, roleAdmin, "admin"), Config{Store: store}) + if recorder.Code != http.StatusServiceUnavailable { + t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String()) + } + body := decodeBody(t, recorder) + if body["code"] != "library_baseline_unprobeable" || !strings.Contains(func() string { + text, _ := body["error"].(string) + return text + }(), "无法读取目标整库状态") { + t.Fatalf("body=%v", body) + } + if store.createN != 0 { + t.Fatal("unreadable target must not create a baseline job") + } + if len(store.versions) != 0 { + t.Fatal("unreadable target must not record any version") + } +} + +// 测试候选只认登记过 test_sealed 的那份快照:归档里同时存在更新的目标基线/ +// 预备份快照时,绝不能把它们当成"拟发布的测试版本"。 +func TestTestSealedCandidateComesFromRegistryNotNewestArchiveEntry(t *testing.T) { + sealed := librarySnapshotView{SnapshotID: "snap-sealed", CreatedAt: time.Date(2026, 10, 29, 8, 54, 0, 0, time.UTC)} + prebackup := librarySnapshotView{SnapshotID: "snap-prebackup", CreatedAt: time.Date(2026, 10, 29, 9, 30, 0, 0, time.UTC)} + byID := map[string]librarySnapshotView{sealed.SnapshotID: sealed, prebackup.SnapshotID: prebackup} + versions := []governance.LibraryVersion{ + {Kind: governance.LibraryVersionProductionPrebackup, SnapshotID: "snap-prebackup"}, + {Kind: governance.LibraryVersionTestSealed, SnapshotID: "snap-sealed"}, + } + candidate, reason := testSealedOf(versions, byID) + if reason != "" { + t.Fatalf("reason=%q", reason) + } + if candidate == nil || candidate.SnapshotID != "snap-sealed" { + t.Fatalf("candidate=%v, want the registered test seal", candidate) + } + + // 没有任何 test_sealed 登记:宁可说"还没有测试封存",也不拿别的快照顶替。 + none, reason := testSealedOf([]governance.LibraryVersion{ + {Kind: governance.LibraryVersionProductionPrebackup, SnapshotID: "snap-prebackup"}, + }, byID) + if none != nil || !strings.Contains(reason, "还没有该目标的测试封存") { + t.Fatalf("candidate=%v reason=%q", none, reason) + } + + // 登记过但它已经不在本机归档:说清是归档问题,不给候选。 + missing, reason := testSealedOf([]governance.LibraryVersion{ + {Kind: governance.LibraryVersionTestSealed, SnapshotID: "snap-gone"}, + }, byID) + if missing != nil || !strings.Contains(reason, "不在本机归档里") { + t.Fatalf("candidate=%v reason=%q", missing, reason) + } +} + +// 非空但无权威版本时,缺项必须指向真实入口,而不是一句"未知"。 +func TestLibrarySyncGateNamesTheBaselineEntryForNonEmptyTarget(t *testing.T) { + gate := librarySyncGateWithGrant(Config{ProductionReleaseRequestsEnabled: true, + ProductionBackupRestoreVerified: true, LibraryProductionDatabaseURL: "postgres://prod.invalid/db", + LibraryProductionInternalKB: "kb-int", LibraryCapacityBudgetBytes: 1 << 30}, + libraryKnownView{State: "unknown"}, libraryKnownView{State: "unknown"}, "snap-1", true) + if gate.SyncAllowed { + t.Fatal("unknown production version must block the sync") + } + joined := strings.Join(gate.Reasons, "|") + if !strings.Contains(joined, "目标非空且没有生效版本记录") || !strings.Contains(joined, "目标权威基线") { + t.Fatalf("reasons=%v", gate.Reasons) + } +} + +// 维护能力是配置事实:配置齐全时必须说"已接线",缺项要列出缺哪个变量。 +func TestLibraryMaintenanceCapabilityReportsWiringInsteadOfOneVagueSentence(t *testing.T) { + wired := libraryMaintenanceCapability(Config{ + LibraryTargetInstance: "kgseal-app", LibraryTargetProject: "kgseal-weknora", + LibraryTargetWriters: []string{"app"}, LibraryTargetIngressURL: "http://kgseal-ingress:18084/health", + LibraryTargetIngressService: "ingress", LibraryTargetJournalRoot: "/var/lib/knowledge-governance/journal", + LibraryTargetDockerSocket: "/var/run/docker.sock", + }) + if wired["configured"] != true { + t.Fatalf("capability=%v", wired) + } + if wired["instance"] != "kgseal-app" { + t.Fatalf("instance=%v", wired["instance"]) + } + broken := libraryMaintenanceCapability(Config{LibraryTargetInstance: "kgseal-app"}) + if broken["configured"] != false { + t.Fatalf("capability=%v", broken) + } + missing, _ := broken["missing"].([]string) + if len(missing) == 0 || !strings.Contains(strings.Join(missing, ","), "LIBRARY_TARGET_WRITERS") { + t.Fatalf("missing=%v", broken["missing"]) + } +} diff --git a/internal/httpapi/library_capture.go b/internal/httpapi/library_capture.go index 08e7217..4aefe34 100644 --- a/internal/httpapi/library_capture.go +++ b/internal/httpapi/library_capture.go @@ -26,13 +26,6 @@ type libraryCaptureRequest struct { } func libraryCaptureEnqueue(w http.ResponseWriter, r *http.Request, config Config) { - if config.LibraryAgentSocket == "" { - writeJSON(w, http.StatusServiceUnavailable, map[string]any{ - "code": "library_capture_unwired", - "reason": "未接线:主机侧封存 agent 没有挂载,页面无法发起封存", - }) - return - } var request libraryCaptureRequest if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&request); err != nil { writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"}) @@ -52,6 +45,17 @@ func libraryCaptureEnqueue(w http.ResponseWriter, r *http.Request, config Config writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid job id"}) return } + // 请求形状先校验(400),再看执行器(503):不只看"配了 socket"——执行器可能 + // 不可达、可能指向别的实例、可能降级。页面与接口必须用同一份判定,否则又会出现 + // "点了才失败"或"封错库"。 + if available, reason, executor := libraryExecutorAvailability(r.Context(), config); !available { + writeJSON(w, http.StatusServiceUnavailable, map[string]any{ + "code": "library_capture_unavailable", + "reason": reason, + "executor": executor, + }) + return + } requestedBy := "" if identity, ok := sessionFrom(r.Context()); ok { requestedBy = identity.Username @@ -97,7 +101,10 @@ func libraryCaptureEnqueue(w http.ResponseWriter, r *http.Request, config Config } } } - requestedID := fmt.Sprintf("library-capture-%s-%d", request.Scope, now.UnixNano()) + // 队列作业 id 落进 jobs.id(varchar(36)):原来拼 "library-capture--" + // 是 45 个字符,真实请求 100% 被 PG 拒(value too long)——只是页面一直停在 + // "未接线"上,没人走到这里。这里按列宽拼短 id,并留出 scope 全称。 + requestedID := fmt.Sprintf("libcap-%s-%d", request.Scope, now.UnixNano()) // The idempotency key is the capture job id: the same logical capture must map // to one queue row. Binding the scope into the key would silently create a // second job instead of reporting a conflicting request, so the conflict is diff --git a/internal/httpapi/library_capture_test.go b/internal/httpapi/library_capture_test.go index 060d56f..2cd6263 100644 --- a/internal/httpapi/library_capture_test.go +++ b/internal/httpapi/library_capture_test.go @@ -17,31 +17,11 @@ func captureRequest(body string) *http.Request { return request } -// An unwired agent must refuse clearly instead of queueing a job nobody can run. -func TestLibraryCaptureRefusedWhenAgentUnwired(t *testing.T) { - store := &fakeJobStore{} - recorder := httptest.NewRecorder() - libraryCaptureEnqueue(recorder, captureRequest(`{"scope":"internal"}`), Config{Store: store}) - if recorder.Code != http.StatusServiceUnavailable { - t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String()) - } - var body map[string]any - if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil { - t.Fatal(err) - } - if body["code"] != "library_capture_unwired" || body["reason"] == "" { - t.Fatalf("unexpected body %v", body) - } - if store.enqueued.Type != "" { - t.Fatal("unwired agent still enqueued a job") - } -} - func TestLibraryCaptureEnqueuesFixedJobShape(t *testing.T) { store := &fakeJobStore{enqueueEchoRequestedID: true} recorder := httptest.NewRecorder() libraryCaptureEnqueue(recorder, captureRequest(`{"scope":"external","job":"lib-external-review"}`), - Config{Store: store, LibraryAgentSocket: "/run/library-agent/agent.sock"}) + captureReadyConfig(store)) if recorder.Code != http.StatusAccepted { t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String()) } @@ -76,7 +56,7 @@ func TestLibraryCaptureRejectsInvalidRequest(t *testing.T) { } { store := &fakeJobStore{} recorder := httptest.NewRecorder() - libraryCaptureEnqueue(recorder, captureRequest(body), Config{Store: store, LibraryAgentSocket: "/tmp/agent.sock"}) + libraryCaptureEnqueue(recorder, captureRequest(body), captureReadyConfig(store)) if recorder.Code != http.StatusBadRequest { t.Fatalf("body %q status=%d", body, recorder.Code) } @@ -86,6 +66,26 @@ func TestLibraryCaptureRejectsInvalidRequest(t *testing.T) { } } +func TestLibraryCaptureQueueIDFitsTheColumn(t *testing.T) { + // jobs.id 是 varchar(36):拼长的 id 会在真实 PG 上 500(value too long)。 + // 页面长期停在"未接线",这条从没被真实请求覆盖过,所以用测试锁住列宽。 + for _, scope := range []string{"internal", "external"} { + store := &fakeJobStore{enqueueEchoRequestedID: true} + recorder := httptest.NewRecorder() + libraryCaptureEnqueue(recorder, captureRequest(`{"scope":"`+scope+`"}`), captureReadyConfig(store)) + if recorder.Code != http.StatusAccepted { + t.Fatalf("%s status=%d body=%s", scope, recorder.Code, recorder.Body.String()) + } + enqueued := store.enqueued + if len(enqueued.ID) > 36 { + t.Fatalf("%s 队列作业 id 超长(%d):%q", scope, len(enqueued.ID), enqueued.ID) + } + if !strings.HasPrefix(enqueued.ID, "libcap-"+scope+"-") { + t.Fatalf("%s 队列作业 id 形状变了:%q", scope, enqueued.ID) + } + } +} + func TestLibraryCaptureJobsAsksStoreForItsOwnType(t *testing.T) { // The global queue is busy with unrelated traffic; the capture job must still // be visible because the query is type-scoped, not filtered after the fact. @@ -125,7 +125,7 @@ func TestLibraryCaptureRejectsScopeConflictForReusedJob(t *testing.T) { Payload: json.RawMessage(`{"scope":"internal","capture_job":"lib-shared","requested_by":"yaozhanghao"}`)}} recorder := httptest.NewRecorder() libraryCaptureEnqueue(recorder, captureRequest(`{"scope":"external","job":"lib-shared"}`), - Config{Store: store, LibraryAgentSocket: "/run/library-agent/agent.sock"}) + captureReadyConfig(store)) if recorder.Code != http.StatusConflict { t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String()) } @@ -145,7 +145,7 @@ func TestLibraryCaptureReplayReportsStoredJob(t *testing.T) { Payload: json.RawMessage(`{"scope":"internal","capture_job":"lib-shared","requested_by":"yaozhanghao"}`)}} recorder := httptest.NewRecorder() libraryCaptureEnqueue(recorder, captureRequest(`{"scope":"internal","job":"lib-shared"}`), - Config{Store: store, LibraryAgentSocket: "/run/library-agent/agent.sock"}) + captureReadyConfig(store)) if recorder.Code != http.StatusAccepted { t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String()) } diff --git a/internal/httpapi/library_deadletters.go b/internal/httpapi/library_deadletters.go new file mode 100644 index 0000000..38efbfd --- /dev/null +++ b/internal/httpapi/library_deadletters.go @@ -0,0 +1,104 @@ +package httpapi + +import ( + "context" + "encoding/json" + "net/http" + "strings" + "time" + + "knowledge-governance/internal/governance" + "knowledge-governance/internal/librarydeadletter" +) + +// 死信解除:上游 task_dead_letters 没有"已解决"标记。这里只写治理库的审计记录, +// 永不删除上游行;解绑的是精确指纹,行一变就不再命中,封存照旧阻断。 +type deadLetterReleaseStore interface { + RecordDeadLetterRelease(context.Context, governance.LibraryDeadLetterRelease) (governance.LibraryDeadLetterRelease, bool, error) + DeadLetterReleases(context.Context, string, int) ([]governance.LibraryDeadLetterRelease, error) +} + +type deadLetterReleaseRequest struct { + DeadLetterID int64 `json:"dead_letter_id"` + Replay librarydeadletter.Replay `json:"replay"` + Note string `json:"note"` + Observation json.RawMessage `json:"observation"` + ObservationDigest string `json:"observation_digest"` +} + +func deadLetterReleaseHandlers(w http.ResponseWriter, r *http.Request, config Config) { + identity, ok := requireSessionIdentity(w, r) + if !ok { + return + } + store, ok := config.Store.(deadLetterReleaseStore) + if !ok { + writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "死信解除存储不可用"}) + return + } + instance := strings.TrimSpace(config.SourceInstance()) + if instance == "" { + writeJSON(w, http.StatusServiceUnavailable, map[string]any{ + "code": "dead_letter_release_unwired", "reason": "未接线:没有配置来源实例身份,无法绑定解除记录", + }) + return + } + if r.Method == http.MethodGet { + releases, err := store.DeadLetterReleases(r.Context(), instance, 100) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "读取死信解除记录失败"}) + return + } + writeJSON(w, http.StatusOK, map[string]any{"instance": instance, "releases": releases}) + return + } + // 写操作需要工程师及以上,并要求显式意图头:避免普通点击意外解除。 + if roleRank(identity.Role) < roleRank(roleEngineer) { + writeJSON(w, http.StatusForbidden, map[string]string{"error": "forbidden"}) + return + } + if r.Header.Get("X-Governance-Action") != "dead-letter-release" { + writeJSON(w, http.StatusForbidden, map[string]string{"error": "write intent required"}) + return + } + var request deadLetterReleaseRequest + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&request); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"}) + return + } + now := time.Now().UTC() + release, err := librarydeadletter.Verify(librarydeadletter.Request{ + Project: instance, DeadLetterID: request.DeadLetterID, Replay: request.Replay, + Note: request.Note, Observation: request.Observation, Digest: request.ObservationDigest, + }, instance, now) + if err != nil { + writeJSON(w, http.StatusConflict, map[string]any{"code": "dead_letter_release_rejected", "error": err.Error()}) + return + } + stored, reused, err := store.RecordDeadLetterRelease(r.Context(), governance.LibraryDeadLetterRelease{ + ID: governance.LibraryDeadLetterReleaseID(instance, release.Fingerprint), + Instance: instance, + TenantID: release.TenantID, + KnowledgeBaseID: release.KnowledgeBaseID, + DeadLetterID: release.DeadLetterID, + Fingerprint: release.Fingerprint, + TaskType: release.TaskType, + Scope: release.Scope, + ScopeID: release.ScopeID, + RelatedID: release.RelatedID, + ReplayQueue: release.Replay.Queue, + ReplayTaskID: release.Replay.TaskID, + ReplayedAt: release.Replay.At, + ReplayEvidence: release.Replay.Evidence, + ObservationDigest: release.Digest, + Observation: release.Observation, + Note: release.Note, + Actor: identity.Username, + CreatedAt: now, + }) + if err != nil { + writeJSON(w, http.StatusConflict, map[string]any{"code": "dead_letter_release_conflict", "error": err.Error()}) + return + } + writeJSON(w, http.StatusOK, map[string]any{"release": stored, "reused": reused}) +} diff --git a/internal/httpapi/library_deadletters_test.go b/internal/httpapi/library_deadletters_test.go new file mode 100644 index 0000000..75c0769 --- /dev/null +++ b/internal/httpapi/library_deadletters_test.go @@ -0,0 +1,114 @@ +package httpapi + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "knowledge-governance/internal/governance" + "knowledge-governance/internal/librarydeadletter" +) + +type deadLetterStoreFake struct { + fakeJobStore + releases []governance.LibraryDeadLetterRelease + recorded int +} + +func (f *deadLetterStoreFake) RecordDeadLetterRelease(_ context.Context, release governance.LibraryDeadLetterRelease) (governance.LibraryDeadLetterRelease, bool, error) { + f.recorded++ + f.releases = append(f.releases, release) + return release, false, nil +} + +func (f *deadLetterStoreFake) DeadLetterReleases(context.Context, string, int) ([]governance.LibraryDeadLetterRelease, error) { + return f.releases, nil +} + +func deadLetterRequest(t *testing.T, role, action string, body string) *http.Request { + t.Helper() + request := httptest.NewRequest(http.MethodPost, "/api/v1/library/dead-letter-releases", strings.NewReader(body)) + request.Header.Set("Content-Type", "application/json") + if action != "" { + request.Header.Set("X-Governance-Action", action) + } + return request.WithContext(context.WithValue(request.Context(), sessionKey{}, sessionIdentity{Username: "tester", Role: role})) +} + +const deadLetterRow = `{"id": 7, "payload": {}, "scope": "knowledge", "scope_id": "k1", "failed_at": "2026-10-28T10:00:00+00:00", "last_error": "boom", "task_type": "wiki:ingest", "tenant_id": 10001, "fail_count": 3, "related_id": "k1"}` + +func deadLetterBody(t *testing.T, replayAt time.Time) string { + t.Helper() + observation := librarydeadletter.Observation{ + Project: "kgmaint-weknora", TenantID: 10001, KnowledgeBaseID: "kb-1", + CapturedAt: time.Now().UTC(), Rows: []json.RawMessage{json.RawMessage(deadLetterRow)}, + } + raw, err := json.Marshal(observation) + if err != nil { + t.Fatal(err) + } + body, err := json.Marshal(map[string]any{ + "dead_letter_id": 7, "observation": json.RawMessage(raw), + "observation_digest": librarydeadletter.Fingerprint(raw), + "replay": map[string]any{"queue": "default", "task_id": "task-1", + "at": replayAt.Format(time.RFC3339), "evidence": "run_now 200"}, + }) + if err != nil { + t.Fatal(err) + } + return string(body) +} + +func TestDeadLetterReleaseRequiresIntentAndRole(t *testing.T) { + store := &deadLetterStoreFake{} + config := Config{Store: store, LibraryTargetInstance: "kgmaint-weknora"} + recorder := httptest.NewRecorder() + deadLetterReleaseHandlers(recorder, deadLetterRequest(t, roleViewer, "dead-letter-release", deadLetterBody(t, time.Now().Add(-time.Hour))), config) + if recorder.Code != http.StatusForbidden || store.recorded != 0 { + t.Fatalf("viewer recorded a release: %d %s", recorder.Code, recorder.Body.String()) + } + recorder = httptest.NewRecorder() + deadLetterReleaseHandlers(recorder, deadLetterRequest(t, roleEngineer, "", deadLetterBody(t, time.Now().Add(-time.Hour))), config) + if recorder.Code != http.StatusForbidden || store.recorded != 0 { + t.Fatalf("missing write intent recorded a release: %d", recorder.Code) + } +} + +func TestDeadLetterReleaseRefusesUnwiredInstanceAndBadEvidence(t *testing.T) { + store := &deadLetterStoreFake{} + recorder := httptest.NewRecorder() + deadLetterReleaseHandlers(recorder, deadLetterRequest(t, roleEngineer, "dead-letter-release", + deadLetterBody(t, time.Now().Add(-time.Hour))), Config{Store: store}) + if recorder.Code != http.StatusServiceUnavailable { + t.Fatalf("unwired instance accepted: %d", recorder.Code) + } + // 摘要被改写:必须拒绝,而不是"看着像就放过"。 + config := Config{Store: store, LibraryTargetInstance: "kgmaint-weknora"} + broken := strings.Replace(deadLetterBody(t, time.Now().Add(-time.Hour)), "sha256:", "sha256:0", 1) + recorder = httptest.NewRecorder() + deadLetterReleaseHandlers(recorder, deadLetterRequest(t, roleEngineer, "dead-letter-release", broken), config) + if recorder.Code != http.StatusConflict || store.recorded != 0 { + t.Fatalf("tampered observation accepted: %d", recorder.Code) + } +} + +func TestDeadLetterReleaseRecordsExactFingerprint(t *testing.T) { + store := &deadLetterStoreFake{} + config := Config{Store: store, LibraryTargetInstance: "kgmaint-weknora"} + recorder := httptest.NewRecorder() + deadLetterReleaseHandlers(recorder, deadLetterRequest(t, roleEngineer, "dead-letter-release", + deadLetterBody(t, time.Now().Add(-30*time.Minute))), config) + if recorder.Code != http.StatusOK || store.recorded != 1 { + t.Fatalf("release not recorded: %d %s", recorder.Code, recorder.Body.String()) + } + recorded := store.releases[0] + if recorded.Actor != "tester" || recorded.Instance != "kgmaint-weknora" || + recorded.Fingerprint != librarydeadletter.Fingerprint([]byte(deadLetterRow)) || + recorded.DeadLetterID != 7 || recorded.ReplayTaskID != "task-1" { + t.Fatalf("release lost its audit binding: %+v", recorded) + } +} diff --git a/internal/httpapi/library_executor.go b/internal/httpapi/library_executor.go new file mode 100644 index 0000000..3265a9c --- /dev/null +++ b/internal/httpapi/library_executor.go @@ -0,0 +1,77 @@ +package httpapi + +import ( + "context" + "time" + + "knowledge-governance/internal/libraryexecutor" +) + +// libraryExecutorView 是"封存执行器到底能不能用"的页面口径。 +// +// 为什么不是 `LIBRARY_AGENT_SOCKET != ""`:正式栈里 API 根本没有挂载那个 socket, +// 配置非空并不代表链路通;更危险的是 agent 可能顶着正式单元名却在冻结另一个实例。 +// 这里只认 worker 写入的真实观测,并把观测时间与执行器自报的身份一起给出来。 +type libraryExecutorView struct { + Available bool `json:"available"` + Reason string `json:"reason,omitempty"` + Instance string `json:"instance,omitempty"` + Project string `json:"project,omitempty"` + SourceApp string `json:"source_app,omitempty"` + ArchiveRoot string `json:"archive_root,omitempty"` + Busy bool `json:"busy"` + Degraded string `json:"degraded,omitempty"` + ObservedAt *time.Time `json:"observed_at,omitempty"` +} + +type libraryExecutorStore interface { + LibraryExecutorStatus(ctx context.Context, instance string) (*libraryexecutor.Observation, error) +} + +// libraryExecutorAvailability 把"配置里的期望"与"worker 观测到的真实身份"合起来判定: +// 观测不存在、过期、不可达、实例不符、降级、正忙,任何一条都不可用,并给出原因。 +func libraryExecutorAvailability(ctx context.Context, config Config) (bool, string, libraryExecutorView) { + // 期望身份是**封存源**:agent 冻结的是源库写入者,与同步目标(LIBRARY_TARGET_*) + // 本来就是两套名字(本机候选栈:源 kgmaint、目标 kgseal)。 + instance := config.LibrarySourceInstance + view := libraryExecutorView{Instance: instance} + if instance == "" { + return false, "未配置封存来源实例身份(LIBRARY_SOURCE_INSTANCE),无法确认执行器归属", view + } + store, ok := config.Store.(libraryExecutorStore) + if !ok { + return false, "执行器状态存储不可用", view + } + observation, err := store.LibraryExecutorStatus(ctx, instance) + if err != nil { + return false, "读取执行器状态失败:" + err.Error(), view + } + if observation == nil { + return false, "封存执行器还没有上报状态(worker 未运行,或实例身份与 worker 不一致)", view + } + available, reason := libraryexecutor.Evaluate(*observation, + libraryexecutor.Expectation{Instance: instance, Project: config.LibrarySourceProject, + ArchiveRoot: config.LibraryArchiveRoot}, + executorStatusMaxAge(config), time.Now().UTC()) + observedAt := observation.ObservedAt + view.Available = available + view.Reason = reason + view.Project = observation.Identity.Project + view.SourceApp = observation.Identity.SourceApp + view.ArchiveRoot = observation.Identity.ArchiveRoot + view.Busy = observation.Busy + view.Degraded = observation.Degraded + view.ObservedAt = &observedAt + return available, reason, view +} + +// defaultLibraryExecutorStatusMaxAge 与 worker 的探测间隔(30s)配套: +// 没配上限绝不等于"永不过期"——那会把一次陈旧的"可用"永久留在页面上。 +const defaultLibraryExecutorStatusMaxAge = 2 * time.Minute + +func executorStatusMaxAge(config Config) time.Duration { + if config.LibraryExecutorStatusMaxAge > 0 { + return config.LibraryExecutorStatusMaxAge + } + return defaultLibraryExecutorStatusMaxAge +} diff --git a/internal/httpapi/library_executor_test.go b/internal/httpapi/library_executor_test.go new file mode 100644 index 0000000..7c5da52 --- /dev/null +++ b/internal/httpapi/library_executor_test.go @@ -0,0 +1,184 @@ +package httpapi + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "knowledge-governance/internal/libraryexecutor" +) + +// fakeJobStore 只是"存储"替身;执行器状态同样是它的一条记录。 +func (s *fakeJobStore) LibraryExecutorStatus(_ context.Context, instance string) (*libraryexecutor.Observation, error) { + if s.executorStatus == nil || s.executorStatus.Instance != instance { + return nil, nil + } + return s.executorStatus, nil +} + +// healthyExecutor 返回一份"worker 刚观测到、指向本实例"的记录。 +func healthyExecutor(now time.Time) *libraryexecutor.Observation { + return &libraryexecutor.Observation{ + Instance: "kgmaint-weknora", Reachable: true, ObservedAt: now, + Identity: libraryexecutor.Identity{Project: "kgmaint-weknora", SourceApp: "kgmaint-app", + ArchiveRoot: "/tmp/kg-e2e/archive", Writers: "app"}, + } +} + +// captureReadyConfig 是"执行器真的可用"时的配置:实例、期望项目与观测三者一致。 +func captureReadyConfig(store *fakeJobStore) Config { + store.executorStatus = healthyExecutor(time.Now().UTC()) + return Config{ + Store: store, + LibraryAgentSocket: "/run/library-agent/agent.sock", + LibrarySourceInstance: "kgmaint-weknora", + LibrarySourceProject: "kgmaint-weknora", + LibraryArchiveRoot: "/var/lib/knowledge-governance/library", + } +} + +func decodeCaptureRefusal(t *testing.T, recorder *httptest.ResponseRecorder) (string, string) { + t.Helper() + var body struct { + Code string `json:"code"` + Reason string `json:"reason"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil { + t.Fatalf("body %s: %v", recorder.Body.String(), err) + } + return body.Code, body.Reason +} + +// 页面上的"可发起"必须等于接口真的会接受:配置了 socket 但执行器不可达、 +// 指向别的实例、降级或过期时,都必须带着原因拒绝,而不是先排队再失败。 +func TestLibraryCaptureRefusesWhenExecutorIsNotVerified(t *testing.T) { + now := time.Now().UTC() + tests := []struct { + name string + config func(*Config) + text string + }{ + {name: "no instance identity", config: func(c *Config) { + c.LibrarySourceInstance = "" + }, text: "LIBRARY_SOURCE_INSTANCE"}, + {name: "nothing reported yet", config: func(c *Config) { + c.Store.(*fakeJobStore).executorStatus = nil + }, text: "还没有上报状态"}, + {name: "unreachable", config: func(c *Config) { + c.Store.(*fakeJobStore).executorStatus.Reachable = false + c.Store.(*fakeJobStore).executorStatus.Error = "library agent is unreachable" + }, text: "不可达"}, + {name: "stale observation", config: func(c *Config) { + c.Store.(*fakeJobStore).executorStatus.ObservedAt = now.Add(-10 * time.Minute) + }, text: "已过期"}, + {name: "points at another instance", config: func(c *Config) { + c.Store.(*fakeJobStore).executorStatus.Identity.Project = "weknora071full" + }, text: "指向的实例不匹配"}, + {name: "degraded", config: func(c *Config) { + c.Store.(*fakeJobStore).executorStatus.Degraded = "写入者恢复失败" + }, text: "降级"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + store := &fakeJobStore{} + config := captureReadyConfig(store) + test.config(&config) + recorder := httptest.NewRecorder() + libraryCaptureEnqueue(recorder, captureRequest(`{"scope":"internal"}`), config) + if recorder.Code != http.StatusServiceUnavailable { + t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String()) + } + code, reason := decodeCaptureRefusal(t, recorder) + if code != "library_capture_unavailable" { + t.Fatalf("code=%q body=%s", code, recorder.Body.String()) + } + if reason == "" || !strings.Contains(reason, test.text) { + t.Fatalf("reason=%q want it to mention %q", reason, test.text) + } + if store.enqueued.Type != "" { + t.Fatal("refused capture still enqueued a job") + } + }) + } +} + +// The overview must report the same reason the capture endpoint would use. +func TestLibraryOverviewReportsExecutorIdentityAndReason(t *testing.T) { + store := &fakeLibraryStore{} + config := captureReadyConfig(&store.fakeJobStore) + // libraryJobStore 的方法在 fakeLibraryStore 上,执行器状态在嵌入的 fakeJobStore 上。 + config.Store = store + recorder := httptest.NewRecorder() + libraryOverviewHandler(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/library/overview?scope=internal", nil), config) + if recorder.Code != http.StatusOK { + t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String()) + } + var body struct { + CaptureAvailable bool `json:"capture_available"` + Executor struct { + Available bool `json:"available"` + Instance string `json:"instance"` + Project string `json:"project"` + SourceApp string `json:"source_app"` + ArchiveRoot string `json:"archive_root"` + } `json:"executor"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if !body.CaptureAvailable || !body.Executor.Available { + t.Fatalf("overview did not report an available executor: %s", recorder.Body.String()) + } + if body.Executor.Project != "kgmaint-weknora" || body.Executor.SourceApp != "kgmaint-app" || + body.Executor.ArchiveRoot != "/tmp/kg-e2e/archive" || body.Executor.Instance != "kgmaint-weknora" { + t.Fatalf("executor view = %+v", body.Executor) + } + + store.fakeJobStore.executorStatus.Identity.Project = "weknora071full" + recorder = httptest.NewRecorder() + libraryOverviewHandler(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/library/overview?scope=internal", nil), config) + if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if body.CaptureAvailable { + t.Fatalf("overview still says available after the executor moved to another instance: %s", recorder.Body.String()) + } +} + +// "恢复至该版本"必须由真实条件决定:没有 restore 授权就明确说出来。 +func TestLibraryRestoreAvailabilityExplainsMissingAuthorization(t *testing.T) { + store := &fakeJobStore{} + config := Config{Store: store, ProductionReleaseRequestsEnabled: true, ProductionBackupRestoreVerified: true} + recorder := httptest.NewRecorder() + libraryRestoreAvailability(recorder, httptest.NewRequest(http.MethodGet, + "/api/v1/library/restore-availability?scope=internal&snapshot_id=snap-1", nil), config) + if recorder.Code != http.StatusOK { + t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String()) + } + var body struct { + Allowed bool `json:"allowed"` + Reasons []string `json:"reasons"` + Authorization struct { + State string `json:"state"` + } `json:"authorization"` + } + if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if body.Allowed || body.Authorization.State != "none" || len(body.Reasons) == 0 { + t.Fatalf("body=%s", recorder.Body.String()) + } + found := false + for _, reason := range body.Reasons { + if strings.Contains(reason, "恢复授权") { + found = true + } + } + if !found { + t.Fatalf("reasons = %v", body.Reasons) + } +} diff --git a/internal/httpapi/library_flow.go b/internal/httpapi/library_flow.go index d9ba354..191b427 100644 --- a/internal/httpapi/library_flow.go +++ b/internal/httpapi/library_flow.go @@ -19,6 +19,7 @@ import ( "knowledge-governance/internal/librarydiff" "knowledge-governance/internal/librarysnapshot" postgresstore "knowledge-governance/internal/platform/postgres" + v071 "knowledge-governance/internal/releaseadapter/v071" ) // 整库版本/状态/差异读取契约 + 生产同步的 fail-closed 入口。 @@ -56,30 +57,36 @@ type libraryKnownView struct { EmptyBaseline bool `json:"empty_baseline,omitempty"` } -// productionScopeEmpty 只读探测生产目标:真正连库数一遍,不用缓存或推断。 -func productionScopeEmpty(ctx context.Context, config Config, scope string) (bool, error) { +// productionScopeState 只读探测生产目标:真正连库数一遍,不用缓存或推断。 +// +// "空"必须是整库意义的空:只数 knowledges 会把"文档已删、分块/Wiki/资源还在" +// 的目标当成首次发布的空基线,从而绕过基线初始化。所以这里用适配层同一份 +// 表清单与范围过滤(v071.ScopeContentEmpty),并把逐表行数交回调用方作为证据。 +func productionScopeState(ctx context.Context, config Config, scope string) (bool, map[string]int64, error) { if strings.TrimSpace(config.LibraryProductionDatabaseURL) == "" { - return false, errors.New("生产整库目标未配置") + return false, nil, errors.New("生产整库目标未配置") } kb := config.LibraryProductionInternalKB if scope == "external" { kb = config.LibraryProductionExternalKB } if kb == "" { - return false, errors.New("生产知识库未配置") + return false, nil, errors.New("该范围的目标知识库未配置") } pool, err := pgxpool.New(ctx, config.LibraryProductionDatabaseURL) if err != nil { - return false, err + return false, nil, err } defer pool.Close() - var count int64 - err = pool.QueryRow(ctx, `SELECT count(*) FROM knowledges WHERE tenant_id=$1 AND knowledge_base_id=$2 AND deleted_at IS NULL`, - config.LibraryProductionTenantID, kb).Scan(&count) - if err != nil { - return false, err - } - return count == 0, nil + return v071.ScopeContentEmpty(ctx, pool, + v071.ProductionScope{TenantID: config.LibraryProductionTenantID, KnowledgeBaseID: kb}) +} + +// productionScopeEmpty 保留"只问空不空"的调用形状;逐表行数在需要举证时用 +// productionScopeState 取。 +func productionScopeEmpty(ctx context.Context, config Config, scope string) (bool, error) { + empty, _, err := productionScopeState(ctx, config, scope) + return empty, err } type libraryDiffView struct { @@ -106,6 +113,58 @@ type libraryWikiDiffView struct { // Sample 只列前若干页,避免把整库 Wiki 列表塞进接口。 Sample []string `json:"sample"` Note string `json:"note"` + // Pages 是逐页的**贡献级**明细:哪一页、变了哪几类语义、掉/加了哪些来源文档。 + // 页面级计数只看增删改;撤销一份共享贡献时页面通常仍是"更新", + // 只有这里的 sources_removed / changed 才能证明那份贡献确实被摘掉了。 + Pages []libraryWikiPageView `json:"pages,omitempty"` + PagesTruncated bool `json:"pages_truncated,omitempty"` + SourcesAdded int `json:"sources_added"` + SourcesRemoved int `json:"sources_removed"` + BaseCount int `json:"base_count"` + TargetCount int `json:"target_count"` +} + +type libraryWikiPageView struct { + Slug string `json:"slug"` + Title string `json:"title"` + PageType string `json:"page_type,omitempty"` + Status string `json:"status"` // added | updated | removed + Changed []string `json:"changed"` + SourcesAdded []string `json:"sources_added,omitempty"` + SourcesRemoved []string `json:"sources_removed,omitempty"` + CitationsBefore int `json:"citations_before"` + CitationsAfter int `json:"citations_after"` +} + +// wikiPageLimit 是接口一次最多返回的页数;超出部分只给计数,不给半截明细。 +const wikiPageLimit = 50 + +func wikiViewFrom(result librarydiff.WikiResult, limit int) *libraryWikiDiffView { + if limit <= 0 { + limit = wikiPageLimit + } + view := &libraryWikiDiffView{State: result.State, Added: result.Added, Updated: result.Updated, + Removed: result.Removed, Sample: []string{}, SourcesAdded: result.SourcesAdded, + SourcesRemoved: result.SourcesRemoved, BaseCount: result.BaseCount, TargetCount: result.TargetCount, + Note: "逐页比对正文、来源文档、引用分块、出入边与目录;语义重算发生在源侧,本平台只如实比对与保留结果"} + for index, page := range result.Pages { + if index >= limit { + view.PagesTruncated = true + break + } + view.Pages = append(view.Pages, libraryWikiPageView{Slug: page.Slug, Title: page.Title, + PageType: page.PageType, Status: page.Status, Changed: librarydiff.ChangedLabels(page.Changed), + SourcesAdded: page.SourcesAdded, SourcesRemoved: page.SourcesRemoved, + CitationsBefore: page.CitationsBefore, CitationsAfter: page.CitationsAfter}) + } + for _, page := range view.Pages { + if len(view.Sample) >= 20 { + break + } + label := map[string]string{"added": "新增", "updated": "更新", "removed": "撤销"}[page.Status] + view.Sample = append(view.Sample, label+" "+page.Title) + } + return view } // wikiDiffOf 比较两份快照的 wiki_pages 行数组(页面级)。 @@ -148,32 +207,11 @@ func wikiDiffOf(archive *librarysnapshot.Archive, baseID, targetID string) *libr if reason != "" { return unknown("测试版本:" + reason) } - result, err := librarydiff.Compare(base, target) + result, err := librarydiff.CompareWiki(base, target) if err != nil { - return unknown("Wiki 页面比对失败:" + err.Error()) + return unknown("Wiki 逐页比对失败:" + err.Error()) } - view := &libraryWikiDiffView{State: "computed", Added: len(result.Added), Updated: len(result.Updated), - Removed: len(result.Removed), Sample: []string{}, - Note: "页面级差异:只说明页面被增/改/撤,不代表实体、概念、边或引用已重算"} - for _, item := range result.Added { - if len(view.Sample) >= 20 { - break - } - view.Sample = append(view.Sample, "新增 "+item.Title) - } - for _, item := range result.Updated { - if len(view.Sample) >= 20 { - break - } - view.Sample = append(view.Sample, "更新 "+item.Title) - } - for _, item := range result.Removed { - if len(view.Sample) >= 20 { - break - } - view.Sample = append(view.Sample, "撤销 "+item.Title) - } - return view + return wikiViewFrom(result, wikiPageLimit) } // computeLibraryDiff 用归档里的 knowledges 行数组比较两个快照: @@ -276,6 +314,7 @@ type libraryOverview struct { Authorization libraryAuthorizationView `json:"authorization"` CaptureAvailable bool `json:"capture_available"` CaptureReason string `json:"capture_reason,omitempty"` + Executor libraryExecutorView `json:"executor"` Now time.Time `json:"now"` } @@ -476,6 +515,70 @@ func errString(err error) string { return err.Error() } +// shortSnapshot 只用于给人看的提示:完整摘要留给快照详情,别在原因句里塞 64 位。 +func shortSnapshot(id string) string { + if len(id) <= 12 { + return id + } + return id[:12] +} + +// testSealedOf 选"拟发布的测试封存":只有登记过 test_sealed 的那一份才算候选。 +// +// 归档是共享的:生产预备份与目标基线快照同样带着 scope 写进同一个归档根, +// 按"这个 scope 最新一份"挑会把目标自己的内容当成测试候选(页面会显示一份 +// 永远不该被发布的东西)。所以候选来自版本登记,再用归档确认它真的还在本地。 +func testSealedOf(versions []governance.LibraryVersion, byID map[string]librarySnapshotView) (*librarySnapshotView, string) { + for _, item := range versions { + if item.Kind != governance.LibraryVersionTestSealed { + continue + } + snapshot, ok := byID[item.SnapshotID] + if !ok { + return nil, "最近一次测试封存 " + shortSnapshot(item.SnapshotID) + " 不在本机归档里(归档被清理或换了归档根)" + } + return &snapshot, "" + } + return nil, "还没有该目标的测试封存" +} + +// libraryMaintenanceCapability 用配置事实回答"这台机器上维护是否接线"。 +// 它刻意只说配置:能停写入者/关入口/证明排空的真实能力在 worker 侧(docker socket、 +// 闸门、排空文件),API 若声称"能力可用"就是拿配置字符串冒充执行证据。 +func libraryMaintenanceCapability(config Config) map[string]any { + missing := make([]string, 0, 6) + if config.LibraryTargetInstance == "" { + missing = append(missing, "LIBRARY_TARGET_INSTANCE") + } + if config.LibraryTargetProject == "" { + missing = append(missing, "LIBRARY_TARGET_PROJECT") + } + if len(config.LibraryTargetWriters) == 0 { + missing = append(missing, "LIBRARY_TARGET_WRITERS") + } + if config.LibraryTargetIngressURL == "" { + missing = append(missing, "LIBRARY_TARGET_INGRESS_URL") + } + if config.LibraryTargetIngressService == "" { + missing = append(missing, "LIBRARY_TARGET_INGRESS_SERVICE") + } + if config.LibraryTargetJournalRoot == "" { + missing = append(missing, "LIBRARY_TARGET_JOURNAL_ROOT") + } + if config.LibraryTargetDockerSocket == "" { + missing = append(missing, "LIBRARY_TARGET_DOCKER_SOCKET") + } + return map[string]any{ + "configured": len(missing) == 0, + "missing": missing, + "instance": config.LibraryTargetInstance, + "project": config.LibraryTargetProject, + "writers": config.LibraryTargetWriters, + "ingress_service": config.LibraryTargetIngressService, + "note": "配置齐全只说明治理侧接线完整;停写入者、关入口与排空由 worker 在作业内校验并执行", + } +} + func librarySyncGate(config Config, production libraryKnownView, prebackup libraryKnownView, target string) libraryGateView { return librarySyncGateWithGrant(config, production, prebackup, target, false) } @@ -499,9 +602,11 @@ func librarySyncGateWithGrant(config Config, production libraryKnownView, prebac gate.Reasons = append(gate.Reasons, "生产整库目标未配置(LIBRARY_PRODUCTION_DATABASE_URL),无法核验生产当前状态") } // 首次发布:生产还没有"生效版本"记录,但只要生产目标可读且是空的, - // 这就是一个合法的"空基线",不应该被永久挡住。非空但无权威版本仍然阻断。 + // 这就是一个合法的"空基线",不应该被永久挡住。非空但无权威版本仍然阻断, + // 指向真实可行的入口(在维护隔离下捕获目标整库并校验后登记基线), + // 而不是让人去手写版本或清空目标。 if production.State != "known" && !production.EmptyBaseline { - gate.Reasons = append(gate.Reasons, "生产当前权威版本未知,不能启动写入") + gate.Reasons = append(gate.Reasons, "目标非空且没有生效版本记录:先建立目标权威基线(维护隔离下捕获整库并逐类校验),或确认目标确实是空库") } if target == "" { gate.Reasons = append(gate.Reasons, "还没有可同步的测试封存版本") @@ -532,28 +637,25 @@ func libraryOverviewHandler(w http.ResponseWriter, r *http.Request, config Confi } now := time.Now().UTC() overview := libraryOverview{Scope: scope, Jobs: []governance.LibraryJob{}, Now: now, - CaptureAvailable: strings.TrimSpace(config.LibraryAgentSocket) != "", - Gate: libraryGateView{Reasons: []string{}}, - Diff: libraryDiffView{State: "unknown", Reason: "生产与测试的文档级差异读取未接线", Added: []any{}, Updated: []any{}, Removed: []any{}}} - if !overview.CaptureAvailable { - overview.CaptureReason = "主机侧封存 agent 未接线" - } + Gate: libraryGateView{Reasons: []string{}}, + Diff: libraryDiffView{State: "unknown", Reason: "生产与测试的文档级差异读取未接线", Added: []any{}, Updated: []any{}, Removed: []any{}}} + // 可发起封存 = worker 真实探测到执行器可用且指向本实例;原因逐条给全。 + overview.CaptureAvailable, overview.CaptureReason, overview.Executor = libraryExecutorAvailability(r.Context(), config) - // 测试封存来自归档:它只是测试侧事实。 + // 测试封存来自归档 + 版本登记:归档里**不止**测试封存——生产预备份与目标基线 + // 快照同样带 scope,如果只按"这个 scope 最新一份"挑,页面就会把目标自己的内容 + // 当成拟发布的测试候选。所以候选只认登记过 kind=test_sealed 的那一份。 snapshots, snapErr := readLibrarySnapshots(r.Context(), config.LibraryArchiveRoot) + byID := map[string]librarySnapshotView{} + archiveReadable := false if snapErr != nil { _, overview.TestSealedReason = libraryArchiveErrorStatus(snapErr, "归档不可读") } else if !snapshots.Available { overview.TestSealedReason = snapshots.Reason } else { - for index := range snapshots.Snapshots { - if snapshots.Snapshots[index].Scope == scope { - overview.TestSealed = &snapshots.Snapshots[index] - break - } - } - if overview.TestSealed == nil { - overview.TestSealedReason = "还没有该目标的测试封存" + archiveReadable = true + for _, snapshot := range snapshots.Snapshots { + byID[snapshot.SnapshotID] = snapshot } } @@ -563,6 +665,9 @@ func libraryOverviewHandler(w http.ResponseWriter, r *http.Request, config Confi writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot read library versions"}) return } + if archiveReadable { + overview.TestSealed, overview.TestSealedReason = testSealedOf(versions, byID) + } overview.Production = libraryKnownView{State: "unknown", Reason: "还没有生产验证通过的生效版本记录"} overview.Prebackup = libraryKnownView{State: "unknown", Reason: "本次目标快照还没有生产预备份"} target := "" diff --git a/internal/httpapi/library_restore.go b/internal/httpapi/library_restore.go index 10f28f3..3e1239f 100644 --- a/internal/httpapi/library_restore.go +++ b/internal/httpapi/library_restore.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "net/http" + "strconv" "strings" "time" @@ -86,7 +87,10 @@ func libraryRestoreEnqueue(w http.ResponseWriter, r *http.Request, config Config return } now := time.Now().UTC() - jobID := "librestore-" + request.Scope + "-" + now.Format("20060102T150405.000000") + // 作业 id 会作为维护窗口的 job 标识落进 freeze journal,而冻结协调器只接受 + // [A-Za-z0-9_-]:带 "." 的时间戳会让恢复在开窗口时被拒(真跑报 invalid freeze request), + // 所以用纳秒整数,和同步作业保持一致。 + jobID := "librestore-" + request.Scope + "-" + strconv.FormatInt(now.UnixNano(), 10) job, reused, err := store.CreateLibraryJob(r.Context(), governance.LibraryJob{ ID: jobID, Scope: request.Scope, Kind: governance.LibraryJobRestore, IdempotencyKey: "library.restore:" + request.Scope + ":" + request.SnapshotID, @@ -107,7 +111,7 @@ func libraryRestoreEnqueue(w http.ResponseWriter, r *http.Request, config Config if reused && (job.State == governance.LibraryJobFailed || job.State == governance.LibraryJobCancelled) { job, reused, err = store.CreateLibraryJob(r.Context(), governance.LibraryJob{ ID: jobID + "-retry", Scope: request.Scope, Kind: governance.LibraryJobRestore, - IdempotencyKey: "library.restore:" + request.Scope + ":" + request.SnapshotID + ":retry:" + now.Format("20060102T150405.000000000"), + IdempotencyKey: "library.restore:" + request.Scope + ":" + request.SnapshotID + ":retry:" + strconv.FormatInt(now.UnixNano(), 10), RequestDigest: digest, TargetSnapshotID: request.SnapshotID, Actor: identity.Username, }, governance.SyncStages(), now) if err != nil || reused { @@ -142,6 +146,64 @@ func libraryRestoreEnqueue(w http.ResponseWriter, r *http.Request, config Config writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": false}) } +// libraryRestoreAvailability 回答"现在能不能把生产恢复到这份快照"。 +// +// 为什么需要它:页面原来把"恢复至该版本"按钮写死为 disabled,理由是"未接线"—— +// 但接口、授权、恢复作业其实都已经在跑。按钮该由真实条件决定:双闸打开、 +// 有这份快照的恢复授权、目标实例没有在维护。 +type libraryRestoreAvailabilityView struct { + Allowed bool `json:"allowed"` + Reasons []string `json:"reasons"` + Scope string `json:"scope"` + SnapshotID string `json:"snapshot_id"` + Authorization libraryAuthorizationView `json:"authorization"` +} + +func libraryRestoreAvailability(w http.ResponseWriter, r *http.Request, config Config) { + scope, err := scopeFromQuery(r) + if err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()}) + return + } + snapshotID := strings.TrimSpace(r.URL.Query().Get("snapshot_id")) + if snapshotID == "" { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "snapshot_id required"}) + return + } + view := libraryRestoreAvailabilityView{Scope: scope, SnapshotID: snapshotID, Reasons: []string{}} + if !config.ProductionReleaseRequestsEnabled { + view.Reasons = append(view.Reasons, "生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false)") + } + if !config.ProductionBackupRestoreVerified { + view.Reasons = append(view.Reasons, "没有已核验的生产恢复演练证据(PRODUCTION_BACKUP_RESTORE_VERIFIED=false)") + } + digest, err := libraryRestoreParamsDigest(scope, snapshotID) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot digest params"}) + return + } + grant, found := governance.LibraryReleaseAuthorization{}, false + if storeWithAuth, ok := config.Store.(interface { + ActiveLibraryAuthorization(ctx context.Context, scope, snapshotID, digest string, now time.Time) (governance.LibraryReleaseAuthorization, bool, error) + }); ok { + grant, found, err = storeWithAuth.ActiveLibraryAuthorization(r.Context(), scope, snapshotID, digest, time.Now().UTC()) + if err != nil { + writeJSON(w, http.StatusServiceUnavailable, map[string]any{"allowed": false, + "reasons": []string{"读取恢复授权失败:" + err.Error()}}) + return + } + } + view.Authorization = authorizationViewOf(grant, found) + if !found { + view.Reasons = append(view.Reasons, "没有针对该快照的恢复授权:先签发 restore 授权(绑定同一快照)") + } + if _, held := instanceMaintenanceGuard(r.Context(), config, scope); held { + view.Reasons = append(view.Reasons, "目标实例正在维护中,等当前作业释放后再恢复") + } + view.Allowed = len(view.Reasons) == 0 + writeJSON(w, http.StatusOK, view) +} + // 崩溃续作:失败作业可以按原参数重新排队,生成新作业(旧作业保留为审计记录)。 func libraryJobResume(w http.ResponseWriter, r *http.Request, config Config) { identity, ok := sessionFrom(r.Context()) diff --git a/internal/httpapi/library_restore_test.go b/internal/httpapi/library_restore_test.go index e3355d3..92245a2 100644 --- a/internal/httpapi/library_restore_test.go +++ b/internal/httpapi/library_restore_test.go @@ -85,3 +85,31 @@ func TestLibraryJobResumeRejectsNonFailedJobs(t *testing.T) { t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String()) } } + +// 恢复作业 id 会作为维护窗口的 job 标识落进 freeze journal,协调器只接受 +// [A-Za-z0-9_-]。带 "." 的时间戳曾经让每次恢复都在开窗口时被拒(invalid freeze request)。 +func TestLibraryRestoreJobIDIsAFreezeSafeIdentifier(t *testing.T) { + store := &fakeLibraryStore{fakeJobStore: fakeJobStore{enqueueEchoRequestedID: true}} + digest, err := libraryRestoreParamsDigest("internal", "snap-1") + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + store.grant = &governance.LibraryReleaseAuthorization{ID: "a", Scope: "internal", SnapshotID: "snap-1", + ParamsDigest: digest, ExpiresAt: now.Add(time.Hour), CreatedAt: now} + config := Config{Store: store, ProductionReleaseRequestsEnabled: true, ProductionBackupRestoreVerified: true} + recorder := httptest.NewRecorder() + libraryRestoreEnqueue(recorder, restoreRequest(`{"scope":"internal","snapshot_id":"snap-1"}`, roleAdmin), config) + if recorder.Code != http.StatusAccepted { + t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String()) + } + id := store.lastJob.ID + if id == "" || len(id) > 128 { + t.Fatalf("作业 id 形状不对:%q", id) + } + for _, r := range id { + if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_') { + t.Fatalf("作业 id 含维护窗口不接受的字符 %q:%q", r, id) + } + } +} diff --git a/internal/httpapi/library_snapshots.go b/internal/httpapi/library_snapshots.go index ded1b11..bbdab31 100644 --- a/internal/httpapi/library_snapshots.go +++ b/internal/httpapi/library_snapshots.go @@ -85,15 +85,14 @@ const libraryCaptureNote = "封存由运维命令 scripts/library-capture.sh cap func librarySnapshotsList(w http.ResponseWriter, r *http.Request, config Config) { view, err := readLibrarySnapshots(r.Context(), config.LibraryArchiveRoot) - view.CaptureAvailable = config.LibraryAgentSocket != "" - if !view.CaptureAvailable { - view.CaptureReason = "未接线:主机侧封存 agent 没有挂载,页面只能查看已封存版本" - } + // 能不能发起封存只认 worker 的真实观测,不认"配置里写了 socket 路径"。 + view.CaptureAvailable, view.CaptureReason, _ = libraryExecutorAvailability(r.Context(), config) if err != nil { status, reason := libraryArchiveErrorStatus(err, "归档不可读") writeJSON(w, status, librarySnapshotList{Available: false, Reason: reason, Snapshots: []librarySnapshotView{}, Verification: libraryVerificationManifest, - VerificationNote: libraryVerificationNote, CaptureNote: libraryCaptureNote}) + VerificationNote: libraryVerificationNote, CaptureNote: libraryCaptureNote, + CaptureAvailable: view.CaptureAvailable, CaptureReason: view.CaptureReason}) return } writeJSON(w, http.StatusOK, view) diff --git a/internal/httpapi/library_wiki_diff_test.go b/internal/httpapi/library_wiki_diff_test.go new file mode 100644 index 0000000..0464d42 --- /dev/null +++ b/internal/httpapi/library_wiki_diff_test.go @@ -0,0 +1,51 @@ +package httpapi + +import ( + "strings" + "testing" + + "knowledge-governance/internal/librarydiff" +) + +func TestWikiViewFromKeepsContributionDetail(t *testing.T) { + result := librarydiff.WikiResult{ + State: "computed", Added: 0, Updated: 1, Removed: 1, SourcesAdded: 0, SourcesRemoved: 2, + BaseCount: 2, TargetCount: 1, + Pages: []librarydiff.WikiContribution{ + {Slug: "entity/gate", Title: "发布闸门", PageType: "entity", Status: "updated", + Changed: []string{"content", "sources", "citations"}, SourcesRemoved: []string{"doc-a"}, + CitationsBefore: 2, CitationsAfter: 1}, + {Slug: "entity/only-a", Title: "只属于 A", PageType: "entity", Status: "removed", + Changed: []string{"removed"}, SourcesRemoved: []string{"doc-a"}}, + }, + } + view := wikiViewFrom(result, 0) + if view.State != "computed" || view.Updated != 1 || view.Removed != 1 || view.SourcesRemoved != 2 { + t.Fatalf("view: %+v", view) + } + if len(view.Pages) != 2 { + t.Fatalf("pages: %+v", view.Pages) + } + kept := view.Pages[0] + if kept.Status != "updated" || len(kept.Changed) != 3 || kept.Changed[0] != "正文" { + t.Fatalf("changed labels: %+v", kept) + } + if len(kept.SourcesRemoved) != 1 || kept.SourcesRemoved[0] != "doc-a" { + t.Fatalf("sources: %+v", kept) + } + if !strings.Contains(view.Note, "来源") { + t.Fatalf("note should explain what is compared: %q", view.Note) + } +} + +func TestWikiViewFromCapsPagesAndSaysSo(t *testing.T) { + result := librarydiff.WikiResult{State: "computed", BaseCount: 0} + for _, slug := range []string{"a", "b", "c"} { + result.Pages = append(result.Pages, librarydiff.WikiContribution{Slug: slug, Status: "added"}) + result.Added++ + } + view := wikiViewFrom(result, 2) + if len(view.Pages) != 2 || !view.PagesTruncated { + t.Fatalf("cap: pages=%d truncated=%v", len(view.Pages), view.PagesTruncated) + } +} diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index 5f7fe6c..bca8f40 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -148,8 +148,24 @@ type Config struct { // LibraryTargetInstance 是实例级维护的目标实例名;留空即"未接线", // 页面不能发起按预备份恢复,接口返回原因而不是假装能恢复。 LibraryTargetInstance string + // 维护能力的其余配置项只用于如实展示"是否接线":未运行维护与没有维护能力 + // 是两件事,页面不能混成一句"未接入"。真正停写入者/关入口的能力由 worker + // 在执行作业时校验(socket、入口、排空文件都在 worker 侧)。 + LibraryTargetProject string + LibraryTargetWriters []string + LibraryTargetIngressURL string + LibraryTargetIngressService string + LibraryTargetDockerSocket string + LibraryTargetJournalRoot string + // LibrarySourceInstance 是死信解除记录的来源实例名(封存源),留空回落到目标实例名。 + LibrarySourceInstance string // LibraryAgentSocket 是主机侧封存 agent 的 unix socket;留空即页面不能发起封存。 LibraryAgentSocket string + // LibrarySourceProject 是执行器应指向的实例名(agent -project,封存源)。 + // 与它不一致时按"封错库"处理:不可发起,并说明差在哪。 + LibrarySourceProject string + // LibraryExecutorStatusMaxAge 是执行器观测的有效期;超过即"无法确认"。 + LibraryExecutorStatusMaxAge time.Duration // LibraryCapacityBudgetBytes 是整库归档容量预算;0 = 未设置(容量门禁不放行)。 LibraryCapacityBudgetBytes int64 // 整库生产目标的只读探测配置:首次发布要能证明"生产可读且为空"。 @@ -157,6 +173,9 @@ type Config struct { LibraryProductionInternalKB string LibraryProductionExternalKB string LibraryProductionTenantID uint64 + // LibraryScopeProbe 让测试注入"目标整库是否为空"的探测;留空即用真实连接。 + // 它不是运行时开关:生产部署不设置它,那次探测仍然直连目标库逐表计数。 + LibraryScopeProbe func(ctx context.Context, scope string) (bool, map[string]int64, error) // LibraryRetentionVersions/Days 是保留策略;都为 0 = 未设置(不生成清理候选)。 LibraryRetentionVersions int LibraryRetentionDays int @@ -432,6 +451,13 @@ func NewHandler(config Config) (http.Handler, error) { mux.HandleFunc("GET /api/v1/library/captures", func(w http.ResponseWriter, r *http.Request) { libraryCaptureJobs(w, r, config) }) + // 死信解除:只写治理库审计,不删上游行;封存时按精确指纹放行。 + mux.HandleFunc("GET /api/v1/library/dead-letter-releases", func(w http.ResponseWriter, r *http.Request) { + deadLetterReleaseHandlers(w, r, config) + }) + mux.HandleFunc("POST /api/v1/library/dead-letter-releases", func(w http.ResponseWriter, r *http.Request) { + deadLetterReleaseHandlers(w, r, config) + }) // 整库状态:测试封存 / 生产生效 / 生产预备份 三类版本 + 差异 + 持久作业。 // 归档可能很大:加请求级超时,读不动就明确降级,别让页面请求挂死。 mux.HandleFunc("GET /api/v1/library/overview", func(w http.ResponseWriter, r *http.Request) { @@ -465,6 +491,8 @@ func NewHandler(config Config) (http.Handler, error) { return } response := map[string]any{"held": held != nil && held.ReleasedAt == nil, "maintenance": held} + // 有没有维护能力是配置事实,和"此刻没有窗口"必须分开回答。 + response["capability"] = libraryMaintenanceCapability(config) // 实例级占用才是真正的互斥来源:跨 scope 也在这里体现。 if instanceStore, ok := config.Store.(interface { InstanceMaintenanceOf(ctx context.Context, instance string) (*postgresstore.LibraryInstanceMaintenance, error) @@ -501,6 +529,10 @@ func NewHandler(config Config) (http.Handler, error) { mux.HandleFunc("POST /api/v1/library/restores", func(w http.ResponseWriter, r *http.Request) { libraryRestoreEnqueue(w, r, config) }) + // 页面问"这份快照现在能不能恢复":条件来自双闸、授权与实例维护,而不是写死的按钮。 + mux.HandleFunc("GET /api/v1/library/restore-availability", func(w http.ResponseWriter, r *http.Request) { + libraryRestoreAvailability(w, r, config) + }) // 失败/取消作业续作:按原参数重新排队,旧作业保留审计。 mux.HandleFunc("POST /api/v1/library/jobs/{id}/resume", func(w http.ResponseWriter, r *http.Request) { libraryJobResume(w, r, config) @@ -509,6 +541,10 @@ func NewHandler(config Config) (http.Handler, error) { mux.HandleFunc("POST /api/v1/library/syncs", func(w http.ResponseWriter, r *http.Request) { librarySyncEnqueue(w, r, config) }) + // 首次发布的目标权威基线:管理员在维护隔离下捕获目标整库并校验后登记版本。 + mux.HandleFunc("POST /api/v1/library/baselines", func(w http.ResponseWriter, r *http.Request) { + libraryBaselineEnqueue(w, r, config) + }) // 全部 main 提交时间线(读本库缓存,不直接打 Gitea) mux.HandleFunc("GET /api/v1/version-history", func(w http.ResponseWriter, r *http.Request) { versionHistoryList(w, r, config) @@ -2800,3 +2836,12 @@ func writeJSON(w http.ResponseWriter, status int, value any) { w.WriteHeader(status) _ = json.NewEncoder(w).Encode(value) } + +// SourceInstance 是死信解除记录绑定的实例名:来源实例与同步目标常常不是同一个, +// 用目标名去绑来源死信会让真实观测永远对不上。 +func (config Config) SourceInstance() string { + if config.LibrarySourceInstance != "" { + return config.LibrarySourceInstance + } + return config.LibraryTargetInstance +} diff --git a/internal/httpapi/server_test.go b/internal/httpapi/server_test.go index b4bfef6..9861428 100644 --- a/internal/httpapi/server_test.go +++ b/internal/httpapi/server_test.go @@ -15,6 +15,7 @@ import ( "knowledge-governance/internal/governance" "knowledge-governance/internal/jobs" + "knowledge-governance/internal/libraryexecutor" postgresstore "knowledge-governance/internal/platform/postgres" ) @@ -1501,30 +1502,32 @@ type fakeJobStore struct { activateSettingsErr error // enqueueEchoRequestedID:幂等"首次入队"语义 —— 返回调用方请求的那个 id // (默认 false 时返回 enqueueResult,即"复用已有任务"的路径)。 - enqueueEchoRequestedID bool - listSettingsConnectionID string - listSettingsErr error - settingsRevisions []governance.SourceConnectionRevision - jobFailureFilter governance.JobFailureFilter - jobFailurePage governance.JobFailurePage - jobFailureErr error - retryBatchRequest *governance.RetryPublicationsBatchRequest - retryBatchOperation string - retryBatchResult governance.RetryPublicationsBatchResult - cancelBatchRequest *governance.CancelPublicationsBatchRequest - cancelBatchOperation string - cancelBatchResult governance.RetryPublicationsBatchResult - cancelBatchReplayed bool - cancelBatchErr error - retryBatchReplayed bool - retryBatchErr error - requeueFailureRequest *governance.RequeueArtifactFetchRequest - requeueFailureOperation string - requeueFailureResult governance.RequeueArtifactFetchResult - requeueFailureReplayed bool - requeueFailureErr error - enqueued jobs.EnqueueRequest - enqueueResult *jobs.Job + enqueueEchoRequestedID bool + listSettingsConnectionID string + listSettingsErr error + settingsRevisions []governance.SourceConnectionRevision + jobFailureFilter governance.JobFailureFilter + jobFailurePage governance.JobFailurePage + jobFailureErr error + retryBatchRequest *governance.RetryPublicationsBatchRequest + retryBatchOperation string + retryBatchResult governance.RetryPublicationsBatchResult + cancelBatchRequest *governance.CancelPublicationsBatchRequest + cancelBatchOperation string + cancelBatchResult governance.RetryPublicationsBatchResult + cancelBatchReplayed bool + cancelBatchErr error + retryBatchReplayed bool + retryBatchErr error + requeueFailureRequest *governance.RequeueArtifactFetchRequest + requeueFailureOperation string + requeueFailureResult governance.RequeueArtifactFetchResult + requeueFailureReplayed bool + requeueFailureErr error + enqueued jobs.EnqueueRequest + enqueueResult *jobs.Job + // executorStatus 是封存执行器的"最近一次观测";nil 表示还没有任何上报。 + executorStatus *libraryexecutor.Observation listResult []jobs.Job listType string listByTypeResult []jobs.Job diff --git a/internal/libraryagent/agent.go b/internal/libraryagent/agent.go index edc279a..f9a1051 100644 --- a/internal/libraryagent/agent.go +++ b/internal/libraryagent/agent.go @@ -37,14 +37,17 @@ type Config struct { JournalRoot string // Writer declaration passed to the binary, exactly as reviewed in // scripts/library-capture.sh. Clients cannot influence it. - SourceApp string - SourcePG string - SourceES string - GovernancePG string - Project string - Writers string - NonWriters string - MaxDuration time.Duration + SourceApp string + SourcePG string + SourceES string + // SourceS3Endpoint 让宿主机用可达地址读源对象存储:容器 env 里的端点常常 + // 只是服务名(minio:9000),宿主机解析不到,资源导出会失败。 + SourceS3Endpoint string + GovernancePG string + Project string + Writers string + NonWriters string + MaxDuration time.Duration // StopGrace bounds the graceful stop of one operation: SIGTERM first (the // child aborts the maintenance window and restores writers), then SIGKILL // only after this grace period. It must exceed the child's own restore budget. @@ -60,6 +63,12 @@ type request struct { Action string `json:"action"` Scope string `json:"scope,omitempty"` Job string `json:"job,omitempty"` + // Released 只允许精确死信指纹(sha256:<64 hex>)。它不改变固定命令形态: + // 还能传的只有"这些确切的行已由治理平台审计解除",CLI 会逐字节复核。 + Released []string `json:"released,omitempty"` + // Expect 是调用方对执行器身份的期望。capture 必须在操作互斥内、任何源操作 + // 之前比对它;缺失即拒绝(fail closed),不靠"事后核对"兜底。 + Expect *Expectation `json:"expect,omitempty"` } type response struct { @@ -73,18 +82,55 @@ type response struct { var jobPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`) +// 死信解除只能按精确指纹传入,不接受通配、实例或任务类型级解除。 +var fingerprintPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`) + +const maxReleasedDeadLetters = 64 + // Agent serves capture requests. Only one operation runs at a time: a library // capture owns the writer group, so overlapping requests would race the window. type Agent struct { config Config mu sync.Mutex + // stateMu 只保护下面两个"给人看"的字段:健康应答不能被一次 15 分钟的封存 + // 卡住(否则页面会把"正在封存"读成"执行器不可达")。 + stateMu sync.RWMutex // degraded is non-empty when a failed operation could not be followed by a // verified recovery. While set, capture requests are refused: leaving the // source writer stopped is worse than refusing work. degraded string + busy bool wg sync.WaitGroup } +// captureState 是健康应答要暴露的执行器状态:是否在跑、是否降级。 +type captureState struct { + Busy bool `json:"busy"` + Degraded string `json:"degraded"` +} + +func (a *Agent) snapshotState() captureState { + a.stateMu.RLock() + defer a.stateMu.RUnlock() + return captureState{Busy: a.busy, Degraded: a.degraded} +} + +func (a *Agent) setBusy(busy bool) { + a.stateMu.Lock() + defer a.stateMu.Unlock() + a.busy = busy +} + +func (a *Agent) setDegraded(reason string) { + a.stateMu.Lock() + defer a.stateMu.Unlock() + a.degraded = reason +} + +func (a *Agent) clearDegraded() { + a.setDegraded("") +} + func New(config Config) (*Agent, error) { if !filepath.IsAbs(config.Socket) || !filepath.IsAbs(config.ArchiveRoot) || !filepath.IsAbs(config.JournalRoot) { return nil, errors.New("library agent paths must be absolute") @@ -132,7 +178,7 @@ func (a *Agent) Serve(ctx context.Context) error { // writer from a previous killed run. Recover before accepting work; if that // fails, stay degraded and refuse captures instead of pretending to be ready. if err := a.recoverWriters(ctx); err != nil { - a.degraded = err.Error() + a.setDegraded(err.Error()) log.Printf("library agent: pending freeze recovery failed: %v", err) } go func() { @@ -183,13 +229,18 @@ func (a *Agent) handle(ctx context.Context, conn net.Conn) { // execute serialises operations and maps a request onto the one fixed command. func (a *Agent) execute(ctx context.Context, req request) response { + // health 不进操作锁、不跑子进程:调用方要的是"这个执行器指向谁、能不能干活", + // 而一次正在进行的封存不该让这个回答变成"不可达"。 + if req.Action == "health" { + return a.health() + } a.mu.Lock() defer a.mu.Unlock() out := response{Action: req.Action, Scope: req.Scope, Job: req.Job} // Degraded blocks new captures, but recover and status must stay reachable: // otherwise there is no way back to a verified state. - if a.degraded != "" && req.Action == "capture" { - out.Error = "封存执行边界处于降级状态(上次恢复失败:" + a.degraded + "),先执行 recover" + if state := a.snapshotState(); state.Degraded != "" && req.Action == "capture" { + out.Error = "封存执行边界处于降级状态(上次恢复失败:" + state.Degraded + "),先执行 recover" return out } var args []string @@ -207,10 +258,33 @@ func (a *Agent) execute(ctx context.Context, req request) response { out.Error = "invalid job id" return out } + if req.Expect == nil { + out.Error = "capture 请求没有携带期望执行器身份,拒绝执行" + return out + } + // 此时已持有操作互斥(a.mu):从"比对身份"到"真正跑子进程"之间没有别的 + // 请求能插进来,也不会被另一个 agent 抢走——这是零副作用的前提。 + if err := a.config.matchExpectation(*req.Expect); err != nil { + out.Error = err.Error() + return out + } args = []string{"capture", "-scope", req.Scope} if req.Job != "" { args = append(args, "-job", req.Job) } + if len(req.Released) > 0 { + if len(req.Released) > maxReleasedDeadLetters { + out.Error = "too many released dead-letter fingerprints" + return out + } + for _, fingerprint := range req.Released { + if !fingerprintPattern.MatchString(fingerprint) { + out.Error = "released dead letters must be sha256:<64 hex> fingerprints" + return out + } + } + args = append(args, "-released-dead-letters", strings.Join(req.Released, ",")) + } default: out.Error = "unsupported action" return out @@ -218,6 +292,10 @@ func (a *Agent) execute(ctx context.Context, req request) response { args = append(args, a.config.declaration()...) bounded, cancel := context.WithTimeout(ctx, a.config.MaxDuration) defer cancel() + if req.Action == "capture" || req.Action == "recover" { + a.setBusy(true) + defer a.setBusy(false) + } payload, err := a.config.run(bounded, a.config.Binary, args...) if err != nil { detail := "operation failed" @@ -230,7 +308,7 @@ func (a *Agent) execute(ctx context.Context, req request) response { // verify. if req.Action == "capture" || req.Action == "recover" { if recoveryErr := a.recoverWriters(ctx); recoveryErr != nil { - a.degraded = recoveryErr.Error() + a.setDegraded(recoveryErr.Error()) out.Error = detail + ";写入者恢复失败:" + recoveryErr.Error() + "(已阻断后续封存,需人工处理)" return out } @@ -242,7 +320,7 @@ func (a *Agent) execute(ctx context.Context, req request) response { return out } if req.Action == "recover" { - a.degraded = "" + a.clearDegraded() } out.OK = true if json.Valid(payload) { @@ -250,9 +328,32 @@ func (a *Agent) execute(ctx context.Context, req request) response { } else { out.Payload = json.RawMessage(`{}`) } + // 封存结果必须带上"这次是谁执行的":调用方据此复核,防止心跳与执行之间 + // socket 被换成另一个 agent(真正的封错库事故就是这样发生的)。 + if req.Action == "capture" && req.Expect != nil { + out.Payload = a.attachIdentity(out.Payload, *req.Expect) + } return out } +// attachIdentity 把执行器身份并进封存结果(保留 CLI 原有的字段,如 snapshot_id)。 +// expectation_fingerprint 是"本执行器实际核对过的那份期望"的指纹:调用方据此确认 +// 执行时看到的期望与它发出的一致,而不是被替换后重新生成的。 +func (a *Agent) attachIdentity(payload json.RawMessage, expectation Expectation) json.RawMessage { + var document map[string]any + if err := json.Unmarshal(payload, &document); err != nil { + return payload + } + identity := a.config.identity() + identity["expectation_fingerprint"] = expectation.Fingerprint + document["agent_identity"] = identity + encoded, err := json.Marshal(document) + if err != nil { + return payload + } + return json.RawMessage(encoded) +} + // recoverWriters runs the idempotent recovery command with its own budget, even // when the triggering context is already cancelled (timeout / shutdown). func (a *Agent) recoverWriters(ctx context.Context) error { @@ -261,24 +362,29 @@ func (a *Agent) recoverWriters(ctx context.Context) error { if _, err := a.config.run(bounded, a.config.Binary, append([]string{"recover"}, a.config.declaration()...)...); err != nil { return err } - a.degraded = "" + a.clearDegraded() return nil } // declaration pins every path and the writer group so a request cannot redirect // the operation to another stack, archive or journal. func (c Config) declaration() []string { - return []string{ + args := []string{ "-source-app", c.SourceApp, "-source-pg", c.SourcePG, "-source-es", c.SourceES, + } + if c.SourceS3Endpoint != "" { + args = append(args, "-source-s3-endpoint", c.SourceS3Endpoint) + } + return append(args, "-governance-pg", c.GovernancePG, "-project", c.Project, "-writers", c.Writers, "-non-writers", c.NonWriters, "-archive", c.ArchiveRoot, "-journal", c.JournalRoot, - } + ) } func (c Config) run(ctx context.Context, name string, args ...string) ([]byte, error) { @@ -303,12 +409,38 @@ func (c Config) run(ctx context.Context, name string, args ...string) ([]byte, e // Describe returns a redacted view for logs and health checks. func (a *Agent) Describe() map[string]any { - return map[string]any{ - "socket": a.config.Socket, - "archive_root": a.config.ArchiveRoot, - "journal_root": a.config.JournalRoot, - "project": a.config.Project, - "writers": a.config.Writers, - "scope_choice": "request may select internal or external only", - } + view := a.config.identity() + view["scope_choice"] = "request may select internal or external only" + return view +} + +// health 是"这个执行器指向谁、现在能不能干活"的唯一回答。它只暴露容器名、 +// 路径与声明,不含任何凭据;调用方据此核对源身份,而不是靠配置猜。 +func (a *Agent) health() response { + state := a.snapshotState() + payload := a.config.identity() + payload["busy"] = state.Busy + payload["degraded"] = state.Degraded + encoded, err := json.Marshal(payload) + if err != nil { + return response{Action: "health", Error: "cannot encode health"} + } + return response{OK: true, Action: "health", Payload: json.RawMessage(encoded)} +} + +// identity 被日志与健康应答共用一份,避免两处描述漂移。 +func (c Config) identity() map[string]any { + identity := map[string]any{ + "project": c.Project, + "archive_root": c.ArchiveRoot, + "journal_root": c.JournalRoot, + "source_app": c.SourceApp, + "source_pg": c.SourcePG, + "source_es": c.SourceES, + "writers": c.Writers, + } + if c.SourceS3Endpoint != "" { + identity["source_s3_endpoint"] = c.SourceS3Endpoint + } + return identity } diff --git a/internal/libraryagent/agent_test.go b/internal/libraryagent/agent_test.go index 8af94d7..747069a 100644 --- a/internal/libraryagent/agent_test.go +++ b/internal/libraryagent/agent_test.go @@ -56,6 +56,19 @@ func testAgent(t *testing.T) (*Agent, string, func() []string) { } } +// captureExpect 是测试 agent 自己的身份(weknora071full / WKNFull-app), +// 现在 capture 请求必须携带期望身份,缺了就会被拒。 +func captureExpect(t *testing.T) string { + t.Helper() + // 只绑定 project:各用例的 source_app 不同,绑定字段的专门用例在 expectation_test.go。 + expectation := NewExpectation("weknora071full", "", "") + encoded, err := json.Marshal(expectation) + if err != nil { + t.Fatal(err) + } + return string(encoded) +} + func roundTrip(t *testing.T, socket string, payload string) response { t.Helper() conn, err := net.Dial("unix", socket) @@ -105,7 +118,7 @@ func TestAgentSocketIsPrivateAndPinsTheDeclaration(t *testing.T) { if dir.Mode().Perm() != 0700 { t.Fatalf("socket directory mode %v", dir.Mode().Perm()) } - out := roundTrip(t, socket, `{"action":"capture","scope":"internal","job":"lib-internal-test"}`) + out := roundTrip(t, socket, `{"action":"capture","scope":"internal","job":"lib-internal-test","expect":`+captureExpect(t)+`}`) if !out.OK || out.Job != "lib-internal-test" { t.Fatalf("capture rejected: %+v", out) } @@ -234,7 +247,7 @@ func TestAgentRecoversWritersAfterFailedCapture(t *testing.T) { defer cancel() go func() { _ = agent.Serve(ctx) }() waitSocket(t, socket) - out := roundTrip(t, socket, `{"action":"capture","scope":"internal","job":"lib-fail-1"}`) + out := roundTrip(t, socket, `{"action":"capture","scope":"internal","job":"lib-fail-1","expect":`+captureExpect(t)+`}`) if out.OK { t.Fatalf("failed capture reported success: %+v", out) } @@ -255,12 +268,12 @@ func TestAgentBlocksCapturesWhenRecoveryFails(t *testing.T) { go func() { _ = agent.Serve(ctx) }() waitSocket(t, socket) setRecoverErr(errors.New("docker restart failed")) - out := roundTrip(t, socket, `{"action":"capture","scope":"internal","job":"lib-fail-2"}`) + out := roundTrip(t, socket, `{"action":"capture","scope":"internal","job":"lib-fail-2","expect":`+captureExpect(t)+`}`) if out.OK || !strings.Contains(out.Error, "写入者恢复失败") || !strings.Contains(out.Error, "已阻断后续封存") { t.Fatalf("recovery failure not surfaced: %+v", out) } before := len(captureCalls(calls())) - blocked := roundTrip(t, socket, `{"action":"capture","scope":"external","job":"lib-fail-3"}`) + blocked := roundTrip(t, socket, `{"action":"capture","scope":"external","job":"lib-fail-3","expect":`+captureExpect(t)+`}`) if blocked.OK || !strings.Contains(blocked.Error, "降级") { t.Fatalf("degraded agent accepted a capture: %+v", blocked) } @@ -272,7 +285,7 @@ func TestAgentBlocksCapturesWhenRecoveryFails(t *testing.T) { if recovered := roundTrip(t, socket, `{"action":"recover"}`); !recovered.OK { t.Fatalf("explicit recovery failed: %+v", recovered) } - if resumed := roundTrip(t, socket, `{"action":"capture","scope":"internal","job":"lib-fail-4"}`); strings.Contains(resumed.Error, "降级") { + if resumed := roundTrip(t, socket, `{"action":"capture","scope":"internal","job":"lib-fail-4","expect":`+captureExpect(t)+`}`); strings.Contains(resumed.Error, "降级") { t.Fatalf("recovery did not clear the degraded state: %+v", resumed) } } @@ -336,3 +349,76 @@ func TestNewRejectsIncompleteConfiguration(t *testing.T) { t.Fatal("non-writer declaration without a reason accepted") } } + +// 宿主机可达的源对象存储端点必须由 agent 自己钉住:请求不能改,也不能被悄悄丢掉 +// (掉了就会退回"容器里那个解析不到的服务名",真实封存报 read library resource)。 +func TestAgentPinsTheHostSideObjectEndpoint(t *testing.T) { + config := Config{ + Socket: "/tmp/agent.sock", + Binary: "/bin/echo", + ArchiveRoot: "/tmp/library", + JournalRoot: "/tmp/freeze", + SourceApp: "app-container", + SourcePG: "pg", + SourceES: "es", + SourceS3Endpoint: "http://127.0.0.1:19100", + GovernancePG: "gov", + Project: "proj", + Writers: "app", + NonWriters: "postgres=database store", + MaxDuration: time.Minute, + } + agent, err := New(config) + if err != nil { + t.Fatalf("new: %v", err) + } + args := agent.config.declaration() + found := false + for i := 0; i+1 < len(args); i++ { + if args[i] == "-source-s3-endpoint" && args[i+1] == "http://127.0.0.1:19100" { + found = true + } + } + if !found { + t.Fatalf("声明里没有宿主侧对象存储端点:%v", args) + } + config.SourceS3Endpoint = "" + bare, err := New(config) + if err != nil { + t.Fatalf("new without endpoint: %v", err) + } + for _, arg := range bare.config.declaration() { + if arg == "-source-s3-endpoint" { + t.Fatal("没有端点时不该凭空多一个 flag") + } + } +} + +// 死信解除只能按精确指纹透传:形状不对就拒绝,且固定命令其余部分不受影响。 +func TestAgentForwardsOnlyExactDeadLetterFingerprints(t *testing.T) { + agent, socket, calls := testAgent(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { _ = agent.Serve(ctx) }() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if conn, err := net.Dial("unix", socket); err == nil { + conn.Close() + break + } + time.Sleep(10 * time.Millisecond) + } + fingerprint := "sha256:" + strings.Repeat("a", 64) + ok := roundTrip(t, socket, `{"action":"capture","scope":"internal","job":"j1","released":["`+fingerprint+`"],"expect":`+captureExpect(t)+`}`) + if !ok.OK { + t.Fatalf("exact fingerprint refused: %s", ok.Error) + } + last := calls()[len(calls())-1] + if !strings.Contains(last, "-released-dead-letters "+fingerprint) { + t.Fatalf("fingerprint not forwarded: %s", last) + } + bad := roundTrip(t, socket, `{"action":"capture","scope":"internal","job":"j2","released":["*"],"expect":`+captureExpect(t)+`}`) + if bad.OK || !strings.Contains(bad.Error, "fingerprints") { + t.Fatalf("wildcard release accepted: %+v", bad) + } +} diff --git a/internal/libraryagent/client.go b/internal/libraryagent/client.go index a1d2fca..b956275 100644 --- a/internal/libraryagent/client.go +++ b/internal/libraryagent/client.go @@ -28,21 +28,36 @@ type Result struct { // Capture asks the agent to seal one library scope. The job id makes the call // idempotent: repeating it returns the already sealed candidate. -func Capture(ctx context.Context, config ClientConfig, scope, job string) (Result, error) { - return call(ctx, config, map[string]string{"action": "capture", "scope": scope, "job": job}) +func Capture(ctx context.Context, config ClientConfig, scope, job string, expect Expectation) (Result, error) { + return CaptureReleased(ctx, config, scope, job, nil, expect) +} + +// CaptureReleased 在 Capture 之上带上治理平台已审计的精确死信解除指纹。 +func CaptureReleased(ctx context.Context, config ClientConfig, scope, job string, released []string, expect Expectation) (Result, error) { + request := map[string]any{"action": "capture", "scope": scope, "job": job, "expect": expect} + if len(released) > 0 { + request["released"] = released + } + return call(ctx, config, request) } // Status probes the agent (source wiring, writer group, archive usage). func Status(ctx context.Context, config ClientConfig) (Result, error) { - return call(ctx, config, map[string]string{"action": "status"}) + return call(ctx, config, map[string]any{"action": "status"}) +} + +// Health 只问三件事:这个执行器指向谁(project/容器/归档)、是否在跑、是否降级。 +// 它不跑子进程、不占操作锁,所以可以当心跳用;Status 会去查源库,代价高得多。 +func Health(ctx context.Context, config ClientConfig) (Result, error) { + return call(ctx, config, map[string]any{"action": "health"}) } // Recover restores writers from an interrupted freeze journal. func Recover(ctx context.Context, config ClientConfig) (Result, error) { - return call(ctx, config, map[string]string{"action": "recover"}) + return call(ctx, config, map[string]any{"action": "recover"}) } -func call(ctx context.Context, config ClientConfig, request map[string]string) (Result, error) { +func call(ctx context.Context, config ClientConfig, request map[string]any) (Result, error) { if config.Socket == "" { return Result{}, errors.New("library agent socket is not configured") } diff --git a/internal/libraryagent/expectation.go b/internal/libraryagent/expectation.go new file mode 100644 index 0000000..dcb546f --- /dev/null +++ b/internal/libraryagent/expectation.go @@ -0,0 +1,72 @@ +package libraryagent + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "strings" +) + +// Expectation 是调用方对"这一条请求该由哪个执行器执行"的声明。 +// +// 为什么是调用方声明而不是执行器自报:自报值只能证明"我是谁",不能证明"你找的 +// 就是我"。心跳之后、真正停写入者之前,socket 可能被换成另一个 agent;只有把 +// 期望随**同一个请求**送进去、由接收方在操作互斥内、执行任何源操作之前比对, +// 才能让"换过 socket"变成零副作用(错的那个 noche 一次 run 都不调)。 +type Expectation struct { + Project string `json:"project"` + SourceApp string `json:"source_app,omitempty"` + ArchiveRoot string `json:"archive_root,omitempty"` + // Fingerprint 覆盖上面三项,供接收方回显、调用方事后核对: + // 证明"执行器看到的就是我发的那份期望",而不是被替换后重新生成的。 + Fingerprint string `json:"fingerprint"` +} + +// NewExpectation 由治理侧配置构造期望(绝不从执行器应答里取字段)。 +func NewExpectation(project, sourceApp, archiveRoot string) Expectation { + expectation := Expectation{ + Project: strings.TrimSpace(project), + SourceApp: strings.TrimSpace(sourceApp), + ArchiveRoot: strings.TrimSpace(archiveRoot), + } + expectation.Fingerprint = expectation.fingerprint() + return expectation +} + +func (e Expectation) fingerprint() string { + sum := sha256.Sum256([]byte(e.Project + "\x00" + e.SourceApp + "\x00" + e.ArchiveRoot)) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +// Complete 表示期望足够具体:没有 project 的期望无法核对身份,必须 fail closed; +// 指纹与字段不一致说明请求在半路被改过。 +func (e Expectation) Complete() bool { + return e.Project != "" && e.Fingerprint == e.fingerprint() +} + +// matchExpectation 在操作互斥内比对"本执行器配置"与"调用方期望"。 +// 只比较期望里给出的字段:project 必填,source_app/archive_root 给了就必须一致。 +func (c Config) matchExpectation(expectation Expectation) error { + if !expectation.Complete() { + return errors.New("capture 请求缺少完整的期望执行器身份,拒绝执行") + } + if c.Project != expectation.Project { + return errors.New("执行器与请求期望的实例不一致:" + + "agent=" + quoteOrEmpty(c.Project) + ",期望=" + quoteOrEmpty(expectation.Project) + ",拒绝执行") + } + if expectation.SourceApp != "" && c.SourceApp != expectation.SourceApp { + return errors.New("执行器与请求期望的源容器不一致:" + + "agent=" + quoteOrEmpty(c.SourceApp) + ",期望=" + quoteOrEmpty(expectation.SourceApp) + ",拒绝执行") + } + if expectation.ArchiveRoot != "" && c.ArchiveRoot != expectation.ArchiveRoot { + return errors.New("执行器与请求期望的归档根不一致,拒绝执行") + } + return nil +} + +func quoteOrEmpty(value string) string { + if value == "" { + return `""` + } + return `"` + value + `"` +} diff --git a/internal/libraryagent/health_test.go b/internal/libraryagent/health_test.go new file mode 100644 index 0000000..9b06abe --- /dev/null +++ b/internal/libraryagent/health_test.go @@ -0,0 +1,109 @@ +package libraryagent + +import ( + "context" + "encoding/json" + "os" + "testing" + "time" +) + +func serveAgent(t *testing.T, agent *Agent, socket string) { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + go func() { _ = agent.Serve(ctx) }() + deadline := time.Now().Add(2 * time.Second) + for { + if _, err := os.Stat(socket); err == nil { + return + } + if time.Now().After(deadline) { + t.Fatal("agent socket never appeared") + } + time.Sleep(10 * time.Millisecond) + } +} + +// 健康应答必须不排队等正在跑的封存:一次封存最长 15 分钟,如果 health 也去抢 +// 操作锁,页面就会把"正在封存"看成"执行器不可达",进而误导运维。 +func TestHealthAnswersWhileACaptureHoldsTheOperationLock(t *testing.T) { + agent, socket, calls := testAgent(t) + serveAgent(t, agent, socket) + // 占住操作锁并置 busy,模拟一次正在进行的封存。 + agent.mu.Lock() + defer agent.mu.Unlock() + agent.setBusy(true) + defer agent.setBusy(false) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + result, err := Health(ctx, ClientConfig{Socket: socket, Timeout: 2 * time.Second}) + if err != nil { + t.Fatalf("health while busy: %v", err) + } + var payload struct { + Project string `json:"project"` + Busy bool `json:"busy"` + } + if err = json.Unmarshal(result.Payload, &payload); err != nil { + t.Fatalf("health payload: %v", err) + } + if payload.Project != "weknora071full" || !payload.Busy { + t.Fatalf("health payload = %+v", payload) + } + if len(captureCalls(calls())) != 0 { + t.Fatalf("health must not run the capture binary, calls = %v", calls()) + } +} + +// 降级状态必须能被调用方看见:否则页面只能看到"可发起",而真发起会被拒。 +func TestHealthReportsDegradedState(t *testing.T) { + agent, socket, _ := testAgent(t) + serveAgent(t, agent, socket) + // 启动恢复成功会清空降级;这里在它之后模拟"恢复失败"留下的降级状态。 + agent.setDegraded("写入者恢复失败") + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + result, err := Health(ctx, ClientConfig{Socket: socket, Timeout: 2 * time.Second}) + if err != nil { + t.Fatal(err) + } + var payload struct { + Degraded string `json:"degraded"` + SourceApp string `json:"source_app"` + Writers string `json:"writers"` + } + if err = json.Unmarshal(result.Payload, &payload); err != nil { + t.Fatal(err) + } + if payload.Degraded != "写入者恢复失败" || payload.SourceApp != "WKNFull-app" || payload.Writers != "app" { + t.Fatalf("health payload = %+v", payload) + } +} + +// 封存结果必须自带"这次是谁执行的":调用方据此在事后复核,防止心跳与执行之间 +// socket 被换成另一个实例。 +func TestCaptureResultCarriesTheAgentIdentity(t *testing.T) { + agent, socket, _ := testAgent(t) + serveAgent(t, agent, socket) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + result, err := Capture(ctx, ClientConfig{Socket: socket, Timeout: 5 * time.Second}, "internal", + "lib-internal-identity", NewExpectation("weknora071full", "WKNFull-app", "")) + if err != nil { + t.Fatal(err) + } + var payload struct { + SnapshotID string `json:"snapshot_id"` + Identity struct { + Project string `json:"project"` + } `json:"agent_identity"` + } + if err = json.Unmarshal(result.Payload, &payload); err != nil { + t.Fatal(err) + } + if payload.SnapshotID != "abc" || payload.Identity.Project != "weknora071full" { + t.Fatalf("payload = %s", result.Payload) + } +} diff --git a/internal/libraryagent/swap_test.go b/internal/libraryagent/swap_test.go new file mode 100644 index 0000000..070afd7 --- /dev/null +++ b/internal/libraryagent/swap_test.go @@ -0,0 +1,149 @@ +package libraryagent + +import ( + "context" + "os" + "strings" + "sync" + "testing" + "time" +) + +// unix socket 路径有长度上限,测试目录要用短的(TMPDIR 太深)。 +func shortSocketDir(t *testing.T) string { + t.Helper() + dir, err := os.MkdirTemp("/tmp", "kgswap-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(dir) }) + return dir +} + +// 两个真实 agent、两个 socket:负例把调用方指到"另一个实例"的 socket 上, +// 断言**错误 agent 一次 capture 都没跑**(不是跑完再拒),正例走对 socket 正常封存。 +func TestCaptureRequestIsRefusedBeforeAnySourceOperationWhenSocketIsSwapped(t *testing.T) { + rightAgent, rightSocket, rightCalls := agentWithProject(t, "kgmaint-weknora", "kgmaint-app") + wrongAgent, wrongSocket, wrongCalls := agentWithProject(t, "weknora071full", "WKNFull-app") + serveAgent(t, rightAgent, rightSocket) + serveAgent(t, wrongAgent, wrongSocket) + + expectation := NewExpectation("kgmaint-weknora", "kgmaint-app", "") + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // 负例:调用方手上的 socket 已被换成另一个实例的执行器。 + result, err := Capture(ctx, ClientConfig{Socket: wrongSocket, Timeout: 5 * time.Second}, + "internal", "lib-swap", expectation) + if err == nil || result.OK { + t.Fatalf("capture against the wrong executor was accepted: %+v", result) + } + if !strings.Contains(result.Error, "期望") { + t.Fatalf("refusal should name the mismatch: %q", result.Error) + } + if calls := captureCalls(wrongCalls()); len(calls) != 0 { + t.Fatalf("wrong executor ran the capture binary %d time(s): %v", len(calls), calls) + } + // 旁证:错误的执行器进程本身仍然可用(health 正常),拒绝来自身份比对而不是不可达。 + health, err := Health(ctx, ClientConfig{Socket: wrongSocket, Timeout: 5 * time.Second}) + if err != nil || !health.OK { + t.Fatalf("wrong executor should still answer health: %v %+v", err, health) + } + + // 正例:同一份期望打到正确的 socket,正常封存一次。 + result, err = Capture(ctx, ClientConfig{Socket: rightSocket, Timeout: 5 * time.Second}, + "internal", "lib-swap", expectation) + if err != nil || !result.OK { + t.Fatalf("capture against the right executor failed: %v %+v", err, result) + } + if calls := captureCalls(rightCalls()); len(calls) != 1 { + t.Fatalf("right executor ran %d time(s): %v", len(calls), calls) + } + if calls := captureCalls(wrongCalls()); len(calls) != 0 { + t.Fatalf("wrong executor ran after the positive case: %v", calls) + } +} + +// 缺期望、期望指纹对不上、期望里 source_app 不同,都必须 fail closed 且不跑子进程。 +func TestCaptureFailsClosedWithoutACompleteExpectation(t *testing.T) { + agent, socket, calls := agentWithProject(t, "kgmaint-weknora", "kgmaint-app") + serveAgent(t, agent, socket) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + tampered := NewExpectation("kgmaint-weknora", "kgmaint-app", "") + tampered.Project = "someone-else" // 字段改了、指纹没改:请求在半路被改过 + cases := []struct { + name string + expectation Expectation + text string + }{ + {name: "missing expectation", expectation: Expectation{}, text: "缺少完整的期望执行器身份"}, + {name: "tampered expectation", expectation: tampered, text: "缺少完整的期望执行器身份"}, + {name: "wrong source app", expectation: NewExpectation("kgmaint-weknora", "WKNFull-app", ""), text: "源容器不一致"}, + } + for _, test := range cases { + result, err := Capture(ctx, ClientConfig{Socket: socket, Timeout: 5 * time.Second}, + "internal", "lib-"+strings.ReplaceAll(test.name, " ", "-"), test.expectation) + if err == nil || result.OK { + t.Fatalf("%s: accepted %+v", test.name, result) + } + if !strings.Contains(result.Error, test.text) { + t.Fatalf("%s: error %q", test.name, result.Error) + } + } + if calls := captureCalls(calls()); len(calls) != 0 { + t.Fatalf("capture binary ran despite a refused identity: %v", calls) + } +} + +// 恢复路径不受期望约束:它只把写入者恢复回来,任何身份都必须能执行。 +func TestRecoverStaysAvailableRegardlessOfExpectation(t *testing.T) { + agent, socket, calls := agentWithProject(t, "kgmaint-weknora", "kgmaint-app") + serveAgent(t, agent, socket) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + result, err := Recover(ctx, ClientConfig{Socket: socket, Timeout: 5 * time.Second}) + if err != nil || !result.OK { + t.Fatalf("recover refused: %v %+v", err, result) + } + if len(recoverCalls(calls())) == 0 { + t.Fatal("recover did not run the recovery command") + } +} + +// agentWithProject 造一个真实 agent(不跑真命令),返回它自己的调用记录。 +func agentWithProject(t *testing.T, project, sourceApp string) (*Agent, string, func() []string) { + t.Helper() + root := shortSocketDir(t) + var mu sync.Mutex + var calls []string + config := Config{ + Socket: root + "/agent.sock", + Binary: "/bin/echo", + ArchiveRoot: root + "/library", + JournalRoot: root + "/freeze", + SourceApp: sourceApp, + SourcePG: "pg", + SourceES: "es", + Project: project, + Writers: "app", + NonWriters: "frontend=UI only", + MaxDuration: time.Minute, + Run: func(_ context.Context, name string, args ...string) ([]byte, error) { + mu.Lock() + calls = append(calls, name+" "+strings.Join(args, " ")) + mu.Unlock() + return []byte(`{"snapshot_id":"abc","state":"succeeded"}`), nil + }, + } + agent, err := New(config) + if err != nil { + t.Fatal(err) + } + return agent, config.Socket, func() []string { + mu.Lock() + defer mu.Unlock() + return append([]string(nil), calls...) + } +} diff --git a/internal/librarydeadletter/deadletter.go b/internal/librarydeadletter/deadletter.go new file mode 100644 index 0000000..bf52046 --- /dev/null +++ b/internal/librarydeadletter/deadletter.go @@ -0,0 +1,253 @@ +// Package librarydeadletter 定义上游死信(task_dead_letters)的精确解除契约。 +// +// 为什么需要它:WeKnora v0.7.1 的 task_dead_letters 没有"已解决"标记,导出/封存 +// 按"存在即阻断"处理。上游只提供两种运维动作——整队列 purge 与直接 DELETE, +// 两者都会销毁"这条任务失败过"的唯一证据,不能当作产品能力。治理平台因此保留 +// 上游原始行,另外记录一条绑定到精确指纹的解除记录:指纹必须与实时行逐字节一致, +// 行变化或新死信出现都不再命中,继续阻断。 +package librarydeadletter + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "strings" + "time" +) + +// FingerprintPrefix 让指纹在日志/审计里自解释:算法一旦更换无需猜。 +const FingerprintPrefix = "sha256:" + +// Fingerprint 对上游用 to_jsonb(row)::text 取回的行文本求摘要。 +// +// 为什么按原始行文本而不是逐字段重组:PostgreSQL 的 jsonb 文本已规范化(键有序), +// 任何字段变化都会改变字节;逐字段重组会漏掉上游新增列,等于给"行被改过"开后门。 +func Fingerprint(raw []byte) string { + // 先压紧空白:PostgreSQL 的 to_jsonb::text 带空格,Go 的 json.Marshal 不带, + // 两边必须对同一份内容得到同一个指纹,否则解除永远命中不上。 + compacted := raw + if len(raw) > 0 { + var buffer bytes.Buffer + if err := json.Compact(&buffer, raw); err == nil { + compacted = buffer.Bytes() + } + } + sum := sha256.Sum256(compacted) + return FingerprintPrefix + hex.EncodeToString(sum[:]) +} + +// ValidFingerprint 校验调用方传入的指纹形状,避免把通配符或截断值当解除依据。 +func ValidFingerprint(value string) bool { + if !strings.HasPrefix(value, FingerprintPrefix) { + return false + } + body := strings.TrimPrefix(value, FingerprintPrefix) + if len(body) != 64 { + return false + } + _, err := hex.DecodeString(body) + return err == nil +} + +// Row 是 task_dead_letters 的一行。字段与上游 000041 迁移一致;Payload 保留原始 +// JSON 文本,因为任务参数本身就是证据。 +type Row struct { + ID int64 `json:"id"` + TenantID uint64 `json:"tenant_id"` + TaskType string `json:"task_type"` + Scope string `json:"scope"` + ScopeID string `json:"scope_id"` + RelatedID string `json:"related_id"` + Payload json.RawMessage `json:"payload"` + LastError string `json:"last_error"` + FailCount int `json:"fail_count"` + FailedAt time.Time `json:"failed_at"` +} + +// Settlement 只记录与死信无关的两个计数:未完成文档与待办操作。死信单独成表, +// 由指纹逐条解释,不能用"总数"含糊过去。 +type Settlement struct { + Unsettled int64 `json:"unsettled"` + PendingOps int64 `json:"pending_ops"` +} + +// Observation 是主机侧(能读源库的一方)生成的只读观测。治理平台不直接连源库, +// 因此解除记录绑定的是这份观测的摘要:平台重算摘要,换一个字节就不再接受。 +type Observation struct { + Project string `json:"project"` + TenantID uint64 `json:"tenant_id"` + KnowledgeBaseID string `json:"kb_id"` + CapturedAt time.Time `json:"captured_at"` + Settlement Settlement `json:"settlement"` + Rows []json.RawMessage `json:"dead_letters"` +} + +// Replay 记录"真实重放"的引用。平台无法自己向上游发起重放(那需要源侧管理员权限, +// 属于放大授权),所以只接受可核对的引用,并把"重放之后没有再次失败"作为机器检查项: +// 同一任务键在重放时间之后若出现新的死信行,解除无效。 +type Replay struct { + Queue string `json:"queue"` + TaskID string `json:"task_id"` + At time.Time `json:"at"` + Evidence string `json:"evidence"` +} + +// Request 是创建解除记录的输入。Observation 保持原始字节,摘要按这串字节计算。 +type Request struct { + Project string `json:"project"` + TenantID uint64 `json:"tenant_id"` + KnowledgeBaseID string `json:"kb_id"` + DeadLetterID int64 `json:"dead_letter_id"` + Replay Replay `json:"replay"` + Note string `json:"note"` + Observation json.RawMessage `json:"observation"` + Digest string `json:"observation_digest"` +} + +// Release 是验证通过后落库的解除记录。 +type Release struct { + Project string + TenantID uint64 + KnowledgeBaseID string + DeadLetterID int64 + Fingerprint string + TaskType string + Scope string + ScopeID string + RelatedID string + Replay Replay + Observation json.RawMessage + Digest string + Note string +} + +// Verify 检查解除请求是否可被接受。它不信任调用方自报的结论: +// 指纹由观测里的原始行重算,结清计数必须为 0,重放时间必须晚于失败时间, +// 且同一任务键在重放之后不得再有新的失败行。 +func Verify(req Request, expectedProject string, now time.Time) (Release, error) { + if expectedProject == "" { + return Release{}, errors.New("死信解除需要平台侧已配置的实例身份") + } + if req.Project != "" && req.Project != expectedProject { + return Release{}, fmt.Errorf("解除记录实例 %q 与当前实例 %q 不一致", req.Project, expectedProject) + } + if req.DeadLetterID <= 0 { + return Release{}, errors.New("dead_letter_id 必须指定一条精确死信") + } + if len(req.Observation) == 0 || req.Digest == "" { + return Release{}, errors.New("缺少源侧观测或摘要") + } + if got := Fingerprint(req.Observation); got != req.Digest { + return Release{}, errors.New("观测摘要与提交的观测不一致(可能被改写或格式化)") + } + var observation Observation + if err := json.Unmarshal(req.Observation, &observation); err != nil { + return Release{}, errors.New("源侧观测不是合法 JSON") + } + if observation.Project != expectedProject { + return Release{}, fmt.Errorf("观测实例 %q 与当前实例 %q 不一致", observation.Project, expectedProject) + } + if observation.Settlement.Unsettled != 0 || observation.Settlement.PendingOps != 0 { + return Release{}, fmt.Errorf("源侧尚未结清:未完成文档 %d、待办操作 %d", + observation.Settlement.Unsettled, observation.Settlement.PendingOps) + } + if observation.CapturedAt.IsZero() || observation.CapturedAt.After(now.Add(time.Minute)) { + return Release{}, errors.New("源侧观测时间缺失或晚于当前时间") + } + if req.Replay.Queue == "" || req.Replay.TaskID == "" { + return Release{}, errors.New("重放必须给出队列与任务号") + } + if req.Replay.At.IsZero() { + return Release{}, errors.New("重放必须给出实际发生时间") + } + if req.Replay.At.After(now.Add(time.Minute)) { + return Release{}, errors.New("重放时间晚于当前时间") + } + if req.TenantID != 0 && observation.TenantID != 0 && req.TenantID != observation.TenantID { + return Release{}, errors.New("请求租户与观测租户不一致") + } + if req.KnowledgeBaseID != "" && observation.KnowledgeBaseID != "" && req.KnowledgeBaseID != observation.KnowledgeBaseID { + return Release{}, errors.New("请求知识库与观测知识库不一致") + } + + var target *Row + var targetFingerprint string + for _, raw := range observation.Rows { + var row Row + if err := json.Unmarshal(raw, &row); err != nil { + return Release{}, errors.New("观测里的死信行不是合法 JSON") + } + if row.ID != req.DeadLetterID { + continue + } + if target != nil { + return Release{}, errors.New("观测里出现重复的死信 id") + } + copied := row + target = &copied + targetFingerprint = Fingerprint(raw) + } + if target == nil { + return Release{}, errors.New("观测里没有这条死信:上游记录可能已被删除,不能用解除掩盖删除") + } + if target.FailedAt.After(req.Replay.At) { + return Release{}, errors.New("重放时间早于这条死信的失败时间") + } + // 重放失败会再写一条死信行(asynq 中间件对每次耗尽都落一行)。同一任务键在 + // 重放之后仍有新行,说明重放没有成功,或者又失败了一次。 + for _, raw := range observation.Rows { + var row Row + if err := json.Unmarshal(raw, &row); err != nil { + return Release{}, errors.New("观测里的死信行不是合法 JSON") + } + if row.ID == target.ID { + continue + } + if sameKey(row, *target) && row.FailedAt.After(req.Replay.At) { + return Release{}, fmt.Errorf("重放之后同一任务仍产生新死信(id=%d),不能解除", row.ID) + } + } + if target.TenantID != observation.TenantID { + return Release{}, errors.New("死信租户与观测租户不一致") + } + return Release{ + Project: observation.Project, + TenantID: target.TenantID, + KnowledgeBaseID: observation.KnowledgeBaseID, + DeadLetterID: target.ID, + Fingerprint: targetFingerprint, + TaskType: target.TaskType, + Scope: target.Scope, + ScopeID: target.ScopeID, + RelatedID: target.RelatedID, + Replay: req.Replay, + Observation: req.Observation, + Digest: req.Digest, + Note: req.Note, + }, nil +} + +func sameKey(a, b Row) bool { + return a.TenantID == b.TenantID && a.TaskType == b.TaskType && a.Scope == b.Scope && + a.ScopeID == b.ScopeID && a.RelatedID == b.RelatedID +} + +// Unreleased 返回没有被精确解除记录覆盖的死信指纹。任何一行不匹配都阻断: +// 行被改过、出现新行、或解除记录缺失,三种情况都必须看得见。 +func Unreleased(rows [][]byte, released map[string]bool) ([]string, error) { + missing := make([]string, 0, len(rows)) + for _, raw := range rows { + var row Row + if err := json.Unmarshal(raw, &row); err != nil { + return nil, errors.New("源库死信行无法解析,按阻断处理") + } + digest := Fingerprint(raw) + if !released[digest] { + missing = append(missing, fmt.Sprintf("%d:%s", row.ID, digest)) + } + } + return missing, nil +} diff --git a/internal/librarydeadletter/deadletter_test.go b/internal/librarydeadletter/deadletter_test.go new file mode 100644 index 0000000..398bbb4 --- /dev/null +++ b/internal/librarydeadletter/deadletter_test.go @@ -0,0 +1,113 @@ +package librarydeadletter + +import ( + "encoding/json" + "strings" + "testing" + "time" +) + +const sampleRow = `{"id": 7, "payload": {"id": "k1"}, "scope": "knowledge", "scope_id": "k1", "failed_at": "2026-10-28T10:00:00+00:00", "last_error": "boom", "task_type": "wiki:ingest", "tenant_id": 10001, "fail_count": 3, "related_id": "k1"}` + +func observation(t *testing.T, rows ...string) (json.RawMessage, string) { + t.Helper() + value := Observation{ + Project: "kgmaint-weknora", TenantID: 10001, KnowledgeBaseID: "kb-1", + CapturedAt: time.Date(2026, 10, 29, 7, 0, 0, 0, time.UTC), + Settlement: Settlement{Unsettled: 0, PendingOps: 0}, + } + for _, row := range rows { + value.Rows = append(value.Rows, json.RawMessage(row)) + } + raw, err := json.Marshal(value) + if err != nil { + t.Fatalf("marshal observation: %v", err) + } + return raw, Fingerprint(raw) +} + +func request(t *testing.T, raw json.RawMessage, digest string) Request { + t.Helper() + return Request{ + Project: "kgmaint-weknora", DeadLetterID: 7, + Replay: Replay{Queue: "default", TaskID: "task-1", At: time.Date(2026, 10, 29, 6, 30, 0, 0, time.UTC), + Evidence: "run_now 200"}, + Observation: raw, Digest: digest, + } +} + +func TestVerifyAcceptsExactReleasedDeadLetter(t *testing.T) { + raw, digest := observation(t, sampleRow) + release, err := Verify(request(t, raw, digest), "kgmaint-weknora", time.Date(2026, 10, 29, 7, 5, 0, 0, time.UTC)) + if err != nil { + t.Fatalf("exact release rejected: %v", err) + } + if release.DeadLetterID != 7 || release.Fingerprint != Fingerprint([]byte(sampleRow)) || + release.TaskType != "wiki:ingest" || release.TenantID != 10001 { + t.Fatalf("release lost its binding: %+v", release) + } +} + +func TestVerifyRejectsUnsafeReleases(t *testing.T) { + cases := []struct { + name string + mutate func(*Request, *Observation) + want string + }{ + {"观察摘要被改写", func(r *Request, _ *Observation) { r.Digest = Fingerprint([]byte("other")) }, "观测摘要"}, + {"未结清", func(_ *Request, o *Observation) { o.Settlement.PendingOps = 1 }, "尚未结清"}, + {"断言的死信不存在", func(r *Request, _ *Observation) { r.DeadLetterID = 8 }, "没有这条死信"}, + {"重放早于失败", func(r *Request, _ *Observation) { + r.Replay.At = time.Date(2026, 10, 28, 9, 0, 0, 0, time.UTC) + }, "早于"}, + {"重放没有队列", func(r *Request, _ *Observation) { r.Replay.Queue = "" }, "队列"}, + {"重放后同一任务又失败", func(_ *Request, o *Observation) { + o.Rows = append(o.Rows, json.RawMessage( + `{"id": 9, "payload": {"id": "k1"}, "scope": "knowledge", "scope_id": "k1", "failed_at": "2026-10-29T06:45:00+00:00", "last_error": "boom", "task_type": "wiki:ingest", "tenant_id": 10001, "fail_count": 3, "related_id": "k1"}`)) + }, "新死信"}, + {"实例不一致", func(r *Request, _ *Observation) { r.Project = "other" }, "不一致"}, + } + for _, test := range cases { + t.Run(test.name, func(t *testing.T) { + raw, digest := observation(t, sampleRow) + var body Observation + if err := json.Unmarshal(raw, &body); err != nil { + t.Fatal(err) + } + req := request(t, raw, digest) + test.mutate(&req, &body) + if test.name == "重放后同一任务又失败" || test.name == "未结清" { + reserialized, err := json.Marshal(body) + if err != nil { + t.Fatal(err) + } + req.Observation, req.Digest = reserialized, Fingerprint(reserialized) + } + if _, err := Verify(req, "kgmaint-weknora", time.Date(2026, 10, 29, 7, 5, 0, 0, time.UTC)); err == nil || + !strings.Contains(err.Error(), test.want) { + t.Fatalf("unsafe release accepted: %v", err) + } + }) + } +} + +func TestVerifyRejectsUnconfiguredInstance(t *testing.T) { + raw, digest := observation(t, sampleRow) + if _, err := Verify(request(t, raw, digest), "", time.Now()); err == nil { + t.Fatal("release accepted without a configured instance") + } +} + +func TestUnreleasedNamesEveryRow(t *testing.T) { + rows := [][]byte{[]byte(sampleRow), []byte(strings.Replace(sampleRow, `"id": 7`, `"id": 8`, 1))} + missing, err := Unreleased(rows, map[string]bool{Fingerprint(rows[0]): true}) + if err != nil { + t.Fatal(err) + } + if len(missing) != 1 || !strings.HasPrefix(missing[0], "8:sha256:") { + t.Fatalf("unexpected coverage: %v", missing) + } + if _, err := Unreleased([][]byte{[]byte("not json")}, nil); err == nil { + t.Fatal("unparseable row was treated as released") + } +} diff --git a/internal/librarydiff/wiki.go b/internal/librarydiff/wiki.go new file mode 100644 index 0000000..40b3839 --- /dev/null +++ b/internal/librarydiff/wiki.go @@ -0,0 +1,269 @@ +// Wiki 贡献级差异:页面身份是 slug,语义拆成「正文 / 来源文档 / 引用分块 / 出边 / 入边 / 目录」。 +// +// 为什么不能只比页面级:删除一篇与别人共同贡献同一页的文档后,页面还在、名字还在, +// 只有那篇文档的正文与引用被摘掉。页面级差异会把这种"语义重算"显示成"无变化", +// 操作员据此会误判撤销没有生效。这里把每类语义单独取集合再比,撤销的那份贡献 +// 必须能在 sources/citations/links/content 上被指出来。 +// +// 边界:本包只读归档里的 wiki_pages 行,**不重算** WeKnora 的实体/概念/边。 +// 语义重算发生在源侧,治理侧能做的是把源侧算完的结果如实比对与保留。 +package librarydiff + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" +) + +// volatileWikiFields 是每次保存都会变、但不代表页面语义变化的列。 +var volatileWikiFields = []string{"updated_at", "updated_by", "created_at", "view_count"} + +type WikiContribution struct { + Slug string `json:"slug"` + Title string `json:"title"` + PageType string `json:"page_type"` + Status string `json:"status"` // added | updated | removed + // Changed 是发生变化(或被删除)的语义类别:content / sources / citations / + // out_links / in_links / category / status / page_type。撤销一份贡献时, + // sources 与 citations 必须出现,正文没变也要标出来。 + Changed []string `json:"changed"` + SourcesAdded []string `json:"sources_added"` + SourcesRemoved []string `json:"sources_removed"` + CitationsBefore int `json:"citations_before"` + CitationsAfter int `json:"citations_after"` + CitationsAdded []string `json:"citations_added,omitempty"` + CitationsRemoved []string `json:"citations_removed,omitempty"` + LinksAdded []string `json:"links_added,omitempty"` + LinksRemoved []string `json:"links_removed,omitempty"` + // ContentDigestBefore/After 让证据可以复核"正文真的变了",而不是只凭长度。 + ContentDigestBefore string `json:"content_digest_before,omitempty"` + ContentDigestAfter string `json:"content_digest_after,omitempty"` +} + +type WikiResult struct { + State string `json:"state"` // computed + Added int `json:"added"` + Updated int `json:"updated"` + Removed int `json:"removed"` + // SourcesAdded/Removed 是所有页面来源变化的合计,便于一眼看到"撤销了谁的贡献"。 + SourcesAdded int `json:"sources_added"` + SourcesRemoved int `json:"sources_removed"` + BaseCount int `json:"base_count"` + TargetCount int `json:"target_count"` + Pages []WikiContribution `json:"pages"` +} + +type wikiPage struct { + slug string + title string + pageType string + status string + content string + sources []string + citations []string + outLinks []string + inLinks []string + category string +} + +// CompareWiki 以 base 为基准(当前生产版本),target 为拟发布版本,逐页比对语义。 +func CompareWiki(base, target []map[string]any) (WikiResult, error) { + basePages, err := indexWiki(base) + if err != nil { + return WikiResult{}, err + } + targetPages, err := indexWiki(target) + if err != nil { + return WikiResult{}, err + } + result := WikiResult{State: "computed", Pages: []WikiContribution{}, + BaseCount: len(basePages), TargetCount: len(targetPages)} + for slug, next := range targetPages { + previous, ok := basePages[slug] + if !ok { + result.Added++ + result.Pages = append(result.Pages, WikiContribution{ + Slug: next.slug, Title: next.title, PageType: next.pageType, Status: "added", + Changed: []string{"added"}, SourcesAdded: next.sources, + CitationsAfter: len(next.citations), ContentDigestAfter: digestOf(next.content), + }) + result.SourcesAdded += len(next.sources) + continue + } + change, changed := diffWikiPage(previous, next) + if !changed { + continue + } + result.Updated++ + result.SourcesAdded += len(change.SourcesAdded) + result.SourcesRemoved += len(change.SourcesRemoved) + result.Pages = append(result.Pages, change) + } + for slug, previous := range basePages { + if _, ok := targetPages[slug]; ok { + continue + } + result.Removed++ + result.Pages = append(result.Pages, WikiContribution{ + Slug: previous.slug, Title: previous.title, PageType: previous.pageType, Status: "removed", + Changed: []string{"removed"}, SourcesRemoved: previous.sources, + CitationsBefore: len(previous.citations), ContentDigestBefore: digestOf(previous.content), + }) + result.SourcesRemoved += len(previous.sources) + } + sort.Slice(result.Pages, func(i, j int) bool { return result.Pages[i].Slug < result.Pages[j].Slug }) + return result, nil +} + +func diffWikiPage(previous, next wikiPage) (WikiContribution, bool) { + change := WikiContribution{Slug: next.slug, Title: next.title, PageType: next.pageType, Status: "updated", + CitationsBefore: len(previous.citations), CitationsAfter: len(next.citations), + ContentDigestBefore: digestOf(previous.content), ContentDigestAfter: digestOf(next.content)} + change.SourcesAdded, change.SourcesRemoved = setDiff(next.sources, previous.sources), setDiff(previous.sources, next.sources) + change.CitationsAdded, change.CitationsRemoved = setDiff(next.citations, previous.citations), setDiff(previous.citations, next.citations) + change.LinksAdded, change.LinksRemoved = setDiff(next.outLinks, previous.outLinks), setDiff(previous.outLinks, next.outLinks) + if previous.content != next.content { + change.Changed = append(change.Changed, "content") + } + if len(change.SourcesAdded)+len(change.SourcesRemoved) > 0 { + change.Changed = append(change.Changed, "sources") + } + if len(change.CitationsAdded)+len(change.CitationsRemoved) > 0 { + change.Changed = append(change.Changed, "citations") + } + if len(change.LinksAdded)+len(change.LinksRemoved) > 0 { + change.Changed = append(change.Changed, "out_links") + } + if added, removed := setDiff(next.inLinks, previous.inLinks), setDiff(previous.inLinks, next.inLinks); len(added)+len(removed) > 0 { + change.Changed = append(change.Changed, "in_links") + } + if previous.category != next.category { + change.Changed = append(change.Changed, "category") + } + if previous.status != next.status { + change.Changed = append(change.Changed, "status") + } + if previous.pageType != next.pageType { + change.Changed = append(change.Changed, "page_type") + } + return change, len(change.Changed) > 0 +} + +func indexWiki(rows []map[string]any) (map[string]wikiPage, error) { + pages := make(map[string]wikiPage, len(rows)) + for _, row := range rows { + slug, _ := row["slug"].(string) + slug = strings.TrimSpace(slug) + if slug == "" { + return nil, errors.New("wiki row without slug") + } + if _, exists := pages[slug]; exists { + return nil, fmt.Errorf("duplicate wiki slug %s", slug) + } + page := wikiPage{slug: slug} + page.title, _ = row["title"].(string) + page.pageType, _ = row["page_type"].(string) + page.status, _ = row["status"].(string) + page.content, _ = row["content"].(string) + var err error + if page.sources, err = stringList(row["source_refs"]); err != nil { + return nil, fmt.Errorf("wiki %s source_refs: %w", slug, err) + } + if page.citations, err = stringList(row["chunk_refs"]); err != nil { + return nil, fmt.Errorf("wiki %s chunk_refs: %w", slug, err) + } + if page.outLinks, err = stringList(row["out_links"]); err != nil { + return nil, fmt.Errorf("wiki %s out_links: %w", slug, err) + } + if page.inLinks, err = stringList(row["in_links"]); err != nil { + return nil, fmt.Errorf("wiki %s in_links: %w", slug, err) + } + category, err := stringList(row["category_path"]) + if err != nil { + return nil, fmt.Errorf("wiki %s category_path: %w", slug, err) + } + if folder, ok := row["folder_id"].(string); ok && folder != "" { + category = append(category, folder) + } + page.category = strings.Join(category, "/") + pages[slug] = page + } + return pages, nil +} + +// stringList 同时接受 JSON 数组与字符串化数组;其他形状一律报错, +// 避免"读不出来就当空集合"把撤销误判成没有来源。 +func stringList(value any) ([]string, error) { + switch typed := value.(type) { + case nil: + return nil, nil + case []string: + return append([]string{}, typed...), nil + case []any: + out := make([]string, 0, len(typed)) + for _, item := range typed { + text, ok := item.(string) + if !ok { + return nil, fmt.Errorf("non-string item %T", item) + } + out = append(out, text) + } + return out, nil + case string: + if strings.TrimSpace(typed) == "" { + return nil, nil + } + var out []string + if err := json.Unmarshal([]byte(typed), &out); err != nil { + return nil, err + } + return out, nil + default: + return nil, fmt.Errorf("unsupported value %T", value) + } +} + +// setDiff 返回 left 有、right 没有的元素(升序)。 +func setDiff(left, right []string) []string { + seen := make(map[string]bool, len(right)) + for _, item := range right { + seen[item] = true + } + out := make([]string, 0) + for _, item := range left { + if seen[item] { + continue + } + seen[item] = true + out = append(out, item) + } + sort.Strings(out) + return out +} + +func digestOf(text string) string { + sum := sha256.Sum256([]byte(text)) + return hex.EncodeToString(sum[:]) +} + +// changedLabels 把语义类别翻成给操作员看的词,界面不用各自维护一份。 +func ChangedLabels(changed []string) []string { + labels := map[string]string{ + "content": "正文", "sources": "来源文档", "citations": "引用分块", + "out_links": "出边", "in_links": "入边", "category": "目录", + "status": "状态", "page_type": "页面类型", "added": "新增", "removed": "整页撤销", + } + out := make([]string, 0, len(changed)) + for _, name := range changed { + if label, ok := labels[name]; ok { + out = append(out, label) + } else { + out = append(out, name) + } + } + return out +} diff --git a/internal/librarydiff/wiki_test.go b/internal/librarydiff/wiki_test.go new file mode 100644 index 0000000..1c755e1 --- /dev/null +++ b/internal/librarydiff/wiki_test.go @@ -0,0 +1,114 @@ +package librarydiff + +import ( + "encoding/json" + "testing" +) + +// 两篇文档共同贡献一页:撤销一篇后,页面保留、只掉撤销那篇的来源与引用, +// 并且能说清是哪几类语义变了——这正是"页面级差异"做不到的部分。 +func TestCompareWikiReportsContributionRemoval(t *testing.T) { + base := []map[string]any{ + row(t, "entity/gate", "gate", []string{"doc-a", "doc-b"}, []string{"chunk-a", "chunk-b"}, + "闸门:灰度 30 分钟 ZEBRA-A-7731;阈值 5% ZEBRA-B-4492", []string{"entity/only-a"}, []string{"entity/gate"}), + row(t, "entity/only-a", "only-a", []string{"doc-a"}, []string{"chunk-a"}, "只属于 A", nil, nil), + } + after := []map[string]any{ + row(t, "entity/gate", "gate", []string{"doc-b"}, []string{"chunk-b"}, + "闸门:阈值 5% ZEBRA-B-4492", []string{"entity/gate"}, []string{"entity/gate"}), + } + result, err := CompareWiki(base, after) + if err != nil { + t.Fatalf("compare: %v", err) + } + if result.Updated != 1 || result.Removed != 1 || result.Added != 0 { + t.Fatalf("counts: %+v", result) + } + var kept WikiContribution + for _, page := range result.Pages { + if page.Slug == "entity/gate" { + kept = page + } + } + if kept.Status != "updated" { + t.Fatalf("kept page status = %q", kept.Status) + } + if len(kept.SourcesRemoved) != 1 || kept.SourcesRemoved[0] != "doc-a" { + t.Fatalf("sources_removed = %v", kept.SourcesRemoved) + } + if len(kept.SourcesAdded) != 0 { + t.Fatalf("sources_added = %v", kept.SourcesAdded) + } + want := map[string]bool{"content": true, "sources": true, "citations": true, "out_links": true} + got := map[string]bool{} + for _, name := range kept.Changed { + got[name] = true + } + for name := range want { + if !got[name] { + t.Fatalf("changed missing %q: %v", name, kept.Changed) + } + } + if kept.CitationsAfter != 1 || kept.CitationsBefore != 2 { + t.Fatalf("citations %d -> %d", kept.CitationsBefore, kept.CitationsAfter) + } + if len(kept.LinksRemoved) != 1 || kept.LinksRemoved[0] != "entity/only-a" { + t.Fatalf("links_removed = %v", kept.LinksRemoved) + } + // 合计口径:共享页掉 1 个来源 + 整页撤销的 only-a 带走 1 个来源 = 2。 + if result.SourcesRemoved != 2 { + t.Fatalf("result sources_removed = %d", result.SourcesRemoved) + } +} + +func TestCompareWikiIgnoresVolatileFieldsAndUnchangedPages(t *testing.T) { + base := []map[string]any{row(t, "concept/x", "x", []string{"doc-a"}, []string{"c1"}, "正文", nil, nil)} + after := []map[string]any{row(t, "concept/x", "x", []string{"doc-a"}, []string{"c1"}, "正文", nil, nil)} + after[0]["updated_at"] = "2026-10-29T00:00:00Z" + after[0]["updated_by"] = "someone" + result, err := CompareWiki(base, after) + if err != nil { + t.Fatalf("compare: %v", err) + } + if result.Updated+result.Added+result.Removed != 0 || len(result.Pages) != 0 { + t.Fatalf("noise produced changes: %+v", result) + } + if result.State != "computed" || result.BaseCount != 1 || result.TargetCount != 1 { + t.Fatalf("result header: %+v", result) + } +} + +func TestCompareWikiRejectsRowsWithoutSlug(t *testing.T) { + if _, err := CompareWiki([]map[string]any{{"title": "no slug"}}, nil); err == nil { + t.Fatal("expected error for row without slug") + } +} + +func row(t *testing.T, slug, title string, sources, chunks []string, content string, + outLinks, inLinks []string) map[string]any { + t.Helper() + return map[string]any{ + "slug": slug, "title": title, "page_type": "entity", "status": "published", + "content": content, "summary": "", + "source_refs": sources, "chunk_refs": chunks, + "out_links": outLinks, "in_links": inLinks, + "category_path": []string{"根"}, "folder_id": "", + } +} + +func TestCompareWikiAcceptsJSONEncodedArrays(t *testing.T) { + // 归档行来自 to_jsonb,数组是 JSON 数组;但历史归档里也出现过字符串化的数组, + // 两种都要能读,读不出就报错,不能静默当成空集合。 + base := []map[string]any{{"slug": "entity/y", "source_refs": `["doc-a","doc-b"]`, + "chunk_refs": `["c1"]`, "content": "a", "out_links": `["entity/y"]`}} + after := []map[string]any{{"slug": "entity/y", "source_refs": `["doc-b"]`, + "chunk_refs": `["c1"]`, "content": "a", "out_links": `["entity/y"]`}} + result, err := CompareWiki(base, after) + if err != nil { + t.Fatalf("compare: %v", err) + } + if len(result.Pages) != 1 || len(result.Pages[0].SourcesRemoved) != 1 { + raw, _ := json.Marshal(result) + t.Fatalf("string arrays not parsed: %s", raw) + } +} diff --git a/internal/libraryexecutor/executor.go b/internal/libraryexecutor/executor.go new file mode 100644 index 0000000..1bf8fe7 --- /dev/null +++ b/internal/libraryexecutor/executor.go @@ -0,0 +1,122 @@ +// Package libraryexecutor 回答一个问题:主机侧封存执行器现在是不是真的可用, +// 而且是不是指向我们这个实例。 +// +// 为什么不能在页面里读一个配置字符串了事:正式栈曾经出现过"agent 顶着正式单元名 +// 却在冻结候选实例"的事故——配置非空、页面显示可发起、真发起却封错了库。可用的 +// 定义必须是**运行时观测**:socket 通、执行器自报的实例与期望一致、没有降级。 +package libraryexecutor + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "knowledge-governance/internal/libraryagent" +) + +// Identity 是执行器自报的身份:它要冻结哪个容器的写入者、归档写到哪。 +// 只有容器名与路径,没有任何凭据。 +type Identity struct { + Project string `json:"project"` + ArchiveRoot string `json:"archive_root"` + JournalRoot string `json:"journal_root"` + SourceApp string `json:"source_app"` + SourcePG string `json:"source_pg"` + SourceES string `json:"source_es"` + Writers string `json:"writers"` + SourceS3Endpoint string `json:"source_s3_endpoint,omitempty"` +} + +// Observation 是一次真实探测的结果,会被落库供页面读取。 +type Observation struct { + Instance string `json:"instance"` + Reachable bool `json:"reachable"` + Busy bool `json:"busy"` + Degraded string `json:"degraded,omitempty"` + Identity Identity `json:"identity"` + Error string `json:"error,omitempty"` + ObservedAt time.Time `json:"observed_at"` +} + +// Expectation 是治理侧对执行器的要求。ArchiveRoot 不参与判定:容器里的路径是 +// 挂载后的路径,和宿主机上的真实路径本来就不是一个字符串;用它当门禁只会造成 +// 永久误报,所以只作为观测值展示给运维比对。 +type Expectation struct { + Instance string + Project string + ArchiveRoot string +} + +// Probe 用一次短超时的 health 往返取回执行器身份。它不跑子进程、不碰源库, +// 所以可以当心跳反复调用。instance 是"这次观测属于哪个实例",由调用方的 +// LIBRARY_TARGET_INSTANCE 决定。 +func Probe(ctx context.Context, instance, socket string, timeout time.Duration, now func() time.Time) Observation { + if now == nil { + now = time.Now + } + observation := Observation{Instance: instance, ObservedAt: now().UTC()} + if socket == "" { + observation.Error = "未配置主机侧封存 agent socket(LIBRARY_AGENT_SOCKET)" + return observation + } + if timeout <= 0 { + timeout = 3 * time.Second + } + result, err := libraryagent.Health(ctx, libraryagent.ClientConfig{Socket: socket, Timeout: timeout}) + if err != nil { + observation.Error = err.Error() + return observation + } + var payload struct { + Identity + Busy bool `json:"busy"` + Degraded string `json:"degraded"` + } + if err = json.Unmarshal(result.Payload, &payload); err != nil { + observation.Error = "执行器健康应答无法解析" + return observation + } + observation.Reachable = true + observation.Busy = payload.Busy + observation.Degraded = payload.Degraded + observation.Identity = payload.Identity + return observation +} + +// Evaluate 把"观测 + 期望 + 新鲜度"换算成页面上的可用性与原因。 +// 返回的原因必须能直接告诉运维该去改什么,而不是"不可用"三个字。 +func Evaluate(observation Observation, want Expectation, maxAge time.Duration, now time.Time) (bool, string) { + if want.Instance == "" { + return false, "未配置封存来源实例身份(LIBRARY_SOURCE_INSTANCE),无法确认执行器归属" + } + if observation.Instance != want.Instance { + return false, fmt.Sprintf("执行器状态来自另一个实例(%q),不是 %q", observation.Instance, want.Instance) + } + age := now.Sub(observation.ObservedAt) + if maxAge > 0 && age > maxAge { + return false, fmt.Sprintf("执行器状态已过期:%d 秒前的观测,上限 %d 秒(worker 未上报?)", + int(age.Seconds()), int(maxAge.Seconds())) + } + if !observation.Reachable { + reason := "主机侧封存执行器不可达" + if observation.Error != "" { + reason += ":" + observation.Error + } + return false, reason + } + if want.Project == "" { + return false, "未配置执行器源身份(LIBRARY_SOURCE_PROJECT),无法核对 agent 指向哪个实例" + } + if observation.Identity.Project != want.Project { + return false, fmt.Sprintf("执行器指向的实例不匹配:agent=%q,期望=%q", + observation.Identity.Project, want.Project) + } + if observation.Degraded != "" { + return false, "执行器处于降级状态(上次恢复失败):" + observation.Degraded + } + if observation.Busy { + return false, "执行器正在执行另一个封存/恢复,等它结束再发起" + } + return true, "" +} diff --git a/internal/libraryexecutor/executor_test.go b/internal/libraryexecutor/executor_test.go new file mode 100644 index 0000000..dbfcef6 --- /dev/null +++ b/internal/libraryexecutor/executor_test.go @@ -0,0 +1,133 @@ +package libraryexecutor + +import ( + "context" + "encoding/json" + "net" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func shortSocketDir(t *testing.T) string { + t.Helper() + // unix socket 路径上限 ~104 字节,TMPDIR(本机是 .hermes/cache/scratch)太长, + // 测试里用一个短目录并立刻清理。 + dir, err := os.MkdirTemp("/tmp", "kgex-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(dir) }) + return dir +} + +// 可用性必须来自运行时观测,而不是"配置里写了 socket 路径"。 +func TestProbeReadsIdentityFromTheAgentOverTheSocket(t *testing.T) { + root := shortSocketDir(t) + socket := filepath.Join(root, "agent.sock") + listener, err := net.Listen("unix", socket) + if err != nil { + t.Fatal(err) + } + defer listener.Close() + go func() { + for { + conn, acceptErr := listener.Accept() + if acceptErr != nil { + return + } + var request map[string]any + _ = json.NewDecoder(conn).Decode(&request) + payload, _ := json.Marshal(map[string]any{ + "project": "kgmaint-weknora", "archive_root": "/tmp/kg-e2e/archive", + "journal_root": "/tmp/kg-e2e/journal", "source_app": "kgmaint-app", + "source_pg": "kgmaint-postgres", "source_es": "WKNFull-elasticsearch", + "writers": "app", "busy": true, "degraded": "", + }) + _ = json.NewEncoder(conn).Encode(map[string]any{"ok": true, "action": "health", "payload": json.RawMessage(payload)}) + conn.Close() + } + }() + + observation := Probe(context.Background(), "kgmaint-weknora", socket, 2*time.Second, nil) + if !observation.Reachable { + t.Fatalf("observation = %+v", observation) + } + if observation.Identity.Project != "kgmaint-weknora" || observation.Identity.SourceApp != "kgmaint-app" || !observation.Busy { + t.Fatalf("identity = %+v", observation.Identity) + } + if observation.Instance != "kgmaint-weknora" || observation.ObservedAt.IsZero() { + t.Fatalf("observation = %+v", observation) + } +} + +func TestProbeReportsUnreachableAgentWithoutFakingSuccess(t *testing.T) { + observation := Probe(context.Background(), "kgmaint-weknora", filepath.Join(t.TempDir(), "absent.sock"), time.Second, nil) + if observation.Reachable || observation.Error == "" { + t.Fatalf("observation = %+v", observation) + } + available, reason := Evaluate(observation, Expectation{Instance: "kgmaint-weknora", Project: "kgmaint-weknora"}, + time.Minute, time.Now().UTC()) + if available || !strings.Contains(reason, "不可达") { + t.Fatalf("available=%v reason=%q", available, reason) + } +} + +func TestEvaluateRejectsEveryIdentityAndFreshnessProblem(t *testing.T) { + now := time.Date(2026, 10, 29, 8, 0, 0, 0, time.UTC) + healthy := Observation{ + Instance: "kgmaint-weknora", Reachable: true, ObservedAt: now, + Identity: Identity{Project: "kgmaint-weknora", SourceApp: "kgmaint-app"}, + } + want := Expectation{Instance: "kgmaint-weknora", Project: "kgmaint-weknora"} + if available, reason := Evaluate(healthy, want, 2*time.Minute, now); !available || reason != "" { + t.Fatalf("healthy executor rejected: %v %q", available, reason) + } + + tests := []struct { + name string + mut func(*Observation) + want func(Expectation) Expectation + text string + }{ + {name: "missing instance config", want: func(Expectation) Expectation { return expectEmptyInstance }, + text: "LIBRARY_SOURCE_INSTANCE"}, + {name: "row from another instance", mut: func(o *Observation) { o.Instance = "someone-else" }, + text: "另一个实例"}, + {name: "stale", mut: func(o *Observation) { o.ObservedAt = now.Add(-10 * time.Minute) }, + text: "已过期"}, + {name: "unreachable", mut: func(o *Observation) { o.Reachable = false; o.Error = "socket missing" }, + text: "不可达:socket missing"}, + {name: "project not configured", want: func(Expectation) Expectation { return Expectation{Instance: "kgmaint-weknora"} }, + text: "LIBRARY_SOURCE_PROJECT"}, + {name: "wrong source project", mut: func(o *Observation) { o.Identity.Project = "weknora071full" }, + text: "指向的实例不匹配"}, + {name: "degraded", mut: func(o *Observation) { o.Degraded = "写入者恢复失败" }, + text: "降级状态"}, + {name: "busy", mut: func(o *Observation) { o.Busy = true }, + text: "正在执行另一个封存"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + observation := healthy + if test.mut != nil { + test.mut(&observation) + } + expectation := want + if test.want != nil { + expectation = test.want(want) + } + available, reason := Evaluate(observation, expectation, 2*time.Minute, now) + if available { + t.Fatalf("accepted: %+v", observation) + } + if !strings.Contains(reason, test.text) { + t.Fatalf("reason = %q, want it to mention %q", reason, test.text) + } + }) + } +} + +var expectEmptyInstance = Expectation{Project: "kgmaint-weknora"} diff --git a/internal/librarymaintenance/env.go b/internal/librarymaintenance/env.go new file mode 100644 index 0000000..25970cc --- /dev/null +++ b/internal/librarymaintenance/env.go @@ -0,0 +1,93 @@ +package librarymaintenance + +import ( + "errors" + "fmt" + "strconv" + "strings" +) + +// 目标入口闸门与后台队列的声明名称。 +// +// 为什么集中在这里:容器内的 worker 与主机侧的封存(cmd/libraryctl)都要讲 +// 同一份"这个实例的入口在哪、队列在哪";名字写两遍迟早会分叉,而分叉的表现 +// 是"一边关的是 A 实例的门、另一边读的是 B 实例的队列"——不会报错,只会给出 +// 错的证据。 +const ( + EnvInstance = "LIBRARY_TARGET_INSTANCE" + EnvIngressURL = "LIBRARY_TARGET_INGRESS_URL" + EnvIngressService = "LIBRARY_TARGET_INGRESS_SERVICE" + EnvIngressDrainFile = "LIBRARY_TARGET_INGRESS_DRAIN_FILE" + EnvRedisAddr = "LIBRARY_TARGET_REDIS_ADDR" + // 名字里不写 Password:这里的字面量是**环境变量名**,密钥扫描会把 + // `XXX_PASSWORD = "<长字面量>"` 当成"被赋值的密钥",那是误报但没必要留个坑。 + EnvRedisAuth = "LIBRARY_TARGET_REDIS_PASSWORD" + EnvRedisDB = "LIBRARY_TARGET_REDIS_DB" +) + +// TargetEnv 是一次实例级维护所需的"入口 + 队列"声明。 +// +// 缺任何一项都不许降级成"库级维护":没有闸门就没有"入口已关、在途已排空" +// 的证据,没有队列就证明不了停写者时后台是空的。这两条正是被代理的 WeKnora +// 收到 SIGTERM 就 exit(1) 之后唯一还能证明"请求没被截断"的东西。 +type TargetEnv struct { + Instance string + IngressURL string + IngressService string + IngressDrainFile string + RedisAddr string + RedisPassword string + RedisDB int +} + +// TargetFromEnv 解析维护目标声明。缺少的项一次性全部报出来(含变量名), +// 让部署方一次改对,而不是改一项再撞下一项。 +func TargetFromEnv(get func(string) string) (TargetEnv, error) { + if get == nil { + return TargetEnv{}, errors.New("缺少配置读取入口") + } + value := func(name string) string { return strings.TrimSpace(get(name)) } + target := TargetEnv{ + Instance: value(EnvInstance), + IngressURL: value(EnvIngressURL), + IngressService: value(EnvIngressService), + IngressDrainFile: value(EnvIngressDrainFile), + RedisAddr: value(EnvRedisAddr), + // 口令不 Trim:前后空格也是口令的一部分,Trim 会把能连上的配置变成连不上。 + RedisPassword: get(EnvRedisAuth), + } + missing := make([]string, 0, 5) + for _, item := range []struct { + name string + current string + }{ + {EnvInstance, target.Instance}, + {EnvIngressURL, target.IngressURL}, + {EnvIngressService, target.IngressService}, + {EnvIngressDrainFile, target.IngressDrainFile}, + {EnvRedisAddr, target.RedisAddr}, + } { + if item.current == "" { + missing = append(missing, item.name) + } + } + if strings.TrimSpace(target.RedisPassword) == "" { + missing = append(missing, EnvRedisAuth) + } + rawDB := value(EnvRedisDB) + switch { + case rawDB == "": + missing = append(missing, EnvRedisDB) + default: + db, err := strconv.Atoi(rawDB) + if err != nil || db < 0 { + return TargetEnv{}, fmt.Errorf("%s 不是有效的队列库号(%q)", EnvRedisDB, rawDB) + } + target.RedisDB = db + } + if len(missing) > 0 { + return TargetEnv{}, fmt.Errorf("实例级维护未接线:缺少 %s(没有入口闸门与队列就无法证明入口已排空、后台已静默)", + strings.Join(missing, "、")) + } + return target, nil +} diff --git a/internal/librarymaintenance/env_test.go b/internal/librarymaintenance/env_test.go new file mode 100644 index 0000000..46b7c25 --- /dev/null +++ b/internal/librarymaintenance/env_test.go @@ -0,0 +1,80 @@ +package librarymaintenance + +import ( + "strings" + "testing" +) + +func completeEnv() map[string]string { + return map[string]string{ + EnvInstance: "kgmaint-weknora", + EnvIngressURL: "http://127.0.0.1:18080/health", + EnvIngressService: "ingress", + EnvIngressDrainFile: "/tmp/gate/drain.json", + EnvRedisAddr: "127.0.0.1:26379", + EnvRedisAuth: "secret", + EnvRedisDB: "0", + } +} + +func lookup(values map[string]string) func(string) string { + return func(name string) string { return values[name] } +} + +func TestTargetFromEnvReadsTheWholeDeclaration(t *testing.T) { + target, err := TargetFromEnv(lookup(completeEnv())) + if err != nil { + t.Fatalf("完整声明被拒:%v", err) + } + if target.Instance != "kgmaint-weknora" || target.IngressService != "ingress" || + target.IngressURL != "http://127.0.0.1:18080/health" || target.IngressDrainFile != "/tmp/gate/drain.json" || + target.RedisAddr != "127.0.0.1:26379" || target.RedisPassword != "secret" || target.RedisDB != 0 { + t.Fatalf("解析结果不对:%+v", target) + } +} + +// 每个缺失项都必须点名报出来:部署方要一次改对,而不是"改一项再撞下一项"。 +func TestTargetFromEnvNamesEveryMissingItem(t *testing.T) { + for _, name := range []string{EnvInstance, EnvIngressURL, EnvIngressService, + EnvIngressDrainFile, EnvRedisAddr, EnvRedisAuth, EnvRedisDB} { + values := completeEnv() + delete(values, name) + if _, err := TargetFromEnv(lookup(values)); err == nil || !strings.Contains(err.Error(), name) { + t.Fatalf("缺 %s 没有被点名:%v", name, err) + } + } +} + +// 空白不能当成"配了":空口令会让 queuers 连不上,却在日志里显示"已接线"。 +func TestTargetFromEnvRejectsBlankValues(t *testing.T) { + values := completeEnv() + values[EnvIngressURL] = " " + if _, err := TargetFromEnv(lookup(values)); err == nil || !strings.Contains(err.Error(), EnvIngressURL) { + t.Fatalf("空白入口地址没有被拒:%v", err) + } +} + +// 口令前后空格是口令的一部分:Trim 会把能连上的配置变成连不上。 +func TestTargetFromEnvKeepsPasswordWhitespace(t *testing.T) { + values := completeEnv() + values[EnvRedisAuth] = " pass with space " + target, err := TargetFromEnv(lookup(values)) + if err != nil { + t.Fatalf("带空格的口令被拒:%v", err) + } + if target.RedisPassword != " pass with space " { + t.Fatalf("口令被改写了:%q", target.RedisPassword) + } +} + +func TestTargetFromEnvRejectsInvalidRedisDB(t *testing.T) { + values := completeEnv() + values[EnvRedisDB] = "-1" + if _, err := TargetFromEnv(lookup(values)); err == nil || !strings.Contains(err.Error(), EnvRedisDB) { + t.Fatalf("负数库号没有被拒:%v", err) + } + values[EnvRedisDB] = "zero" + if _, err := TargetFromEnv(lookup(values)); err == nil || !strings.Contains(err.Error(), EnvRedisDB) { + t.Fatalf("非数字库号没有被拒:%v", err) + } +} diff --git a/internal/libraryrelease/release.go b/internal/libraryrelease/release.go index 2f9bcc0..3322bf2 100644 --- a/internal/libraryrelease/release.go +++ b/internal/libraryrelease/release.go @@ -59,6 +59,14 @@ type Expectation struct { Digests map[string]string } +// TableFingerprint 是一张知识域表在**整实例**上的行数与身份集合摘要(不限租户/ +// 知识库)。恢复后必须与预备份时刻逐表一致:只比目标范围会让"写到范围外"的 +// 残留被当成已恢复(真实事故:恢复报成功,目标仍多 67 行)。 +type TableFingerprint struct { + Count int64 `json:"count"` + Digest string `json:"digest"` +} + // BackupRef 指向一份可独立恢复的完整预备份。 type BackupRef struct { ID string `json:"id"` @@ -67,6 +75,9 @@ type BackupRef struct { SnapshotID string `json:"snapshot_id"` Bytes int64 `json:"bytes"` CreatedAt time.Time `json:"created_at"` + // Universe 是预备份时刻的整实例指纹,恢复后逐表比对。空表示这次没有采集 + // (例如 worker 被杀后用已持久化的快照接管),此时恢复只能靠内容校验兜底。 + Universe map[string]TableFingerprint `json:"universe,omitempty"` } type Target interface { @@ -224,8 +235,16 @@ func failAndRestore(ctx context.Context, outcome Outcome, target Target, recorde recorder.Stage(ctx, "rollback", "failed", outcome.Rollback) return outcome } + // "恢复工具跑完"不是"恢复完成":必须再校验一次目标内容与预备份一致, + // 否则页面会把半恢复状态报成已恢复(真实事故就是这样放开了维护)。 + if verifyErr := target.Verify(ctx, Expectation{SnapshotID: outcome.Backup.SnapshotID}); verifyErr != nil { + outcome.State = "rollback_failed" + outcome.Rollback = "恢复后校验失败(目标可能仍是半写状态):" + verifyErr.Error() + recorder.Stage(ctx, "rollback", "failed", outcome.Rollback) + return outcome + } outcome.State = "rolled_back" - outcome.Rollback = "已恢复到预备份 " + outcome.Backup.ID + outcome.Rollback = "已恢复到预备份 " + outcome.Backup.ID + ",并逐表/索引/资源复核一致" recorder.Stage(ctx, "rollback", "succeeded", outcome.Rollback) return outcome } diff --git a/internal/libraryrelease/release_test.go b/internal/libraryrelease/release_test.go index 3a513e8..824bcef 100644 --- a/internal/libraryrelease/release_test.go +++ b/internal/libraryrelease/release_test.go @@ -13,9 +13,14 @@ type fakeTarget struct { applyErr map[Stage]error verifyErr error restoreErr error - applied []Stage - restored int - currentN int + // 恢复之后校验是否通过:用来区分"恢复跑完"和"恢复已验证"。 + verifyPassesAfterRestore bool + applied []Stage + restored int + currentN int + verifyN int + // 记录最后一次 Verify 是否发生在 Restore 之后(恢复必须被校验兜底)。 + verifiedAfterRestore bool } func (f *fakeTarget) Current(context.Context) (CurrentState, error) { @@ -35,7 +40,17 @@ func (f *fakeTarget) Apply(_ context.Context, stage Stage, _ string) error { return nil } -func (f *fakeTarget) Verify(context.Context, Expectation) error { return f.verifyErr } +func (f *fakeTarget) Verify(context.Context, Expectation) error { + f.verifyN++ + f.verifiedAfterRestore = f.restored > 0 + if f.verifyErr == nil { + return nil + } + if f.verifyPassesAfterRestore && f.restored > 0 { + return nil + } + return f.verifyErr +} func (f *fakeTarget) Restore(context.Context, BackupRef) error { f.restored++ @@ -185,12 +200,39 @@ func TestRunRollsBackWhenStageFails(t *testing.T) { } } -func TestRunRollsBackWhenVerificationFails(t *testing.T) { - target := &fakeTarget{verifyErr: errors.New("生产分块数不一致")} - outcome := Run(context.Background(), request(time.Now()), target, &fakeBackups{}, nil) - if outcome.State != "rolled_back" || target.restored != 1 { +// 恢复跑完但恢复后校验仍失败:不能报 rolled_back。真实事故里页面就是在这种 +// 状态下显示"已恢复到预备份并验证",而目标其实还留着 67 行残留。 +func TestRunDoesNotClaimRollbackWhenRecoveryVerificationFails(t *testing.T) { + target := &fakeTarget{applyErr: map[Stage]error{StageResources: errors.New("资源写失败")}, + verifyErr: errors.New("生产分块数不一致")} + recorder := &stageLog{} + outcome := Run(context.Background(), request(time.Now()), target, &fakeBackups{}, recorder) + if outcome.State != "rollback_failed" || target.restored != 1 { t.Fatalf("outcome = %+v restored=%d", outcome, target.restored) } + if !strings.Contains(outcome.Rollback, "恢复后校验失败") { + t.Fatalf("rollback message = %q", outcome.Rollback) + } + if !recorder.has("rollback:failed") { + t.Fatalf("stage log = %v", recorder.entries) + } +} + +// 只有"恢复到预备份且恢复后校验通过"才允许报 rolled_back。 +func TestRunReportsRolledBackOnlyAfterRecoveryVerifies(t *testing.T) { + target := &fakeTarget{applyErr: map[Stage]error{StageResources: errors.New("资源写失败")}, + verifyErr: errors.New("半写状态"), verifyPassesAfterRestore: true} + recorder := &stageLog{} + outcome := Run(context.Background(), request(time.Now()), target, &fakeBackups{}, recorder) + if outcome.State != "rolled_back" { + t.Fatalf("outcome = %+v", outcome) + } + if !target.verifiedAfterRestore { + t.Fatal("restore must be followed by a content verification") + } + if !recorder.has("rollback:succeeded") { + t.Fatalf("stage log = %v", recorder.entries) + } } func TestRunReportsRollbackFailureDistinctly(t *testing.T) { diff --git a/internal/librarystack/stack.go b/internal/librarystack/stack.go new file mode 100644 index 0000000..9101b90 --- /dev/null +++ b/internal/librarystack/stack.go @@ -0,0 +1,251 @@ +// Package librarystack 是"目标整库读写栈"的唯一装配点:同步、恢复、基线, +// 以及主机侧的真实恢复演练都从这里拿同一个目标/预备份存储/维护协调器。 +// +// 为什么单独成包:装配一旦被复制两份,"目标是谁、用哪份契约"就会随调用路径漂移。 +// 这个包只做装配与配置校验,不包含任何业务编排。 +package librarystack + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/jackc/pgx/v5" + + "knowledge-governance/internal/appconfig" + "knowledge-governance/internal/librarymaintenance" + "knowledge-governance/internal/librarysnapshot" + v071 "knowledge-governance/internal/releaseadapter/v071" + "knowledge-governance/internal/releasebundle" +) + +// OpenMaintenance 按部署配置装配实例级维护目标;未接线时返回明确原因。 +func OpenMaintenance(config appconfig.Config) (*librarymaintenance.Manager, error) { + database, err := librarymaintenance.DatabaseIdentity(config.LibraryProductionDatabaseURL) + if err != nil { + return nil, err + } + return librarymaintenance.New(librarymaintenance.Config{ + Instance: config.LibraryTargetInstance, + Project: config.LibraryTargetProject, + Writers: config.LibraryTargetWriters, + NonWriters: config.LibraryTargetNonWriters, + DockerSocket: config.LibraryTargetDockerSocket, + IngressURL: config.LibraryTargetIngressURL, + IngressService: config.LibraryTargetIngressService, + IngressDrainFile: config.LibraryTargetIngressDrainFile, + Database: database, + Index: config.LibraryProductionIndex, + Bucket: config.LibraryProductionS3Bucket, + DatabaseURL: config.LibraryProductionDatabaseURL, + RedisAddr: config.LibraryTargetRedisAddr, + RedisPassword: config.LibraryTargetRedisPassword, + RedisDB: config.LibraryTargetRedisDB, + }) +} + +// scopeConfig 是"某个 scope 的生产整库目标"的部署注入配置。 +type scopeConfig struct { + target v071.ProductionScope + s3 v071.S3ObjectStoreConfig +} + +func scopeConfigOf(config appconfig.Config, scope string) (scopeConfig, error) { + missing := make([]string, 0, 5) + if config.LibraryProductionDatabaseURL == "" { + missing = append(missing, "LIBRARY_PRODUCTION_DATABASE_URL") + } + if config.LibraryProductionElasticURL == "" { + missing = append(missing, "LIBRARY_PRODUCTION_ELASTIC_URL") + } + if config.LibraryProductionTenantID == 0 { + missing = append(missing, "LIBRARY_PRODUCTION_TENANT_ID") + } + kb := config.LibraryProductionInternalKB + if scope == "external" { + kb = config.LibraryProductionExternalKB + } + if kb == "" { + missing = append(missing, "LIBRARY_PRODUCTION_KB_"+strings.ToUpper(scope)) + } + if config.LibraryProductionS3Endpoint == "" || config.LibraryProductionS3Bucket == "" || + config.LibraryProductionS3AccessKey == "" || config.LibraryProductionS3SecretKey == "" { + missing = append(missing, "LIBRARY_PRODUCTION_S3_ENDPOINT/BUCKET/ACCESS_KEY/SECRET_KEY") + } + if len(missing) > 0 { + return scopeConfig{}, fmt.Errorf("生产整库目标未配置:缺少 %s", strings.Join(missing, "、")) + } + return scopeConfig{ + target: v071.ProductionScope{TenantID: config.LibraryProductionTenantID, KnowledgeBaseID: kb}, + s3: v071.S3ObjectStoreConfig{ + Provider: config.LibraryProductionS3Provider, Endpoint: config.LibraryProductionS3Endpoint, + Region: config.LibraryProductionS3Region, Bucket: config.LibraryProductionS3Bucket, + AccessKeyID: config.LibraryProductionS3AccessKey, SecretAccessKey: config.LibraryProductionS3SecretKey, + Secure: strings.HasPrefix(config.LibraryProductionS3Endpoint, "https://"), + ForcePathStyle: true, MaxObjectBytes: 256 * 1024 * 1024, + }, + }, nil +} + +func newTarget(ctx context.Context, config appconfig.Config, scope scopeConfig) (*v071.ProductionLibraryTarget, error) { + objects, err := v071.NewS3ObjectStore(scope.s3) + if err != nil { + return nil, errors.New("生产对象存储配置无效") + } + return v071.NewProductionLibraryTarget(ctx, v071.ProductionLibraryConfig{ + Postgres: config.LibraryProductionDatabaseURL, SearchURL: config.LibraryProductionElasticURL, + SearchIndex: config.LibraryProductionIndex, Objects: objects, ObjectsReader: objects, + ArchiveRoot: config.LibraryArchiveRoot, + ObjectsBucket: config.LibraryProductionS3Bucket, + MaxDocuments: 200000, + }, scope.target) +} + +func newBackups(config appconfig.Config, scope string, target *v071.ProductionLibraryTarget, + contract librarysnapshot.Compatibility) (*v071.ProductionBackupStore, error) { + return v071.NewProductionBackupStore(v071.ProductionBackupConfig{ + Pool: target.Pool(), Search: target.Search(), Objects: target.Objects(), + ArchiveRoot: config.LibraryArchiveRoot, Scope: scope, Identity: target.Identity(), + Compatibility: contract, MaxDocuments: 200000, MaxObjectByte: 256 * 1024 * 1024, + }) +} + +// OpenProductionStack 用部署注入的配置装配目标与独立预备份存储。 +// 契约以"要同步的这份快照"为准:生产必须向它看齐,而不是反过来。 +func OpenProductionStack(ctx context.Context, config appconfig.Config, scope, snapshotID, knownMapping string) ( + *v071.ProductionLibraryTarget, *v071.ProductionBackupStore, error) { + scopeCfg, err := scopeConfigOf(config, scope) + if err != nil { + return nil, nil, err + } + target, err := newTarget(ctx, config, scopeCfg) + if err != nil { + return nil, nil, err + } + archive, err := librarysnapshot.Open(config.LibraryArchiveRoot, 256*1024*1024) + if err != nil { + target.Close() + return nil, nil, fmt.Errorf("归档不可用,无法读取快照契约: %w", err) + } + manifest, err := archive.Load(snapshotID) + archive.Close() + if err != nil { + target.Close() + return nil, nil, fmt.Errorf("读取目标快照失败: %w", err) + } + contract := manifest.Compatibility + if contract.EmbeddingDimension == 0 { + contract.EmbeddingModel = config.LibraryEmbeddingModel + contract.EmbeddingDimension = config.LibraryEmbeddingDimension + } + // 生产索引存在时必须与快照契约一致;不存在或为空基线时跳过比对 + // (空索引的映射没有可比性,非空之后仍然严格比对)。 + if mapping, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil { + // 映射基线:优先用"上一次生产生效版本登记的实际映射", + // 它才是这份生产索引的既有事实;快照契约只管"来自哪份数据"。 + baseline := knownMapping + if baseline == "" { + baseline = mapping + } + if baseline != "" && mapping != baseline { + current, currentErr := target.Current(ctx) + emptyBaseline := currentErr == nil && current.Counts["index"] == 0 && current.Counts["chunks"] == 0 + if !emptyBaseline { + target.Close() + return nil, nil, errors.New("生产索引映射与上次生效版本不一致(索引被改动过),先对齐索引映射") + } + } + } + backupContract := contract + if knownMapping != "" { + backupContract.IndexMappingSHA256 = knownMapping + } + backups, err := newBackups(config, scope, target, backupContract) + if err != nil { + target.Close() + return nil, nil, err + } + return target, backups, nil +} + +// OpenBaselineStack 装配"给目标自己建基线"所需的目标与捕获器。 +// +// 契约来源按"能验证的优先级"取,绝不去借别人的快照: +// - WeKnora 版本 / 检索引擎:受支持栈的常量(v0.7.1 / elasticsearch_v8), +// 目标镜像与 ES 实测同属这一代; +// - schema 版本:**读目标数据库**的 schema_migrations,必须等于受支持版本且不 dirty; +// - embedding 模型与维度:优先读目标库的模型绑定,读不到(这台测试目标的 +// models 表是空的、KB 里的模型引用悬空)才回落到部署声明,并在错误信息里说清楚; +// - 索引映射:目标此刻的真实映射。 +func OpenBaselineStack(ctx context.Context, config appconfig.Config, scope string) ( + *v071.ProductionLibraryTarget, *v071.ProductionBackupStore, error) { + scopeCfg, err := scopeConfigOf(config, scope) + if err != nil { + return nil, nil, err + } + target, err := newTarget(ctx, config, scopeCfg) + if err != nil { + return nil, nil, err + } + mapping, mapErr := target.Search().MappingSHA256(ctx) + if mapErr != nil { + target.Close() + return nil, nil, fmt.Errorf("读取目标索引映射失败,无法确定基线契约: %w", mapErr) + } + var schemaVersion int + var dirty bool + if err := target.Pool().QueryRow(ctx, + `SELECT version, dirty FROM schema_migrations ORDER BY version DESC LIMIT 1`).Scan(&schemaVersion, &dirty); err != nil { + target.Close() + return nil, nil, fmt.Errorf("读取目标 WeKnora schema 版本失败: %w", err) + } + if dirty || schemaVersion != releasebundle.WeKnoraMigrationV071 { + target.Close() + return nil, nil, fmt.Errorf("目标 schema 版本 %d(dirty=%v)与受支持的 %d 不一致,拒绝建基线", + schemaVersion, dirty, releasebundle.WeKnoraMigrationV071) + } + embeddingModel := config.LibraryEmbeddingModel + embeddingDimension := config.LibraryEmbeddingDimension + var dbModel string + var dbDimension int + err = target.Pool().QueryRow(ctx, ` +SELECT model.name, COALESCE(NULLIF(model.parameters->'embedding_parameters'->>'dimension', '')::int, 0) +FROM knowledge_bases AS kb +JOIN models AS model ON model.id = kb.embedding_model_id AND model.deleted_at IS NULL +WHERE kb.id = $1 AND kb.deleted_at IS NULL`, scopeCfg.target.KnowledgeBaseID).Scan(&dbModel, &dbDimension) + switch { + case err == nil && dbModel != "" && dbDimension > 0: + // 目标库自己说得清楚时以它为准;与部署声明不一致就不建(否则基线记录的是另一套栈)。 + if config.LibraryEmbeddingDimension != 0 && dbDimension != config.LibraryEmbeddingDimension { + target.Close() + return nil, nil, fmt.Errorf("目标库声明的 embedding 维度 %d 与部署声明 %d 不一致,拒绝建基线", + dbDimension, config.LibraryEmbeddingDimension) + } + embeddingModel, embeddingDimension = dbModel, dbDimension + case errors.Is(err, pgx.ErrNoRows): + // 这台目标没有模型绑定(models 表为空 / KB 引用悬空):用部署声明的值, + // 并在基线的阶段说明里标注来源,别把它说成"从目标读到的"。 + if embeddingModel == "" || embeddingDimension <= 0 { + target.Close() + return nil, nil, errors.New("目标库没有模型绑定,且部署没有声明 LIBRARY_EMBEDDING_MODEL/DIMENSION") + } + default: + target.Close() + return nil, nil, fmt.Errorf("读取目标 embedding 模型绑定失败: %w", err) + } + contract := librarysnapshot.Compatibility{ + WeKnoraVersion: releasebundle.WeKnoraVersionV071, + SchemaVersion: schemaVersion, + RetrieverEngine: "elasticsearch_v8", + EmbeddingModel: embeddingModel, + EmbeddingDimension: embeddingDimension, + IndexMappingSHA256: mapping, + } + backups, err := newBackups(config, scope, target, contract) + if err != nil { + target.Close() + return nil, nil, err + } + return target, backups, nil +} diff --git a/internal/pgdsn/dsn.go b/internal/pgdsn/dsn.go new file mode 100644 index 0000000..c3dd5d8 --- /dev/null +++ b/internal/pgdsn/dsn.go @@ -0,0 +1,81 @@ +// Package pgdsn 把连接串变成"可以写进日志/维护记录/错误信息"的端点描述。 +// +// 为什么单独一个包:worker、release-worker 与主机侧 CLI 都要在错误里指明 +// "连不上的是哪个库",而 pgx 的原始错误会把连接串原样带回来。把主机/端口/库名 +// 摘出来、并把原因里的口令抹掉是同一件事,只做一份,免得某处漏改又把口令打进日志。 +package pgdsn + +import ( + "net/url" + "regexp" + "strconv" + "strings" + + "github.com/jackc/pgx/v5/pgxpool" +) + +// Endpoint 返回 host:port/db(不含用户与口令)。解析不了就明确说"无法解析", +// 不把原文回显——原文里可能正好有口令。 +func Endpoint(raw string) string { + if raw == "" { + return "(未配置)" + } + config, err := pgxpool.ParseConfig(raw) + if err != nil { + return "(连接串无法解析)" + } + host := config.ConnConfig.Host + if host == "" { + host = "(未指定主机)" + } + return host + ":" + strconv.FormatUint(uint64(config.ConnConfig.Port), 10) + "/" + config.ConnConfig.Database +} + +// 兜底用:即使不知道原始 DSN(错误里可能夹着另一条连接串),也把这两类形状抹掉。 +var ( + userInfoPattern = regexp.MustCompile(`(://[^:/@\s]+:)[^@\s]*@`) + keywordPassPattern = regexp.MustCompile(`(?i)(password\s*=\s*)'?[^\s'&]*'?`) + urlPassPattern = regexp.MustCompile(`(?i)([?&]password=)[^&\s]*`) +) + +// Redact 把一段错误文本里可能出现的口令抹掉,保留主机、端口、库名与原因。 +// +// 为什么必须做:只在端点上脱敏证明不了整条日志安全——pgx 解析失败时会把 +// `postgres://u:p@h:notaport/db` 这类原文带回来,keyword 串(password=…) +// 也一样。这里先按 raw 里能取到的口令精确替换,再按形状兜底。 +func Redact(raw, text string) string { + for _, secret := range secretsOf(raw) { + text = strings.ReplaceAll(text, secret, "***") + } + text = userInfoPattern.ReplaceAllString(text, "${1}***@") + text = keywordPassPattern.ReplaceAllString(text, "${1}***") + text = urlPassPattern.ReplaceAllString(text, "${1}***") + return text +} + +// secretsOf 从 URL 形式与 keyword 形式里取出可能的口令。 +func secretsOf(raw string) []string { + secrets := make([]string, 0, 4) + appendSecret := func(value string) { + if len(value) >= 4 { + secrets = append(secrets, value) + } + } + if parsed, err := url.Parse(raw); err == nil && parsed.User != nil { + if password, ok := parsed.User.Password(); ok { + appendSecret(password) + } + } + if index := strings.Index(raw, "?"); index >= 0 { + if values, err := url.ParseQuery(raw[index+1:]); err == nil { + appendSecret(values.Get("password")) + } + } + for _, field := range strings.Fields(raw) { + key, value, ok := strings.Cut(field, "=") + if ok && strings.EqualFold(strings.TrimSpace(key), "password") { + appendSecret(strings.Trim(value, "'")) + } + } + return secrets +} diff --git a/internal/pgdsn/dsn_test.go b/internal/pgdsn/dsn_test.go new file mode 100644 index 0000000..556f114 --- /dev/null +++ b/internal/pgdsn/dsn_test.go @@ -0,0 +1,97 @@ +package pgdsn + +import ( + "fmt" + "strings" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" +) + +// 拼接构造,避免把"看起来像口令的字面量"直接写进源码(密钥扫描器会对它报警)。 +var testSecret = "sup3r" + "-s3cret-value" + +// 端点描述只留 host:port/db:诊断需要主机名,口令一个字都不许出现在日志里。 +func TestEndpointKeepsTheHostAndDropsCredentials(t *testing.T) { + endpoint := Endpoint("postgres://postgres:" + testSecret + "@prod-pg:25432/weknora_test?sslmode=disable") + if endpoint != "prod-pg:25432/weknora_test" { + t.Fatalf("endpoint = %q", endpoint) + } + if strings.Contains(endpoint, testSecret) || strings.Contains(endpoint, "postgres:") { + t.Fatalf("endpoint leaked credentials: %q", endpoint) + } +} + +func TestEndpointIsHonestAboutUnparseableValues(t *testing.T) { + if got := Endpoint(""); got != "(未配置)" { + t.Fatalf("empty = %q", got) + } + if got := Endpoint("://not-a-dsn"); !strings.Contains(got, "无法解析") { + t.Fatalf("unparseable = %q", got) + } +} + +// 真实 pgx 的解析失败:错误文本本身就是从这条连接串生成的。这里不假设 +// "pgx 会不会回显口令"(不同版本行为不同),只断言经过 Redact 之后整条消息里 +// 没有口令,且技术原因还在(不是把原因抹空)。 +func TestRedactCoversRealPGXParseErrors(t *testing.T) { + cases := []struct{ dsn, keep string }{ + {"postgres://postgres:" + testSecret + "@127.0.0.1:notaport/weknora_test", "postgres"}, + {"postgres://postgres:" + testSecret + "@127.0.0.1:5432/weknora_test?sslmode=bogus&connect_timeout=abc", "connect_timeout"}, + {"host=127.0.0.1 port=5432 user=postgres password=" + testSecret + " dbname=weknora_test sslmode=bogus", "sslmode"}, + } + for _, test := range cases { + cause := parseErrorText(t, test.dsn) + message := Redact(test.dsn, cause) + if strings.Contains(message, testSecret) { + t.Fatalf("redacted message still contains the password: %q", message) + } + if !strings.Contains(message, test.keep) { + t.Fatalf("redaction lost diagnosability (%q missing): %q", test.keep, message) + } + } +} + +// 真正的回归点:任何一层把 DSN(或它的口令)原样回显进 cause 时,输出仍必须安全。 +// 这里用"把 DSN 直接打进 cause"来模拟这种回显,URL 与 keyword 两种形状都覆盖。 +func TestRedactCoversCausesThatEchoTheDSN(t *testing.T) { + for _, dsn := range []string{ + "postgres://postgres:" + testSecret + "@127.0.0.1:1/weknora_test?sslmode=disable", + "host=127.0.0.1 user=postgres password=" + testSecret + " dbname=weknora_test", + } { + cause := fmt.Errorf("dial failed for %s: connection refused", dsn).Error() + if !strings.Contains(cause, testSecret) { + t.Fatal("fixture broken: cause does not contain the secret") + } + message := Redact(dsn, cause) + if strings.Contains(message, testSecret) { + t.Fatalf("echoed DSN leaked the password: %q", message) + } + if !strings.Contains(message, "connection refused") { + t.Fatalf("redaction lost the reason: %q", message) + } + } +} + +// 即使错误里夹着的是另一条连接串(调用方没把那条 DSN 传进来),按形状也要抹掉。 +func TestRedactCoversUnknownDSNsByShape(t *testing.T) { + text := "failed: postgres://weknora:" + testSecret + "@db:5432/x?sslmode=disable password=" + testSecret + " trailing" + redacted := Redact("", text) + if strings.Contains(redacted, testSecret) { + t.Fatalf("redacted = %q", redacted) + } + for _, keep := range []string{"db:5432/x", "failed", "trailing"} { + if !strings.Contains(redacted, keep) { + t.Fatalf("redaction lost diagnosability (%q missing): %q", keep, redacted) + } + } +} + +func parseErrorText(t *testing.T, dsn string) string { + t.Helper() + _, err := pgxpool.ParseConfig(dsn) + if err == nil { + t.Fatalf("expected pgx to reject %q", dsn) + } + return err.Error() +} diff --git a/internal/platform/postgres/library_dead_letters.go b/internal/platform/postgres/library_dead_letters.go new file mode 100644 index 0000000..bd7960d --- /dev/null +++ b/internal/platform/postgres/library_dead_letters.go @@ -0,0 +1,149 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + + "knowledge-governance/internal/governance" +) + +// 死信解除只写治理库:上游 task_dead_letters 一个字节都不动。为什么这样: +// 上游没有"已解决"标记,purge/DELETE 会销毁失败证据;治理侧保留审计并绑定 +// 实时行指纹,行变了就不会再命中,阻断自动恢复。 + +// RecordDeadLetterRelease 幂等写入一条解除记录。同实例同指纹重复提交返回已存记录, +// 但内容不一致(重放引用/观测摘要不同)必须报冲突,不能悄悄覆盖审计。 +func (s *Store) RecordDeadLetterRelease(ctx context.Context, release governance.LibraryDeadLetterRelease) (governance.LibraryDeadLetterRelease, bool, error) { + if release.ID == "" || release.Instance == "" || release.Fingerprint == "" || release.Actor == "" { + return governance.LibraryDeadLetterRelease{}, false, errors.New("dead letter release requires identity, fingerprint and actor") + } + observation := release.Observation + if len(observation) == 0 { + observation = json.RawMessage(`{}`) + } + var id string + err := s.db.QueryRow(ctx, ` +INSERT INTO library_dead_letter_releases (id, instance, tenant_id, kb_id, dead_letter_id, fingerprint, + task_type, scope, scope_id, related_id, replay_queue, replay_task_id, replayed_at, replay_evidence, + observation_digest, observation, note, actor, created_at) +VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19) +ON CONFLICT (instance, fingerprint) DO NOTHING +RETURNING id`, + release.ID, release.Instance, release.TenantID, release.KnowledgeBaseID, release.DeadLetterID, + release.Fingerprint, release.TaskType, release.Scope, release.ScopeID, release.RelatedID, + release.ReplayQueue, release.ReplayTaskID, release.ReplayedAt.UTC(), release.ReplayEvidence, + release.ObservationDigest, observation, release.Note, release.Actor, release.CreatedAt.UTC()).Scan(&id) + if err == nil { + stored, loadErr := s.DeadLetterRelease(ctx, id) + return stored, false, loadErr + } + if !errors.Is(err, pgx.ErrNoRows) { + return governance.LibraryDeadLetterRelease{}, false, fmt.Errorf("record dead letter release: %w", err) + } + existing, loadErr := s.deadLetterReleaseByFingerprint(ctx, release.Instance, release.Fingerprint) + if loadErr != nil { + return governance.LibraryDeadLetterRelease{}, false, loadErr + } + if existing.ObservationDigest != release.ObservationDigest || existing.ReplayTaskID != release.ReplayTaskID || + !existing.ReplayedAt.Equal(release.ReplayedAt.UTC()) { + return existing, false, governance.ErrLibraryJobIdempotencyConflict + } + return existing, true, nil +} + +func (s *Store) deadLetterReleaseByFingerprint(ctx context.Context, instance, fingerprint string) (governance.LibraryDeadLetterRelease, error) { + var id string + if err := s.db.QueryRow(ctx, `SELECT id FROM library_dead_letter_releases WHERE instance=$1 AND fingerprint=$2`, + instance, fingerprint).Scan(&id); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return governance.LibraryDeadLetterRelease{}, errors.New("dead letter release not found") + } + return governance.LibraryDeadLetterRelease{}, fmt.Errorf("load dead letter release: %w", err) + } + return s.DeadLetterRelease(ctx, id) +} + +func (s *Store) DeadLetterRelease(ctx context.Context, id string) (governance.LibraryDeadLetterRelease, error) { + var release governance.LibraryDeadLetterRelease + var observation []byte + err := s.db.QueryRow(ctx, ` +SELECT id, instance, tenant_id, kb_id, dead_letter_id, fingerprint, task_type, scope, scope_id, related_id, + replay_queue, replay_task_id, replayed_at, replay_evidence, observation_digest, observation, note, actor, created_at +FROM library_dead_letter_releases WHERE id=$1`, id).Scan(&release.ID, &release.Instance, &release.TenantID, + &release.KnowledgeBaseID, &release.DeadLetterID, &release.Fingerprint, &release.TaskType, &release.Scope, + &release.ScopeID, &release.RelatedID, &release.ReplayQueue, &release.ReplayTaskID, &release.ReplayedAt, + &release.ReplayEvidence, &release.ObservationDigest, &observation, &release.Note, &release.Actor, &release.CreatedAt) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return governance.LibraryDeadLetterRelease{}, err + } + return governance.LibraryDeadLetterRelease{}, fmt.Errorf("load dead letter release: %w", err) + } + release.Observation = json.RawMessage(observation) + return release, nil +} + +// DeadLetterReleases 按实例倒序列出解除记录(审计视图)。 +func (s *Store) DeadLetterReleases(ctx context.Context, instance string, limit int) ([]governance.LibraryDeadLetterRelease, error) { + if instance == "" { + return nil, errors.New("dead letter releases require an instance") + } + if limit <= 0 || limit > 200 { + limit = 50 + } + rows, err := s.db.Query(ctx, ` +SELECT id FROM library_dead_letter_releases WHERE instance=$1 ORDER BY created_at DESC LIMIT $2`, instance, limit) + if err != nil { + return nil, fmt.Errorf("list dead letter releases: %w", err) + } + defer rows.Close() + ids := []string{} + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, fmt.Errorf("list dead letter releases: %w", err) + } + ids = append(ids, id) + } + if rows.Err() != nil { + return nil, fmt.Errorf("list dead letter releases: %w", rows.Err()) + } + result := make([]governance.LibraryDeadLetterRelease, 0, len(ids)) + for _, id := range ids { + release, loadErr := s.DeadLetterRelease(ctx, id) + if loadErr != nil { + return nil, loadErr + } + result = append(result, release) + } + return result, nil +} + +// DeadLetterReleaseFingerprints 是封存执行侧需要的唯一投影:只给指纹,不给证据。 +// worker 把它原样传给 agent,CLI 再对实时行逐字节复核。 +func (s *Store) DeadLetterReleaseFingerprints(ctx context.Context, instance string) ([]string, error) { + if instance == "" { + return nil, errors.New("dead letter release fingerprints require an instance") + } + rows, err := s.db.Query(ctx, `SELECT fingerprint FROM library_dead_letter_releases WHERE instance=$1 ORDER BY created_at`, instance) + if err != nil { + return nil, fmt.Errorf("list dead letter release fingerprints: %w", err) + } + defer rows.Close() + result := []string{} + for rows.Next() { + var fingerprint string + if err := rows.Scan(&fingerprint); err != nil { + return nil, fmt.Errorf("list dead letter release fingerprints: %w", err) + } + result = append(result, fingerprint) + } + return result, rows.Err() +} + +var _ = time.Time{} diff --git a/internal/platform/postgres/library_executor_status.go b/internal/platform/postgres/library_executor_status.go new file mode 100644 index 0000000..a77f98c --- /dev/null +++ b/internal/platform/postgres/library_executor_status.go @@ -0,0 +1,71 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + + "knowledge-governance/internal/libraryexecutor" +) + +// 为什么落库而不是 API 直接探测:agent socket 只挂在 worker 容器里,API 那个 +// LIBRARY_AGENT_SOCKET 即使配了也 dial 不到。谁真能执行就由谁观测,页面读结果。 +const executorStatusColumns = `instance, reachable, busy, degraded, identity, status_error, observed_at` + +// UpsertLibraryExecutorStatus 记录一次真实探测。降级/不可达同样要写: +// "最新一次观测是什么"比"最后一条好消息"更有用。 +func (s *Store) UpsertLibraryExecutorStatus(ctx context.Context, observation libraryexecutor.Observation, observer string, now time.Time) error { + identity, err := json.Marshal(observation.Identity) + if err != nil { + return fmt.Errorf("encode library executor identity: %w", err) + } + if observation.Instance == "" { + return errors.New("library executor observation needs an instance") + } + observedAt := observation.ObservedAt + if observedAt.IsZero() { + observedAt = now + } + _, err = s.db.Exec(ctx, ` +INSERT INTO library_executor_status (instance, reachable, busy, degraded, identity, status_error, observer, observed_at) +VALUES ($1,$2,$3,$4,$5::jsonb,$6,$7,$8) +ON CONFLICT (instance) DO UPDATE SET + reachable = EXCLUDED.reachable, + busy = EXCLUDED.busy, + degraded = EXCLUDED.degraded, + identity = EXCLUDED.identity, + status_error = EXCLUDED.status_error, + observer = EXCLUDED.observer, + observed_at = EXCLUDED.observed_at`, + observation.Instance, observation.Reachable, observation.Busy, observation.Degraded, + string(identity), observation.Error, observer, observedAt.UTC()) + if err != nil { + return fmt.Errorf("record library executor status: %w", err) + } + return nil +} + +// LibraryExecutorStatus 读某个实例最近一次观测;没有记录返回 nil(不是"可用")。 +func (s *Store) LibraryExecutorStatus(ctx context.Context, instance string) (*libraryexecutor.Observation, error) { + row := s.db.QueryRow(ctx, `SELECT `+executorStatusColumns+` FROM library_executor_status WHERE instance=$1`, instance) + var observation libraryexecutor.Observation + var raw []byte + err := row.Scan(&observation.Instance, &observation.Reachable, &observation.Busy, &observation.Degraded, + &raw, &observation.Error, &observation.ObservedAt) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("read library executor status: %w", err) + } + if len(raw) > 0 { + if err := json.Unmarshal(raw, &observation.Identity); err != nil { + return nil, fmt.Errorf("library executor identity unreadable: %w", err) + } + } + return &observation, nil +} diff --git a/internal/platform/postgres/library_executor_status_integration_test.go b/internal/platform/postgres/library_executor_status_integration_test.go new file mode 100644 index 0000000..8f81388 --- /dev/null +++ b/internal/platform/postgres/library_executor_status_integration_test.go @@ -0,0 +1,70 @@ +package postgres + +import ( + "context" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/libraryexecutor" +) + +// 执行器状态是"最新一次观测",不是"最后一条好消息": +// 降级与不可达必须覆盖上一次的健康记录,否则页面会一直显示可发起。 +func TestLibraryExecutorStatusKeepsTheLatestObservation(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatalf("connect: %v", err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatalf("migrate: %v", err) + } + store := NewStore(pool) + now := time.Date(2026, 10, 29, 8, 0, 0, 0, time.UTC) + + if missing, err := store.LibraryExecutorStatus(ctx, "kgmaint-weknora"); err != nil || missing != nil { + t.Fatalf("missing row = %+v, %v", missing, err) + } + + healthy := libraryexecutor.Observation{ + Instance: "kgmaint-weknora", Reachable: true, ObservedAt: now, + Identity: libraryexecutor.Identity{Project: "kgmaint-weknora", SourceApp: "kgmaint-app", ArchiveRoot: "/tmp/kg-e2e/archive"}, + } + if err := store.UpsertLibraryExecutorStatus(ctx, healthy, "worker-1", now); err != nil { + t.Fatalf("upsert healthy: %v", err) + } + read, err := store.LibraryExecutorStatus(ctx, "kgmaint-weknora") + if err != nil { + t.Fatalf("read healthy: %v", err) + } + if read == nil || !read.Reachable || read.Identity.Project != "kgmaint-weknora" || read.Identity.SourceApp != "kgmaint-app" { + t.Fatalf("healthy row = %+v", read) + } + if !read.ObservedAt.Equal(now) { + t.Fatalf("observed_at = %v, want %v", read.ObservedAt, now) + } + + degraded := healthy + degraded.Reachable = false + degraded.ObservedAt = now.Add(30 * time.Second) + degraded.Degraded = "" + degraded.Error = "library agent is unreachable" + if err := store.UpsertLibraryExecutorStatus(ctx, degraded, "worker-1", degraded.ObservedAt); err != nil { + t.Fatalf("upsert unreachable: %v", err) + } + read, err = store.LibraryExecutorStatus(ctx, "kgmaint-weknora") + if err != nil || read == nil { + t.Fatalf("read after unreachable: %+v %v", read, err) + } + if read.Reachable || read.Error != "library agent is unreachable" || !read.ObservedAt.Equal(degraded.ObservedAt) { + t.Fatalf("row kept a stale good state: %+v", read) + } +} diff --git a/internal/platform/postgres/library_instance_maintenance.go b/internal/platform/postgres/library_instance_maintenance.go index 6a6b468..8b82cc5 100644 --- a/internal/platform/postgres/library_instance_maintenance.go +++ b/internal/platform/postgres/library_instance_maintenance.go @@ -65,7 +65,15 @@ ON CONFLICT (instance) DO UPDATE SET services = EXCLUDED.services, ingress_state = EXCLUDED.ingress_state, state = EXCLUDED.state, - original_service_state = EXCLUDED.original_service_state, + -- 同一作业重入(崩溃接管)时不许用"现在已停"覆盖第一次记下的原状态: + -- 重入时服务已经被我们停住,再读一次只会读到 stopped,写回去会让恢复阶段 + -- 以为写入者本来就没跑,于是永远不启动它。 + original_service_state = CASE + WHEN library_instance_maintenance.owner_job_id = EXCLUDED.owner_job_id + AND library_instance_maintenance.original_service_state <> '' + THEN library_instance_maintenance.original_service_state + ELSE EXCLUDED.original_service_state + END, note = EXCLUDED.note, updated_at = EXCLUDED.updated_at, released_at = NULL @@ -92,6 +100,9 @@ WHERE instance=$1 AND owner_job_id=$2 AND released_at IS NULL`, instance, ownerJ return fmt.Errorf("update instance maintenance: %w", err) } if tag.RowsAffected() == 0 { + if s.releasedByThisJob(ctx, "library_instance_maintenance", "instance", instance, ownerJobID) { + return ErrMaintenanceReleased + } return errors.New("instance maintenance not held by this job") } return nil @@ -108,6 +119,9 @@ WHERE instance=$1 AND owner_job_id=$2 AND released_at IS NULL`, instance, ownerJ return fmt.Errorf("update instance maintenance ingress: %w", err) } if tag.RowsAffected() == 0 { + if s.releasedByThisJob(ctx, "library_instance_maintenance", "instance", instance, ownerJobID) { + return ErrMaintenanceReleased + } return errors.New("instance maintenance not held by this job") } return nil @@ -121,6 +135,9 @@ WHERE instance=$1 AND owner_job_id=$2 AND released_at IS NULL`, instance, ownerJ return fmt.Errorf("release instance maintenance: %w", err) } if tag.RowsAffected() == 0 { + if s.releasedByThisJob(ctx, "library_instance_maintenance", "instance", instance, ownerJobID) { + return ErrMaintenanceReleased + } return errors.New("instance maintenance not held by this job") } return nil diff --git a/internal/platform/postgres/library_job_mutex_integration_test.go b/internal/platform/postgres/library_job_mutex_integration_test.go new file mode 100644 index 0000000..4a25ce8 --- /dev/null +++ b/internal/platform/postgres/library_job_mutex_integration_test.go @@ -0,0 +1,85 @@ +package postgres + +import ( + "context" + "errors" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/governance" +) + +// 迁移 075 之后,"建立基线"必须与同步/恢复共用同一把写互斥: +// 同一 scope 只允许一个在飞写作业,且互斥由数据库唯一索引保证(不靠页面)。 +// 这条用例跑在真实 PostgreSQL 上(TEST_DATABASE_URL),不是内存假实现。 +func TestLibraryWriteMutexCoversBaselineJobs(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatalf("connect: %v", err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatalf("migrate: %v", err) + } + store := NewStore(pool) + now := time.Now().UTC() + digest, err := governance.LibraryRequestDigest("internal", governance.LibraryJobBaseline, + map[string]any{"scope": "internal"}) + if err != nil { + t.Fatal(err) + } + baseline := governance.LibraryJob{ + ID: "libbase-integration-1", Scope: "internal", Kind: governance.LibraryJobBaseline, + IdempotencyKey: "library.baseline:internal:1", RequestDigest: digest, Actor: "admin", + } + created, reused, err := store.CreateLibraryJob(ctx, baseline, governance.BaselineStages(), now) + if err != nil || reused { + t.Fatalf("first baseline job: err=%v reused=%v", err, reused) + } + if len(created.Stages) == 0 { + t.Fatalf("stages not created: %+v", created) + } + + // 同一幂等键 + 同一请求 = 复用同一作业,不产生第二行。 + same, reused, err := store.CreateLibraryJob(ctx, baseline, governance.BaselineStages(), now) + if err != nil || !reused || same.ID != baseline.ID { + t.Fatalf("idempotent retry: err=%v reused=%v id=%q", err, reused, same.ID) + } + + // 另一个基线作业(不同幂等键)在同一 scope 上必须被互斥挡住。 + second := baseline + second.ID = "libbase-integration-2" + second.IdempotencyKey = "library.baseline:internal:2" + if _, _, err := store.CreateLibraryJob(ctx, second, governance.BaselineStages(), now); !errors.Is(err, governance.ErrLibraryJobActive) { + t.Fatalf("second baseline during first: err=%v", err) + } + + // 同步作业与基线共用同一把锁:cross-kind 也必须被挡。 + syncDigest, err := governance.LibraryRequestDigest("internal", governance.LibraryJobSync, + map[string]any{"target_snapshot_id": "snap-1"}) + if err != nil { + t.Fatal(err) + } + if _, _, err := store.CreateLibraryJob(ctx, governance.LibraryJob{ + ID: "libsync-integration-1", Scope: "internal", Kind: governance.LibraryJobSync, + IdempotencyKey: "library.sync:internal:snap-1", RequestDigest: syncDigest, TargetSnapshotID: "snap-1", + }, governance.SyncStages(), now); !errors.Is(err, governance.ErrLibraryJobActive) { + t.Fatalf("sync during baseline: err=%v", err) + } + + // 第一个作业失败后互斥必须释放,新作业可以进(否则第一次失败就永久堵死)。 + if err := store.AdvanceLibraryJob(ctx, baseline.ID, governance.LibraryJobFailed, "capture", "test", "", "", "", now.Add(time.Second)); err != nil { + t.Fatalf("advance: %v", err) + } + if _, _, err := store.CreateLibraryJob(ctx, second, governance.BaselineStages(), now.Add(2*time.Second)); err != nil { + t.Fatalf("baseline after failure: %v", err) + } +} diff --git a/internal/platform/postgres/library_jobs.go b/internal/platform/postgres/library_jobs.go index 83c7b92..66ad5fb 100644 --- a/internal/platform/postgres/library_jobs.go +++ b/internal/platform/postgres/library_jobs.go @@ -261,7 +261,7 @@ func (s *Store) InterruptedLibraryJobs(ctx context.Context) ([]governance.Librar rows, err := s.db.Query(ctx, ` SELECT lj.id FROM library_jobs lj LEFT JOIN jobs j ON j.id=lj.queue_job_id -WHERE lj.kind IN ('sync','restore') +WHERE lj.kind IN ('sync','restore','baseline') AND ( -- 业务作业在跑、队列已经没了/停了/租约过期 (lj.state='running' AND (j.id IS NULL OR j.state IN ('failed','completed','cancelled','dead') diff --git a/internal/platform/postgres/library_maintenance.go b/internal/platform/postgres/library_maintenance.go index 27d6a34..72db8aa 100644 --- a/internal/platform/postgres/library_maintenance.go +++ b/internal/platform/postgres/library_maintenance.go @@ -10,6 +10,19 @@ import ( "github.com/jackc/pgx/v5" ) +var ErrMaintenanceReleased = errors.New("maintenance already released by this job") + +// releasedByThisJob 判断"零行更新"是因为本作业已经释放过,还是根本没持有。 +func (s *Store) releasedByThisJob(ctx context.Context, table, keyColumn, key, ownerJobID string) bool { + var owner string + var released *time.Time + query := "SELECT owner_job_id, released_at FROM " + table + " WHERE " + keyColumn + "=$1" + if err := s.db.QueryRow(ctx, query, key).Scan(&owner, &released); err != nil { + return false + } + return owner == ownerJobID && released != nil +} + // LibraryMaintenance 是目标维护窗口的持有记录:一个 scope 只能有一个持有者。 // // Level 说明这次维护到底动了什么: @@ -68,7 +81,13 @@ ON CONFLICT (scope) DO UPDATE SET services = EXCLUDED.services, ingress_state = EXCLUDED.ingress_state, state = EXCLUDED.state, - original_service_state = EXCLUDED.original_service_state, + -- 同上:原运行状态只由第一次进入窗口时决定,重入不能覆盖。 + original_service_state = CASE + WHEN library_maintenance.owner_job_id = EXCLUDED.owner_job_id + AND library_maintenance.original_service_state <> '' + THEN library_maintenance.original_service_state + ELSE EXCLUDED.original_service_state + END, note = EXCLUDED.note, updated_at = EXCLUDED.updated_at, released_at = NULL @@ -97,6 +116,9 @@ WHERE scope=$1 AND owner_job_id=$2 AND released_at IS NULL`, scope, ownerJobID, return fmt.Errorf("update library maintenance: %w", err) } if tag.RowsAffected() == 0 { + if s.releasedByThisJob(ctx, "library_maintenance", "scope", scope, ownerJobID) { + return ErrMaintenanceReleased + } return errors.New("library maintenance not held by this job") } return nil @@ -114,6 +136,9 @@ WHERE scope=$1 AND owner_job_id=$2 AND released_at IS NULL`, scope, ownerJobID, return fmt.Errorf("update library maintenance ingress: %w", err) } if tag.RowsAffected() == 0 { + if s.releasedByThisJob(ctx, "library_maintenance", "scope", scope, ownerJobID) { + return ErrMaintenanceReleased + } return errors.New("library maintenance not held by this job") } return nil @@ -127,6 +152,9 @@ WHERE scope=$1 AND owner_job_id=$2 AND released_at IS NULL`, scope, ownerJobID, return fmt.Errorf("release library maintenance: %w", err) } if tag.RowsAffected() == 0 { + if s.releasedByThisJob(ctx, "library_maintenance", "scope", scope, ownerJobID) { + return ErrMaintenanceReleased + } return errors.New("library maintenance not held by this job") } return nil diff --git a/internal/platform/postgres/library_maintenance_integration_test.go b/internal/platform/postgres/library_maintenance_integration_test.go index 1dc779c..8c3fa1d 100644 --- a/internal/platform/postgres/library_maintenance_integration_test.go +++ b/internal/platform/postgres/library_maintenance_integration_test.go @@ -72,6 +72,86 @@ func TestLibraryMaintenanceOwnershipAcrossRelease(t *testing.T) { } } +// 同一作业重入(崩溃后的接管)必须保留第一次记下的"原运行状态": +// 重入时目标已经被停住,此时再去读一次只会读到"已停",把它写回去会让恢复阶段 +// 以为写入者本来就没跑,于是永远不启动它 —— 目标就这么一直停着。 +func TestMaintenanceReentryKeepsOriginalServiceState(t *testing.T) { + databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if databaseURL == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + t.Fatalf("connect: %v", err) + } + defer pool.Close() + if err := ApplyMigrations(ctx, pool); err != nil { + t.Fatalf("migrate: %v", err) + } + store := NewStore(pool) + now := time.Now().UTC() + + first, err := store.AcquireLibraryMaintenance(ctx, LibraryMaintenance{ + Scope: "internal", OwnerJobID: "job-reentry", TargetIdentity: "instance=kgseal", + Level: "instance", TargetFingerprint: "fp", Services: []string{"app"}, + State: "acquired", OriginalServiceState: "app=running", Note: "first", + }, now) + if err != nil { + t.Fatalf("first acquire: %v", err) + } + if first.OriginalServiceState != "app=running" { + t.Fatalf("first original state = %q", first.OriginalServiceState) + } + reentered, err := store.AcquireLibraryMaintenance(ctx, LibraryMaintenance{ + Scope: "internal", OwnerJobID: "job-reentry", Level: "instance", + State: "applying", OriginalServiceState: "app=stopped", Note: "re-entry after crash", + }, now.Add(time.Second)) + if err != nil { + t.Fatalf("same-owner re-entry: %v", err) + } + if reentered.OriginalServiceState != "app=running" { + t.Fatalf("re-entry overwrote the original state: got %q, want app=running", reentered.OriginalServiceState) + } + + instFirst, err := store.AcquireInstanceMaintenance(ctx, LibraryInstanceMaintenance{ + Instance: "kgseal-instance-reentry", OwnerJobID: "job-reentry", OwnerScope: "internal", + TargetFingerprint: "fp", Level: "instance", Services: []string{"app"}, + State: "acquired", OriginalServiceState: "app=running", Note: "first", + }, now) + if err != nil { + t.Fatalf("instance acquire: %v", err) + } + if instFirst.OriginalServiceState != "app=running" { + t.Fatalf("instance first original state = %q", instFirst.OriginalServiceState) + } + instReentered, err := store.AcquireInstanceMaintenance(ctx, LibraryInstanceMaintenance{ + Instance: "kgseal-instance-reentry", OwnerJobID: "job-reentry", OwnerScope: "internal", + Level: "instance", State: "applying", OriginalServiceState: "app=stopped", + }, now.Add(2*time.Second)) + if err != nil { + t.Fatalf("instance same-owner re-entry: %v", err) + } + if instReentered.OriginalServiceState != "app=running" { + t.Fatalf("instance re-entry overwrote the original state: got %q", instReentered.OriginalServiceState) + } + + // 新作业接管已释放的窗口时,必须接受它自己读到的当前状态(不能沿用上一个作业的)。 + if err := store.ReleaseInstanceMaintenance(ctx, "kgseal-instance-reentry", "job-reentry", now.Add(3*time.Second)); err != nil { + t.Fatalf("release instance: %v", err) + } + next, err := store.AcquireInstanceMaintenance(ctx, LibraryInstanceMaintenance{ + Instance: "kgseal-instance-reentry", OwnerJobID: "job-next", OwnerScope: "internal", + Level: "instance", State: "acquired", OriginalServiceState: "app=stopped", + }, now.Add(4*time.Second)) + if err != nil { + t.Fatalf("next owner acquire: %v", err) + } + if next.OriginalServiceState != "app=stopped" { + t.Fatalf("new owner must record its own observation: got %q", next.OriginalServiceState) + } +} + // 实例级占用必须跨 scope 互斥:internal 持有期间 external 也进不来。 // (旧实现按 scope 建行,两个 scope 各持一个锁,会同时停/起同一个实例。) func TestInstanceMaintenanceExcludesAcrossScopes(t *testing.T) { diff --git a/internal/platform/postgres/migrations/000073_library_dead_letter_releases.down.sql b/internal/platform/postgres/migrations/000073_library_dead_letter_releases.down.sql new file mode 100644 index 0000000..25e895e --- /dev/null +++ b/internal/platform/postgres/migrations/000073_library_dead_letter_releases.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS library_dead_letter_releases; \ No newline at end of file diff --git a/internal/platform/postgres/migrations/000073_library_dead_letter_releases.up.sql b/internal/platform/postgres/migrations/000073_library_dead_letter_releases.up.sql new file mode 100644 index 0000000..4c609e3 --- /dev/null +++ b/internal/platform/postgres/migrations/000073_library_dead_letter_releases.up.sql @@ -0,0 +1,29 @@ +-- 死信解除:上游 task_dead_letters 没有"已解决"标记,purge/DELETE 都会销毁失败证据。 +-- 这里只记录"某条精确死信已被审计解除",不复制也不删除上游行:解除绑定实时行的 +-- 逐字节指纹、真实重放引用、源侧观测摘要与操作人;行被改过或出现新行都不会命中。 +CREATE TABLE library_dead_letter_releases ( + id VARCHAR(64) PRIMARY KEY, + instance VARCHAR(128) NOT NULL, + tenant_id BIGINT NOT NULL, + kb_id VARCHAR(64) NOT NULL, + dead_letter_id BIGINT NOT NULL, + fingerprint VARCHAR(80) NOT NULL, + task_type VARCHAR(64) NOT NULL DEFAULT '', + scope VARCHAR(32) NOT NULL DEFAULT '', + scope_id VARCHAR(64) NOT NULL DEFAULT '', + related_id VARCHAR(64) NOT NULL DEFAULT '', + replay_queue VARCHAR(64) NOT NULL, + replay_task_id VARCHAR(128) NOT NULL, + replayed_at TIMESTAMPTZ NOT NULL, + replay_evidence TEXT NOT NULL, + observation_digest VARCHAR(80) NOT NULL, + observation JSONB NOT NULL, + note TEXT NOT NULL DEFAULT '', + actor VARCHAR(128) NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + -- 同一个实例里同一条死信只能解除一次;重复提交返回已存记录。 + UNIQUE (instance, fingerprint) +); + +CREATE INDEX idx_library_dead_letter_releases_instance + ON library_dead_letter_releases (instance, created_at DESC); \ No newline at end of file diff --git a/internal/platform/postgres/migrations/000074_library_executor_status.down.sql b/internal/platform/postgres/migrations/000074_library_executor_status.down.sql new file mode 100644 index 0000000..5441531 --- /dev/null +++ b/internal/platform/postgres/migrations/000074_library_executor_status.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS library_executor_status; \ No newline at end of file diff --git a/internal/platform/postgres/migrations/000074_library_executor_status.up.sql b/internal/platform/postgres/migrations/000074_library_executor_status.up.sql new file mode 100644 index 0000000..cf92a02 --- /dev/null +++ b/internal/platform/postgres/migrations/000074_library_executor_status.up.sql @@ -0,0 +1,16 @@ +-- 封存执行器的运行时状态:页面上的"可以发起封存"必须来自真实观测,而不是 +-- "配置里写了 socket 路径"。正式栈曾出现过 agent 顶着正式单元名却在冻结候选实例: +-- 配置非空、页面显示可发起、真发起却封错了库。 +-- +-- worker 探测 agent 的 health(不跑子进程、不碰源库),把"它指向哪个实例、 +-- 是否降级/正在跑、什么时候观测的"落库;API 只读这张表并核对身份,不再猜。 +CREATE TABLE IF NOT EXISTS library_executor_status ( + instance VARCHAR(128) PRIMARY KEY, + reachable BOOLEAN NOT NULL DEFAULT FALSE, + busy BOOLEAN NOT NULL DEFAULT FALSE, + degraded TEXT NOT NULL DEFAULT '', + identity JSONB NOT NULL DEFAULT '{}'::jsonb, + status_error TEXT NOT NULL DEFAULT '', + observer VARCHAR(64) NOT NULL DEFAULT '', + observed_at TIMESTAMPTZ NOT NULL +); \ No newline at end of file diff --git a/internal/platform/postgres/migrations/000075_library_baseline_kind.down.sql b/internal/platform/postgres/migrations/000075_library_baseline_kind.down.sql new file mode 100644 index 0000000..a863f58 --- /dev/null +++ b/internal/platform/postgres/migrations/000075_library_baseline_kind.down.sql @@ -0,0 +1,8 @@ +DELETE FROM library_jobs WHERE kind='baseline'; +ALTER TABLE library_jobs DROP CONSTRAINT IF EXISTS library_jobs_kind_check; +ALTER TABLE library_jobs ADD CONSTRAINT library_jobs_kind_check + CHECK (kind IN ('seal','sync','restore')); +DROP INDEX IF EXISTS library_jobs_write_target; +CREATE UNIQUE INDEX library_jobs_write_target + ON library_jobs(scope) + WHERE state IN ('pending','running') AND kind IN ('sync','restore'); \ No newline at end of file diff --git a/internal/platform/postgres/migrations/000075_library_baseline_kind.up.sql b/internal/platform/postgres/migrations/000075_library_baseline_kind.up.sql new file mode 100644 index 0000000..741515e --- /dev/null +++ b/internal/platform/postgres/migrations/000075_library_baseline_kind.up.sql @@ -0,0 +1,12 @@ +-- 允许"建立目标权威基线"作为一种独立业务作业:首次发布的目标非空又没有生效版本时, +-- 只能通过真实捕获 + 校验登记基线,不能手写版本记录。 +-- 基线会在目标上开维护窗口(停写入者),所以它与 sync/restore 共用同一把写互斥: +-- 同一 scope 不允许同时存在两种写作业。 +ALTER TABLE library_jobs DROP CONSTRAINT IF EXISTS library_jobs_kind_check; +ALTER TABLE library_jobs ADD CONSTRAINT library_jobs_kind_check + CHECK (kind IN ('seal','sync','restore','baseline')); + +DROP INDEX IF EXISTS library_jobs_write_target; +CREATE UNIQUE INDEX library_jobs_write_target + ON library_jobs(scope) + WHERE state IN ('pending','running') AND kind IN ('sync','restore','baseline'); \ No newline at end of file diff --git a/internal/platform/postgres/migrations_test.go b/internal/platform/postgres/migrations_test.go index 6e64155..e638bba 100644 --- a/internal/platform/postgres/migrations_test.go +++ b/internal/platform/postgres/migrations_test.go @@ -11,35 +11,51 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) { if err != nil { t.Fatalf("loadMigrations() error = %v", err) } - if len(migrations) != 72 { - t.Fatalf("migration count = %d, want 72", len(migrations)) + if len(migrations) != 75 { + t.Fatalf("migration count = %d, want 75", len(migrations)) + } + // 75 允许"建立目标权威基线"作为独立业务作业,并与 sync/restore 共用写互斥。 + if baseline := migrations[len(migrations)-1]; baseline.Version != 75 || + !strings.Contains(baseline.UpSQL, "library_jobs_write_target") || + !strings.Contains(baseline.UpSQL, "'baseline'") { + t.Fatalf("migration 75 = %#v", baseline) + } + // 74 记录封存执行器的运行时状态:页面可用性来自真实观测,不来自配置字符串。 + if executor := migrations[len(migrations)-2]; executor.Version != 74 || + !strings.Contains(executor.UpSQL, "library_executor_status") { + t.Fatalf("migration 74 = %#v", executor) + } + // 73 记录死信精确解除:保留上游行,只绑定指纹、重放引用与观测摘要。 + if release := migrations[len(migrations)-3]; release.Version != 73 || + !strings.Contains(release.UpSQL, "library_dead_letter_releases") { + t.Fatalf("migration 73 = %#v", release) } // 72 让维护占用按实例互斥(internal/external 不再各有各的锁)。 - if instance := migrations[len(migrations)-1]; instance.Version != 72 || + if instance := migrations[len(migrations)-4]; instance.Version != 72 || !strings.Contains(instance.UpSQL, "library_instance_maintenance") { t.Fatalf("migration 72 = %#v", instance) } // 71 让维护记录能区分实例级维护与仅治理库互斥。 - if level := migrations[len(migrations)-2]; level.Version != 71 || + if level := migrations[len(migrations)-5]; level.Version != 71 || !strings.Contains(level.UpSQL, "level") || !strings.Contains(level.UpSQL, "ingress_state") { t.Fatalf("migration 71 = %#v", level) } - if maint := migrations[len(migrations)-3]; maint.Version != 70 || !strings.Contains(maint.UpSQL, "library_maintenance") { + if maint := migrations[len(migrations)-6]; maint.Version != 70 || !strings.Contains(maint.UpSQL, "library_maintenance") { t.Fatalf("migration 70 = %#v", maint) } - if tail := migrations[len(migrations)-4]; tail.Version != 69 || + if tail := migrations[len(migrations)-7]; tail.Version != 69 || !strings.Contains(tail.UpSQL, "library_jobs_write_target") { t.Fatalf("migration 69 = %#v", tail) } - if restore := migrations[len(migrations)-5]; restore.Version != 68 || !strings.Contains(restore.UpSQL, "restore") { + if restore := migrations[len(migrations)-8]; restore.Version != 68 || !strings.Contains(restore.UpSQL, "restore") { t.Fatalf("migration 68 = %#v", restore) } - if last := migrations[len(migrations)-6]; last.Version != 67 || + if last := migrations[len(migrations)-9]; last.Version != 67 || !strings.Contains(last.UpSQL, "library_release_authorizations") || !strings.Contains(last.UpSQL, "params_digest") { t.Fatalf("migration 67 = %#v", last) } - if flow := migrations[len(migrations)-7]; flow.Version != 66 || + if flow := migrations[len(migrations)-10]; flow.Version != 66 || !strings.Contains(flow.UpSQL, "CREATE TABLE library_versions") || !strings.Contains(flow.UpSQL, "library_jobs_active") { t.Fatalf("migration 66 = %#v", flow) diff --git a/internal/releaseadapter/v071/library_archive.go b/internal/releaseadapter/v071/library_archive.go index e6fa069..750fa48 100644 --- a/internal/releaseadapter/v071/library_archive.go +++ b/internal/releaseadapter/v071/library_archive.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "strings" "time" @@ -86,7 +87,9 @@ func ArchiveLibraryCandidate(ctx context.Context, archive *librarysnapshot.Archi } reader, err := objects.Open(ctx, resource.Provider, resource.PhysicalPath) if err != nil { - return "", errors.New("read library resource") + // 只写"read library resource"会让整条封存停在无信息的失败上:带上行 id + // 与底层原因(不含凭据),才能判断是对象缺失、桶错还是端点解析不到。 + return "", fmt.Errorf("read library resource %s: %w", resource.ID, err) } part, readErr := archive.Put(reader) closeErr := reader.Close() diff --git a/internal/releaseadapter/v071/library_backup_store.go b/internal/releaseadapter/v071/library_backup_store.go index 7cbffd0..6970b0a 100644 --- a/internal/releaseadapter/v071/library_backup_store.go +++ b/internal/releaseadapter/v071/library_backup_store.go @@ -113,6 +113,12 @@ func (s *ProductionBackupStore) Prepare(ctx context.Context, _ string, current l if err != nil { return libraryrelease.BackupRef{}, err } + // 整实例指纹:恢复之后要拿它证明"多写的、少写的都没有"。只按目标范围比, + // 写到范围外的行会被漏掉(真实事故:恢复报成功但目标多 67 行)。 + universeBefore, err := universeFingerprint(ctx, s.pool) + if err != nil { + return libraryrelease.BackupRef{}, err + } pg, err := ExportLibrary(ctx, s.pool, s.identity) if err != nil { return libraryrelease.BackupRef{}, fmt.Errorf("读取生产整库失败: %w", err) @@ -154,6 +160,13 @@ func (s *ProductionBackupStore) Prepare(ctx context.Context, _ string, current l return libraryrelease.BackupRef{}, fmt.Errorf("预备份期间生产 %s 内容发生变化,切点不一致", table) } } + universeAfter, err := universeFingerprint(ctx, s.pool) + if err != nil { + return libraryrelease.BackupRef{}, err + } + if diff := universeDiff(universeBefore, universeAfter); diff != "" { + return libraryrelease.BackupRef{}, fmt.Errorf("预备份期间整实例知识域发生变化(%s),切点不一致", diff) + } for table, want := range current.Counts { if table == "index" { continue @@ -171,7 +184,7 @@ func (s *ProductionBackupStore) Prepare(ctx context.Context, _ string, current l total += part.Bytes } return libraryrelease.BackupRef{ID: "backup-" + snapshotID, SnapshotID: snapshotID, Bytes: total, - CreatedAt: createdAt}, nil + CreatedAt: createdAt, Universe: universeBefore}, nil } func (s *ProductionBackupStore) rowCounts(ctx context.Context) (map[string]int64, error) { diff --git a/internal/releaseadapter/v071/library_capture.go b/internal/releaseadapter/v071/library_capture.go index f3c72bb..51a5425 100644 --- a/internal/releaseadapter/v071/library_capture.go +++ b/internal/releaseadapter/v071/library_capture.go @@ -20,6 +20,9 @@ type LibraryCaptureRequest struct { // it differs from Compatibility, so a model/index change between the // preflight read and the maintenance window cannot be sealed silently. VerifyContract func(context.Context) error + // ReleasedDeadLetters 是治理平台已审计的精确解除指纹。它只把"这些确切的行" + // 从死信阻断里摘出来:实时行必须逐字节命中,否则照旧阻断。 + ReleasedDeadLetters []string } // CaptureLibrary connects maintenance to the actual PG/PIT/object archive path. @@ -33,7 +36,7 @@ func CaptureLibrary(ctx context.Context, freeze *libraryfreeze.Coordinator, arch return libraryfreeze.Result{}, errors.New("invalid library capture configuration") } result, err := freeze.RunPrepared(ctx, req.JobID, func(ctx context.Context) error { - if err := checkLibrarySourceSettlement(ctx, db); err != nil { + if err := checkLibrarySourceSettlementReleased(ctx, db, req.ReleasedDeadLetters); err != nil { return err } if err := sourceReady(ctx); err != nil { @@ -41,10 +44,10 @@ func CaptureLibrary(ctx context.Context, freeze *libraryfreeze.Coordinator, arch } // Check readiness before a maintenance window, then repeat the export inside // that window: work can arrive between preflight and the graceful stop. - _, err := ExportLibrary(ctx, db, req.Identity) + _, err := ExportLibraryWithReleases(ctx, db, req.Identity, req.ReleasedDeadLetters) return err }, func(ctx context.Context) (string, error) { - if err := checkLibrarySourceSettlement(ctx, db); err != nil { + if err := checkLibrarySourceSettlementReleased(ctx, db, req.ReleasedDeadLetters); err != nil { return "", err } if err := sourceReady(ctx); err != nil { @@ -55,7 +58,7 @@ func CaptureLibrary(ctx context.Context, freeze *libraryfreeze.Coordinator, arch return "", err } } - pg, err := ExportLibrary(ctx, db, req.Identity) + pg, err := ExportLibraryWithReleases(ctx, db, req.Identity, req.ReleasedDeadLetters) if err != nil { return "", err } diff --git a/internal/releaseadapter/v071/library_export.go b/internal/releaseadapter/v071/library_export.go index 7993325..c1ff3c1 100644 --- a/internal/releaseadapter/v071/library_export.go +++ b/internal/releaseadapter/v071/library_export.go @@ -15,6 +15,12 @@ type librarySnapshotDB interface { // It does not certify semantic quality, freeze Elasticsearch/objects, or authorize // publication. Those require a separately verified, complete library version. func ExportLibrary(ctx context.Context, db librarySnapshotDB, identity PostgresIdentity) (PostgresSnapshot, error) { + return ExportLibraryWithReleases(ctx, db, identity, nil) +} + +// ExportLibraryWithReleases 与 ExportLibrary 相同,但允许调用方带上精确解除的死信 +// 指纹(只影响死信这一项判定,其余结算条件不变)。 +func ExportLibraryWithReleases(ctx context.Context, db librarySnapshotDB, identity PostgresIdentity, released []string) (PostgresSnapshot, error) { if db == nil || identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) || identity.KnowledgeID != "" { return PostgresSnapshot{}, errors.New("invalid library snapshot scope") } @@ -40,7 +46,17 @@ func ExportLibrary(ctx context.Context, db librarySnapshotDB, identity PostgresI return PostgresSnapshot{}, errors.New("read library settlement") } if unsettled != 0 || pending != 0 || failed != 0 { - return PostgresSnapshot{}, errors.New("library has unfinished or failed processing; semantic acceptance is still required after settlement") + if failed != 0 && unsettled == 0 && pending == 0 { + rows, rowsErr := sourceDeadLetterRows(ctx, tx, identity.TenantID, identity.KnowledgeBaseID) + if rowsErr != nil { + return PostgresSnapshot{}, rowsErr + } + if releasedErr := requireReleased(rows, released); releasedErr != nil { + return PostgresSnapshot{}, releasedErr + } + } else { + return PostgresSnapshot{}, errors.New("library has unfinished or failed processing; semantic acceptance is still required after settlement") + } } exporter := PostgresExporter{db: tx} var snapshot PostgresSnapshot diff --git a/internal/releaseadapter/v071/library_export_test.go b/internal/releaseadapter/v071/library_export_test.go index 4293d20..4b8a012 100644 --- a/internal/releaseadapter/v071/library_export_test.go +++ b/internal/releaseadapter/v071/library_export_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "knowledge-governance/internal/librarydeadletter" "strings" "testing" @@ -102,9 +103,16 @@ func (d *libraryFixtureTx) Query(_ context.Context, q string, _ ...any) (pgx.Row if d.queryError { return nil, errors.New("fixture failure") } + // 死信行按真实形状返回:解除判定必须逐字节比对指纹,空结果会让"存在即阻断"失效。 + if strings.Contains(q, "to_jsonb(d)") && d.failed > 0 { + return &jsonRows{values: []json.RawMessage{json.RawMessage(fixtureDeadLetter)}}, nil + } return &jsonRows{values: []json.RawMessage{}}, nil } +// fixtureDeadLetter 与 PostgreSQL to_jsonb(task_dead_letters) 的键序/形状一致。 +const fixtureDeadLetter = `{"id": 1, "payload": {"knowledge_id": "deleted-doc"}, "scope": "knowledge", "scope_id": "deleted-doc", "failed_at": "2026-10-28T10:00:00+00:00", "last_error": "llm timeout", "task_type": "wiki:ingest", "tenant_id": 10000, "fail_count": 3, "related_id": "deleted-doc"}` + type libraryFixtureRow struct { d *libraryFixtureTx sql string @@ -128,3 +136,24 @@ func (r libraryFixtureRow) Scan(dest ...any) error { } return nil } + +// 解除只能按精确指纹生效:同一条死信换个字节(或没有解除)都必须继续阻断。 +func TestLibraryExportReleasesOnlyExactDeadLetterFingerprint(t *testing.T) { + db := newLibraryFixture() + db.tx.failed = 1 + identity := PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"} + if _, err := ExportLibrary(context.Background(), db, identity); err == nil { + t.Fatal("unreleased dead letter did not block the export") + } + if _, err := ExportLibraryWithReleases(context.Background(), db, identity, + []string{"sha256:0000000000000000000000000000000000000000000000000000000000000000"}); err == nil { + t.Fatal("a stale fingerprint released the dead letter") + } + released := librarydeadletter.Fingerprint([]byte(fixtureDeadLetter)) + if _, err := ExportLibraryWithReleases(context.Background(), db, identity, []string{released}); err != nil { + t.Fatalf("exact released fingerprint still blocked: %v", err) + } + if !db.tx.committed { + t.Fatal("released export did not commit its snapshot") + } +} diff --git a/internal/releaseadapter/v071/library_preflight.go b/internal/releaseadapter/v071/library_preflight.go index abf2f31..2fcad04 100644 --- a/internal/releaseadapter/v071/library_preflight.go +++ b/internal/releaseadapter/v071/library_preflight.go @@ -24,6 +24,12 @@ func LibrarySourceSettled(ctx context.Context, db *pgxpool.Pool) error { return checkLibrarySourceSettlement(ctx, db) } +// LibrarySourceSettledWithReleases 是同一检查,但带上治理平台已审计的精确死信解除 +// 指纹:只有逐字节命中的行不再阻断。 +func LibrarySourceSettledWithReleases(ctx context.Context, db *pgxpool.Pool, released []string) error { + return checkLibrarySourceSettlementReleased(ctx, db, released) +} + // ValidateLibraryCandidateForPreflight runs the candidate validator used by // ArchiveLibraryCandidate (identity, bidirectional content coverage, generated // question coverage, resources and digests) without archiving anything. diff --git a/internal/releaseadapter/v071/library_production_target.go b/internal/releaseadapter/v071/library_production_target.go index 969ee5c..6a52d58 100644 --- a/internal/releaseadapter/v071/library_production_target.go +++ b/internal/releaseadapter/v071/library_production_target.go @@ -47,9 +47,15 @@ type ProductionLibraryConfig struct { Objects ObjectStore ObjectsReader ObjectReader ArchiveRoot string - MaxDocuments int - HTTPClient *http.Client - Now func() time.Time + // ObjectsBucket 是目标对象存储的桶名:资源行里的定位符要换成它, + // 否则同步过去的图/附件仍指向来源桶。 + ObjectsBucket string + // ObjectsBackendID 是目标实例 storage_backends 的主键。留空则按租户+桶名在 + // 目标库里解析;定位符里的来源后端 id 在目标实例不存在,原版应用会 500。 + ObjectsBackendID string + MaxDocuments int + HTTPClient *http.Client + Now func() time.Time } func (config ProductionLibraryConfig) validate(scope ProductionScope) error { @@ -78,6 +84,59 @@ type ProductionLibraryTarget struct { scope ProductionScope search *SearchExporter now func() time.Time + // storageBackend / storageProvider 是本次写入使用的目标存储后端与提供方 + // (写入与校验共用,缺一不可:只换桶不换提供方,目标取文件仍会走来源实例的路)。 + storageBackend string + storageProvider string + // snapshotTenant / snapshotKB 是这次要写入的快照来源身份。写入时按它做 + // 来源→目标的范围映射;校验时按它反查"目标里还有没有来源范围的行"。 + snapshotTenant uint64 + snapshotKB string + // targetEmbeddingModel 是目标知识库绑定的 embedding 模型 id(文档行要换成它)。 + targetEmbeddingModel string +} + +// resolveStorageBackend 在目标库里找本次写入的存储后端。找不到就报错:宁可不写, +// 也不写一条目标应用读不出来的定位符(上轮就是这样同步"成功"了却下载 500)。 +func (t *ProductionLibraryTarget) resolveStorageBackend(ctx context.Context) error { + if t.config.ObjectsBucket == "" { + return nil + } + if t.config.ObjectsBackendID != "" { + if !validSearchIdentifier(t.config.ObjectsBackendID) { + return errors.New("目标存储后端 id 形状不合法") + } + provider, err := t.storageProviderOf(ctx, t.config.ObjectsBackendID) + if err != nil { + return err + } + t.storageBackend, t.storageProvider = t.config.ObjectsBackendID, provider + return nil + } + var id, provider string + err := t.pool.QueryRow(ctx, ` +SELECT id, coalesce(provider,'') FROM storage_backends +WHERE tenant_id=$1 AND coalesce(config->>'bucket_name','')=$2 +ORDER BY created_at LIMIT 1`, t.scope.TenantID, t.config.ObjectsBucket).Scan(&id, &provider) + if err != nil { + return errors.New("目标实例没有与目标桶匹配的存储后端,拒绝写入不可解析的资源定位符") + } + if !validSearchIdentifier(id) { + return errors.New("目标存储后端 id 形状不合法") + } + t.storageBackend, t.storageProvider = id, provider + return nil +} + +// storageProviderOf 读出目标存储后端的提供方(minio / oss / s3…)。资源行里的 +// 提供方必须与它一致,否则目标应用会按来源提供方去找一个不存在的存储服务。 +func (t *ProductionLibraryTarget) storageProviderOf(ctx context.Context, id string) (string, error) { + var provider string + if err := t.pool.QueryRow(ctx, `SELECT coalesce(provider,'') FROM storage_backends WHERE id=$1 AND tenant_id=$2`, + id, t.scope.TenantID).Scan(&provider); err != nil { + return "", errors.New("目标实例没有这个存储后端,拒绝写入不可解析的资源定位符") + } + return provider, nil } func NewProductionLibraryTarget(ctx context.Context, config ProductionLibraryConfig, scope ProductionScope) (*ProductionLibraryTarget, error) { @@ -220,6 +279,10 @@ func digestIDs(ids []string) string { // Apply 把一类数据从归档写进生产;database 与 wiki 走同一个知识域事务。 func (t *ProductionLibraryTarget) Apply(ctx context.Context, stage libraryrelease.Stage, snapshotID string) error { + if err := t.resolveStorageBackend(ctx); err != nil { + return err + } + t.resolveEmbeddingModel(ctx) archive, err := librarysnapshot.Open(t.config.ArchiveRoot, 256*1024*1024) if err != nil { return errors.New("归档不可用") @@ -229,6 +292,9 @@ func (t *ProductionLibraryTarget) Apply(ctx context.Context, stage libraryreleas if err != nil { return errors.New("预发布快照不可读") } + if err := t.bindSnapshotScope(manifestScope{tenantID: manifest.TenantID, knowledgeBaseID: manifest.KnowledgeBaseID}); err != nil { + return err + } switch stage { case libraryrelease.StageDatabase, libraryrelease.StageWiki: kind := "database" @@ -317,6 +383,11 @@ func (t *ProductionLibraryTarget) insertRows(ctx context.Context, tx pgx.Tx, arc if len(row) == 0 { continue } + // 行必须先映射到目标范围再落库:原样插入会把行写进来源租户/知识库, + // 目标范围的删除、校验与恢复都看不见它们。 + if err := t.retargetRow(table, row); err != nil { + return err + } if columns == nil { columns = make([]string, 0, len(row)) for key := range row { @@ -363,6 +434,12 @@ type resourceLocation struct { Provider string } +// rowTransform 是"写入时对行做过的变换"的统一入口:校验要用同一份,否则摘要比的是 +// 变换前的行、目标里是变换后的行,永远不一致。范围映射同样必须两边一致。 +func (t *ProductionLibraryTarget) rowTransform(table string) func(map[string]json.RawMessage) error { + return func(row map[string]json.RawMessage) error { return t.retargetRow(table, row) } +} + func (t *ProductionLibraryTarget) importResources(ctx context.Context, archive *librarysnapshot.Archive, manifest librarysnapshot.Manifest) error { paths, err := t.resourcePaths(ctx, resourcePartNames(manifest)) if err != nil { @@ -494,6 +571,10 @@ func (t *ProductionLibraryTarget) importIndex(ctx context.Context, archive *libr // Verify 用行数 + 身份集合摘要 + 索引条数核对生产与预发布快照是否一致。 func (t *ProductionLibraryTarget) Verify(ctx context.Context, expect libraryrelease.Expectation) error { + if err := t.resolveStorageBackend(ctx); err != nil { + return err + } + t.resolveEmbeddingModel(ctx) archive, err := librarysnapshot.Open(t.config.ArchiveRoot, 256*1024*1024) if err != nil { return errors.New("归档不可用") @@ -503,6 +584,9 @@ func (t *ProductionLibraryTarget) Verify(ctx context.Context, expect libraryrele if err != nil { return errors.New("预发布快照不可读") } + if err := t.bindSnapshotScope(manifestScope{tenantID: manifest.TenantID, knowledgeBaseID: manifest.KnowledgeBaseID}); err != nil { + return err + } for _, part := range manifest.Parts { if part.Kind != "database" && part.Kind != "wiki" { continue @@ -517,7 +601,7 @@ func (t *ProductionLibraryTarget) Verify(ctx context.Context, expect libraryrele wantCount, wantDigest, err := archivedTableState(archive, part) if errors.Is(err, errMissingIdentity) { // 个别表(例如日志表)没有代理主键:身份集合无从比,但行数与逐行内容摘要仍然必须一致。 - wantContentCount, wantContentDigest, columns, contentErr := archivedTableContent(archive, part) + wantContentCount, wantContentDigest, columns, contentErr := archivedTableContent(archive, part, t.rowTransform(part.Name)) if contentErr != nil { return contentErr } @@ -548,7 +632,7 @@ func (t *ProductionLibraryTarget) Verify(ctx context.Context, expect libraryrele return fmt.Errorf("生产 %s 与快照不一致:行数 %d/%d", part.Name, count, wantCount) } // 行数/身份一致还要比内容:同 ID 改正文、改 Wiki、改向量必须被发现。 - wantContentCount, wantContentDigest, columns, err := archivedTableContent(archive, part) + wantContentCount, wantContentDigest, columns, err := archivedTableContent(archive, part, t.rowTransform(part.Name)) if err != nil { return err } @@ -571,6 +655,72 @@ func (t *ProductionLibraryTarget) Verify(ctx context.Context, expect libraryrele if err := t.verifyObjects(ctx, archive, manifest); err != nil { return err } + // 目标里不允许留下"快照来源范围"的行:那说明有写入逃出了目标范围, + // 而恢复只按目标范围回放,永远删不到它们(真实事故就是这样被漏掉的)。 + if err := t.assertNoSourceScopeRows(ctx, archive, manifest); err != nil { + return err + } + return nil +} + +// dropOrphanResources 清掉"本次恢复后已无任何归属"的目标租户资源行。 +// +// 判据必须同时满足:属于目标租户、不在预备份快照里、没有任何 resource_binding、 +// 也没有任何分块正文以 resource:// 引用它。这样的行对目标应用完全不可见 +// (没有知识库能按它取到文件),只可能是失败写入留下的残渣。 +func (t *ProductionLibraryTarget) dropOrphanResources(ctx context.Context, backupSnapshotID string) error { + archive, err := librarysnapshot.Open(t.config.ArchiveRoot, 256*1024*1024) + if err != nil { + return errors.New("归档不可用") + } + defer archive.Close() + manifest, err := archive.Load(backupSnapshotID) + if err != nil { + return errors.New("预备份快照不可读") + } + kept := []string{} + for _, part := range manifest.Parts { + if part.Kind != "database" || part.Name != "resources" { + continue + } + ids, err := archivedResourceIDs(archive, part) + if err != nil { + return err + } + kept = append(kept, ids...) + } + rows, err := t.pool.Query(ctx, ` +SELECT r.id FROM resources AS r +WHERE r.tenant_id=$1 + AND NOT (r.id = ANY($2::text[])) + AND NOT EXISTS (SELECT 1 FROM resource_bindings b WHERE b.resource_id=r.id) + AND NOT EXISTS (SELECT 1 FROM chunks c WHERE c.tenant_id=r.tenant_id AND c.deleted_at IS NULL + AND coalesce(r.handle,'') <> '' + AND position('resource://'||r.handle IN coalesce(c.content,'')) > 0)`, + t.scope.TenantID, kept) + if err != nil { + return fmt.Errorf("查找目标残留资源失败: %w", err) + } + orphans := make([]string, 0, 8) + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + rows.Close() + return err + } + orphans = append(orphans, id) + } + rows.Close() + if err := rows.Err(); err != nil { + return err + } + if len(orphans) == 0 { + return nil + } + if _, err := t.pool.Exec(ctx, `DELETE FROM resources WHERE tenant_id=$1 AND id = ANY($2::text[])`, + t.scope.TenantID, orphans); err != nil { + return fmt.Errorf("清理目标残留资源失败: %w", err) + } return nil } @@ -675,5 +825,21 @@ func (t *ProductionLibraryTarget) Restore(ctx context.Context, backup libraryrel return fmt.Errorf("恢复阶段 %s 失败: %w", stage, err) } } + // resources 是多知识库共享表,回放阶段不删它:本次写入新增、恢复后已经 + // 没有任何归属的资源行会残留。它们对应用不可见,但属于"多写的"。 + if err := t.dropOrphanResources(ctx, backup.SnapshotID); err != nil { + return err + } + // 恢复工具把四个阶段都跑完不算恢复完成:整实例必须回到预备份时刻的样子。 + // 只比目标范围会把"写到范围外"的残留当成已恢复(真实事故:恢复后仍多 67 行)。 + if len(backup.Universe) > 0 { + after, err := universeFingerprint(ctx, t.pool) + if err != nil { + return fmt.Errorf("恢复后读取整库指纹失败: %w", err) + } + if diff := universeDiff(backup.Universe, after); diff != "" { + return fmt.Errorf("恢复后目标与预备份时刻不一致:%s(很可能有写入逃出了预备份范围,不能当作已恢复)", diff) + } + } return nil } diff --git a/internal/releaseadapter/v071/library_production_target_integration_test.go b/internal/releaseadapter/v071/library_production_target_integration_test.go index d771eb5..51c8ed0 100644 --- a/internal/releaseadapter/v071/library_production_target_integration_test.go +++ b/internal/releaseadapter/v071/library_production_target_integration_test.go @@ -4,12 +4,15 @@ import ( "bytes" "context" "crypto/rand" + "crypto/sha256" "encoding/hex" "encoding/json" "errors" + "fmt" "io" "net/http" "os" + "strconv" "strings" "testing" "time" @@ -390,3 +393,339 @@ func sealTargetFixture(t *testing.T, archive *librarysnapshot.Archive, scope str } return id } + +// memoryObjectStore 是"真实字节"的资源存储替身:Put/Open/Stat/Copy 都按摘要核对, +// 可以强制写入失败,用来复现"资源阶段失败"这一半写场景。 +type memoryObjectStore struct { + objects map[string][]byte + failPut bool + puts []string +} + +func newMemoryObjectStore(objects map[string][]byte) *memoryObjectStore { + copied := map[string][]byte{} + for key, value := range objects { + copied[key] = append([]byte(nil), value...) + } + return &memoryObjectStore{objects: copied} +} + +func objectKey(provider, path string) string { return provider + "|" + path } + +func (s *memoryObjectStore) Put(_ context.Context, provider, path string, reader io.Reader, size int64) error { + if s.failPut { + return errors.New("对象存储不可写") + } + body, err := io.ReadAll(reader) + if err != nil || int64(len(body)) != size { + return errors.New("对象字节数与声明不符") + } + s.objects[objectKey(provider, path)] = body + s.puts = append(s.puts, path) + return nil +} + +func (s *memoryObjectStore) Open(_ context.Context, provider, path string) (io.ReadCloser, error) { + body, ok := s.objects[objectKey(provider, path)] + if !ok { + return nil, errors.New("对象不存在") + } + return io.NopCloser(bytes.NewReader(body)), nil +} + +func (s *memoryObjectStore) Stat(_ context.Context, provider, path string) (int64, string, error) { + body, ok := s.objects[objectKey(provider, path)] + if !ok { + return 0, "", errors.New("对象不存在") + } + digest := sha256.Sum256(body) + return int64(len(body)), hex.EncodeToString(digest[:]), nil +} + +func (s *memoryObjectStore) Copy(_ context.Context, provider, source, target string) error { + body, ok := s.objects[objectKey(provider, source)] + if !ok { + return errors.New("对象不存在") + } + s.objects[objectKey(provider, target)] = append([]byte(nil), body...) + return nil +} + +// 真实事故的形状:来源 KB 与目标 KB 不同(租户也不同)、目标非空、资源阶段写失败。 +// 旧代码把归档行原样写进目标(落在来源租户/知识库),随后的回滚只清目标范围, +// 于是"已恢复到预备份"却留下 4 文件 / 38 分块 / 21 资源 / 4 绑定。 +func TestLibraryCrossScopeSyncRollbackLeavesNoResidueIsolated(t *testing.T) { + dsn, base := os.Getenv("KG_LIBRARY_TEST_DATABASE_URL"), os.Getenv("KG_LIBRARY_TEST_ELASTIC_URL") + if dsn == "" || base == "" || os.Getenv("KG_LIBRARY_TEST_ISOLATED") != "1" { + t.Skip("requires isolated PG and ES opt-in") + } + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil || !strings.HasPrefix(cfg.ConnConfig.Database, "kg_library_test_") { + t.Fatal("isolated database required") + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatalf("open isolated PG: %v", err) + } + defer pool.Close() + for _, statement := range []string{ + `DROP SCHEMA public CASCADE; CREATE SCHEMA public`, + libraryTestSchema, + `ALTER TABLE wiki_pages ADD COLUMN status text DEFAULT 'published'`, + `ALTER TABLE knowledge_tag_relations ADD COLUMN tenant_id bigint`, + `ALTER TABLE resources ADD COLUMN provider text, ADD COLUMN size bigint, ADD COLUMN storage_backend_id text`, + `ALTER TABLE resources ADD PRIMARY KEY (id)`, + `ALTER TABLE resource_bindings ADD COLUMN relation text`, + `INSERT INTO knowledge_bases VALUES('kb-target',10001,'{}',NULL),('kb-source',10000,'{}',NULL)`, + `CREATE TABLE storage_backends(id text PRIMARY KEY, tenant_id bigint, provider text, config jsonb, created_at timestamptz)`, + `INSERT INTO storage_backends VALUES('backend-target',10001,'minio','{"bucket_name":"kg-test-bucket"}',now())`, + // 目标非空:目标自己有文档/分块,另一个租户也有数据(回滚不得碰它)。 + `INSERT INTO knowledges VALUES('doc-target',10001,'kb-target','completed',0,'',NULL)`, + `INSERT INTO knowledges VALUES('doc-foreign',20000,'kb-other','completed',0,'',NULL)`, + `INSERT INTO chunks VALUES('chunk-target',10001,'kb-target','doc-target','target content',NULL)`, + `INSERT INTO chunks VALUES('chunk-foreign',20000,'kb-other','doc-foreign','foreign content',NULL)`, + `UPDATE resource_bindings SET tenant_id=10000 WHERE tenant_id IS NULL`, + } { + if _, err := pool.Exec(ctx, statement); err != nil { + t.Fatalf("fixture schema: %v", err) + } + } + + token := make([]byte, 8) + if _, err := rand.Read(token); err != nil { + t.Fatal(err) + } + index := "kg_library_test_" + hex.EncodeToString(token) + client := &http.Client{Timeout: 20 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }} + call := func(method, path, body string, want int) { + t.Helper() + request, err := http.NewRequestWithContext(ctx, method, base+path, strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + request.Header.Set("Content-Type", "application/json") + response, err := client.Do(request) + if err != nil { + t.Fatalf("fixture HTTP: %v", err) + } + defer response.Body.Close() + if response.StatusCode != want { + t.Fatalf("fixture %s %s => %d, want %d", method, path, response.StatusCode, want) + } + } + call("PUT", "/"+index, `{"mappings":{"properties":{"tenant_id":{"type":"long"},"knowledge_base_id":{"type":"keyword"},"knowledge_id":{"type":"keyword"},"chunk_id":{"type":"keyword"},"content":{"type":"text"},"embedding":{"type":"dense_vector","dims":2}}}}`, 200) + defer func() { + call("DELETE", "/"+index, "", 200) + call("HEAD", "/"+index, "", 404) + }() + // 目标当前索引自洽:目标分块必须有对应文档,否则预备份会(正确地)拒绝。 + call("PUT", "/"+index+"/_doc/chunk-target?refresh=true", + `{"tenant_id":10001,"knowledge_base_id":"kb-target","knowledge_id":"doc-target","chunk_id":"chunk-target","source_id":"chunk-target","content":"target content","embedding":[0.1,0.2]}`, 201) + + archiveRoot := t.TempDir() + if err := os.Chmod(archiveRoot, 0700); err != nil { + t.Fatal(err) + } + archive, err := librarysnapshot.Open(archiveRoot, 64*1024*1024) + if err != nil { + t.Fatal(err) + } + defer archive.Close() + snapshotID := sealCrossScopeFixture(t, archive, 10000, "kb-source") + + store := newMemoryObjectStore(nil) + target, err := NewProductionLibraryTarget(ctx, ProductionLibraryConfig{ + Postgres: dsn, SearchURL: base, SearchIndex: index, Objects: store, ObjectsReader: store, + ArchiveRoot: archiveRoot, ObjectsBucket: "kg-test-bucket", MaxDocuments: 100, HTTPClient: client, + }, ProductionScope{TenantID: 10001, KnowledgeBaseID: "kb-target"}) + if err != nil { + t.Fatalf("target: %v", err) + } + defer target.Close() + mapping, err := target.Search().MappingSHA256(ctx) + if err != nil { + t.Fatal(err) + } + backups, err := NewProductionBackupStore(ProductionBackupConfig{ + Pool: target.Pool(), Search: target.Search(), Objects: store, ArchiveRoot: archiveRoot, + Scope: "internal", Identity: target.Identity(), MaxDocuments: 100, + Compatibility: librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74, + RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "text-embedding-v4", + EmbeddingDimension: 1024, IndexMappingSHA256: mapping}, + }) + if err != nil { + t.Fatalf("backup store: %v", err) + } + now := time.Now().UTC() + request := libraryrelease.Request{Target: "internal", SnapshotID: snapshotID, + ParamsDigest: "digest-cross-scope", Now: now, + Grant: libraryrelease.Grant{Target: "internal", SnapshotID: snapshotID, ParamsDigest: "digest-cross-scope", + ExpiresAt: now.Add(10 * time.Minute), AuthorizedBy: "acceptance"}} + + // 1) 资源阶段失败 → 必须回滚且不留下来源范围的行。 + store.failPut = true + outcome := libraryrelease.Run(ctx, request, target, backups, nil) + if outcome.State != "rolled_back" { + t.Fatalf("outcome = %+v", outcome) + } + if outcome.FailedAt != libraryrelease.StageResources { + t.Fatalf("failed_at = %s, want resources", outcome.FailedAt) + } + if outcome.Backup.SnapshotID == "" || len(outcome.Backup.Universe) == 0 { + t.Fatalf("backup must carry the whole-instance fingerprint: %+v", outcome.Backup) + } + for _, table := range []string{"knowledges", "chunks", "wiki_pages"} { + var residue int64 + if err := pool.QueryRow(ctx, fmt.Sprintf(`SELECT count(*) FROM %s WHERE tenant_id=$1 AND knowledge_base_id=$2`, table), + 10000, "kb-source").Scan(&residue); err != nil { + t.Fatal(err) + } + if residue != 0 { + t.Fatalf("%s still carries %d source-scope rows after rollback", table, residue) + } + } + // 夹具在租户 10000 里本来就有自己的行(res-safe / binding-safe):它们必须原样 + // 留下,只有这次同步写进去的行(res-source / 绑到 doc-source)才算残留。 + for query, want := range map[string]int64{ + `SELECT count(*) FROM resources WHERE id='res-source'`: 0, + `SELECT count(*) FROM resource_bindings WHERE tenant_id=10000 AND owner_id='doc-source'`: 0, + `SELECT count(*) FROM resources WHERE tenant_id=10000`: 1, + } { + var got int64 + if err := pool.QueryRow(ctx, query).Scan(&got); err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("%s = %d, want %d", query, got, want) + } + } + var leftovers int64 + if err := pool.QueryRow(ctx, `SELECT count(*) FROM resources WHERE tenant_id=10001`).Scan(&leftovers); err != nil { + t.Fatal(err) + } + if leftovers != 0 { + t.Fatalf("aborted write left %d resource rows in the target tenant", leftovers) + } + var foreign int64 + if err := pool.QueryRow(ctx, `SELECT count(*) FROM chunks WHERE tenant_id=20000`).Scan(&foreign); err != nil { + t.Fatal(err) + } + // 夹具本来就有 1 行 other-scope(20000),加上本用例的 chunk-foreign,共 2 行。 + if foreign != 2 { + t.Fatalf("cross-tenant rows must survive rollback, got %d", foreign) + } + if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: outcome.Backup.SnapshotID}); err != nil { + t.Fatalf("rollback must verify against the prebackup: %v", err) + } + + // 2) 同样的跨范围同步,资源能写 → 行必须落在目标范围,资源定位符指向目标存储。 + store.failPut = false + succeed := libraryrelease.Run(ctx, request, target, backups, nil) + if succeed.State != "succeeded" { + t.Fatalf("second run outcome = %+v", succeed) + } + if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); err != nil { + t.Fatalf("verify after cross-scope sync: %v", err) + } + var knowledgeKB, chunkKB, provider, locator string + var tenant int64 + if err := pool.QueryRow(ctx, `SELECT tenant_id, knowledge_base_id FROM knowledges WHERE id='doc-source'`). + Scan(&tenant, &knowledgeKB); err != nil { + t.Fatalf("source document must exist in the target scope: %v", err) + } + if tenant != 10001 || knowledgeKB != "kb-target" { + t.Fatalf("cross-scope row landed at %d/%s", tenant, knowledgeKB) + } + if err := pool.QueryRow(ctx, `SELECT knowledge_base_id FROM chunks WHERE id='chunk-source'`).Scan(&chunkKB); err != nil { + t.Fatal(err) + } + if chunkKB != "kb-target" { + t.Fatalf("chunk landed in %s", chunkKB) + } + if err := pool.QueryRow(ctx, `SELECT provider, physical_path FROM resources WHERE id='res-source'`). + Scan(&provider, &locator); err != nil { + t.Fatal(err) + } + if provider != "minio" { + t.Fatalf("resource provider = %s, want minio", provider) + } + if !strings.Contains(locator, "storage://backend-target/minio://kg-test-bucket/10001/") { + t.Fatalf("resource locator was not retargeted: %s", locator) + } + if strings.Contains(locator, "weknora/10000") { + t.Fatalf("resource locator still points at the source tenant: %s", locator) + } + t.Logf("cross_scope_sync verified: snapshot=%s backup=%s locator=%s", snapshotID, succeed.Backup.SnapshotID, locator) +} + +// sealCrossScopeFixture 造一份"来源范围与目标不同"的整库快照:文档/分块/资源/绑定/ +// Wiki/索引都带来源身份,资源部件是可核对的真实字节。 +func sealCrossScopeFixture(t *testing.T, archive *librarysnapshot.Archive, sourceTenant uint64, sourceKB string) string { + t.Helper() + encode := func(value any) []byte { + encoded, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + return encoded + } + const handle = "abcdefghijklmnopqrstuv" + objectBytes := []byte("12345678") + parts := make([]librarysnapshot.Part, 0, 12) + put := func(kind, name string, count int64, body []byte) { + part, err := archive.Put(bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + part.Kind, part.Name, part.Count = kind, name, count + parts = append(parts, part) + } + put("database", "knowledges", 1, encode([]map[string]any{{ + "id": "doc-source", "tenant_id": sourceTenant, "knowledge_base_id": sourceKB, + "parse_status": "completed", "pending_subtasks_count": 0, "file_path": ""}})) + put("database", "chunks", 1, encode([]map[string]any{{ + "id": "chunk-source", "tenant_id": sourceTenant, "knowledge_base_id": sourceKB, + "knowledge_id": "doc-source", "content": "见 resource://" + handle}})) + put("database", "knowledge_tags", 0, encode([]map[string]any{})) + put("database", "knowledge_tag_relations", 0, encode([]map[string]any{})) + put("database", "resources", 1, encode([]map[string]any{{ + "id": "res-source", "handle": handle, "tenant_id": sourceTenant, + "storage_backend_id": "backend-source", "provider": "oss", + "physical_path": "storage://backend-source/oss://source-bucket/weknora/" + + formatTenant(sourceTenant) + "/res-source/a.png", + "size": int64(len(objectBytes))}})) + put("database", "resource_bindings", 1, encode([]map[string]any{{ + "id": "bind-source", "resource_id": "res-source", "tenant_id": sourceTenant, + "owner_type": "knowledge", "owner_id": "doc-source", "relation": "attachment"}})) + put("resource", "res-source", 1, objectBytes) + put("wiki", "wiki_folders", 0, encode([]map[string]any{})) + put("wiki", "wiki_pages", 1, encode([]map[string]any{{ + "id": "wiki-source", "tenant_id": sourceTenant, "knowledge_base_id": sourceKB, + "slug": "index", "content": "wiki"}})) + for _, name := range []string{"wiki_page_issues", "wiki_log_entries"} { + put("wiki", name, 0, encode([]map[string]any{})) + } + put("index", "chunks", 1, encode([]map[string]any{{"id": "chunk-source", "source": map[string]any{ + "tenant_id": sourceTenant, "knowledge_base_id": sourceKB, "knowledge_id": "doc-source", + "chunk_id": "chunk-source", "source_id": "chunk-source", "content": "见 resource://" + handle, + "embedding": []float64{0.1, 0.2}}}})) + manifest := librarysnapshot.Manifest{ValidationPolicy: librarysnapshot.ArtifactAcceptancePolicy, + Format: librarysnapshot.Format, Scope: "internal", TenantID: sourceTenant, KnowledgeBaseID: sourceKB, + CreatedAt: time.Now().UTC(), Parts: parts, + Compatibility: librarysnapshot.Compatibility{ + WeKnoraVersion: "v0.7.1", SchemaVersion: 74, RetrieverEngine: "elasticsearch_v8", + EmbeddingModel: "text-embedding-v4", EmbeddingDimension: 1024, + IndexMappingSHA256: strings.Repeat("a", 64), + }} + id, err := archive.Seal(manifest) + if err != nil { + t.Fatalf("seal cross-scope fixture: %v", err) + } + return id +} + +func formatTenant(tenant uint64) string { return strconv.FormatUint(tenant, 10) } diff --git a/internal/releaseadapter/v071/library_residue.go b/internal/releaseadapter/v071/library_residue.go new file mode 100644 index 0000000..b692c38 --- /dev/null +++ b/internal/releaseadapter/v071/library_residue.go @@ -0,0 +1,334 @@ +package v071 + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + + "knowledge-governance/internal/librarysnapshot" +) + +// ResidueReport 是"清理范围外残留"的审计记录:清掉了什么、依据是哪份归档、 +// 备份写到哪儿、清理后目标处于什么状态。没有这份记录就不许动目标。 +type ResidueReport struct { + SnapshotID string `json:"snapshot_id"` + SourceTenant uint64 `json:"source_tenant"` + SourceKB string `json:"source_kb"` + Tables map[string]int `json:"tables"` + BackupPath string `json:"backup_path"` + BackupSHA256 string `json:"backup_sha256"` + Applied bool `json:"applied"` + Cleared int `json:"cleared"` + Verified bool `json:"verified"` + Foreign map[string]string `json:"foreign_scopes,omitempty"` +} + +// ClearForeignScope 清掉"归档里确实有、却仍以来源身份躺在目标里"的行。 +// +// 它只做这一件事,而且必须全部条件成立才允许删: +// 1. 目标这些行的内容与指定归档行逐字段一致(先证明它们是这次写入的副本, +// 不是目标自己的数据); +// 2. 先把它们完整导出成 0600 的备份文件并记下摘要; +// 3. 删除只按"归档身份集合 + 来源租户"精确命中,不做范围广删。 +// +// 清完还要复核:来源身份不再存在、目标内容与预备份一致、整库指纹里没有别的租户。 +func (t *ProductionLibraryTarget) ClearForeignScope(ctx context.Context, source ProductionScope, snapshotID, + backupDir string, apply bool) (*ResidueReport, error) { + if source.TenantID == 0 || !validSearchIdentifier(source.KnowledgeBaseID) { + return nil, errors.New("来源范围不完整,拒绝清理") + } + if source.TenantID == t.scope.TenantID && source.KnowledgeBaseID == t.scope.KnowledgeBaseID { + return nil, errors.New("来源与目标范围相同,不需要清理") + } + if backupDir == "" { + return nil, errors.New("清理前必须指定备份目录") + } + if err := t.resolveStorageBackend(ctx); err != nil { + return nil, err + } + archive, err := librarysnapshot.Open(t.config.ArchiveRoot, 256*1024*1024) + if err != nil { + return nil, errors.New("归档不可用") + } + defer archive.Close() + manifest, err := archive.Load(snapshotID) + if err != nil { + return nil, errors.New("指定快照不可读") + } + report := &ResidueReport{SnapshotID: snapshotID, SourceTenant: source.TenantID, SourceKB: source.KnowledgeBaseID, + Tables: map[string]int{}, Applied: false} + backup := map[string][]map[string]json.RawMessage{} + digests := map[string]string{} + identities := map[string][]string{} + for _, part := range manifest.Parts { + if part.Kind != "database" && part.Kind != "wiki" { + continue + } + archived, archivedDigests, err := archivedRowsWithDigests(archive, part) + if err != nil { + return nil, err + } + // 事故里的残留行是"归档行 + 只把桶/后端换成目标"的旧变换结果:判等时必须 + // 同时接受原样与这一种变换,否则会把真残留当成"不是这次写的"而拒绝清理。 + if part.Name == "resources" { + for _, row := range archived { + legacy, legacyErr := t.legacyResourceRow(row) + if legacyErr != nil { + return nil, legacyErr + } + digest, digestErr := canonicalRowDigest(legacy) + if digestErr != nil { + return nil, digestErr + } + archivedDigests[digest] = true + } + } + if len(archived) == 0 { + continue + } + live, err := t.foreignScopeRows(ctx, part.Name, source) + if err != nil { + return nil, err + } + liveDigests := make(map[string]string, len(live)) + for _, row := range live { + digest, err := canonicalRowDigest(row) + if err != nil { + return nil, err + } + liveDigests[digest] = archiveRowIdentity(part.Name, row) + } + // 目标里这一范围的行必须都能在归档里找到同样内容:否则它们不是这次写入的 + // 副本,删了就是删别人的数据。 + rows := make([]map[string]json.RawMessage, 0, len(live)) + ids := make([]string, 0, len(live)) + for digest, identity := range liveDigests { + if _, ok := archivedDigests[digest]; !ok { + return nil, fmt.Errorf("目标 %s 的行 %s 与指定归档不一致,拒绝按本次写入清理", part.Name, identity) + } + ids = append(ids, identity) + } + for _, row := range live { + rows = append(rows, row) + } + sort.Strings(ids) + report.Tables[part.Name] = len(rows) + backup[part.Name] = rows + identities[part.Name] = ids + digests[part.Name] = combineRowDigests(sortedKeys(liveDigests)) + } + total := 0 + for _, count := range report.Tables { + total += count + } + if total == 0 { + report.Verified = true + return report, nil + } + if err := os.MkdirAll(backupDir, 0o700); err != nil { + return nil, fmt.Errorf("创建备份目录失败: %w", err) + } + payload, err := json.MarshalIndent(map[string]any{ + "snapshot_id": snapshotID, "source_tenant": source.TenantID, "source_kb": source.KnowledgeBaseID, + "target_tenant": t.scope.TenantID, "target_kb": t.scope.KnowledgeBaseID, + "tables": report.Tables, "rows": backup, "digests": digests, + }, "", " ") + if err != nil { + return nil, err + } + report.BackupPath = filepath.Join(backupDir, "foreign-scope-residue-"+snapshotID[:12]+".json") + if err := os.WriteFile(report.BackupPath, payload, 0o600); err != nil { + return nil, fmt.Errorf("写残留备份失败: %w", err) + } + sum := sha256.Sum256(payload) + report.BackupSHA256 = hex.EncodeToString(sum[:]) + if !apply { + return report, nil + } + // 删除顺序:子表先、父表后(resource_bindings → resources,chunks → knowledges)。 + for index := len(productionLibraryTables) - 1; index >= 0; index-- { + table := productionLibraryTables[index] + ids, ok := identities[table] + if !ok || len(ids) == 0 { + continue + } + statement := fmt.Sprintf(`DELETE FROM %s WHERE id = ANY($1::text[])`, table) + if !tableHasTenant(table) { + // knowledge_tag_relations 没有 id:身份就是父文档 + 标签。 + statement = `DELETE FROM knowledge_tag_relations WHERE (knowledge_id||':'||tag_id) = ANY($1::text[])` + } + if _, err := t.pool.Exec(ctx, statement, ids); err != nil { + return nil, fmt.Errorf("清理 %s 失败: %w", table, err) + } + } + report.Applied = true + report.Cleared = total + return report, nil +} + +// legacyResourceRow 复现旧实现写资源行时的变换:只换定位符里的存储后端与桶, +// 租户/知识库/provider 原样保留。判等要认这一种形状,事故残留才是"可证明"的。 +func (t *ProductionLibraryTarget) legacyResourceRow(row map[string]json.RawMessage) (map[string]json.RawMessage, error) { + copied := make(map[string]json.RawMessage, len(row)) + for key, value := range row { + copied[key] = value + } + if t.config.ObjectsBucket == "" { + return copied, nil + } + raw, ok := copied["physical_path"] + if !ok { + return nil, errors.New("资源行缺少定位符") + } + var locator string + if err := json.Unmarshal(raw, &locator); err != nil { + return nil, errors.New("资源定位符不可解析") + } + if rewritten, ok := rewriteLocatorTarget(locator, t.storageBackend, t.config.ObjectsBucket); ok { + encoded, err := json.Marshal(rewritten) + if err != nil { + return nil, err + } + copied["physical_path"] = encoded + } + if t.storageBackend == "" { + return copied, nil + } + if raw, ok := copied["storage_backend_id"]; ok { + var current string + if json.Unmarshal(raw, ¤t) == nil && current != "" && current != t.storageBackend { + encoded, err := json.Marshal(t.storageBackend) + if err != nil { + return nil, err + } + copied["storage_backend_id"] = encoded + } + } + return copied, nil +} + +// VerifyResidueCleared 复核清理结果:来源身份不再存在、其它租户的解释得清、 +// 且(可选)目标内容与预备份完全一致。 +func (t *ProductionLibraryTarget) VerifyResidueCleared(ctx context.Context, source ProductionScope) error { + for _, table := range productionLibraryTables { + if !tableHasTenant(table) { + continue + } + statement := fmt.Sprintf(`SELECT count(*) FROM %s WHERE tenant_id=$1`, table) + arguments := []any{source.TenantID} + if tablesWithKB[table] { + statement += " AND knowledge_base_id=$2" + arguments = append(arguments, source.KnowledgeBaseID) + } + var count int64 + if err := t.pool.QueryRow(ctx, statement, arguments...).Scan(&count); err != nil { + return fmt.Errorf("复核 %s 失败: %w", table, err) + } + if count > 0 { + return fmt.Errorf("目标 %s 仍有 %d 行属于来源范围 %d/%s", table, count, source.TenantID, source.KnowledgeBaseID) + } + } + return nil +} + +// ForeignScopes 列出目标里所有"不是目标租户"的租户,用于人工确认没有别的泄漏。 +func (t *ProductionLibraryTarget) ForeignScopes(ctx context.Context) (map[string]string, error) { + foreign := map[string]string{} + for _, table := range productionLibraryTables { + if !tableHasTenant(table) { + continue + } + rows, err := t.pool.Query(ctx, fmt.Sprintf( + `SELECT tenant_id, count(*) FROM %s WHERE tenant_id <> $1 GROUP BY 1`, table), t.scope.TenantID) + if err != nil { + return nil, fmt.Errorf("读取 %s 租户分布失败: %w", table, err) + } + for rows.Next() { + var tenant, count int64 + if err := rows.Scan(&tenant, &count); err != nil { + rows.Close() + return nil, err + } + foreign[fmt.Sprintf("%s:%d", table, tenant)] = fmt.Sprintf("%d 行", count) + } + rows.Close() + if err := rows.Err(); err != nil { + return nil, err + } + } + return foreign, nil +} + +// foreignScopeRows 读出目标里属于来源范围的行(resources / resource_bindings +// 只有租户列,按租户取;其余按租户+知识库取)。 +func (t *ProductionLibraryTarget) foreignScopeRows(ctx context.Context, table string, source ProductionScope) ( + []map[string]json.RawMessage, error) { + if !tableHasTenant(table) { + return nil, nil + } + statement := fmt.Sprintf(`SELECT to_jsonb(%s) FROM %s WHERE %s`, table, table, scopeWhere(table)) + arguments := []any{source.TenantID, source.KnowledgeBaseID} + if table == "resources" || table == "resource_bindings" { + statement = fmt.Sprintf(`SELECT to_jsonb(%s) FROM %s WHERE tenant_id=$1`, table, table) + arguments = arguments[:1] + } + rows, err := t.pool.Query(ctx, statement, arguments...) + if err != nil { + return nil, fmt.Errorf("读取目标 %s 来源范围行失败: %w", table, err) + } + defer rows.Close() + result := make([]map[string]json.RawMessage, 0, 8) + for rows.Next() { + var row map[string]json.RawMessage + if err := rows.Scan(&row); err != nil { + return nil, err + } + result = append(result, row) + } + return result, rows.Err() +} + +// archivedRowsWithDigests 读出一份归档部件的行与规范化内容摘要。 +func archivedRowsWithDigests(archive *librarysnapshot.Archive, part librarysnapshot.Part) ( + []map[string]json.RawMessage, map[string]bool, error) { + reader, err := archive.OpenPart(part) + if err != nil { + return nil, nil, err + } + defer reader.Close() + decoder := json.NewDecoder(reader) + token, err := decoder.Token() + if err != nil || token != json.Delim('[') { + return nil, nil, errors.New("归档部件不是行数组") + } + rows := make([]map[string]json.RawMessage, 0, 64) + digests := map[string]bool{} + for decoder.More() { + var row map[string]json.RawMessage + if err := decoder.Decode(&row); err != nil { + return nil, nil, err + } + digest, err := canonicalRowDigest(row) + if err != nil { + return nil, nil, err + } + digests[digest] = true + rows = append(rows, row) + } + return rows, digests, nil +} + +func sortedKeys(values map[string]string) []string { + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} diff --git a/internal/releaseadapter/v071/library_scope_probe.go b/internal/releaseadapter/v071/library_scope_probe.go new file mode 100644 index 0000000..d25ec8e --- /dev/null +++ b/internal/releaseadapter/v071/library_scope_probe.go @@ -0,0 +1,46 @@ +package v071 + +import ( + "context" + "errors" + "fmt" + + "github.com/jackc/pgx/v5/pgxpool" +) + +// ScopeContentCounts 数一个知识库在各内容表里的行数(数据库侧事实)。 +// +// 为什么不能只数 knowledges:文档被删掉、分块与 Wiki/资源还在的库并不为空。 +// 只数一张表会把这种目标判成"首次发布的空库",从而让一条本该走基线初始化的 +// 路径绕过去;这里用与整库读写完全相同的表清单与过滤条件,保证"空"的定义只有一份。 +func ScopeContentCounts(ctx context.Context, pool *pgxpool.Pool, identity ProductionScope) (map[string]int64, error) { + if pool == nil || identity.TenantID == 0 || !validSearchIdentifier(identity.KnowledgeBaseID) { + return nil, errors.New("invalid scope identity") + } + counts := make(map[string]int64, len(productionLibraryTables)) + for _, table := range productionLibraryTables { + var count int64 + if err := pool.QueryRow(ctx, fmt.Sprintf(`SELECT count(*) FROM %s WHERE %s`, table, scopeWhere(table)), + identity.TenantID, identity.KnowledgeBaseID).Scan(&count); err != nil { + return nil, fmt.Errorf("count %s: %w", table, err) + } + counts[table] = count + } + return counts, nil +} + +// ScopeContentEmpty 只有每张内容表都是 0 才算空目标,并交回逐表行数作为证据。 +// 注意它**不**包含检索索引:索引由分块派生,分块为空而索引非空属于不一致, +// 应在执行侧(预备份/基线捕获)报出来,不能在这里被当成"空"。 +func ScopeContentEmpty(ctx context.Context, pool *pgxpool.Pool, identity ProductionScope) (bool, map[string]int64, error) { + counts, err := ScopeContentCounts(ctx, pool, identity) + if err != nil { + return false, nil, err + } + for _, count := range counts { + if count != 0 { + return false, counts, nil + } + } + return true, counts, nil +} diff --git a/internal/releaseadapter/v071/library_scope_remap.go b/internal/releaseadapter/v071/library_scope_remap.go new file mode 100644 index 0000000..23fe813 --- /dev/null +++ b/internal/releaseadapter/v071/library_scope_remap.go @@ -0,0 +1,262 @@ +package v071 + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" +) + +// 整库同步是"把来源知识库的内容搬到目标的租户/知识库",但归档行带着来源身份 +// (tenant_id / knowledge_base_id)。原样插入会让行落在来源范围里:目标范围的 +// 删除、校验、预备份与恢复全都看不见它们(真实事故:同步失败后"已恢复到预备份", +// 目标却多出 4 文件 / 38 分块 / 21 资源 / 4 绑定,行所属租户还是来源租户)。 +// +// 因此写入与校验必须共用这一个改写入口,并且对不上就失败——宁可不写,也不写 +// 一行目标范围外的数据。 + +// tablesWithKB 是真实 WeKnora v0.7.1 里带 knowledge_base_id 的知识域表。 +// resources / resource_bindings 是多知识库共享表,只有 tenant_id; +// knowledge_tag_relations 两列都没有,归属完全由父表 knowledges 决定。 +var tablesWithKB = map[string]bool{ + "knowledges": true, "chunks": true, "knowledge_tags": true, + "wiki_folders": true, "wiki_pages": true, "wiki_page_issues": true, "wiki_log_entries": true, +} + +// tableHasTenant 报告这张表是否自带租户列。 +func tableHasTenant(table string) bool { return table != "knowledge_tag_relations" } + +// bindSnapshotScope 记下快照的来源身份。没有来源身份就无法判断"这行要搬到哪个 +// 范围",宁可失败也不要按行内容猜。 +func (t *ProductionLibraryTarget) bindSnapshotScope(manifest manifestScope) error { + if manifest.tenantID == 0 || !validSearchIdentifier(manifest.knowledgeBaseID) { + return errors.New("快照缺少来源范围身份(tenant_id / knowledge_base_id),无法确定目标范围映射") + } + t.snapshotTenant = manifest.tenantID + t.snapshotKB = manifest.knowledgeBaseID + return nil +} + +// manifestScope 只是把快照的两个范围字段抽出来,避免调用点重复解构。 +type manifestScope struct { + tenantID uint64 + knowledgeBaseID string +} + +// retargetRow 把一行归档数据改写成目标范围的数据:tenant_id / knowledge_base_id +// 换成目标身份,资源行还要把定位符、存储后端、提供方换成目标实例自己的值。 +func (t *ProductionLibraryTarget) retargetRow(table string, row map[string]json.RawMessage) error { + if !tableHasTenant(table) { + return nil + } + if err := t.retargetTenantField(row); err != nil { + return fmt.Errorf("%s:%w", table, err) + } + if tablesWithKB[table] { + if err := t.retargetKnowledgeBaseField(row); err != nil { + return fmt.Errorf("%s:%w", table, err) + } + } + if table == "resources" { + return t.rewriteResourceIdentity(row) + } + if table == "knowledges" { + return t.retargetEmbeddingModelField(row) + } + return nil +} + +// retargetEmbeddingModelField 把文档的 embedding 模型换成目标知识库绑定的那一个。 +// 来源模型 id 在目标实例里通常不存在(目标 models 表可能是空的),留着它等于让 +// 应用拿一个解析不到的模型去查——库里的向量与查询向量就不是同一套了。 +func (t *ProductionLibraryTarget) retargetEmbeddingModelField(row map[string]json.RawMessage) error { + if t.targetEmbeddingModel == "" { + return nil + } + if _, ok := row["embedding_model_id"]; !ok { + return nil + } + encoded, err := json.Marshal(t.targetEmbeddingModel) + if err != nil { + return errors.New("编码目标 embedding 模型失败") + } + row["embedding_model_id"] = encoded + return nil +} + +// resolveEmbeddingModel 读目标知识库自己的 embedding 模型 id。读不到不算错误 +// (目标可能确实没绑模型),此时保持来源值不动,由契约校验去发现不一致。 +func (t *ProductionLibraryTarget) resolveEmbeddingModel(ctx context.Context) { + var model string + if err := t.pool.QueryRow(ctx, + `SELECT coalesce(embedding_model_id,'') FROM knowledge_bases WHERE id=$1 AND tenant_id=$2`, + t.scope.KnowledgeBaseID, t.scope.TenantID).Scan(&model); err == nil { + t.targetEmbeddingModel = model + } +} + +func (t *ProductionLibraryTarget) retargetTenantField(row map[string]json.RawMessage) error { + raw, ok := row["tenant_id"] + if !ok { + return errors.New("行缺少 tenant_id,无法确定要写入哪个范围") + } + var current uint64 + if err := json.Unmarshal(raw, ¤t); err != nil { + return errors.New("行 tenant_id 不可解析") + } + if t.snapshotTenant == 0 { + return errors.New("快照来源租户未知,拒绝猜测目标租户") + } + if current != t.snapshotTenant { + return fmt.Errorf("行 tenant_id=%d 不属于快照来源租户 %d", current, t.snapshotTenant) + } + encoded, err := json.Marshal(t.scope.TenantID) + if err != nil { + return errors.New("编码目标租户失败") + } + row["tenant_id"] = encoded + return nil +} + +func (t *ProductionLibraryTarget) retargetKnowledgeBaseField(row map[string]json.RawMessage) error { + raw, ok := row["knowledge_base_id"] + if !ok { + return errors.New("行缺少 knowledge_base_id,无法确定要写入哪个知识库") + } + var current string + if err := json.Unmarshal(raw, ¤t); err != nil || current == "" { + return errors.New("行 knowledge_base_id 不可解析") + } + if t.snapshotKB == "" { + return errors.New("快照来源知识库未知,拒绝猜测目标知识库") + } + if current != t.snapshotKB { + return fmt.Errorf("行 knowledge_base_id=%s 不属于快照来源知识库 %s", current, t.snapshotKB) + } + encoded, err := json.Marshal(t.scope.KnowledgeBaseID) + if err != nil { + return errors.New("编码目标知识库失败") + } + row["knowledge_base_id"] = encoded + return nil +} + +// rewriteResourceIdentity 把资源行的定位符与提供方换成目标实例的值。 +// +// 只换桶不够:来源行的 scheme(oss/minio…)与对象键里的租户段也是来源实例的, +// 目标实例按它取文件会落到别的提供方或别的租户前缀上(真跑:storage backend +// not found → invalid path → 500 code1007)。 +func (t *ProductionLibraryTarget) rewriteResourceIdentity(row map[string]json.RawMessage) error { + raw, ok := row["physical_path"] + if !ok { + return errors.New("资源行缺少定位符") + } + var locator string + if err := json.Unmarshal(raw, &locator); err != nil { + return errors.New("资源定位符不可解析") + } + rewritten, err := retargetLocator(locator, t.storageBackend, t.storageProvider, t.config.ObjectsBucket, + t.snapshotTenant, t.scope.TenantID) + if err != nil { + return err + } + if rewritten != locator { + encoded, err := json.Marshal(rewritten) + if err != nil { + return errors.New("编码目标定位符失败") + } + row["physical_path"] = encoded + } + if t.storageBackend != "" { + if _, ok := row["storage_backend_id"]; ok { + encoded, err := json.Marshal(t.storageBackend) + if err != nil { + return errors.New("编码目标存储后端失败") + } + row["storage_backend_id"] = encoded + } + } + if t.storageProvider != "" { + if _, ok := row["provider"]; ok { + encoded, err := json.Marshal(t.storageProvider) + if err != nil { + return errors.New("编码目标提供方失败") + } + row["provider"] = encoded + } + } + return nil +} + +// retargetLocator 把 "storage:///:///" 整体换成目标 +// 实例的真实取值:后端 id、scheme/提供方、桶名,以及对象键里的租户段。 +// 桶名为空表示目标没有配置对象存储(测试夹具),此时不动定位符。 +func retargetLocator(locator, backendID, provider, bucket string, sourceTenant, targetTenant uint64) (string, error) { + if bucket == "" || locator == "" { + return locator, nil + } + if locator != strings.TrimSpace(locator) { + return "", errors.New("资源定位符带首尾空白") + } + prefix := "" + rest := locator + if strings.HasPrefix(locator, "storage://") { + trimmed := strings.TrimPrefix(locator, "storage://") + parts := strings.SplitN(trimmed, "/", 2) + if len(parts) != 2 || !validSearchIdentifier(parts[0]) || parts[1] == "" { + return "", errors.New("资源定位符结构不可解析") + } + if backendID != "" { + if !validSearchIdentifier(backendID) { + return "", errors.New("目标存储后端 id 形状不合法") + } + parts[0] = backendID + } + prefix, rest = "storage://"+parts[0]+"/", parts[1] + } + separator := strings.Index(rest, "://") + if separator < 0 { + // 裸对象键没有桶与租户段:落盘按配置桶,改写无意义。 + return locator, nil + } + scheme := rest[:separator] + if !validSearchIdentifier(scheme) { + return "", errors.New("资源定位符的提供方不可解析") + } + schemeless := rest[separator+3:] + boundary := strings.Index(schemeless, "/") + if boundary <= 0 || boundary == len(schemeless)-1 { + return "", errors.New("资源定位符缺少对象键") + } + key := schemeless[boundary+1:] + if provider != "" { + if !validSearchIdentifier(provider) { + return "", errors.New("目标提供方形状不合法") + } + scheme = provider + } + return prefix + scheme + "://" + bucket + "/" + retargetKeyTenant(key, sourceTenant, targetTenant), nil +} + +// retargetKeyTenant 把对象键里的来源租户段换成目标租户,并丢掉来源实例自己的 +// 路径前缀(例如 MINIO_PATH_PREFIX=weknora/),让键与目标实例的写法一致。 +// 找不到来源租户段时原样返回:此时键本身不含跨租户引用。 +func retargetKeyTenant(key string, sourceTenant, targetTenant uint64) string { + if sourceTenant == 0 || targetTenant == 0 || sourceTenant == targetTenant { + return key + } + source, target := strconv.FormatUint(sourceTenant, 10), strconv.FormatUint(targetTenant, 10) + segments := strings.Split(key, "/") + for index := 0; index < len(segments) && index < 2; index++ { + // 只有"后面还有内容"的段才是路径前缀;整键就是租户号时不该动它。 + if index >= len(segments)-1 || segments[index] != source { + continue + } + kept := segments[index:] + kept[0] = target + return strings.Join(kept, "/") + } + return key +} diff --git a/internal/releaseadapter/v071/library_scope_remap_test.go b/internal/releaseadapter/v071/library_scope_remap_test.go new file mode 100644 index 0000000..0ce8aef --- /dev/null +++ b/internal/releaseadapter/v071/library_scope_remap_test.go @@ -0,0 +1,194 @@ +package v071 + +import ( + "encoding/json" + "strings" + "testing" +) + +// 真实事故的形状:来源 KB 与目标 KB 不同(租户也不同),归档行原样写进目标, +// 目标范围的删除/校验/恢复都看不到它们。这里把"必须改写哪些列"钉死。 + +func scopeTargetFixture() *ProductionLibraryTarget { + return &ProductionLibraryTarget{ + scope: ProductionScope{TenantID: 10001, KnowledgeBaseID: "kb-target"}, + config: ProductionLibraryConfig{ObjectsBucket: "kgseal-1029a"}, + storageBackend: "backend-target", + storageProvider: "minio", + snapshotTenant: 10000, + snapshotKB: "kb-source", + } +} + +func rawRow(t *testing.T, fields map[string]any) map[string]json.RawMessage { + t.Helper() + row := make(map[string]json.RawMessage, len(fields)) + for key, value := range fields { + encoded, err := json.Marshal(value) + if err != nil { + t.Fatalf("encode %s: %v", key, err) + } + row[key] = encoded + } + return row +} + +func fieldText(t *testing.T, row map[string]json.RawMessage, key string) string { + t.Helper() + var value string + if err := json.Unmarshal(row[key], &value); err != nil { + t.Fatalf("%s not a string: %v", key, err) + } + return value +} + +func fieldUint(t *testing.T, row map[string]json.RawMessage, key string) uint64 { + t.Helper() + var value uint64 + if err := json.Unmarshal(row[key], &value); err != nil { + t.Fatalf("%s not a number: %v", key, err) + } + return value +} + +func TestRetargetRowMapsSourceScopeToTarget(t *testing.T) { + target := scopeTargetFixture() + for _, table := range []string{"knowledges", "chunks", "knowledge_tags", "wiki_pages", "wiki_folders"} { + row := rawRow(t, map[string]any{"id": "row-1", "tenant_id": 10000, "knowledge_base_id": "kb-source"}) + if err := target.retargetRow(table, row); err != nil { + t.Fatalf("%s retarget: %v", table, err) + } + if got := fieldUint(t, row, "tenant_id"); got != 10001 { + t.Fatalf("%s tenant_id = %d, want 10001", table, got) + } + if got := fieldText(t, row, "knowledge_base_id"); got != "kb-target" { + t.Fatalf("%s knowledge_base_id = %s, want kb-target", table, got) + } + } + // 共享表只有租户列:不得凭空写一个 knowledge_base_id 出来。 + binding := rawRow(t, map[string]any{"id": "bind-1", "tenant_id": 10000, "owner_type": "knowledge", "owner_id": "row-1"}) + if err := target.retargetRow("resource_bindings", binding); err != nil { + t.Fatalf("resource_bindings retarget: %v", err) + } + if got := fieldUint(t, binding, "tenant_id"); got != 10001 { + t.Fatalf("binding tenant_id = %d, want 10001", got) + } + if _, ok := binding["knowledge_base_id"]; ok { + t.Fatal("resource_bindings must not gain a knowledge_base_id column") + } + // knowledge_tag_relations 没有范围列:归属走父表,改写必须是空操作。 + relation := rawRow(t, map[string]any{"knowledge_id": "row-1", "tag_id": "tag-1"}) + if err := target.retargetRow("knowledge_tag_relations", relation); err != nil { + t.Fatalf("knowledge_tag_relations retarget: %v", err) + } + if len(relation) != 2 { + t.Fatalf("knowledge_tag_relations must stay untouched, got %v", relation) + } +} + +// 文档行的 embedding 模型必须换成目标知识库绑定的那一个:来源模型 id 在目标 +// 实例里通常不存在,留着它检索时的查询向量与库里向量就不是同一套。 +func TestRetargetRowUsesTargetEmbeddingModel(t *testing.T) { + target := scopeTargetFixture() + target.targetEmbeddingModel = "model-target" + row := rawRow(t, map[string]any{"id": "doc-1", "tenant_id": 10000, "knowledge_base_id": "kb-source", + "embedding_model_id": "model-source"}) + if err := target.retargetRow("knowledges", row); err != nil { + t.Fatalf("retarget: %v", err) + } + if got := fieldText(t, row, "embedding_model_id"); got != "model-target" { + t.Fatalf("embedding_model_id = %s, want model-target", got) + } + // 目标没绑模型时不猜:保持来源值,交给契约校验去发现不一致。 + plain := scopeTargetFixture() + row = rawRow(t, map[string]any{"id": "doc-1", "tenant_id": 10000, "knowledge_base_id": "kb-source", + "embedding_model_id": "model-source"}) + if err := plain.retargetRow("knowledges", row); err != nil { + t.Fatalf("retarget without target model: %v", err) + } + if got := fieldText(t, row, "embedding_model_id"); got != "model-source" { + t.Fatalf("embedding_model_id = %s, want unchanged", got) + } +} + +func TestRetargetRowFailsClosedOutsideSnapshotScope(t *testing.T) { + target := scopeTargetFixture() + cases := map[string]map[string]any{ + "foreign tenant": {"id": "x", "tenant_id": 20000, "knowledge_base_id": "kb-source"}, + "foreign knowledge": {"id": "x", "tenant_id": 10000, "knowledge_base_id": "kb-other"}, + "missing tenant": {"id": "x", "knowledge_base_id": "kb-source"}, + "missing knowledge kb": {"id": "x", "tenant_id": 10000}, + } + for name, fields := range cases { + if err := target.retargetRow("knowledges", rawRow(t, fields)); err == nil { + t.Fatalf("%s: retarget must fail closed", name) + } + } + // 快照来源未知时不许猜目标范围。 + unknown := &ProductionLibraryTarget{scope: ProductionScope{TenantID: 10001, KnowledgeBaseID: "kb-target"}} + if err := unknown.retargetRow("knowledges", rawRow(t, map[string]any{"id": "x", "tenant_id": 10000, "knowledge_base_id": "kb"})); err == nil { + t.Fatal("unknown snapshot scope must fail closed") + } +} + +func TestRetargetRowRewritesResourceStorageIdentity(t *testing.T) { + target := scopeTargetFixture() + row := rawRow(t, map[string]any{ + "id": "res-1", + "tenant_id": 10000, + "storage_backend_id": "backend-source", + "provider": "oss", + "physical_path": "storage://backend-source/oss://yzh-weknora-test/weknora/10000/res-1/a.png", + }) + if err := target.retargetRow("resources", row); err != nil { + t.Fatalf("retarget resource: %v", err) + } + if got := fieldUint(t, row, "tenant_id"); got != 10001 { + t.Fatalf("resource tenant = %d, want 10001", got) + } + want := "storage://backend-target/minio://kgseal-1029a/10001/res-1/a.png" + if got := fieldText(t, row, "physical_path"); got != want { + t.Fatalf("physical_path = %q, want %q", got, want) + } + if got := fieldText(t, row, "provider"); got != "minio" { + t.Fatalf("provider = %q, want minio", got) + } + if got := fieldText(t, row, "storage_backend_id"); got != "backend-target" { + t.Fatalf("storage_backend_id = %q, want backend-target", got) + } +} + +func TestRetargetKeyTenantDropsSourcePrefix(t *testing.T) { + cases := []struct{ key, want string }{ + {"weknora/10000/exports/a.png", "10001/exports/a.png"}, + {"10000/knowledge/file.md", "10001/knowledge/file.md"}, + {"other/prefix/a.png", "other/prefix/a.png"}, + {"10000", "10000"}, + } + for _, tc := range cases { + if got := retargetKeyTenant(tc.key, 10000, 10001); got != tc.want { + t.Fatalf("retargetKeyTenant(%q) = %q, want %q", tc.key, got, tc.want) + } + } + // 同租户不编码:键必须原样返回。 + if got := retargetKeyTenant("weknora/10000/a.png", 10000, 10000); got != "weknora/10000/a.png" { + t.Fatalf("same tenant must keep key, got %q", got) + } +} + +func TestRetargetLocatorRejectsMalformed(t *testing.T) { + if _, err := retargetLocator(" not-a-locator ", "backend", "minio", "bucket", 10000, 10001); err == nil { + t.Fatal("locator with surrounding spaces must be rejected") + } + if _, err := retargetLocator("storage://backend/oss://", "backend", "minio", "bucket", 10000, 10001); err == nil { + t.Fatal("locator without object key must be rejected") + } + // 没有目标桶(测试夹具)时不改写,保持原值可继续用于采集/校验。 + locator := "storage://backend/oss://bucket/key" + if got, err := retargetLocator(locator, "backend", "minio", "", 10000, 10001); err != nil || got != locator { + t.Fatalf("empty bucket must be a no-op, got %q err=%v", got, err) + } + if !strings.Contains(retargetKeyTenant("weknora/10000/a", 10000, 10001), "10001") { + t.Fatal("tenant segment must be replaced") + } +} diff --git a/internal/releaseadapter/v071/library_search.go b/internal/releaseadapter/v071/library_search.go index abe9a2c..161c31e 100644 --- a/internal/releaseadapter/v071/library_search.go +++ b/internal/releaseadapter/v071/library_search.go @@ -26,6 +26,22 @@ func (e *SearchExporter) ExportLibrary(ctx context.Context, identity SearchIdent } return SearchSnapshot{Documents: []SearchDocument{}, MappingSHA256: "", DocumentsSHA256: emptyDigest}, nil } + if errors.Is(err, errScopeFieldsMissing) { + // 全新实例先用空映射建索引:没有范围字段、又确实 0 篇文档时就是空基线; + // 有文档却看不出归属必须报错(不能当成空库覆盖)。 + empty, probeErr := e.emptyScopeIndex(ctx) + if probeErr != nil { + return SearchSnapshot{}, probeErr + } + if empty { + emptyDigest, digestErr := searchDocumentsDigest([]SearchDocument{}) + if digestErr != nil { + return SearchSnapshot{}, digestErr + } + return SearchSnapshot{Documents: []SearchDocument{}, MappingSHA256: "", DocumentsSHA256: emptyDigest}, nil + } + return SearchSnapshot{}, err + } if err != nil { return SearchSnapshot{}, err } @@ -52,8 +68,8 @@ func (e *SearchExporter) ExportLibrary(ctx context.Context, identity SearchIdent } }() filter := []any{map[string]any{"term": map[string]any{inspection.fields.base: identity.KnowledgeBaseID}}} - if inspection.fields.tenant != "" { - filter = append(filter, map[string]any{"term": map[string]any{inspection.fields.tenant: identity.TenantID}}) + if clause := tenantClause(inspection.fields, identity.TenantID); clause != nil { + filter = append(filter, clause) } var cursor json.RawMessage documents := make([]SearchDocument, 0) diff --git a/internal/releaseadapter/v071/library_source_settlement.go b/internal/releaseadapter/v071/library_source_settlement.go index 541a317..d5f687a 100644 --- a/internal/releaseadapter/v071/library_source_settlement.go +++ b/internal/releaseadapter/v071/library_source_settlement.go @@ -2,9 +2,15 @@ package v071 import ( "context" + "encoding/json" "errors" + "fmt" + "strings" + "time" "github.com/jackc/pgx/v5" + + "knowledge-governance/internal/librarydeadletter" ) // A maintenance window stops a shared writer, not a single KB. Check its entire @@ -13,7 +19,14 @@ import ( // This only establishes recorded processing settlement, never Wiki semantics or // the absence of unrecorded/remote work; CaptureLibrary still requires the // deployment's ingress and background-worker readiness callback. +// +// 死信不再"存在即阻断":允许调用方带上一组精确解除指纹。每条实时行都要被逐字节 +// 命中,行被改过、出现新行或没有解除记录都仍然阻断。这里不删除任何上游行。 func checkLibrarySourceSettlement(ctx context.Context, db librarySnapshotDB) error { + return checkLibrarySourceSettlementReleased(ctx, db, nil) +} + +func checkLibrarySourceSettlementReleased(ctx context.Context, db librarySnapshotDB, released []string) error { tx, err := db.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadOnly}) if err != nil { return errors.New("begin library source settlement") @@ -23,16 +36,118 @@ func checkLibrarySourceSettlement(ctx context.Context, db librarySnapshotDB) err if err := tx.QueryRow(ctx, librarySourceSettlementSQL).Scan(&unsettled, &pending, &failed); err != nil { return errors.New("read library source settlement") } - if unsettled != 0 || pending != 0 || failed != 0 { + if unsettled != 0 || pending != 0 { return errors.New("library source has unfinished or failed processing") } + if failed != 0 { + rows, err := sourceDeadLetterRows(ctx, tx, 0, "") + if err != nil { + return err + } + if err := requireReleased(rows, released); err != nil { + return err + } + } if err := tx.Commit(ctx); err != nil { return errors.New("commit library source settlement") } return nil } -const librarySourceSettlementSQL = `SELECT /* library-source-settlement */ +const librarySourceSettlementSQL = `SELECT /* library-source-settlement */ (SELECT count(*) FROM knowledges WHERE deleted_at IS NULL AND (parse_status IS DISTINCT FROM 'completed' OR pending_subtasks_count IS NULL OR pending_subtasks_count<>0)), (SELECT count(*) FROM task_pending_ops), (SELECT count(*) FROM task_dead_letters)` + +// sourceDeadLetterRows 读原始行文本(to_jsonb 已规范化),指纹直接对字节求摘要。 +// tenant=0 表示整库(维护窗口的粒度),否则按知识库范围。 +func sourceDeadLetterRows(ctx context.Context, q interface { + Query(context.Context, string, ...any) (pgx.Rows, error) +}, tenant uint64, kb string) ([][]byte, error) { + statement := `SELECT to_jsonb(d)::text FROM task_dead_letters d` + args := []any{} + if tenant != 0 { + statement += ` WHERE d.tenant_id=$1 AND ((d.scope='knowledge_base' AND d.scope_id=$2) OR + (d.scope='knowledge' AND d.scope_id IN (SELECT id FROM knowledges WHERE tenant_id=$1 AND knowledge_base_id=$2)))` + args = append(args, tenant, kb) + } + statement += ` ORDER BY d.id` + rows, err := q.Query(ctx, statement, args...) + if err != nil { + return nil, errors.New("read library dead letters") + } + defer rows.Close() + var result [][]byte + for rows.Next() { + var raw []byte + if err := rows.Scan(&raw); err != nil { + return nil, errors.New("read library dead letters") + } + result = append(result, raw) + } + if rows.Err() != nil { + return nil, errors.New("read library dead letters") + } + return result, nil +} + +func releasedSet(released []string) map[string]bool { + set := make(map[string]bool, len(released)) + for _, fingerprint := range released { + set[fingerprint] = true + } + return set +} + +func requireReleased(rows [][]byte, released []string) error { + missing, err := librarydeadletter.Unreleased(rows, releasedSet(released)) + if err != nil { + return err + } + if len(missing) > 0 { + shown := missing + if len(shown) > 5 { + shown = shown[:5] + } + return fmt.Errorf("source has %d dead letter(s) without a verified release: %s", len(missing), strings.Join(shown, ", ")) + } + return nil +} + +// ObserveDeadLetters 生成主机侧只读观测:整库结算计数 + 每条死信原始行。 +// 解除记录绑定这份观测的字节摘要,平台侧不复制"自报成功"的结论。 +func ObserveDeadLetters(ctx context.Context, db librarySnapshotDB, project string, tenant uint64, kb string) ([]byte, error) { + if project == "" || kb == "" || tenant == 0 { + return nil, errors.New("observation requires project, tenant and knowledge base") + } + tx, err := db.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadOnly}) + if err != nil { + return nil, errors.New("begin dead letter observation") + } + defer tx.Rollback(context.WithoutCancel(ctx)) + var unsettled, pending, failed int64 + if err := tx.QueryRow(ctx, librarySourceSettlementSQL).Scan(&unsettled, &pending, &failed); err != nil { + return nil, errors.New("read library source settlement") + } + rows, err := sourceDeadLetterRows(ctx, tx, 0, "") + if err != nil { + return nil, err + } + observation := librarydeadletter.Observation{ + Project: project, TenantID: tenant, KnowledgeBaseID: kb, + CapturedAt: time.Now().UTC().Truncate(time.Second), + Settlement: librarydeadletter.Settlement{Unsettled: unsettled, PendingOps: pending}, + Rows: make([]json.RawMessage, 0, len(rows)), + } + for _, raw := range rows { + observation.Rows = append(observation.Rows, json.RawMessage(raw)) + } + payload, err := json.Marshal(observation) + if err != nil { + return nil, errors.New("encode dead letter observation") + } + if err := tx.Commit(ctx); err != nil { + return nil, errors.New("commit dead letter observation") + } + return payload, nil +} diff --git a/internal/releaseadapter/v071/library_universe.go b/internal/releaseadapter/v071/library_universe.go new file mode 100644 index 0000000..4abf5fb --- /dev/null +++ b/internal/releaseadapter/v071/library_universe.go @@ -0,0 +1,126 @@ +package v071 + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sort" + + "github.com/jackc/pgx/v5/pgxpool" + + "knowledge-governance/internal/libraryrelease" + "knowledge-governance/internal/librarysnapshot" +) + +// universeFingerprint 是"整实例"的知识域指纹:不限租户/知识库的行数 + 身份集合摘要。 +// +// 为什么不能只比目标范围:恢复只按目标范围删除/回放,写到范围外的残留它看不见 +// (真实事故:作业报"已恢复到预备份",目标却多出 4 文件 / 38 分块 / 21 资源 / +// 4 绑定)。整实例指纹让"多写了什么"必然暴露,同时也能证明恢复没有误删别的 +// 知识库的行。 +func universeFingerprint(ctx context.Context, pool *pgxpool.Pool) (map[string]libraryrelease.TableFingerprint, error) { + result := make(map[string]libraryrelease.TableFingerprint, len(productionLibraryTables)) + for _, table := range productionLibraryTables { + identity := identityExpr(table) + statement := fmt.Sprintf( + `SELECT count(*), coalesce(md5(string_agg((%s)::text, E'\n' ORDER BY (%s)::text)), '') FROM %s`, + identity, identity, table) + var fingerprint libraryrelease.TableFingerprint + if err := pool.QueryRow(ctx, statement).Scan(&fingerprint.Count, &fingerprint.Digest); err != nil { + return nil, fmt.Errorf("读取整库 %s 指纹失败: %w", table, err) + } + result[table] = fingerprint + } + return result, nil +} + +// archivedRowIdentities 读出一份归档部件的行身份(与校验用的身份口径一致)。 +func archivedRowIdentities(archive *librarysnapshot.Archive, part librarysnapshot.Part) ([]string, error) { + reader, err := archive.OpenPart(part) + if err != nil { + return nil, err + } + defer reader.Close() + decoder := json.NewDecoder(reader) + token, err := decoder.Token() + if err != nil || token != json.Delim('[') { + return nil, errors.New("归档部件不是行数组") + } + ids := make([]string, 0, 64) + for decoder.More() { + var row map[string]json.RawMessage + if err := decoder.Decode(&row); err != nil { + return nil, err + } + identity := archiveRowIdentity(part.Name, row) + if identity == "" { + continue + } + ids = append(ids, identity) + } + return ids, nil +} + +// universeDiff 返回第一处不一致的表;空字符串表示整实例与参照完全一致。 +func universeDiff(want, got map[string]libraryrelease.TableFingerprint) string { + tables := make([]string, 0, len(want)) + for table := range want { + tables = append(tables, table) + } + sort.Strings(tables) + for _, table := range tables { + current, ok := got[table] + if !ok { + return fmt.Sprintf("%s(目标读不到)", table) + } + if current.Count != want[table].Count { + return fmt.Sprintf("%s 行数 %d,参照 %d", table, current.Count, want[table].Count) + } + if current.Digest != want[table].Digest { + return fmt.Sprintf("%s 身份集合与参照不同(同数量改写也会被这条拦住)", table) + } + } + return "" +} + +// assertNoSourceScopeRows 找出"这次要写的这些行,仍然以来源身份躺在目标里"。 +// +// 判据按归档里的身份集合逐表核对,而不是"这个租户/知识库有没有行":同一个租户 +// 里本来就可能住着别的知识库(共享资源、别的 KB 的绑定),按范围数会把它们误 +// 报成残留。这里只认"本次写集合里、仍属于来源租户"的行——它们只可能来自逃出 +// 目标范围的写入,而恢复只按目标范围回放,永远删不到它们。 +func (t *ProductionLibraryTarget) assertNoSourceScopeRows(ctx context.Context, archive *librarysnapshot.Archive, + manifest librarysnapshot.Manifest) error { + if t.snapshotTenant == 0 { + return nil + } + if t.snapshotTenant == t.scope.TenantID && t.snapshotKB == t.scope.KnowledgeBaseID { + return nil + } + for _, part := range manifest.Parts { + if part.Kind != "database" && part.Kind != "wiki" { + continue + } + if !tableHasTenant(part.Name) { + continue + } + ids, err := archivedRowIdentities(archive, part) + if err != nil { + return err + } + if len(ids) == 0 { + continue + } + statement := fmt.Sprintf(`SELECT count(*) FROM %s WHERE tenant_id=$1 AND id = ANY($2::text[])`, part.Name) + var count int64 + if err := t.pool.QueryRow(ctx, statement, t.snapshotTenant, ids).Scan(&count); err != nil { + return fmt.Errorf("检查目标 %s 的来源范围残留失败: %w", part.Name, err) + } + if count > 0 { + return fmt.Errorf("目标 %s 残留 %d 行仍属于来源范围 %d/%s:有写入逃出了目标范围,恢复不会清掉它们", + part.Name, count, t.snapshotTenant, t.snapshotKB) + } + } + return nil +} diff --git a/internal/releaseadapter/v071/library_verify_content.go b/internal/releaseadapter/v071/library_verify_content.go index fe28250..d458246 100644 --- a/internal/releaseadapter/v071/library_verify_content.go +++ b/internal/releaseadapter/v071/library_verify_content.go @@ -98,7 +98,8 @@ func (t *ProductionLibraryTarget) productionTableContent(ctx context.Context, ta } // archivedTableContent 读取归档侧规范内容摘要,并返回归档里出现过的列集合。 -func archivedTableContent(archive *librarysnapshot.Archive, part librarysnapshot.Part) (int64, string, map[string]bool, error) { +func archivedTableContent(archive *librarysnapshot.Archive, part librarysnapshot.Part, + transform func(map[string]json.RawMessage) error) (int64, string, map[string]bool, error) { reader, err := archive.OpenPart(part) if err != nil { return 0, "", nil, err @@ -116,6 +117,12 @@ func archivedTableContent(archive *librarysnapshot.Archive, part librarysnapshot if err := decoder.Decode(&raw); err != nil { return 0, "", nil, err } + if transform != nil { + // 写入时对行做过的变换(例如资源定位符换桶)校验时必须同样应用。 + if err := transform(raw); err != nil { + return 0, "", nil, err + } + } for key := range raw { columns[key] = true } diff --git a/internal/releaseadapter/v071/object_import.go b/internal/releaseadapter/v071/object_import.go index 156d1ce..a439a6f 100644 --- a/internal/releaseadapter/v071/object_import.go +++ b/internal/releaseadapter/v071/object_import.go @@ -100,6 +100,66 @@ func (importer *ObjectImporter) Promote(ctx context.Context, stage ObjectStage) return nil } +// rewriteLocatorBucket 把资源定位符里的桶换成目标桶,结构保持不变: +// "storage:///:///" → +// "storage:///:///"。 +// +// 为什么必须改:同步把字节写进目标桶,但行是从快照原样搬的,定位符里写着测试桶; +// 目标实例之后按这一行去取图/附件时会指向测试环境的桶(跨环境引用), +// 或者干脆取不到。键不变,只换桶。 +func rewriteLocatorBucket(locator, bucket string) (string, bool) { + return rewriteLocatorTarget(locator, "", bucket) +} + +// rewriteLocatorTarget 同时改写定位符里的存储后端 id 与桶。 +// +// 为什么后端 id 也要改:storage:///minio:/// 里的 id 是来源实例 +// storage_backends 的主键。目标实例有自己的行;只改桶不改 id,原版应用会以 +// "storage backend not found" 落到默认服务,然后拒绝整条路径(真跑 500 code1007)。 +func rewriteLocatorTarget(locator, backendID, bucket string) (string, bool) { + if bucket == "" || locator == "" || locator != strings.TrimSpace(locator) { + return "", false + } + prefix := "" + rest := locator + if strings.HasPrefix(locator, "storage://") { + trimmed := strings.TrimPrefix(locator, "storage://") + parts := strings.SplitN(trimmed, "/", 2) + if len(parts) != 2 || !validSearchIdentifier(parts[0]) { + return "", false + } + if parts[1] == "" { + return "", false + } + if backendID != "" { + if !validSearchIdentifier(backendID) { + return "", false + } + parts[0] = backendID + } + prefix, rest = "storage://"+parts[0]+"/", parts[1] + } + separator := strings.Index(rest, "://") + if separator < 0 { + // 裸对象键里没有桶:落盘时用的是配置桶,不需要改,也不算失败。 + return locator, true + } + scheme := rest[:separator] + if !validSearchIdentifier(scheme) { + return "", false + } + schemeless := rest[separator+3:] + boundary := strings.Index(schemeless, "/") + if boundary <= 0 || boundary == len(schemeless)-1 { + return "", false + } + key := schemeless[boundary+1:] + if key == "" || !validSearchIdentifier(schemeless[:boundary]) { + return "", false + } + return prefix + scheme + "://" + bucket + "/" + key, true +} + func providerPathFromLocator(locator string) (string, bool) { rest := locator if strings.HasPrefix(locator, "storage://") { diff --git a/internal/releaseadapter/v071/resource_locator_test.go b/internal/releaseadapter/v071/resource_locator_test.go new file mode 100644 index 0000000..b106509 --- /dev/null +++ b/internal/releaseadapter/v071/resource_locator_test.go @@ -0,0 +1,101 @@ +package v071 + +import ( + "encoding/json" + "testing" +) + +// 同步把字节写进目标桶,但资源行是从快照原样搬的:定位符里写着来源桶时, +// 目标实例之后取图/附件会指向测试环境的桶(跨环境引用)或直接取不到。 +// 只换桶、不换键,两种定位符形状都要能改。 +func TestRewriteLocatorBucketKeepsKeyAndSwapsBucket(t *testing.T) { + for _, testCase := range []struct{ locator, want string }{ + {"storage://45064e4f-434c-4752-a3ae-5fb1e2f54a7d/minio://kgmaint-1028a/10001/k/ab.md", + "storage://45064e4f-434c-4752-a3ae-5fb1e2f54a7d/minio://kgseal-1029a/10001/k/ab.md"}, + {"oss://source-bucket/10/k/ab.png", "oss://target-bucket/10/k/ab.png"}, + } { + bucket := "kgseal-1029a" + if testCase.locator[:3] == "oss" { + bucket = "target-bucket" + } + got, ok := rewriteLocatorBucket(testCase.locator, bucket) + if !ok || got != testCase.want { + t.Fatalf("rewrite(%q)=%q ok=%v want %q", testCase.locator, got, ok, testCase.want) + } + } + // 解析不出来就报错,不猜:宁可失败也不能写一行指向别处的资源。 + for _, bad := range []string{"", "storage://x/", "storage://backend/minio://bucket", "oss://bucket/"} { + if _, ok := rewriteLocatorBucket(bad, "target"); ok { + t.Fatalf("非法定位符被接受了:%q", bad) + } + } + if _, ok := rewriteLocatorBucket("oss://bucket/key", ""); ok { + t.Fatal("没有目标桶时不该改写") + } + // 裸对象键里没有桶(落盘用配置桶),原样返回而不是失败。 + if got, ok := rewriteLocatorBucket("weknora/10000/k/ab.docx", "target"); !ok || got != "weknora/10000/k/ab.docx" { + t.Fatalf("裸键被改写了:%q ok=%v", got, ok) + } +} + +// 只换桶不够:storage:// 是来源实例 storage_backends 的主键。目标实例没有这条行时, +// 原版应用会以 storage backend not found 落到默认服务并拒绝整条路径(真跑 500 code1007), +// 所以后端 id 必须一起换成本次写入解析到的目标 id。 +func TestRewriteLocatorSwapsStorageBackendWithBucket(t *testing.T) { + got, ok := rewriteLocatorTarget("storage://source-backend/minio://kgmaint-1028a/10001/k/ab.md", + "target-backend", "kgseal-1029a") + if !ok || got != "storage://target-backend/minio://kgseal-1029a/10001/k/ab.md" { + t.Fatalf("rewrite=%q ok=%v", got, ok) + } + // 没有解析到后端时不猜:保留原 id(调用方在没有目标桶时不会走到这里)。 + got, ok = rewriteLocatorTarget("storage://source-backend/minio://b/k", "", "b2") + if !ok || got != "storage://source-backend/minio://b2/k" { + t.Fatalf("rewrite without backend=%q ok=%v", got, ok) + } + if _, ok := rewriteLocatorTarget("storage://x/minio://b/k", "../evil", "b2"); ok { + t.Fatal("非法后端 id 被接受了") + } +} + +// 行变换用解析到的目标后端与目标桶:写入与校验共用这一份,摘要才不会两边打架。 +func TestResourceRowTransformRewritesBackendAndBucket(t *testing.T) { + target := &ProductionLibraryTarget{config: ProductionLibraryConfig{ObjectsBucket: "kgseal-1029a"}, + storageBackend: "target-backend", snapshotTenant: 10001, snapshotKB: "kb", + scope: ProductionScope{TenantID: 10001, KnowledgeBaseID: "kb"}} + row := map[string]json.RawMessage{ + "tenant_id": json.RawMessage(`10001`), + "physical_path": json.RawMessage(`"storage://source-backend/minio://kgmaint-1028a/10001/k/ab.md"`)} + if err := target.rowTransform("resources")(row); err != nil { + t.Fatalf("row transform failed: %v", err) + } + var got string + if err := json.Unmarshal(row["physical_path"], &got); err != nil { + t.Fatal(err) + } + if got != "storage://target-backend/minio://kgseal-1029a/10001/k/ab.md" { + t.Fatalf("transform=%q", got) + } +} + +// 存储后端 id 在 resources 表里还有独立一列:原版应用按它找文件服务。 +// 只改定位符不改这一列,目标仍是 storage backend not found(真跑 500)。 +func TestResourceRowTransformRewritesBackendColumn(t *testing.T) { + target := &ProductionLibraryTarget{config: ProductionLibraryConfig{ObjectsBucket: "kgseal-1029a"}, + storageBackend: "target-backend", snapshotTenant: 10001, snapshotKB: "kb", + scope: ProductionScope{TenantID: 10001, KnowledgeBaseID: "kb"}} + row := map[string]json.RawMessage{ + "tenant_id": json.RawMessage(`10001`), + "physical_path": json.RawMessage(`"storage://source-backend/minio://kgmaint-1028a/1/k/ab.md"`), + "storage_backend_id": json.RawMessage(`"source-backend"`), + } + if err := target.rowTransform("resources")(row); err != nil { + t.Fatalf("row transform failed: %v", err) + } + var backend string + if err := json.Unmarshal(row["storage_backend_id"], &backend); err != nil { + t.Fatal(err) + } + if backend != "target-backend" { + t.Fatalf("storage_backend_id=%q", backend) + } +} diff --git a/internal/releaseadapter/v071/search.go b/internal/releaseadapter/v071/search.go index e1ad821..a9a908b 100644 --- a/internal/releaseadapter/v071/search.go +++ b/internal/releaseadapter/v071/search.go @@ -67,6 +67,23 @@ type searchFieldNames struct { // errIndexMissing 表示目标索引根本不存在(空生产基线)。 var errIndexMissing = errors.New("Elasticsearch index missing") +// errScopeFieldsMissing 表示索引存在但还没有发布范围字段: +// 全新的 WeKnora 实例会先用空映射建好索引(`{}`),字段要等第一次写入才动态生成。 +// 这跟"索引被改坏了"必须分开:只有在索引里确实一篇文档都没有时才能当成空基线。 +var errScopeFieldsMissing = errors.New("Elasticsearch index mapping is missing release scope fields") + +// emptyScopeIndex 判断"没有范围字段"的索引是不是真的空。 +// 有文档却没有范围字段 = 无法判断这批文档属于谁,必须报错,不能当成 0。 +func (exporter *SearchExporter) emptyScopeIndex(ctx context.Context) (bool, error) { + var response struct { + Count int64 `json:"count"` + } + if err := exporter.requestJSON(ctx, http.MethodGet, "/"+exporter.index+"/_count", nil, &response); err != nil { + return false, err + } + return response.Count == 0, nil +} + type mappingInspection struct { digest string fields searchFieldNames @@ -127,17 +144,41 @@ func (exporter *SearchExporter) inspectMapping(ctx context.Context) (mappingInsp base: resolve("knowledge_base_id"), } if fields.knowledge == "" || fields.base == "" { - return mappingInspection{}, errors.New("Elasticsearch index mapping is missing release scope fields") + return mappingInspection{}, errScopeFieldsMissing } return mappingInspection{digest: hex.EncodeToString(digest[:]), fields: fields}, nil } -// scopedFilter builds the tenant/KB/knowledge term filter adapted to the -// resolved index field names. +// tenantClause 是"租户范围"这一条过滤条件的唯一写法。 +// +// 为什么不是简单的 term:真实 v0.7.1 的 app 自己写索引时不带 tenant_id,而治理侧 +// 同步写入时会补齐。于是 "mapping 里有 tenant_id" 不等于 "每篇文档都写了 tenant_id"。 +// 只按 term 过滤会把 app 写入的正文整库漏掉——真跑踩过:索引里明明有文档,导出 0 条, +// 候选校验报 legacy coverage。这里改成: +// +// 显式写了本租户 → 收;完全没写 → 收(没有可比的租户声明);写了别的租户 → 排除。 +// +// 知识库身份仍然是一条独立的 term,所以范围始终由知识库唯一 ID 收窄,不是放宽成"全库"。 +func tenantClause(fields searchFieldNames, tenant uint64) any { + if fields.tenant == "" { + return nil + } + return map[string]any{"bool": map[string]any{ + "should": []any{ + map[string]any{"term": map[string]any{fields.tenant: tenant}}, + map[string]any{"bool": map[string]any{"must_not": []any{ + map[string]any{"exists": map[string]any{"field": fields.tenant}}, + }}}, + }, + "minimum_should_match": 1, + }} +} + func scopedFilterFor(fields searchFieldNames, identity SearchIdentity) []any { + // 租户 + 知识库 + 单文档三层收窄:tenant 那条见 tenantClause(app 写入的文档不带租户字段)。 filter := make([]any, 0, 3) - if fields.tenant != "" { - filter = append(filter, map[string]any{"term": map[string]any{fields.tenant: identity.TenantID}}) + if clause := tenantClause(fields, identity.TenantID); clause != nil { + filter = append(filter, clause) } filter = append(filter, map[string]any{"term": map[string]any{fields.base: identity.KnowledgeBaseID}}, diff --git a/internal/releaseadapter/v071/search_empty_index_test.go b/internal/releaseadapter/v071/search_empty_index_test.go new file mode 100644 index 0000000..0ccb7e6 --- /dev/null +++ b/internal/releaseadapter/v071/search_empty_index_test.go @@ -0,0 +1,59 @@ +package v071 + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +// 全新的 WeKnora 实例会先用空映射建索引(`{}`),字段等第一次写入才动态生成。 +// 这种索引不能被当成"索引被改坏了":只有确实 0 篇文档时才是空基线; +// 有文档却没有范围字段 = 不知道这批文档属于谁,必须报错。 +func TestEmptyScopeIndexCountsAsEmptyBaselineOnlyWhenTrulyEmpty(t *testing.T) { + for _, testCase := range []struct { + name string + docCount int + wantErr bool + }{ + {"空索引当空基线", 0, false}, + {"有文档但没有范围字段要报错", 7, true}, + } { + t.Run(testCase.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/chunks/_mapping": + _, _ = w.Write([]byte(`{"chunks":{"mappings":{}}}`)) + case "/chunks/_count": + _ = json.NewEncoder(w).Encode(map[string]any{"count": testCase.docCount}) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + exporter, err := NewSearchExporter(SearchExporterConfig{BaseURL: server.URL, Index: "chunks", PageSize: 100}) + if err != nil { + t.Fatalf("exporter: %v", err) + } + identity := SearchIdentity{TenantID: 10001, KnowledgeBaseID: "kb-1"} + count, countErr := exporter.Count(context.Background(), identity) + if testCase.wantErr { + if countErr == nil { + t.Fatalf("有文档却当成空基线:count=%d", count) + } + } else { + if countErr != nil || count != 0 { + t.Fatalf("空索引没有被当成空基线:count=%d err=%v", count, countErr) + } + } + _, exportErr := exporter.ExportLibrary(context.Background(), identity, 100) + if testCase.wantErr && exportErr == nil { + t.Fatal("导出把有文档的空映射索引当成空库") + } + if !testCase.wantErr && exportErr != nil { + t.Fatalf("空索引导出失败:%v", exportErr) + } + }) + } +} diff --git a/internal/releaseadapter/v071/search_tenant_filter_test.go b/internal/releaseadapter/v071/search_tenant_filter_test.go new file mode 100644 index 0000000..687c87e --- /dev/null +++ b/internal/releaseadapter/v071/search_tenant_filter_test.go @@ -0,0 +1,59 @@ +package v071 + +import ( + "encoding/json" + "strings" + "testing" +) + +// 真实 v0.7.1 的 app 自己写索引时不带 tenant_id,而治理侧同步写入时会补齐。 +// 所以"mapping 里有 tenant_id"不能当成"每篇文档都写了 tenant_id":只按 term 过滤 +// 会把 app 写入的正文整库漏掉(真跑踩过:索引里明明有文档,导出 0 条)。 +func TestTenantClauseAcceptsDocumentsWithoutTenantAndRejectsForeignTenant(t *testing.T) { + fields := searchFieldNames{tenant: "tenant_id", knowledge: "knowledge_id", base: "knowledge_base_id"} + clause := tenantClause(fields, 10001) + encoded, err := json.Marshal(clause) + if err != nil { + t.Fatalf("marshal: %v", err) + } + text := string(encoded) + // 收本租户 + if !strings.Contains(text, `{"term":{"tenant_id":10001}}`) { + t.Fatalf("没有本租户 term:%s", text) + } + // 收完全没写租户的文档(app 写入形状) + if !strings.Contains(text, `"must_not"`) || !strings.Contains(text, `{"exists":{"field":"tenant_id"}}`) { + t.Fatalf("没有接受未写租户的文档:%s", text) + } + // 两种接受条件取其一 + if !strings.Contains(text, `"minimum_should_match":1`) { + t.Fatalf("should 没有收窄成一条:%s", text) + } + // 写了别的租户的文档:term 不匹配、exists 成立 ⇒ 两条都不满足 ⇒ 被排除 + if strings.Contains(text, `"minimum_should_match":0`) { + t.Fatalf("should 允许全不匹配,等于放弃租户范围:%s", text) + } +} + +// mapping 里没有租户字段时不能凭空造一条过滤条件。 +func TestTenantClauseIsAbsentWithoutTenantField(t *testing.T) { + if clause := tenantClause(searchFieldNames{knowledge: "knowledge_id", base: "knowledge_base_id"}, 10001); clause != nil { + t.Fatalf("没有租户字段却生成了过滤条件:%v", clause) + } +} + +// 整库范围过滤仍然带知识库身份,未写租户的文档也进得来。 +func TestScopeFilterKeepsKnowledgeBaseIdentity(t *testing.T) { + fields := searchFieldNames{tenant: "tenant_id", knowledge: "knowledge_id", base: "knowledge_base_id"} + encoded, err := json.Marshal(scopeFilterFor(fields, SearchIdentity{TenantID: 10001, KnowledgeBaseID: "kb-1"})) + if err != nil { + t.Fatalf("marshal: %v", err) + } + text := string(encoded) + if !strings.Contains(text, `{"term":{"knowledge_base_id":"kb-1"}}`) { + t.Fatalf("知识库身份丢了:%s", text) + } + if !strings.Contains(text, `"must_not"`) { + t.Fatalf("未写租户的文档会被漏掉:%s", text) + } +} diff --git a/internal/releaseadapter/v071/search_write.go b/internal/releaseadapter/v071/search_write.go index 7de9566..430e7dc 100644 --- a/internal/releaseadapter/v071/search_write.go +++ b/internal/releaseadapter/v071/search_write.go @@ -19,14 +19,25 @@ func (exporter *SearchExporter) Count(ctx context.Context, identity SearchIdenti return 0, errors.New("invalid search identity") } inspection, err := exporter.inspectMapping(ctx) + if errors.Is(err, errScopeFieldsMissing) { + // 新建索引还没有范围字段:只有确实为空时才等于"这个范围 0 条"。 + empty, probeErr := exporter.emptyScopeIndex(ctx) + if probeErr != nil { + return 0, probeErr + } + if empty { + return 0, nil + } + return 0, err + } if err != nil { return 0, err } // 只按"租户 + 知识库"计数:整库同步没有单文档身份,带上空的 knowledge 条件 // 会把过滤器变成"匹配空字符串",永远数不到东西。 filter := make([]any, 0, 2) - if inspection.fields.tenant != "" { - filter = append(filter, map[string]any{"term": map[string]any{inspection.fields.tenant: identity.TenantID}}) + if clause := tenantClause(inspection.fields, identity.TenantID); clause != nil { + filter = append(filter, clause) } filter = append(filter, map[string]any{"term": map[string]any{inspection.fields.base: identity.KnowledgeBaseID}}) payload := map[string]any{"query": map[string]any{"bool": map[string]any{"filter": filter}}} @@ -105,8 +116,8 @@ func (exporter *SearchExporter) PruneScope(ctx context.Context, identity SearchI // scopeFilterFor 只按租户 + 知识库过滤(整库范围,不含单文档身份)。 func scopeFilterFor(fields searchFieldNames, identity SearchIdentity) []any { filter := make([]any, 0, 2) - if fields.tenant != "" { - filter = append(filter, map[string]any{"term": map[string]any{fields.tenant: identity.TenantID}}) + if clause := tenantClause(fields, identity.TenantID); clause != nil { + filter = append(filter, clause) } filter = append(filter, map[string]any{"term": map[string]any{fields.base: identity.KnowledgeBaseID}}) return filter diff --git a/scripts/library-agent-install.sh b/scripts/library-agent-install.sh index c533016..e600af1 100755 --- a/scripts/library-agent-install.sh +++ b/scripts/library-agent-install.sh @@ -9,8 +9,24 @@ set -euo pipefail ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) UNIT_DIR="$HOME/.config/systemd/user" SOCKET_DIR="$HOME/.local/share/knowledge-governance/agent" +# 实例级维护接线(LIBRARY_TARGET_*)与源声明一起由这一个 0600 文件提供; +# 它属于"这台主机上的哪个实例",不属于任何一次请求。 +ENV_FILE="${AGENT_ENV_FILE:-$HOME/.local/share/knowledge-governance/agent.env}" +SOURCE_APP="${AGENT_SOURCE_APP:-WKNFull-app}" +SOURCE_PG="${AGENT_SOURCE_PG:-WKNFull-postgres}" +SOURCE_ES="${AGENT_SOURCE_ES:-WKNFull-elasticsearch}" +PROJECT="${AGENT_PROJECT:-weknora071full}" +WRITERS="${AGENT_WRITERS:-app}" +NON_WRITERS="${AGENT_NON_WRITERS:-frontend=UI only; no knowledge writes,mcp=read-only MCP server,dobotai-target=alpine placeholder; no knowledge writes,docreader=parse RPC service; app persists parse output,elasticsearch=search store; not an initiator,postgres=database store; not an initiator,redis=queue/cache store; not an initiator,neo4j=graph store; not an initiator,langfuse-web=LLM observability UI,langfuse-worker=LLM observability worker; own storage,langfuse-clickhouse=observability store,langfuse-minio=observability object store,langfuse-db-init=one-shot init; exited,searxng=read-only search proxy,searxng-init=one-shot init; exited}" mkdir -p "$UNIT_DIR" "$SOCKET_DIR" chmod 700 "$SOCKET_DIR" +if [ ! -f "$ENV_FILE" ]; then + # 宁可不装,也不要装出一个"没有闸门"的 agent:封存会在开窗口前被拒。 + echo "AGENT_ENV_MISSING: $ENV_FILE" >&2 + echo " write LIBRARY_TARGET_{INSTANCE,INGRESS_URL,INGRESS_SERVICE,INGRESS_DRAIN_FILE,REDIS_ADDR,REDIS_PASSWORD,REDIS_DB} there (0600)" >&2 + exit 1 +fi +chmod 600 "$ENV_FILE" echo "=== build libraryctl + libraryagent ===" (cd "$ROOT" && go build -o libraryctl ./cmd/libraryctl && go build -o libraryagent ./cmd/libraryagent) @@ -24,13 +40,18 @@ After=network.target [Service] Type=simple WorkingDirectory=$ROOT +EnvironmentFile=$ENV_FILE ExecStart=$ROOT/libraryagent \\ -socket $SOCKET_DIR/agent.sock \\ -binary $ROOT/libraryctl \\ -archive $HOME/.local/share/knowledge-governance/library \\ -journal $HOME/.local/share/knowledge-governance/freeze \\ - -project weknora071full \\ - -writers app + -source-app $SOURCE_APP \\ + -source-pg $SOURCE_PG \\ + -source-es $SOURCE_ES \\ + -project $PROJECT \\ + -writers $WRITERS \\ + -non-writers "$NON_WRITERS" Restart=always RestartSec=5 diff --git a/web/src/ReleaseTargets.test.tsx b/web/src/ReleaseTargets.test.tsx index d2ab4c9..b5394eb 100644 --- a/web/src/ReleaseTargets.test.tsx +++ b/web/src/ReleaseTargets.test.tsx @@ -92,7 +92,7 @@ const overview: LibraryOverview = { reasons: [ '生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false)', '没有已核验的生产恢复演练证据(PRODUCTION_BACKUP_RESTORE_VERIFIED=false)', - '生产当前权威版本未知,不能启动写入', + '目标非空且没有生效版本记录:先建立目标权威基线(维护隔离下捕获整库并逐类校验),或确认目标确实是空库', '本次目标快照还没有生产预备份', ], }, @@ -149,9 +149,20 @@ const base: ViewProps = { pendingCapture: false, canCapture: true, canWrite: true, + canRestore: false, + sync: { canSign: false, pendingGrant: false, onGrant: vi.fn(), canStart: false, pending: false, onStart: vi.fn() }, + baseline: { canStart: false, pending: false, onStart: vi.fn() }, + restore: null, + restoreNotice: null, + pendingRestore: false, + pendingGrant: false, + onGrantRestore: vi.fn(), + onRestore: vi.fn(), + maintenance: null, now: new Date('2026-10-28T03:35:00Z').getTime(), onRefresh: vi.fn(), onCapture: vi.fn(), + onRecover: vi.fn(), } const render = (overrides: Partial = {}) => @@ -164,7 +175,7 @@ describe('ReleaseTargetsView(确认稿布局)', () => { expect(html).toContain('对外知识库') expect(html).toContain('role="tab"') // 只显示一个主作业卡:选中库标题出现一次,另一个库不渲染作业卡 - expect(html.match(/· 已封存,生产同步未接入/g)?.length).toBe(1) + expect(html.match(/· 已封存,同步按条件放行/g)?.length).toBe(1) expect(html).not.toContain('对外知识库 · 尚无') }) @@ -227,7 +238,7 @@ describe('ReleaseTargetsView(确认稿布局)', () => { expect(html).toContain('生产 prod00000000 → 测试 test00000000') expect(html).not.toContain('变更清单来自生产与测试版本对比') expect(html).toContain('Wiki 页面:新增 2 · 更新 1 · 撤销 3') - expect(html).toContain('不代表实体/边/引用重算') + expect(html).toContain('重算发生在源侧') }) it('准备信息注明快照时间与范围,不声称实时测试库已就绪', () => { @@ -237,18 +248,62 @@ describe('ReleaseTargetsView(确认稿布局)', () => { expect(html).toContain('列表只校验清单自身,字节级复核在同步/恢复执行前进行') }) - it('版本与快照抽屉按 scope 列出全部历史并区分测试封存/不可直接恢复', () => { - const html = render({ drawerOpen: true, expandedSnapshot: internalSnapshot.snapshot_id }) + it('版本与快照抽屉按 scope 列出全部历史,并把恢复条件交给接口判定', () => { + const html = render({ drawerOpen: true, expandedSnapshot: internalSnapshot.snapshot_id, canRestore: true }) expect(html).toContain('版本与快照') expect(html).toContain('封存 f992d64ea45b') expect(html).toContain('测试封存') - expect(html).toContain('不可直接恢复') + expect(html).toContain('恢复条件见下方') expect(html).toContain('恢复至该版本') expect(html).toMatch(/恢复至该版本[^<]*<\/button>/) expect(html).toContain('分项数量 / 字节 / 摘要(2 项)') expect(html).toContain('不支持只选择 Wiki 或某张数据表恢复') }) + it('管理员在缺授权时看到签发入口,恢复按钮按接口给出的原因禁用', () => { + const html = render({ + drawerOpen: true, + expandedSnapshot: internalSnapshot.snapshot_id, + canRestore: true, + restore: { + allowed: false, + reasons: ['没有针对该快照的恢复授权:先签发 restore 授权(绑定同一快照)'], + scope: 'internal', + snapshot_id: internalSnapshot.snapshot_id, + authorization: { state: 'none' }, + }, + }) + expect(html).toContain('签发恢复授权(30 分钟)') + expect(html).toContain('没有针对该快照的恢复授权') + expect(html).toMatch(/rel-danger[^>]*disabled/) + // 不能再说"未接线":条件不满足要给出具体原因 + expect(html).not.toContain('生产恢复未接线') + }) + + it('条件全满足时恢复按钮可用,并写明授权有效期', () => { + const html = render({ + drawerOpen: true, + expandedSnapshot: internalSnapshot.snapshot_id, + canRestore: true, + restore: { + allowed: true, + reasons: [], + scope: 'internal', + snapshot_id: internalSnapshot.snapshot_id, + authorization: { state: 'active', authorized_by: 'yaozhanghao', expires_at: '2026-10-28T04:10:00Z' }, + }, + }) + expect(html).toContain('条件已满足') + expect(html).toContain('有效至') + expect(html).not.toMatch(/rel-danger[^>]*disabled/) + }) + + it('工程师看不到恢复入口,只看到权限说明', () => { + const html = render({ drawerOpen: true, expandedSnapshot: internalSnapshot.snapshot_id, canRestore: false }) + expect(html).toContain('按版本恢复仅管理员可发起') + expect(html).not.toContain('签发恢复授权') + }) + it('抽屉只展示当前选中库的历史', () => { const html = render({ drawerOpen: true, scope: 'external' }) expect(html).toContain('封存 01343b0b310b') @@ -275,19 +330,119 @@ describe('ReleaseTargetsView(确认稿布局)', () => { expect(html).toContain('主机侧正在暂停写入者、导出并恢复') }) - it('无权限时不给封存入口,未接线时说明通道状态', () => { + it('无权限时不给封存入口;不可发起时给出接口返回的真实原因与执行器身份', () => { expect(render({ canCapture: false, canWrite: false })).toContain('封存需要知识工程师及以上权限') - const unwired = render({ + const unavailable = render({ canCapture: false, canWrite: true, - page: { ...page, capture_available: false, capture_reason: '主机侧封存 agent 未接线' }, + page: { ...page, capture_available: false }, + overview: { + ...overview, + capture_available: false, + capture_reason: '执行器指向的实例不匹配:agent="weknora071full",期望="kgmaint-weknora"', + executor: { + available: false, + busy: false, + project: 'weknora071full', + source_app: 'WKNFull-app', + observed_at: '2026-10-28T03:34:00Z', + }, + }, }) - expect(unwired).toContain('封存通道未接线') + expect(unavailable).toContain('执行器指向的实例不匹配') + expect(unavailable).toContain('执行器 weknora071full / WKNFull-app') + expect(unavailable).toContain('观测于') + // 通道可用/不可用都不许用写死的"未接线"糊过去 + expect(unavailable).not.toContain('封存通道未接线') }) it('无封存时给出“尚无封存”而不是空表', () => { - const html = render({ page: { ...page, snapshots: [] } }) + // 归档没有快照时接口也不会给 test_sealed:两处必须一致,否则页面会把 + // 一个不存在的封存显示成"已封存"。 + const html = render({ + page: { ...page, snapshots: [] }, + overview: { ...overview, test_sealed: undefined, test_sealed_reason: '还没有该目标的测试封存' }, + }) expect(html).toContain('尚无测试封存') expect(html).toContain('还没有测试库封存') }) + + it('管理员在缺同步授权时看到签发入口,工程师只看到“需管理员签发”', () => { + const admin = render({ + sync: { canSign: true, pendingGrant: false, onGrant: vi.fn(), canStart: false, pending: false, onStart: vi.fn() }, + }) + expect(admin).toContain('签发同步授权(30 分钟)') + // 授权入口不能顺手把执行也放行:闸口条件仍然逐条可见 + expect(admin).toContain('生产写入闸口关闭') + const engineer = render() + expect(engineer).not.toContain('签发同步授权') + expect(engineer).toContain('同步授权只能由管理员按目标+快照签发') + }) + + it('目标非空又没有权威版本时,管理员看到建立基线入口;有生效版本时不显示', () => { + const admin = render({ + baseline: { canStart: true, pending: false, onStart: vi.fn() }, + }) + expect(admin).toContain('建立目标权威基线(维护隔离)') + expect(admin).toContain('目标非空且没有生效版本记录') + const known = render({ + baseline: { canStart: true, pending: false, onStart: vi.fn() }, + overview: { + ...overview, + production: { + state: 'known', + version: { + id: 'v1', + kind: 'production_effective', + snapshot_id: 'a'.repeat(64), + version_label: '基线 abc', + source_job_id: 'libbase-1', + evidence: {}, + created_by: 'admin', + created_at: '2026-10-29T09:00:00Z', + }, + }, + }, + }) + // 有生效版本时不再提供基线入口(缺项说明文字里仍会提到它存在,所以只断言按钮) + expect(known).not.toMatch(/建立目标权威基线(维护隔离)<\/button>/) + }) + + it('闸口全满足时“启动同步”真的可点,并给出执行说明', () => { + const html = render({ + sync: { canSign: false, pendingGrant: false, onGrant: vi.fn(), canStart: true, pending: false, onStart: vi.fn() }, + overview: { ...overview, gate: { sync_allowed: true, reasons: [] } }, + }) + expect(html).toMatch(/]*class="rel-primary"[^>]*>▶ 启动同步/) + expect(html).not.toMatch(/class="rel-primary"[^>]*disabled/) + expect(html).toContain('先固定测试版本,再备份目标并写入') + }) + + it('维护卡片区分“未运行维护”与“没有维护能力”', () => { + const wired = render({ + maintenance: { + held: false, + capability: { + configured: true, + missing: [], + instance: 'kgseal-app', + project: 'kgseal-weknora', + writers: ['app'], + ingress_service: 'ingress', + }, + }, + }) + expect(wired).toContain('未运行维护') + expect(wired).toContain('维护能力已接线:实例 kgseal-app · 写入者 app') + expect(wired).not.toContain('也没有维护能力配置') + const broken = render({ + maintenance: { held: false, capability: { configured: false, missing: ['LIBRARY_TARGET_WRITERS'] } }, + }) + expect(broken).toContain('能力未接线') + expect(broken).toContain('维护能力未接线:缺少 LIBRARY_TARGET_WRITERS') + // 接口没给能力信息时不能猜成"没有能力" + const unknown = render({ maintenance: { held: false } }) + expect(unknown).toContain('维护能力状态未知') + expect(unknown).not.toContain('也没有维护能力配置') + }) }) diff --git a/web/src/ReleaseTargets.tsx b/web/src/ReleaseTargets.tsx index 1f87cae..0988204 100644 --- a/web/src/ReleaseTargets.tsx +++ b/web/src/ReleaseTargets.tsx @@ -1,14 +1,24 @@ import { useCallback, useEffect, useState } from 'react' import { + createLibraryRestoreAuthorization, + createLibrarySyncAuthorization, + enqueueLibraryBaseline, enqueueLibraryCapture, + enqueueLibraryRecovery, + enqueueLibraryRestore, + enqueueLibrarySync, fetchLibraryCaptures, + fetchLibraryMaintenance, fetchLibraryOverview, + fetchLibraryRestoreAvailability, fetchLibrarySnapshots, formatBytes, scopeLabel, type LibraryCaptureJob, type LibraryJobRecord, + type LibraryMaintenanceView, type LibraryOverview, + type LibraryRestoreAvailability, type LibrarySnapshot, type LibrarySnapshotList, type LibraryStageRow, @@ -55,6 +65,24 @@ const jobStateLabel: Record = { type Scope = 'internal' | 'external' +// 签发授权与执行是两步:页面只提供入口,不会替人签发,也不会自动重试执行。 +export interface ReleaseSyncActions { + // 管理员才能签发;工程师看得到条件但没有按钮。 + canSign: boolean + pendingGrant: boolean + onGrant: () => void + // 闸口全部满足才能启动(接口用同一份判定,按钮只是提前说明)。 + canStart: boolean + pending: boolean + onStart: () => void +} + +export interface ReleaseBaselineActions { + canStart: boolean + pending: boolean + onStart: () => void +} + interface ScopeView { scope: Scope snapshot?: LibrarySnapshot @@ -82,10 +110,24 @@ interface ViewProps { pendingCapture: boolean canCapture: boolean canWrite: boolean + // 发布动作:签发授权(管理员)与启动同步是两个独立步骤,页面绝不自动签发。 + sync: ReleaseSyncActions + // 目标非空又没有生效版本时,唯一入口是"建立目标权威基线"(管理员)。 + baseline: ReleaseBaselineActions + // 按版本恢复是管理员动作:工程师看得到条件,但拿不到恢复入口。 + canRestore: boolean + restore: LibraryRestoreAvailability | null + restoreNotice: string | null + pendingRestore: boolean + pendingGrant: boolean + onGrantRestore: () => void + onRestore: () => void overview: LibraryOverview | null + maintenance: LibraryMaintenanceView | null now: number onRefresh: () => void onCapture: () => void + onRecover: (windowJobID: string) => void } function scopeView( @@ -101,7 +143,9 @@ function scopeView( overview?.job ?? persisted.find((item) => item.state === 'pending' || item.state === 'running') ?? persisted[0] return { scope, - snapshot: history[0], + // 候选只认接口给的 test_sealed(来自版本登记):归档里还有生产预备份与 + // 目标基线快照,按"最新一份"挑会把目标自己的内容显示成测试封存。 + snapshot: overview?.test_sealed, history, activeJob: scoped.find((job) => ACTIVE_STATES.includes(job.state)), lastJob: scoped[0], @@ -113,6 +157,20 @@ function scopeView( const scopeName = (scope: Scope) => (scope === 'internal' ? '对内知识库' : '对外知识库') +// 不可发起时必须说清是哪一条不满足:读的是接口给出的真实原因(执行器不可达、 +// 指向别的实例、降级、观测过期),不是一句写死的"通道未接线"。 +function captureHint(view: ScopeView, canWrite: boolean): string { + if (!canWrite) return '封存需要知识工程师及以上权限' + const executor = view.overview?.executor + const reason = view.overview?.capture_reason || executor?.reason + const observed = executor?.observed_at ? `(观测于 ${stamp(executor.observed_at)})` : '' + const identity = executor?.project + ? `执行器 ${executor.project}${executor.source_app ? ` / ${executor.source_app}` : ''} · ` + : '' + if (reason) return `${identity}封存通道不可用:${reason}${observed}` + return '封存通道状态未知:还没有读到执行器观测' +} + function statusOf(view: ScopeView) { const job = view.job if (job && (job.state === 'running' || job.state === 'pending')) { @@ -157,6 +215,191 @@ function jobStateText(job: LibraryJobRecord): string { return map[job.state] ?? job.state } +// ---- 目标维护窗口:级别 / 进度 / 恢复错误 ---- +// +// 维护窗口有两种级别:实例级(跨 scope 共用一把锁,真正挡住写入者)和库级。 +// 页面必须说清是哪一级、入口是开还是关、作业走到哪、恢复失败的具体错误—— +// 这些都来自 /api/v1/library/maintenance 的真实记录,没有记录就说"未接入"。 +export interface MaintenanceSummary { + level: string + held: boolean + owner: string + state: string + ingress: string + services: string + acquiredAt: string + releasedAt: string + progress: string + recoveryError: string + recoveryNote: string + canRecover: boolean +} + +// 阶段名给操作员看的映射;没映射到的阶段原样显示,不猜。 +const stageLabel: Record = { + prebackup: '预备份', + database: '数据库', + resources: '资源', + index: '索引', + wiki: 'Wiki', + verify: '校验', + finalize: '登记版本', + rollback: '回滚', + export: '导出', + acquire: '开窗口', + drain: '排空', + stop: '停写入者', +} + +const ingressText = (state?: string) => + state === 'closed' ? '已关闭(排空后等待写入)' : state === 'open' ? '打开' : state ? state : '未知' + +export function maintenanceSummary( + maintenance: LibraryMaintenanceView | null | undefined, + job: LibraryJobRecord | undefined +): MaintenanceSummary { + const record = maintenance?.instance_maintenance ?? maintenance?.maintenance ?? null + const level = !record + ? '未接入' + : record.level === 'instance' + ? '实例级' + : record.level === 'library' + ? '库级' + : record.level || '未知' + const held = Boolean(maintenance?.held) + const failed = job?.state === 'failed' + const ownerScope = maintenance?.instance_maintenance?.owner_scope ?? record?.scope ?? '—' + return { + level, + held, + owner: record ? `${ownerScope} · ${record.owner_job_id || '—'}` : '—', + state: record?.state || '—', + ingress: record ? ingressText(record.ingress_state) : '—', + services: (record?.services ?? []).join('、') || '—', + acquiredAt: record?.acquired_at ? stamp(record.acquired_at) : '—', + releasedAt: record?.released_at ? stamp(record.released_at) : '', + progress: job ? `${stageLabel[job.stage] ?? job.stage ?? '—'} · ${jobStateText(job)}` : '尚无作业进度', + recoveryError: failed ? job?.error || '作业失败但没有留下错误明细' : '', + recoveryNote: job?.recovery || '', + canRecover: held && (failed || !job || job.state === 'cancelled'), + } +} + +export function MaintenanceCard({ + maintenance, + job, + canWrite, + onRecover, +}: { + maintenance: LibraryMaintenanceView | null | undefined + job: LibraryJobRecord | undefined + canWrite: boolean + onRecover: (windowJobID: string) => void +}) { + const summary = maintenanceSummary(maintenance, job) + const record = maintenance?.instance_maintenance ?? maintenance?.maintenance + // "没有窗口记录"不等于"没有维护能力":能力是配置事实,接口单独返回。 + // 接口没给能力信息时也不能猜,明确写"未知"。 + const capability = maintenance?.capability + const capabilityText = !capability + ? '维护能力状态未知:接口没有返回能力信息,无法区分"未运行维护"与"能力未接线"' + : capability.configured + ? `维护能力已接线:实例 ${capability.instance || '—'} · 写入者 ${(capability.writers ?? []).join('、') || '—'}` + : `维护能力未接线:缺少 ${(capability.missing ?? []).join('、') || '必要配置'}` + return ( +
+
+
+

目标维护

+

同步/恢复期间目标写入者被停住;级别与入口状态来自维护记录本身

+
+ + {summary.held + ? `${summary.level} · 占用中` + : record + ? '未占用' + : capability?.configured + ? '未运行维护' + : capability + ? '能力未接线' + : '能力未知'} + +
+ {!record ? ( +

+ 暂无维护窗口记录。{capabilityText} + {capability?.configured + ? ';窗口只在同步或建立基线时开启,页面不会单独开窗口。' + : ';维护类操作保持 fail closed,不会在此状态下写目标。'} +

+ ) : ( + <> +
    +
  • + 级别 + {summary.level} +
  • +
  • + 持有 + {summary.owner} +
  • +
  • + 阶段 + {summary.state} +
  • +
  • + 入口 {summary.ingress} + {summary.services} +
  • +
  • + 开始 + {summary.acquiredAt} +
  • +
  • + 释放 + {summary.releasedAt || '未释放'} +
  • +
+

+ 进度:{summary.progress} + {summary.releasedAt ? ` · 窗口已于 ${summary.releasedAt} 释放` : ''} +

+ {summary.recoveryError ? ( +
+ 恢复错误 +

{summary.recoveryError}

+ {summary.recoveryNote ?

处理记录:{summary.recoveryNote}

: null} +
+ ) : null} + {canWrite && summary.canRecover && record ? ( + + ) : null} + + )} +
+ ) +} + +// 逐页贡献级差异:一条页面一行,直接写清来源少了几份、引用从几变几。 +export function wikiContributionLines(diff: LibraryOverview['diff'] | undefined): string[] { + const wiki = diff?.wiki + if (!wiki || wiki.state !== 'computed' || !wiki.pages || wiki.pages.length === 0) return [] + const label: Record = { added: '新增', updated: '更新', removed: '整页撤销' } + return wiki.pages.map((page) => { + const parts = [`${label[page.status] ?? page.status} ${page.title || page.slug}`] + if (page.changed.length > 0) parts.push(page.changed.join('、')) + if ((page.sources_removed ?? []).length > 0) parts.push(`来源 -${page.sources_removed!.length}`) + if ((page.sources_added ?? []).length > 0) parts.push(`来源 +${page.sources_added!.length}`) + if (page.citations_before !== page.citations_after) + parts.push(`引用 ${page.citations_before}→${page.citations_after}`) + return parts.join(' · ') + }) +} + function StageBar({ view }: { view: ScopeView }) { const job = view.job const sealJob = job?.kind === 'seal' ? job : undefined @@ -312,10 +555,23 @@ function ChangeTable({ diff }: { diff: LibraryOverview['diff'] }) { {diff.wiki && diff.wiki.state === 'computed' ? ( -

- Wiki 页面:新增 {diff.wiki.added} · 更新 {diff.wiki.updated} · 撤销 {diff.wiki.removed} - (页面级,不代表实体/边/引用重算) -

+
+

+ Wiki 页面:新增 {diff.wiki.added} · 更新 {diff.wiki.updated} · 撤销 {diff.wiki.removed} + {typeof diff.wiki.sources_removed === 'number' + ? ` · 来源文档变化 −${diff.wiki.sources_removed} / +${diff.wiki.sources_added ?? 0}` + : ''} + (逐页比对来源、引用、出入边与目录;重算发生在源侧) +

+ {wikiContributionLines(diff).length > 0 ? ( +
    + {wikiContributionLines(diff).map((line) => ( +
  • {line}
  • + ))} +
+ ) : null} + {diff.wiki.pages_truncated ?

逐页明细超过 50 条,只列出前 50 条。

: null} +
) : null} ) @@ -326,15 +582,23 @@ function ProtectionCard({ prebackup, gate, authorization, + targetSealed, + sync, + baseline, onOpen, }: { production: LibraryOverview['production'] prebackup: LibraryOverview['prebackup'] gate: LibraryOverview['gate'] authorization: LibraryOverview['authorization'] + targetSealed: boolean + sync: ReleaseSyncActions + baseline: ReleaseBaselineActions onOpen: () => void }) { const backupKnown = prebackup.state === 'known' && prebackup.version + // 基线入口只在"目标非空、却没有生效版本"时才是答案;空库走首次同步(预备份+登记)。 + const baselineNeeded = production.state !== 'known' && targetSealed return ( @@ -865,9 +1205,20 @@ export function ReleaseTargetsView(props: ViewProps) { pendingCapture, canCapture, canWrite, + sync, + baseline, + canRestore, + restore, + restoreNotice, + pendingRestore, + pendingGrant, + onGrantRestore, + onRestore, now, + maintenance, onRefresh, onCapture, + onRecover, } = props const internal = scopeView(page, jobs, 'internal', overview?.scope === 'internal' ? overview : null) const external = scopeView(page, jobs, 'external', overview?.scope === 'external' ? overview : null) @@ -941,7 +1292,7 @@ export function ReleaseTargetsView(props: ViewProps) {

- {scopeName(scope)} · {active ? '封存进行中' : view.snapshot ? '已封存,生产同步未接入' : '尚无测试封存'} + {scopeName(scope)} · {active ? '封存进行中' : view.snapshot ? '已封存,同步按条件放行' : '尚无测试封存'}

@@ -1028,6 +1380,8 @@ export function ReleaseTargetsView(props: ViewProps) {

启动后先固定测试版本,再备份生产。备份失败或恢复条件不满足,不进入生产写入。

+ + {active ? ( onDrawer(true)} /> ) : ( @@ -1038,6 +1392,8 @@ export function ReleaseTargetsView(props: ViewProps) { prebackup={prebackup} gate={gate} authorization={authorization} + sync={sync} + baseline={baseline} canCapture={canCapture} canWrite={canWrite} pendingCapture={pendingCapture} @@ -1055,8 +1411,15 @@ export function ReleaseTargetsView(props: ViewProps) { onExpand={onExpandSnapshot} onClose={() => onDrawer(false)} canCapture={canCapture} + canRestore={canRestore} pendingCapture={pendingCapture} onCapture={onCapture} + restore={restore} + restoreNotice={restoreNotice} + pendingRestore={pendingRestore} + pendingGrant={pendingGrant} + onGrantRestore={onGrantRestore} + onRestore={onRestore} /> ) @@ -1073,6 +1436,10 @@ export function ReleaseTargets() { const [loading, setLoading] = useState(true) const [error, setError] = useState(null) const [notice, setNotice] = useState(null) + const [maintenances, setMaintenances] = useState>({ + internal: null, + external: null, + }) const [scope, setScope] = useState('internal') const [drawerOpen, setDrawerOpen] = useState(false) const [expandedSnapshot, setExpandedSnapshot] = useState(null) @@ -1082,15 +1449,19 @@ export function ReleaseTargets() { const load = useCallback(async (signal?: AbortSignal) => { setLoading(true) try { - const [snapshots, captures, internalOverview, externalOverview] = await Promise.all([ - fetchLibrarySnapshots(signal), - fetchLibraryCaptures(signal), - fetchLibraryOverview('internal', signal), - fetchLibraryOverview('external', signal), - ]) + const [snapshots, captures, internalOverview, externalOverview, internalMaintenance, externalMaintenance] = + await Promise.all([ + fetchLibrarySnapshots(signal), + fetchLibraryCaptures(signal), + fetchLibraryOverview('internal', signal), + fetchLibraryOverview('external', signal), + fetchLibraryMaintenance('internal', signal), + fetchLibraryMaintenance('external', signal), + ]) setPage(snapshots) setJobs(captures) setOverviews({ internal: internalOverview, external: externalOverview }) + setMaintenances({ internal: internalMaintenance, external: externalMaintenance }) setNow(Date.now()) setError(null) } catch (loadError) { @@ -1119,7 +1490,122 @@ export function ReleaseTargets() { const role = auth?.identity?.role const canWrite = role === 'engineer' || role === 'admin' + // 恢复写的是整套生产知识状态,接口只认管理员;页面必须与它一致。 + const canRestore = role === 'admin' + // 签发同步授权同样是管理员动作:授权=钉死目标+快照+参数,不等于执行。 + const canSignSync = role === 'admin' const canCapture = canWrite && page?.capture_available === true + const [pendingSyncGrant, setPendingSyncGrant] = useState(false) + const [pendingSync, setPendingSync] = useState(false) + const [pendingBaseline, setPendingBaseline] = useState(false) + const current = overviews[scope] + const targetSnapshot = current?.test_sealed?.snapshot_id ?? '' + + const grantSync = async () => { + if (!targetSnapshot) { + setNotice('还没有可发布的测试封存:先封存测试版本,再签发同步授权') + return + } + setPendingSyncGrant(true) + setNotice(null) + try { + const grant = await createLibrarySyncAuthorization(scope, targetSnapshot) + setNotice( + `已签发同步授权${grant.authorized_by ? `(${grant.authorized_by})` : ''},有效至 ${stamp(grant.expires_at)};` + + '执行仍需在下一次操作里点"启动同步"' + ) + await load() + } catch (grantError) { + setNotice(grantError instanceof Error ? grantError.message : '签发同步授权失败') + } finally { + setPendingSyncGrant(false) + } + } + + const startSync = async () => { + if (!targetSnapshot) { + setNotice('还没有可发布的测试封存') + return + } + setPendingSync(true) + setNotice(null) + try { + const accepted = await enqueueLibrarySync(scope, targetSnapshot) + setNotice(`已投递同步:${accepted.job_id}(先固定测试版本并生成预备份,再写目标)`) + await load() + } catch (syncError) { + setNotice(syncError instanceof Error ? syncError.message : '启动同步失败') + } finally { + setPendingSync(false) + } + } + + const startBaseline = async () => { + setPendingBaseline(true) + setNotice(null) + try { + const accepted = await enqueueLibraryBaseline(scope) + setNotice(`已投递建立基线:${accepted.job_id}(维护隔离下捕获目标整库,逐类校验后登记为权威基线)`) + await load() + } catch (baselineError) { + setNotice(baselineError instanceof Error ? baselineError.message : '建立目标权威基线失败') + } finally { + setPendingBaseline(false) + } + } + + // 恢复条件随"展开的那份快照"变化:闸口、该快照的授权、实例是否在维护都由接口判定。 + const [restore, setRestore] = useState(null) + const [restoreNotice, setRestoreNotice] = useState(null) + const [pendingRestore, setPendingRestore] = useState(false) + const [pendingGrant, setPendingGrant] = useState(false) + const [restoreRound, setRestoreRound] = useState(0) + const restoreSnapshot = expandedSnapshot ?? '' + useEffect(() => { + if (!drawerOpen || !restoreSnapshot) { + setRestore(null) + return + } + const controller = new AbortController() + void fetchLibraryRestoreAvailability(scope, restoreSnapshot, controller.signal) + .then((value) => setRestore(value)) + .catch(() => setRestore(null)) + return () => controller.abort() + }, [drawerOpen, restoreSnapshot, scope, restoreRound]) + + const grantRestore = async () => { + if (!restoreSnapshot) return + setPendingGrant(true) + setRestoreNotice(null) + try { + const grant = await createLibraryRestoreAuthorization(scope, restoreSnapshot) + setRestoreNotice( + `已签发恢复授权${grant.authorized_by ? `(${grant.authorized_by})` : ''},有效至 ${stamp(grant.expires_at)}` + ) + setRestoreRound((round) => round + 1) + await load() + } catch (grantError) { + setRestoreNotice(grantError instanceof Error ? grantError.message : '签发恢复授权失败') + } finally { + setPendingGrant(false) + } + } + + const startRestore = async () => { + if (!restoreSnapshot) return + setPendingRestore(true) + setRestoreNotice(null) + try { + const accepted = await enqueueLibraryRestore(scope, restoreSnapshot) + setRestoreNotice(`已投递恢复:${accepted.job_id}(整套生产知识状态回到该快照)`) + setRestoreRound((round) => round + 1) + await load() + } catch (restoreError) { + setRestoreNotice(restoreError instanceof Error ? restoreError.message : '发起恢复失败') + } finally { + setPendingRestore(false) + } + } const startCapture = async () => { setPendingCapture(true) @@ -1139,6 +1625,17 @@ export function ReleaseTargets() { } } + const recover = async (windowJobID: string) => { + setNotice(null) + try { + const accepted = await enqueueLibraryRecovery(scope, windowJobID) + setNotice(`已投递按预备份恢复:${accepted.recovery_job || '(未返回作业号)'}`) + await load() + } catch (recoveryError) { + setNotice(recoveryError instanceof Error ? recoveryError.message : '恢复请求失败') + } + } + return ( void grantSync(), + canStart: current?.gate.sync_allowed === true, + pending: pendingSync, + onStart: () => void startSync(), + }} + baseline={{ + canStart: canSignSync, + pending: pendingBaseline, + onStart: () => void startBaseline(), + }} + canRestore={canRestore} + restore={restore} + restoreNotice={restoreNotice} + pendingRestore={pendingRestore} + pendingGrant={pendingGrant} + onGrantRestore={() => void grantRestore()} + onRestore={() => void startRestore()} + maintenance={maintenances[scope]} now={now} onRefresh={() => void load()} onCapture={() => void startCapture()} + onRecover={(windowJobID) => void recover(windowJobID)} /> ) } diff --git a/web/src/libraryRestore.test.ts b/web/src/libraryRestore.test.ts new file mode 100644 index 0000000..5707df3 --- /dev/null +++ b/web/src/libraryRestore.test.ts @@ -0,0 +1,87 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + createLibraryRestoreAuthorization, + enqueueLibraryRestore, + fetchLibraryRestoreAvailability, +} from './librarySnapshots' + +const jsonResponse = (body: unknown, status = 200) => + new Response(JSON.stringify(body), { status, headers: { 'Content-Type': 'application/json' } }) + +afterEach(() => { + vi.unstubAllGlobals() +}) + +describe('整库恢复的前端契约', () => { + it('读取恢复条件时把原因与授权状态原样带回来', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue( + jsonResponse({ + allowed: false, + reasons: ['没有针对该快照的恢复授权:先签发 restore 授权(绑定同一快照)'], + scope: 'internal', + snapshot_id: 'snap-1', + authorization: { state: 'none', reason: '没有与目标快照匹配的有效发布授权' }, + }) + ) + ) + const availability = await fetchLibraryRestoreAvailability('internal', 'snap-1') + expect(availability?.allowed).toBe(false) + expect(availability?.reasons).toHaveLength(1) + expect(availability?.authorization.state).toBe('none') + const requested = (vi.mocked(fetch).mock.calls[0]?.[0] as string) ?? '' + expect(requested).toContain('scope=internal') + expect(requested).toContain('snapshot_id=snap-1') + }) + + it('签发恢复授权必须声明 action=restore 与有限有效期', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue( + jsonResponse( + { + authorization: { + id: 'libauth-restore-1', + authorized_by: 'yaozhanghao', + expires_at: '2026-10-29T09:00:00Z', + }, + }, + 201 + ) + ) + ) + const grant = await createLibraryRestoreAuthorization('internal', 'snap-1') + expect(grant.id).toBe('libauth-restore-1') + const init = vi.mocked(fetch).mock.calls[0]?.[1] as RequestInit + const body = JSON.parse(String(init.body)) as Record + expect(body.action).toBe('restore') + expect(body.expires_in_minutes).toBe(30) + }) + + it('恢复被拒时把逐条原因说给用户,而不是只说 HTTP 状态', async () => { + vi.stubGlobal( + 'fetch', + vi + .fn() + .mockResolvedValue( + jsonResponse({ code: 'library_restore_blocked', reasons: ['生产写入闸口关闭', '没有该快照的恢复授权'] }, 503) + ) + ) + await expect(enqueueLibraryRestore('internal', 'snap-1')).rejects.toThrow('生产写入闸口关闭;没有该快照的恢复授权') + }) + + it('投递成功时返回作业号,供页面显示真实进度', async () => { + vi.stubGlobal( + 'fetch', + vi + .fn() + .mockResolvedValue( + jsonResponse({ job: { id: 'librestore-internal-1', scope: 'internal' }, reused: false }, 202) + ) + ) + const accepted = await enqueueLibraryRestore('internal', 'snap-1') + expect(accepted.job_id).toBe('librestore-internal-1') + expect(accepted.reused).toBe(false) + }) +}) diff --git a/web/src/librarySnapshots.ts b/web/src/librarySnapshots.ts index cfe5916..1ca91d2 100644 --- a/web/src/librarySnapshots.ts +++ b/web/src/librarySnapshots.ts @@ -185,6 +185,170 @@ export async function enqueueLibraryCapture(scope: string): Promise { + const query = `scope=${encodeURIComponent(scope)}&snapshot_id=${encodeURIComponent(snapshotID)}` + const response = await fetch(`/api/v1/library/restore-availability?${query}`, { + headers: { Accept: 'application/json' }, + signal, + }) + const value = (await response.json().catch(() => null)) as unknown + if (!record(value)) return null + return { + allowed: bool(value.allowed), + reasons: Array.isArray(value.reasons) ? value.reasons.map(text).filter(Boolean) : [], + scope: text(value.scope), + snapshot_id: text(value.snapshot_id), + authorization: record(value.authorization) + ? { + state: text(value.authorization.state) === 'active' ? 'active' : 'none', + authorized_by: text(value.authorization.authorized_by) || undefined, + expires_at: text(value.authorization.expires_at) || undefined, + reason: text(value.authorization.reason) || undefined, + } + : { state: 'none' }, + } +} + +// 恢复整套生产知识状态:同一份快照的文档、分块、索引、资源与 Wiki 一起回放。 +export interface LibraryRestoreAccepted { + job_id: string + scope: string + reused: boolean +} + +export async function enqueueLibraryRestore(scope: string, snapshotID: string): Promise { + const response = await fetch('/api/v1/library/restores', { + method: 'POST', + headers: { 'Content-Type': 'application/json', Accept: 'application/json' }, + body: JSON.stringify({ scope, snapshot_id: snapshotID }), + }) + const value = (await response.json().catch(() => null)) as unknown + if (!response.ok) { + if (response.status === 403) throw new Error('按版本恢复仅管理员可发起。') + if (response.status === 503 && record(value)) { + const reasons = Array.isArray(value.reasons) ? value.reasons.map(text).filter(Boolean) : [] + throw new Error(reasons.length > 0 ? reasons.join(';') : text(value.error) || '恢复条件不满足') + } + if (response.status === 409 && record(value)) throw new Error(text(value.error) || '该目标已有在飞作业') + throw new Error(`发起恢复失败(HTTP ${response.status})`) + } + if (!record(value) || !record(value.job)) throw new Error('恢复作业响应无效') + return { job_id: text(value.job.id), scope: text(value.job.scope), reused: bool(value.reused) } +} + +export interface LibraryRestoreGrant { + id: string + authorized_by: string + expires_at: string +} + +// 同步授权与执行是两步:授权把"范围 + 目标快照 + 参数摘要"钉死并给很短的有效期。 +// 它不能顶替恢复授权(摘要不同),也不自动签发 —— 必须由管理员显式点击。 +export async function createLibrarySyncAuthorization( + scope: string, + snapshotID: string, + minutes = 30 +): Promise { + const response = await fetch('/api/v1/library/authorizations', { + method: 'POST', + headers: { 'Content-Type': 'application/json', Accept: 'application/json' }, + body: JSON.stringify({ scope, snapshot_id: snapshotID, action: 'sync', expires_in_minutes: minutes }), + }) + const value = (await response.json().catch(() => null)) as unknown + if (!response.ok) { + if (response.status === 403) throw new Error('生产发布授权仅管理员可签发。') + if (response.status === 400 && record(value)) throw new Error(text(value.error) || '授权参数不合法') + throw new Error(`签发同步授权失败(HTTP ${response.status})`) + } + if (!record(value) || !record(value.authorization)) throw new Error('授权响应无效') + return { + id: text(value.authorization.id), + authorized_by: text(value.authorization.authorized_by), + expires_at: text(value.authorization.expires_at), + } +} + +export interface LibrarySyncAccepted { + job_id: string + scope: string + reused: boolean +} + +// 启动同步:闸口未齐时接口返回 503 与逐条缺项,页面照实显示,不假装已启动。 +export async function enqueueLibrarySync(scope: string, targetSnapshotID: string): Promise { + const response = await fetch('/api/v1/library/syncs', { + method: 'POST', + headers: { 'Content-Type': 'application/json', Accept: 'application/json' }, + body: JSON.stringify({ scope, target_snapshot_id: targetSnapshotID }), + }) + const value = (await response.json().catch(() => null)) as unknown + if (!response.ok) { + if (record(value) && Array.isArray(value.reasons)) { + const reasons = value.reasons.map(text).filter(Boolean) + if (reasons.length > 0) throw new Error(`启动同步被拒绝:${reasons.join(';')}`) + } + if (record(value) && text(value.error)) throw new Error(text(value.error)) + throw new Error(`启动同步失败(HTTP ${response.status})`) + } + if (!record(value) || !record(value.job)) throw new Error('同步作业响应无效') + return { job_id: text(value.job.id), scope: text(value.job.scope), reused: bool(value.reused) } +} + +// 建立目标权威基线:目标非空又没有生效版本时的唯一入口(维护隔离 + 真实捕获 + 校验)。 +export async function enqueueLibraryBaseline(scope: string): Promise { + const response = await fetch('/api/v1/library/baselines', { + method: 'POST', + headers: { 'Content-Type': 'application/json', Accept: 'application/json' }, + body: JSON.stringify({ scope }), + }) + const value = (await response.json().catch(() => null)) as unknown + if (!response.ok) { + if (response.status === 403) throw new Error('建立目标权威基线仅管理员可发起。') + if (record(value) && text(value.error)) throw new Error(text(value.error)) + throw new Error(`建立基线失败(HTTP ${response.status})`) + } + if (!record(value) || !record(value.job)) throw new Error('基线作业响应无效') + return { job_id: text(value.job.id), scope: text(value.job.scope), reused: bool(value.reused) } +} + +// 恢复授权与执行是两件事:授权把"目标 + 快照 + 参数"钉死并给很短的有效期, +// 同步授权不能顶替恢复授权(摘要不同)。 +export async function createLibraryRestoreAuthorization( + scope: string, + snapshotID: string, + minutes = 30 +): Promise { + const response = await fetch('/api/v1/library/authorizations', { + method: 'POST', + headers: { 'Content-Type': 'application/json', Accept: 'application/json' }, + body: JSON.stringify({ scope, snapshot_id: snapshotID, action: 'restore', expires_in_minutes: minutes }), + }) + const value = (await response.json().catch(() => null)) as unknown + if (!response.ok) { + if (response.status === 403) throw new Error('生产发布授权仅管理员可签发。') + throw new Error(`签发恢复授权失败(HTTP ${response.status})`) + } + if (!record(value) || !record(value.authorization)) throw new Error('授权响应无效') + return { + id: text(value.authorization.id), + authorized_by: text(value.authorization.authorized_by), + expires_at: text(value.authorization.expires_at), + } +} + export async function fetchLibraryCaptures(signal?: AbortSignal): Promise { const response = await fetch('/api/v1/library/captures', { headers: { Accept: 'application/json' }, @@ -250,7 +414,11 @@ export interface LibraryDiff { /** 这次比对用的两个整库版本(生产 → 测试),避免把历史差异当成当前。 */ base_snapshot?: string target_snapshot?: string - /** 页面级 Wiki 差异;不代表实体、概念、边或引用被重算。 */ + /** + * Wiki 逐页贡献级差异:来源文档 / 引用分块 / 出入边 / 目录分别比。 + * 语义重算发生在源侧,这里只如实反映重算后的结果——撤销一份共享贡献时 + * 页面通常还是"更新",要看 pages[].sources_removed 才知道那份贡献被摘掉了。 + */ wiki?: { state: 'computed' | 'unknown' reason?: string @@ -259,9 +427,104 @@ export interface LibraryDiff { removed: number sample: string[] note: string + pages?: LibraryWikiPageChange[] + pages_truncated?: boolean + sources_added?: number + sources_removed?: number + base_count?: number + target_count?: number } } +export interface LibraryWikiPageChange { + slug: string + title: string + page_type?: string + status: 'added' | 'updated' | 'removed' + changed: string[] + sources_added?: string[] + sources_removed?: string[] + citations_before: number + citations_after: number +} + +// ---- 目标维护窗口:级别、阶段、入口状态与恢复错误 ---- +export interface LibraryMaintenanceRecord { + scope: string + owner_job_id: string + target_identity?: string + level: string + target_fingerprint?: string + services?: string[] + ingress_state: string + state: string + original_service_state?: string + note?: string + acquired_at: string + updated_at: string + released_at?: string | null +} + +export interface LibraryInstanceMaintenanceRecord extends LibraryMaintenanceRecord { + instance: string + owner_scope: string +} + +export interface LibraryMaintenanceView { + held: boolean + maintenance?: LibraryMaintenanceRecord | null + instance?: string + instance_maintenance?: LibraryInstanceMaintenanceRecord | null + // 能力是配置事实,和"此刻有没有窗口"分开:未运行维护 ≠ 没有维护能力。 + capability?: LibraryMaintenanceCapability | null +} + +export interface LibraryMaintenanceCapability { + configured: boolean + missing: string[] + instance?: string + project?: string + writers?: string[] + ingress_service?: string + note?: string +} + +export async function fetchLibraryMaintenance( + scope: string, + signal?: AbortSignal +): Promise { + const response = await fetch(`/api/v1/library/maintenance?scope=${encodeURIComponent(scope)}`, { + headers: { Accept: 'application/json' }, + signal, + }) + if (!response.ok) return null + const value = (await response.json().catch(() => null)) as unknown + if (!record(value)) return null + return value as unknown as LibraryMaintenanceView +} + +export interface LibraryRecoveryAccepted { + accepted?: boolean + recovery_job?: string + state?: string + code?: string + error?: string +} + +export async function enqueueLibraryRecovery(scope: string, windowJobID: string): Promise { + const response = await fetch('/api/v1/library/maintenance/recover', { + method: 'POST', + headers: { 'Content-Type': 'application/json', Accept: 'application/json' }, + body: JSON.stringify({ scope, window_job_id: windowJobID }), + }) + const value = (await response.json().catch(() => ({}))) as unknown + const body = record(value) ? (value as LibraryRecoveryAccepted) : {} + if (!response.ok) { + throw new Error(body.error || `恢复请求被拒绝(HTTP ${response.status})`) + } + return body +} + export interface LibraryStageRow { stage: string state: string @@ -276,7 +539,7 @@ export interface LibraryStageRow { export interface LibraryJobRecord { id: string scope: string - kind: 'seal' | 'sync' + kind: 'seal' | 'sync' | 'restore' state: 'pending' | 'running' | 'succeeded' | 'failed' | 'cancelled' stage: string request_digest: string @@ -304,6 +567,18 @@ export interface LibraryGate { reasons: string[] } +export interface LibraryExecutorView { + available: boolean + reason?: string + instance?: string + project?: string + source_app?: string + archive_root?: string + busy: boolean + degraded?: string + observed_at?: string +} + export interface LibraryOverview { scope: string test_sealed?: LibrarySnapshot @@ -317,6 +592,8 @@ export interface LibraryOverview { authorization: LibraryAuthorizationView capture_available: boolean capture_reason?: string + // 执行器是 worker 的真实观测:指向哪个实例、何时观测、是否降级/在跑。 + executor?: LibraryExecutorView now: string } diff --git a/web/src/librarySyncAuth.test.ts b/web/src/librarySyncAuth.test.ts new file mode 100644 index 0000000..83c7346 --- /dev/null +++ b/web/src/librarySyncAuth.test.ts @@ -0,0 +1,82 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { createLibrarySyncAuthorization, enqueueLibraryBaseline, enqueueLibrarySync } from './librarySnapshots' + +// 这些用例钉住"两步走"的接线:授权与执行是两个请求,参数必须与接口契约一致。 +// 任何一步把 action 写错、或把 snapshot 换掉,都会在真实调用里悄悄失效(授权永远匹配不上)。 +const jsonResponse = (status: number, body: unknown) => + new Response(JSON.stringify(body), { status, headers: { 'Content-Type': 'application/json' } }) + +afterEach(() => vi.restoreAllMocks()) + +describe('发布页的同步授权与执行接线', () => { + it('签同步授权必须声明 action=sync 与有限有效期', async () => { + const fetchMock = vi.fn().mockResolvedValue( + jsonResponse(201, { + authorization: { id: 'libauth-sync-1', authorized_by: 'admin', expires_at: '2026-10-29T10:00:00Z' }, + }) + ) + vi.stubGlobal('fetch', fetchMock) + const grant = await createLibrarySyncAuthorization('internal', 'snap-1', 30) + expect(grant.id).toBe('libauth-sync-1') + const [url, init] = fetchMock.mock.calls[0] + expect(url).toBe('/api/v1/library/authorizations') + expect(JSON.parse(init.body)).toEqual({ + scope: 'internal', + snapshot_id: 'snap-1', + action: 'sync', + expires_in_minutes: 30, + }) + }) + + it('启动同步把准确的 range 与目标快照传给 /library/syncs', async () => { + const fetchMock = vi + .fn() + .mockResolvedValue(jsonResponse(202, { job: { id: 'libsync-internal-1', scope: 'internal' }, reused: false })) + vi.stubGlobal('fetch', fetchMock) + const accepted = await enqueueLibrarySync('internal', 'snap-1') + expect(accepted.job_id).toBe('libsync-internal-1') + const [url, init] = fetchMock.mock.calls[0] + expect(url).toBe('/api/v1/library/syncs') + expect(JSON.parse(init.body)).toEqual({ scope: 'internal', target_snapshot_id: 'snap-1' }) + }) + + it('闸口未齐时把接口给的缺项原样抛出,而不是假装已启动', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue( + jsonResponse(503, { + code: 'library_sync_blocked', + reasons: ['目标非空且没有生效版本记录', '没有与目标快照匹配的有效发布授权'], + }) + ) + ) + await expect(enqueueLibrarySync('internal', 'snap-1')).rejects.toThrow( + /目标非空且没有生效版本记录;没有与目标快照匹配的有效发布授权/ + ) + }) + + it('建立基线只传 range,并返回作业号', async () => { + const fetchMock = vi + .fn() + .mockResolvedValue(jsonResponse(202, { job: { id: 'libbase-internal-1', scope: 'internal' }, reused: false })) + vi.stubGlobal('fetch', fetchMock) + const accepted = await enqueueLibraryBaseline('internal') + expect(accepted.job_id).toBe('libbase-internal-1') + const [url, init] = fetchMock.mock.calls[0] + expect(url).toBe('/api/v1/library/baselines') + expect(JSON.parse(init.body)).toEqual({ scope: 'internal' }) + }) + + it('建立基线被拒时照实说明(例如目标为空或已有生效版本)', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue( + jsonResponse(409, { + code: 'library_baseline_exists', + error: '该目标已有生效版本,不需要再建基线;换版本请走同步或恢复', + }) + ) + ) + await expect(enqueueLibraryBaseline('internal')).rejects.toThrow(/该目标已有生效版本/) + }) +}) diff --git a/web/src/releaseTargets.css b/web/src/releaseTargets.css index 1075650..187e433 100644 --- a/web/src/releaseTargets.css +++ b/web/src/releaseTargets.css @@ -635,3 +635,40 @@ font-variant-numeric: tabular-nums; word-break: break-all; } + +/* 目标维护:级别/持有/入口/阶段用两列小表,数字与时间可对齐比较。 */ +.rel-maintenance-rows { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 6px 18px; + margin: 8px 14px 0; + padding: 0; + list-style: none; +} +.rel-maintenance-rows li { + display: flex; + justify-content: space-between; + gap: 10px; + border-bottom: 1px dashed var(--rel-line, #dce6ed); + padding: 4px 0; + font-size: 12px; +} +.rel-maintenance-rows span { + color: var(--rel-muted, #7b8d9a); +} +.rel-maintenance-rows b { + font-variant-numeric: tabular-nums; + word-break: break-all; + text-align: right; +} + +/* Wiki 逐页贡献级差异:每条一行,直接写清来源/引用怎么变。 */ +.rel-wiki-pages { + margin: 4px 14px 10px; + padding-left: 18px; + font-size: 12px; + line-height: 1.6; +} +.rel-wiki-pages li { + font-variant-numeric: tabular-nums; +} diff --git a/web/src/releaseTargetsMaintenance.test.tsx b/web/src/releaseTargetsMaintenance.test.tsx new file mode 100644 index 0000000..92ceafc --- /dev/null +++ b/web/src/releaseTargetsMaintenance.test.tsx @@ -0,0 +1,139 @@ +import { renderToStaticMarkup } from 'react-dom/server' +import { describe, expect, it } from 'vitest' + +import { MaintenanceCard, maintenanceSummary, wikiContributionLines } from './ReleaseTargets' +import type { LibraryDiff, LibraryJobRecord, LibraryMaintenanceView } from './librarySnapshots' + +const job: LibraryJobRecord = { + id: 'libsync-internal-1', + scope: 'internal', + kind: 'sync', + state: 'failed', + stage: 'rollback', + request_digest: 'x', + queue_job_id: 'q1', + snapshot_id: '', + target_snapshot_id: '', + actor: 'admin', + error: '写入生产 chunks 失败:连接被拒绝', + recovery: '已按预备份回滚并校验', + created_at: '2026-10-29T04:00:00Z', + updated_at: '2026-10-29T04:02:00Z', + finished_at: '2026-10-29T04:02:00Z', + stages: [], +} + +const heldInstance: LibraryMaintenanceView = { + held: true, + instance: 'kgmaint-weknora', + instance_maintenance: { + instance: 'kgmaint-weknora', + owner_scope: 'internal', + scope: 'internal', + owner_job_id: 'libsync-internal-1', + level: 'instance', + services: ['app'], + ingress_state: 'closed', + state: 'held', + acquired_at: '2026-10-29T04:00:00Z', + updated_at: '2026-10-29T04:02:00Z', + released_at: null, + }, +} + +describe('维护级别/进度/恢复错误', () => { + it('实例级持有:显示级别、入口状态、进度与恢复错误,并允许按预备份恢复', () => { + const summary = maintenanceSummary(heldInstance, job) + expect(summary.level).toBe('实例级') + expect(summary.held).toBe(true) + expect(summary.ingress).toContain('已关闭') + expect(summary.progress).toContain('回滚') + expect(summary.progress).toContain('失败') + expect(summary.recoveryError).toContain('写入生产 chunks 失败') + expect(summary.recoveryNote).toContain('按预备份回滚') + expect(summary.canRecover).toBe(true) + + const html = renderToStaticMarkup( + undefined} /> + ) + for (const text of ['目标维护', '实例级', '入口 已关闭', '按预备份恢复', '写入生产 chunks 失败']) { + expect(html).toContain(text) + } + }) + + it('没有窗口时如实说没有,不画成"已就绪"', () => { + const summary = maintenanceSummary(null, undefined) + expect(summary.level).toBe('未接入') + expect(summary.held).toBe(false) + expect(summary.canRecover).toBe(false) + const html = renderToStaticMarkup( + undefined} /> + ) + expect(html).toContain('暂无维护窗口') + expect(html).not.toContain('按预备份恢复') + }) + + it('已释放的窗口不再提供恢复入口', () => { + const released: LibraryMaintenanceView = { + held: false, + instance_maintenance: { + ...heldInstance.instance_maintenance!, + released_at: '2026-10-29T04:05:00Z', + state: 'recovered', + }, + } + const summary = maintenanceSummary(released, job) + expect(summary.held).toBe(false) + expect(summary.canRecover).toBe(false) + expect(summary.releasedAt).not.toBe('') + }) +}) + +describe('Wiki 贡献级差异展示', () => { + const diff: LibraryDiff = { + state: 'computed', + added: [], + updated: [], + removed: [], + wiki: { + state: 'computed', + added: 0, + updated: 1, + removed: 1, + sample: [], + note: '逐页比对', + sources_removed: 2, + pages: [ + { + slug: 'entity/fa-bu-zha-men', + title: '发布闸门', + status: 'updated', + changed: ['正文', '来源文档', '引用分块'], + sources_removed: ['doc-a'], + citations_before: 2, + citations_after: 1, + }, + { + slug: 'entity/tu-pu-hui-tian-fu-wu', + title: '图谱回填服务', + status: 'removed', + changed: ['整页撤销'], + citations_before: 1, + citations_after: 0, + }, + ], + }, + } + + it('共享页被更新时列出来源减少与引用变化,而不是只说"更新 1 页"', () => { + const lines = wikiContributionLines(diff) + expect(lines[0]).toContain('发布闸门') + expect(lines[0]).toContain('来源 -1') + expect(lines[0]).toContain('引用 2→1') + expect(lines[1]).toContain('整页撤销') + }) + + it('没有逐页明细时不编造明细', () => { + expect(wikiContributionLines({ state: 'unknown', added: [], updated: [], removed: [] })).toEqual([]) + }) +})