Compare commits
79 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| e8bfdbbc06 | |||
| ef3c1fbf84 | |||
| 2da96da592 | |||
| 10182b027e | |||
| 060872a144 | |||
| 0f7b8987b4 | |||
| 5e850e96f5 | |||
| 719876d039 | |||
| 9d445f4953 | |||
| 2eff9447e1 | |||
| 146ac832a2 | |||
| 77bbc3ac6b | |||
| eaa388f76b | |||
| cf11343cf4 | |||
| 7511c59987 | |||
| b81fcd07e3 | |||
| 1a0f99e346 | |||
| 7675e2cd88 | |||
| 5bf4aae30e | |||
| 1c80b07f2c | |||
| dfbf3336b2 | |||
| 600cca02e5 | |||
| e735ae603d | |||
| 2a2e98bd8e | |||
| 92684c6f60 | |||
| 537a6df46d | |||
| 0d9091e464 | |||
| e0fc2c32b8 | |||
| 6042955498 | |||
| a55319d972 | |||
| 017765763a | |||
| bd452e85b3 | |||
| 71f23c6763 | |||
| b4add6c9b0 | |||
| f9457fab52 | |||
| 5e1482e349 | |||
| 4421dd38ce | |||
| 22fb33aa75 | |||
| 0d180aa401 | |||
| 0a87dc8e03 | |||
| 96472c1dde | |||
| bad9034cf7 | |||
| f330a75641 | |||
| 19ec03c417 | |||
| 6209072e43 | |||
| e61f05c98c | |||
| a9d5adf78f | |||
| e8f8069473 | |||
| b7d6661a4e | |||
| 97da6ceba8 | |||
| 7cdbb74c8f | |||
| 1aa60c159e | |||
| 245cefd6b0 | |||
| 1e2bb385c7 | |||
| bf7f5fa03c | |||
| 24608e132b | |||
| 80254756f1 | |||
| 37f2dc73f9 | |||
| 9e31a6f956 | |||
| 1c58f1ef7e | |||
| 3b7ad9061c | |||
| 882e38591d | |||
| a1112d368a | |||
| ca32f2e661 | |||
| 513aa9e6f5 | |||
| 3bce6c7a3f | |||
| 18a4b83f78 | |||
| 7b2ab642f7 | |||
| 0a7c9f8510 | |||
| e09fddcb74 | |||
| fad26166a4 | |||
| 47be4ae502 | |||
| 6827fa922f | |||
| c31a7df4ca | |||
| 770252b55a | |||
| c256f2985c | |||
| f1cddfb601 | |||
| d2ff089b95 | |||
| 12b37690ec |
@@ -108,9 +108,23 @@ if [ -n "$BEFORE" ] && [ "$BEFORE" != "$ZERO" ]; then
|
||||
if git cat-file -e "${BEFORE}^{commit}" 2>/dev/null; then
|
||||
echo "✅ 事件 before ${BEFORE:0:12} 已在克隆里(本次推送范围可直接算)"
|
||||
else
|
||||
gitauth fetch -q --depth 50 origin "$BEFORE" 2>/dev/null \
|
||||
&& echo "✅ 已补取事件 before ${BEFORE:0:12}" \
|
||||
|| echo "⚠️ 取不到事件 before ${BEFORE:0:12}:范围校验会明确失败,不会静默收缩"
|
||||
# ⚠️ 为什么重试一次:浅克隆里用 `fetch --depth` 取一个**未公开**的对象(撤销/改写推送之后
|
||||
# 的旧提交就是这种)时,git 会在这次 fetch 中途重写 .git/shallow,随后发现自己读到的
|
||||
# 与开始时不同就整体中止:`fatal: shallow file has changed since we read it`。
|
||||
# 实测(run 305–317,含原地重跑)这一步必中止,而**同一仓库里紧接着重试同一条命令即成功**。
|
||||
# before 取不到 ⇒ commits 检查算不出范围 ⇒ 流水线红,且红的原因与代码质量无关。
|
||||
# 重试只负责"把 before 取进克隆",不收缩范围;两次都失败仍明确报警,由 commits 检查判红。
|
||||
FETCH_OK=""
|
||||
FETCH_ERR=""
|
||||
for _attempt in 1 2; do
|
||||
FETCH_ERR="$(gitauth fetch -q --depth 50 origin "$BEFORE" 2>&1 >/dev/null || true)"
|
||||
if git cat-file -e "${BEFORE}^{commit}" 2>/dev/null; then FETCH_OK=1; break; fi
|
||||
done
|
||||
if [ -n "$FETCH_OK" ]; then
|
||||
echo "✅ 已补取事件 before ${BEFORE:0:12}"
|
||||
else
|
||||
echo "⚠️ 取不到事件 before ${BEFORE:0:12}:范围校验会明确失败,不会静默收缩(最后一次错误:${FETCH_ERR:0:160})"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
echo "(事件里没有 before / before 为全 0:新分支推送,范围按默认分支为基准判)"
|
||||
|
||||
+5642
-12
File diff suppressed because it is too large
Load Diff
+2
-2
@@ -50,7 +50,7 @@ RUN set -eux; \
|
||||
dws version
|
||||
WORKDIR /app
|
||||
COPY --from=build /out/ /app/
|
||||
COPY scripts/source_catalog_import.py scripts/live_catalog_refresh.py scripts/artifact_fetch.py \
|
||||
scripts/axls_to_xlsx.py scripts/dws_error_map.py /app/scripts/
|
||||
COPY scripts/source_catalog_import.py scripts/live_catalog_refresh.py scripts/dws_dingtalk_scan.py \
|
||||
scripts/artifact_fetch.py scripts/axls_to_xlsx.py scripts/dws_error_map.py /app/scripts/
|
||||
USER 65532:65532
|
||||
CMD ["/app/api"]
|
||||
|
||||
@@ -52,6 +52,7 @@
|
||||
"/api/v1/qa-ops/master/weekly-mail/confirmations/{week}": {"get":{"operationId":"getQAWeeklyMailConfirmation","tags":["QA operations"],"summary":"读取该周是否已确认自动发送;current=false表示确认后正文已被修改,需要重新确认","parameters":[{"name":"week","in":"path","required":true,"schema":{"type":"string","pattern":"^[0-9]{4}-W[0-9]{2}$"}}],"responses":{"400":{"$ref":"#/components/responses/BadRequest"},"500":{"$ref":"#/components/responses/InternalError"},"200":{"$ref":"#/components/responses/Envelope"},"400":{"$ref":"#/components/responses/BadRequest"},"401":{"description":"需要登录"},"403":{"description":"仅管理员可修改发送设置"},"409":{"description":"发送设置已被其他管理员修改"},"503":{"description":"邮件设置存储不可用,未标记为已发送"}}},"post":{"operationId":"confirmQAWeeklyMail","tags":["QA operations"],"summary":"确认本周周报可自动发送,绑定当前草稿版本与正文哈希;后续改动使确认失效,不发送改动后的正文","parameters":[{"name":"week","in":"path","required":true,"schema":{"type":"string","pattern":"^[0-9]{4}-W[0-9]{2}$"}},{"name":"X-Governance-Action","in":"header","required":true,"schema":{"const":"qa-weekly-mail"}}],"responses":{"400":{"$ref":"#/components/responses/BadRequest"},"500":{"$ref":"#/components/responses/InternalError"},"200":{"$ref":"#/components/responses/Envelope"},"400":{"$ref":"#/components/responses/BadRequest"},"401":{"description":"需要登录"},"403":{"description":"仅管理员可修改发送设置"},"409":{"description":"发送设置已被其他管理员修改"},"503":{"description":"邮件设置存储不可用,未标记为已发送"}}},"delete":{"operationId":"unconfirmQAWeeklyMail","tags":["QA operations"],"summary":"取消该周的自动发送确认","parameters":[{"name":"week","in":"path","required":true,"schema":{"type":"string","pattern":"^[0-9]{4}-W[0-9]{2}$"}},{"name":"X-Governance-Action","in":"header","required":true,"schema":{"const":"qa-weekly-mail"}}],"responses":{"400":{"$ref":"#/components/responses/BadRequest"},"500":{"$ref":"#/components/responses/InternalError"},"200":{"$ref":"#/components/responses/Envelope"},"400":{"$ref":"#/components/responses/BadRequest"},"401":{"description":"需要登录"},"403":{"description":"仅管理员可修改发送设置"},"409":{"description":"发送设置已被其他管理员修改"},"503":{"description":"邮件设置存储不可用,未标记为已发送"}}}},
|
||||
"/api/v1/qa-ops/master/weekly-mail/sends": {"get":{"operationId":"listQAWeeklyMailSends","tags":["QA operations"],"summary":"读取周报邮件发送记录:周次、草稿版本、发送中/已发送/失败及错误摘要,不返回正文或收件人","parameters":[{"name":"week","in":"query","required":false,"schema":{"type":"string","pattern":"^[0-9]{4}-W[0-9]{2}$"}}],"responses":{"400":{"$ref":"#/components/responses/BadRequest"},"500":{"$ref":"#/components/responses/InternalError"},"200":{"$ref":"#/components/responses/Envelope"},"400":{"$ref":"#/components/responses/BadRequest"},"401":{"description":"需要登录"},"403":{"description":"仅管理员可修改发送设置"},"409":{"description":"发送设置已被其他管理员修改"},"503":{"description":"邮件设置存储不可用,未标记为已发送"}}}},
|
||||
"/api/v1/qa-ops/master/weekly-mail/test": {"post":{"operationId":"sendQAWeeklyMailTest","tags":["QA operations"],"summary":"管理员用当前已保存的邮件设置立即发送一封测试邮件,正文取该周已保存草稿;不写入发送记录、不改变确认状态、不把该周标记为已发送","parameters":[{"name":"X-Governance-Action","in":"header","required":true,"schema":{"const":"qa-weekly-mail"}}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"type":"object","additionalProperties":false,"required":["week"],"properties":{"week":{"type":"string","pattern":"^[0-9]{4}-W[0-9]{2}$"}}}}}},"responses":{"400":{"$ref":"#/components/responses/BadRequest"},"500":{"$ref":"#/components/responses/InternalError"},"200":{"$ref":"#/components/responses/Envelope"},"401":{"description":"需要登录"},"403":{"description":"仅管理员可测试发送"},"502":{"description":"SMTP 拒绝或不可达,返回脱敏原因"},"503":{"description":"邮件设置存储不可用"}}}},
|
||||
"/api/v1/qa-ops/master/weekly-mail/send/{week}": {"get":{"operationId":"previewQAWeeklyMailSend","tags":["QA operations"],"summary":"管理员补发前核对:按周、已确认版本与正文hash返回真实主送/抄送人数、版本与是否已发送;不投递、不写记录","parameters":[{"name":"week","in":"path","required":true,"schema":{"type":"string","pattern":"^[0-9]{4}-W[0-9]{2}$"}},{"name":"revision","in":"query","required":true,"schema":{"type":"integer","minimum":1}},{"name":"body_sha256","in":"query","required":true,"schema":{"type":"string","pattern":"^[0-9a-f]{64}$"}}],"responses":{"200":{"$ref":"#/components/responses/Envelope"},"400":{"$ref":"#/components/responses/BadRequest"},"401":{"description":"需要登录"},"403":{"description":"仅管理员"},"409":{"description":"未确认/正文或版本已变化/当前周未到设定时点/上次结果未知"},"500":{"$ref":"#/components/responses/InternalError"},"503":{"description":"邮件设置存储不可用"}}},"post":{"operationId":"sendQAWeeklyMailFormal","tags":["QA operations"],"summary":"管理员正式补发周报:精确指定周、已确认版本与正文hash;已结束周可发、当前周须到设定时点、未来周拒绝;正文/看板图片/主送/抄送与自动发送共用,写入正式发送记录并返回真实message_id;已发送幂等、结果未知不重发","parameters":[{"name":"week","in":"path","required":true,"schema":{"type":"string","pattern":"^[0-9]{4}-W[0-9]{2}$"}},{"name":"X-Governance-Action","in":"header","required":true,"schema":{"const":"qa-weekly-mail"}}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"type":"object","additionalProperties":false,"required":["revision","body_sha256"],"properties":{"revision":{"type":"integer","minimum":1},"body_sha256":{"type":"string","pattern":"^[0-9a-f]{64}$"}}}}}},"responses":{"200":{"$ref":"#/components/responses/Envelope"},"400":{"$ref":"#/components/responses/BadRequest"},"401":{"description":"需要登录"},"403":{"description":"仅管理员且需同源操作意图"},"409":{"description":"未确认/正文或版本已变化/当前周未到设定时点/上次结果未知/授权码缺失"},"500":{"$ref":"#/components/responses/InternalError"},"502":{"description":"SMTP 拒绝或不可达,返回脱敏原因且不写已发送"},"503":{"description":"邮件设置存储不可用"}}}},
|
||||
"/api/v1/qa-ops/master/weekly-mail/confirmations/{week}/image": {"post":{"operationId":"saveQAWeeklyMailImage","tags":["QA operations"],"summary":"管理员保存该周确认对应的统计看板 PNG;发送时作为内嵌图片(cid)放在正文末尾,正文改动使确认失效后不再使用旧图","parameters":[{"name":"week","in":"path","required":true,"schema":{"type":"string","pattern":"^[0-9]{4}-W[0-9]{2}$"}},{"name":"X-Governance-Action","in":"header","required":true,"schema":{"const":"qa-weekly-mail"}}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"type":"object","additionalProperties":false,"required":["png_base64"],"properties":{"png_base64":{"type":"string","maxLength":12000000}}}}}},"responses":{"400":{"$ref":"#/components/responses/BadRequest"},"500":{"$ref":"#/components/responses/InternalError"},"200":{"$ref":"#/components/responses/Envelope"},"401":{"description":"需要登录"},"403":{"description":"仅管理员可上传"},"429":{"description":"请求过快"},"503":{"description":"邮件设置存储不可用"}}}},
|
||||
"/api/v1/qa-ops/departments/statistics": {
|
||||
"post": {
|
||||
@@ -1368,6 +1369,45 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/library/jobs/{id}/reconcile-capture": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"Runtime"
|
||||
],
|
||||
"operationId": "reconcileLibraryCapture",
|
||||
"summary": "仅在 completed capture queue、精确 test_sealed 版本与归档复核一致时,修复该封存作业终态;不重新封存",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "id",
|
||||
"in": "path",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "已修复或已成功(幂等)"
|
||||
},
|
||||
"400": {
|
||||
"$ref": "#/components/responses/BadRequest"
|
||||
},
|
||||
"403": {
|
||||
"description": "需要管理员权限"
|
||||
},
|
||||
"404": {
|
||||
"description": "作业不存在"
|
||||
},
|
||||
"409": {
|
||||
"description": "队列/阶段/版本/归档证据不一致,拒绝修复"
|
||||
},
|
||||
"500": {
|
||||
"$ref": "#/components/responses/InternalError"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/library/restore-availability": {
|
||||
"get": {
|
||||
"tags": [
|
||||
@@ -1655,6 +1695,93 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/library/verifications": {
|
||||
"get": {
|
||||
"tags": [
|
||||
"Runtime"
|
||||
],
|
||||
"operationId": "getLibraryRestoreVerification",
|
||||
"summary": "当前整库目标的恢复验证证据:verified 只由真实演练记录回答(按 target_id 绑定,换目标自动失效),并返回完整目标与失败原因",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "scope",
|
||||
"in": "query",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"internal",
|
||||
"external"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "目标与证据:verified/reason/latest/history/target"
|
||||
},
|
||||
"400": {
|
||||
"$ref": "#/components/responses/BadRequest"
|
||||
},
|
||||
"500": {
|
||||
"$ref": "#/components/responses/InternalError"
|
||||
},
|
||||
"503": {
|
||||
"description": "恢复验证存储不可用或证据读取失败"
|
||||
}
|
||||
}
|
||||
},
|
||||
"post": {
|
||||
"tags": [
|
||||
"Runtime"
|
||||
],
|
||||
"operationId": "startLibraryRestoreVerification",
|
||||
"summary": "发起一次真实恢复验证(管理员):维护隔离下捕获目标 → 校验基线 → 写 canary 证明校验有效 → 按预备份恢复 → 复核,通过后才产生同步闸口认的证据",
|
||||
"responses": {
|
||||
"202": {
|
||||
"description": "已创建 verify 作业(worker 在维护窗口内真实执行)"
|
||||
},
|
||||
"400": {
|
||||
"$ref": "#/components/responses/BadRequest"
|
||||
},
|
||||
"403": {
|
||||
"description": "不是管理员"
|
||||
},
|
||||
"409": {
|
||||
"description": "目标正在维护或已有在飞写作业(同步/恢复/基线/验证互斥)"
|
||||
},
|
||||
"500": {
|
||||
"$ref": "#/components/responses/InternalError"
|
||||
},
|
||||
"503": {
|
||||
"description": "生产写入闸口关闭或恢复验证存储不可用"
|
||||
}
|
||||
},
|
||||
"requestBody": {
|
||||
"required": true,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"scope"
|
||||
],
|
||||
"properties": {
|
||||
"scope": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"internal",
|
||||
"external"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/library/snapshots/{id}": {
|
||||
"get": {
|
||||
"tags": [
|
||||
|
||||
+41
-23
@@ -8,7 +8,6 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@@ -26,6 +25,7 @@ import (
|
||||
"knowledge-governance/internal/qaopsorg"
|
||||
"knowledge-governance/internal/qaopsquality"
|
||||
"knowledge-governance/internal/qawemail"
|
||||
"knowledge-governance/internal/readiness"
|
||||
"knowledge-governance/internal/serviceverify"
|
||||
"knowledge-governance/internal/version"
|
||||
)
|
||||
@@ -90,26 +90,6 @@ func run(ctx context.Context, getenv func(string) string) error {
|
||||
return errors.New("read production release authorization token")
|
||||
}
|
||||
}
|
||||
// P10: with a mounted API key the probe verifies a real knowledge-base read
|
||||
// (readiness); without one it degrades to the liveness-only HTTP check.
|
||||
var weknoraReadiness func(context.Context) error
|
||||
var knowledgeBaseLister httpapi.KnowledgeBaseLister
|
||||
if strings.TrimSpace(config.TestWeKnoraBaseURL) != "" {
|
||||
if apiKey, keyErr := weknora.ReadAPIKeyFile(config.TestWeKnoraAPIKeyFile); keyErr == nil {
|
||||
probeTimeout := config.TestWeKnoraHTTPTimeout
|
||||
if probeTimeout > 10*time.Second || probeTimeout <= 0 {
|
||||
probeTimeout = 10 * time.Second
|
||||
}
|
||||
if probeClient, clientErr := weknora.NewClient(weknora.ClientConfig{BaseURL: config.TestWeKnoraBaseURL, APIKey: apiKey, Timeout: probeTimeout, AllowDockerHTTP: config.TestWeKnoraAllowDockerHTTP}); clientErr == nil {
|
||||
knowledgeBaseLister = probeClient
|
||||
routingKB := config.InternalRoutingKnowledgeBaseID
|
||||
weknoraReadiness = func(probeCtx context.Context) error {
|
||||
_, _, err := probeClient.ListKnowledge(probeCtx, routingKB, 1, 1)
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Production WeKnora is probed for liveness only: the console must show the
|
||||
// real state of that endpoint, while credentials and the publish target stay
|
||||
// in the release runtime config (RELEASE_CONFIG_FILE).
|
||||
@@ -131,7 +111,43 @@ func run(ctx context.Context, getenv func(string) string) error {
|
||||
}
|
||||
return settings.TestWeKnoraBaseURL
|
||||
}
|
||||
monitorProvider, err := monitoring.NewRealProvider(monitoring.RealProviderConfig{DB: pool, WeKnoraBaseURL: config.TestWeKnoraBaseURL, TestWeKnoraResolver: testEndpointResolver, ProductionWeKnoraBaseURL: config.ProductionWeKnoraBaseURL, ProductionWeKnoraResolver: productionEndpointResolver, MinerUBaseURL: config.MinerUBaseURL, WeKnoraReadiness: weknoraReadiness, WorkerStaleAfter: config.WorkerLivenessStaleAfter})
|
||||
// P10 readiness 与执行链看同一目标:网页保存的端点、当前持久化绑定、同一份
|
||||
// API Key。启动时只捕获 resolver,不捕获任何一项的值(F13:旧 env 绑定 404)。
|
||||
readinessResolver := newWeKnoraReadinessResolver(
|
||||
func(resolveCtx context.Context) string {
|
||||
if saved := testEndpointResolver(resolveCtx); saved != "" {
|
||||
return saved
|
||||
}
|
||||
return config.TestWeKnoraBaseURL
|
||||
},
|
||||
config.InternalRoutingKnowledgeBaseID,
|
||||
func(resolveCtx context.Context) (string, error) {
|
||||
internal, _, _, err := actionStore.LoadRoutingKBBindings(resolveCtx)
|
||||
return internal, err
|
||||
},
|
||||
func() (string, error) { return weknora.ReadAPIKeyFile(config.TestWeKnoraAPIKeyFile) },
|
||||
)
|
||||
readinessProber := readiness.Prober{
|
||||
Resolve: readinessResolver,
|
||||
Timeout: config.TestWeKnoraHTTPTimeout,
|
||||
AllowDockerHTTP: config.TestWeKnoraAllowDockerHTTP,
|
||||
}
|
||||
weknoraProbe := func(probeCtx context.Context) (string, error) {
|
||||
result := readinessProber.Check(probeCtx)
|
||||
return result.Target, result.Err
|
||||
}
|
||||
knowledgeBaseLister := &weKnoraLister{
|
||||
endpoint: func(resolveCtx context.Context) string {
|
||||
if saved := testEndpointResolver(resolveCtx); saved != "" {
|
||||
return saved
|
||||
}
|
||||
return config.TestWeKnoraBaseURL
|
||||
},
|
||||
readKey: func() (string, error) { return weknora.ReadAPIKeyFile(config.TestWeKnoraAPIKeyFile) },
|
||||
timeout: config.TestWeKnoraHTTPTimeout,
|
||||
allowDockerHTTP: config.TestWeKnoraAllowDockerHTTP,
|
||||
}
|
||||
monitorProvider, err := monitoring.NewRealProvider(monitoring.RealProviderConfig{DB: pool, WeKnoraBaseURL: config.TestWeKnoraBaseURL, TestWeKnoraResolver: testEndpointResolver, ProductionWeKnoraBaseURL: config.ProductionWeKnoraBaseURL, ProductionWeKnoraResolver: productionEndpointResolver, MinerUBaseURL: config.MinerUBaseURL, WeKnoraReadiness: weknoraProbe, WorkerStaleAfter: config.WorkerLivenessStaleAfter})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -190,7 +206,7 @@ func run(ctx context.Context, getenv func(string) string) error {
|
||||
QAOpsTenantKey: qaSettings.TenantKey,
|
||||
QAOpsEnabled: qaSettings.Enabled,
|
||||
QAOpsProblem: qaSettings.Problem,
|
||||
Store: postgresstore.NewStore(pool).WithLibraryTarget(config.LibraryTargetIdentity()),
|
||||
Store: postgresstore.NewStore(pool).WithLibraryTarget(config.LibraryTargetIdentity()).WithRoutingDefaults(config.InternalRoutingKnowledgeBaseID, config.ExternalRoutingKnowledgeBaseID),
|
||||
Monitor: monitorProvider,
|
||||
InternalRoutingKnowledgeBaseID: config.InternalRoutingKnowledgeBaseID,
|
||||
ExternalRoutingKnowledgeBaseID: config.ExternalRoutingKnowledgeBaseID,
|
||||
@@ -232,6 +248,8 @@ func run(ctx context.Context, getenv func(string) string) error {
|
||||
LibraryProductionInternalKB: config.LibraryProductionInternalKB,
|
||||
LibraryProductionExternalKB: config.LibraryProductionExternalKB,
|
||||
LibraryProductionTenantID: config.LibraryProductionTenantID,
|
||||
LibraryProductionIndex: config.LibraryProductionIndex,
|
||||
LibraryProductionS3Bucket: config.LibraryProductionS3Bucket,
|
||||
LibraryRetentionVersions: config.LibraryRetentionVersions,
|
||||
LibraryRetentionDays: config.LibraryRetentionDays,
|
||||
BackupHost: backuphost.New(backuphost.Config{
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
"knowledge-governance/internal/readiness"
|
||||
)
|
||||
|
||||
// newWeKnoraReadinessResolver 把「探测目标」收敛到与业务同一组真源:
|
||||
// 端点来自网页保存的服务配置(endpoint 内部已做 env 兜底),知识库来自
|
||||
// 当前持久化绑定(未绑定才用 env 引导值),凭据每次重新读同一份 key 文件。
|
||||
// 任何一项读取失败都必须显式报错——静默退回旧值正是 F13 的成因。
|
||||
func newWeKnoraReadinessResolver(
|
||||
endpoint func(context.Context) string,
|
||||
envKnowledgeBaseID string,
|
||||
loadBinding func(context.Context) (string, error),
|
||||
readKey func() (string, error),
|
||||
) func(context.Context) (readiness.Target, error) {
|
||||
resolver := readiness.Resolver{
|
||||
Endpoint: endpoint,
|
||||
DefaultKB: envKnowledgeBaseID,
|
||||
LoadBinding: func(resolveCtx context.Context) (string, error) {
|
||||
if loadBinding == nil {
|
||||
return "", nil
|
||||
}
|
||||
stored, err := loadBinding(resolveCtx)
|
||||
if err != nil {
|
||||
log.Printf("readiness: load routing bindings: %v", err)
|
||||
}
|
||||
return stored, err
|
||||
},
|
||||
ReadKey: func() (string, error) {
|
||||
if readKey == nil {
|
||||
return "", nil
|
||||
}
|
||||
value, err := readKey()
|
||||
if err != nil {
|
||||
log.Printf("readiness: read WeKnora API key: %v", err)
|
||||
}
|
||||
return value, err
|
||||
},
|
||||
}
|
||||
return resolver.Resolve
|
||||
}
|
||||
|
||||
// weKnoraLister 让「系统设置」的知识库下拉与保存校验同样跟随当前端点/凭据,
|
||||
// 而不是复用启动时的客户端(否则改端点后选项仍来自旧实例)。
|
||||
type weKnoraLister struct {
|
||||
endpoint func(context.Context) string
|
||||
readKey func() (string, error)
|
||||
timeout time.Duration
|
||||
allowDockerHTTP bool
|
||||
}
|
||||
|
||||
func (lister *weKnoraLister) ListKnowledgeBases(ctx context.Context) ([]weknora.KnowledgeBase, error) {
|
||||
baseURL := ""
|
||||
if lister.endpoint != nil {
|
||||
baseURL = strings.TrimSpace(lister.endpoint(ctx))
|
||||
}
|
||||
if baseURL == "" {
|
||||
return nil, fmt.Errorf("WeKnora 端点未配置")
|
||||
}
|
||||
apiKey, err := lister.readKey()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
timeout := lister.timeout
|
||||
if timeout <= 0 || timeout > 10*time.Second {
|
||||
timeout = 10 * time.Second
|
||||
}
|
||||
client, err := weknora.NewClient(weknora.ClientConfig{BaseURL: baseURL, APIKey: apiKey, Timeout: timeout, AllowDockerHTTP: lister.allowDockerHTTP})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return client.ListKnowledgeBases(ctx)
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/readiness"
|
||||
)
|
||||
|
||||
// F13 的接线回归:探测目标必须来自「网页保存的端点 + 当前持久化绑定」,
|
||||
// env 只能是未绑定时的引导兜底;任何一项都不许在启动时固化。
|
||||
func TestReadinessResolverUsesPersistedBindingOverEnv(t *testing.T) {
|
||||
const envKB = "1b52c140-2e18-4676-afc5-4c7d8d190a8c"
|
||||
const storedKB = "0c84a098-d3ed-4b82-a165-e6de9691d6a8"
|
||||
resolve := newWeKnoraReadinessResolver(
|
||||
func(context.Context) string { return "https://weknora.test" },
|
||||
envKB,
|
||||
func(context.Context) (string, error) { return storedKB, nil },
|
||||
func() (string, error) { return "key-1", nil },
|
||||
)
|
||||
target, err := resolve(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if target.KnowledgeBaseID != storedKB || target.BaseURL != "https://weknora.test" || target.APIKey != "key-1" {
|
||||
t.Fatalf("resolved target = %#v", target)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadinessResolverFallsBackToEnvOnlyWhenUnbound(t *testing.T) {
|
||||
const envKB = "1b52c140-2e18-4676-afc5-4c7d8d190a8c"
|
||||
resolve := newWeKnoraReadinessResolver(
|
||||
func(context.Context) string { return "https://weknora.test" },
|
||||
envKB,
|
||||
func(context.Context) (string, error) { return "", nil },
|
||||
func() (string, error) { return "key-1", nil },
|
||||
)
|
||||
target, err := resolve(context.Background())
|
||||
if err != nil || target.KnowledgeBaseID != envKB {
|
||||
t.Fatalf("unbound fallback = %#v, %v", target, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadinessResolverReportsBindingAndCredentialFailures(t *testing.T) {
|
||||
bindingDown := newWeKnoraReadinessResolver(
|
||||
func(context.Context) string { return "https://weknora.test" },
|
||||
"env-kb",
|
||||
func(context.Context) (string, error) { return "", errors.New("db unavailable") },
|
||||
func() (string, error) { return "key-1", nil },
|
||||
)
|
||||
target, err := bindingDown(context.Background())
|
||||
if err == nil {
|
||||
t.Fatalf("binding read failure must not silently probe env: %#v", target)
|
||||
}
|
||||
keyDown := newWeKnoraReadinessResolver(
|
||||
func(context.Context) string { return "https://weknora.test" },
|
||||
"env-kb",
|
||||
func(context.Context) (string, error) { return "", nil },
|
||||
func() (string, error) { return "", errors.New("key file missing") },
|
||||
)
|
||||
if _, err := keyDown(context.Background()); err == nil {
|
||||
t.Fatal("missing API key must surface, not degrade to liveness-only")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadinessResolverMarksUnconfiguredEndpoint(t *testing.T) {
|
||||
resolve := newWeKnoraReadinessResolver(
|
||||
func(context.Context) string { return "" },
|
||||
"env-kb",
|
||||
func(context.Context) (string, error) { return "", nil },
|
||||
func() (string, error) { return "key-1", nil },
|
||||
)
|
||||
// 端点缺失时归类为「未配置」,不是「服务不可用」。
|
||||
result := readiness.Prober{Resolve: resolve}.Check(context.Background())
|
||||
if !errors.Is(result.Err, readiness.ErrNotConfigured) {
|
||||
t.Fatalf("blank endpoint probe = %v, want ErrNotConfigured", result.Err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKnowledgeBaseListerFollowsTheCurrentEndpoint(t *testing.T) {
|
||||
base := func(name string) *httptest.Server {
|
||||
return httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
if request.URL.Path != "/api/v1/knowledge-bases" {
|
||||
writer.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
_, _ = writer.Write([]byte(`{"success":true,"data":[{"id":"` + name + `","name":"` + name + `"}]}`))
|
||||
}))
|
||||
}
|
||||
first, second := base("kb-a"), base("kb-b")
|
||||
defer first.Close()
|
||||
defer second.Close()
|
||||
endpoint := first.URL
|
||||
lister := &weKnoraLister{
|
||||
endpoint: func(context.Context) string { return endpoint },
|
||||
readKey: func() (string, error) { return "key-1", nil },
|
||||
timeout: time.Second,
|
||||
}
|
||||
bases, err := lister.ListKnowledgeBases(context.Background())
|
||||
if err != nil || len(bases) != 1 || bases[0].ID != "kb-a" {
|
||||
t.Fatalf("initial endpoint bases = %#v, %v", bases, err)
|
||||
}
|
||||
endpoint = second.URL
|
||||
bases, err = lister.ListKnowledgeBases(context.Background())
|
||||
if err != nil || len(bases) != 1 || bases[0].ID != "kb-b" {
|
||||
t.Fatalf("rotated endpoint bases = %#v, %v", bases, err)
|
||||
}
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
v071 "knowledge-governance/internal/releaseadapter/v071"
|
||||
)
|
||||
|
||||
func writeDrillCanary(ctx context.Context, root string, ledger *libraryrelease.WriteSet, id string, mark func(context.Context) error, write func(context.Context) error) error {
|
||||
ledger.Rows = append(ledger.Rows, libraryrelease.WriteSetRow{Table: "knowledge_tags", ID: id, Existing: false})
|
||||
if err := ledger.Seal(); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := v071.PersistWriteSet(root, ledger); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := mark(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return write(ctx)
|
||||
}
|
||||
@@ -1,60 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCanaryPersistsRecoveryEvidenceBeforeWrite(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
ledger := libraryrelease.WriteSet{Version: 1, JobID: "drill", Target: "internal", TenantID: 10000, KnowledgeBaseID: "kb", BackupSnapshotID: "backup", TargetSnapshotID: "backup", CreatedAt: time.Now().UTC(), Universe: map[string]libraryrelease.TableFingerprint{"knowledge_tags": {Count: 0, Digest: "empty"}}}
|
||||
marked := false
|
||||
written := false
|
||||
err := writeDrillCanary(context.Background(), root, &ledger, "canary-id", func(context.Context) error { marked = true; return nil }, func(context.Context) error {
|
||||
if !marked {
|
||||
t.Fatal("write preceded risk marker")
|
||||
}
|
||||
if len(ledger.CreatedIDs("knowledge_tags")) != 1 {
|
||||
t.Fatal("canary absent from write set")
|
||||
}
|
||||
matches, e := filepath.Glob(filepath.Join(root, "writesets", "*.json"))
|
||||
if e != nil || len(matches) != 1 {
|
||||
t.Fatalf("durable ledger missing: %v %v", matches, e)
|
||||
}
|
||||
if _, e := os.Stat(matches[0]); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
written = true
|
||||
return nil
|
||||
})
|
||||
if err != nil || !written {
|
||||
t.Fatalf("err=%v written=%v", err, written)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanaryNeverWritesWithoutDurableLedger(t *testing.T) {
|
||||
root := filepath.Join(t.TempDir(), "blocked")
|
||||
if err := os.WriteFile(root, []byte("not a directory"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ledger := libraryrelease.WriteSet{Version: 1, JobID: "drill", Target: "internal", TenantID: 10000, KnowledgeBaseID: "kb", BackupSnapshotID: "backup", TargetSnapshotID: "backup", CreatedAt: time.Now().UTC(), Universe: map[string]libraryrelease.TableFingerprint{"knowledge_tags": {Count: 0, Digest: "empty"}}}
|
||||
marked, written := false, false
|
||||
err := writeDrillCanary(context.Background(), root, &ledger, "canary-id", func(context.Context) error { marked = true; return nil }, func(context.Context) error { written = true; return nil })
|
||||
if err == nil || marked || written {
|
||||
t.Fatalf("err=%v marked=%v written=%v", err, marked, written)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanaryNeverWritesWhenRiskMarkerFails(t *testing.T) {
|
||||
ledger := libraryrelease.WriteSet{Version: 1, JobID: "drill", Target: "internal", TenantID: 10000, KnowledgeBaseID: "kb", BackupSnapshotID: "backup", TargetSnapshotID: "backup", CreatedAt: time.Now().UTC(), Universe: map[string]libraryrelease.TableFingerprint{"knowledge_tags": {Count: 0, Digest: "empty"}}}
|
||||
written := false
|
||||
err := writeDrillCanary(context.Background(), t.TempDir(), &ledger, "canary-id", func(context.Context) error { return errors.New("journal unavailable") }, func(context.Context) error { written = true; return nil })
|
||||
if err == nil || written {
|
||||
t.Fatalf("err=%v written=%v", err, written)
|
||||
}
|
||||
}
|
||||
+15
-203
@@ -1,4 +1,4 @@
|
||||
// command librarydrill 在目标实例上做一次**真实**的恢复演练:
|
||||
// command librarydrill 是恢复演练的**命令行入口**,实现在 internal/libraryverify:
|
||||
//
|
||||
// 维护隔离(关入口/停写入者/证明排空)
|
||||
// → 捕获目标整库(真预备份,落归档)
|
||||
@@ -8,16 +8,14 @@
|
||||
// → 逐表/索引/资源/Wiki 再校验一次,必须完全一致
|
||||
// → 关窗口(按原运行状态恢复服务、开入口、验证健康)
|
||||
//
|
||||
// 它只读/写**被测目标自己**,绝不触碰封存源;所有凭据来自进程环境
|
||||
// (与 worker 同一份 LIBRARY_* 配置),不进命令行、不打印。
|
||||
// 任一步失败都不伪装成功:窗口保持维护、报告写明失败阶段。
|
||||
// 为什么只剩薄封装:页面要能点一次"恢复验证",worker 必须在进程内跑同一份实现。
|
||||
// 两份实现意味着"CLI 通过、页面失败"或反过来,演练结论就不可信了。
|
||||
// 它只读/写**被测目标自己**,绝不触碰封存源;所有凭据来自进程环境。
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
@@ -26,35 +24,12 @@ import (
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/appconfig"
|
||||
"knowledge-governance/internal/libraryfreeze"
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
"knowledge-governance/internal/librarystack"
|
||||
"knowledge-governance/internal/libraryverify"
|
||||
)
|
||||
|
||||
type drillVerification struct {
|
||||
SnapshotID string `json:"snapshot_id"`
|
||||
Verified bool `json:"verified"`
|
||||
Error string `json:"error,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
type drillReport struct {
|
||||
Status string `json:"status"` // passed | failed | maintenance_held
|
||||
Scope string `json:"scope"`
|
||||
Target string `json:"target"`
|
||||
SnapshotID string `json:"snapshot_id"`
|
||||
Project string `json:"project"`
|
||||
Counts map[string]int64 `json:"counts"`
|
||||
BaselineCheck *drillVerification `json:"baseline_check"`
|
||||
DriftCheck *drillVerification `json:"drift_check"` // 必须失败:证明确实能发现差异
|
||||
RestoreCheck *drillVerification `json:"restore_check"`
|
||||
Canary string `json:"canary"`
|
||||
Failure string `json:"failure,omitempty"`
|
||||
Stage string `json:"stage,omitempty"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
FinishedAt time.Time `json:"finished_at"`
|
||||
Until map[string]string `json:"until,omitempty"`
|
||||
}
|
||||
// 报告类型直接复用共享实现:命令行与页面/worker 的证据字段完全一致。
|
||||
type drillVerification = libraryverify.Verification
|
||||
type drillReport = libraryverify.Report
|
||||
|
||||
func main() {
|
||||
scope := flag.String("scope", "internal", "internal|external")
|
||||
@@ -88,18 +63,7 @@ func recoverWindows() error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("load config: %w", err)
|
||||
}
|
||||
manager, err := librarystack.OpenMaintenance(config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer manager.Close()
|
||||
coordinator, control, err := manager.Open(config.LibraryTargetJournalRoot)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer control.Close()
|
||||
defer coordinator.Close()
|
||||
if err := coordinator.Recover(context.Background()); err != nil {
|
||||
if err := libraryverify.RecoverWindows(config); err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("maintenance windows recovered; target should be back to its original service state")
|
||||
@@ -107,170 +71,18 @@ func recoverWindows() error {
|
||||
}
|
||||
|
||||
func run(scope, jobID string) (*drillReport, error) {
|
||||
if scope != "internal" && scope != "external" {
|
||||
return nil, errors.New("scope must be internal or external")
|
||||
}
|
||||
config, err := appconfig.Load(os.Getenv)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load config: %w", err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
report := &drillReport{Status: "failed", Scope: scope, Project: config.LibraryTargetProject,
|
||||
Target: config.LibraryTargetInstance, StartedAt: time.Now().UTC()}
|
||||
fail := func(stage string, cause error, held bool) (*drillReport, error) {
|
||||
report.Stage = stage
|
||||
report.Failure = cause.Error()
|
||||
report.FinishedAt = time.Now().UTC()
|
||||
if held {
|
||||
report.Status = "maintenance_held"
|
||||
}
|
||||
return report, cause
|
||||
}
|
||||
|
||||
target, backups, err := librarystack.OpenBaselineStack(ctx, config, scope)
|
||||
if err != nil {
|
||||
return fail("open_target", err, false)
|
||||
}
|
||||
defer target.Close()
|
||||
manager, err := librarystack.OpenMaintenance(config)
|
||||
if err != nil {
|
||||
return fail("open_maintenance", err, false)
|
||||
}
|
||||
defer manager.Close()
|
||||
if err := manager.VerifyTopology(ctx); err != nil {
|
||||
return fail("topology", err, false)
|
||||
}
|
||||
coordinator, control, err := manager.Open(config.LibraryTargetJournalRoot)
|
||||
if err != nil {
|
||||
return fail("open_coordinator", err, false)
|
||||
}
|
||||
defer control.Close()
|
||||
defer coordinator.Close()
|
||||
|
||||
if jobID == "" {
|
||||
jobID = "drill-" + scope + "-" + time.Now().UTC().Format("20060102T150405")
|
||||
jobID = fmt.Sprintf("drill-%s-%d", scope, time.Now().UTC().UnixNano())
|
||||
}
|
||||
var (
|
||||
snapshotID string
|
||||
counts map[string]int64
|
||||
canary string
|
||||
contentTouched bool
|
||||
baseline drillVerification
|
||||
drift drillVerification
|
||||
restored drillVerification
|
||||
)
|
||||
work := func(ctx context.Context, window *libraryfreeze.Window) (string, error) {
|
||||
start := time.Now()
|
||||
current, err := target.Current(ctx)
|
||||
if err != nil {
|
||||
return "", errors.New("读取目标当前整库状态失败:" + err.Error())
|
||||
}
|
||||
counts = current.Counts
|
||||
report.Counts = counts
|
||||
// 1) 真预备份(落归档、内容寻址、导出后复核内容未变)。
|
||||
backup, err := backups.Prepare(ctx, "", current)
|
||||
if err != nil {
|
||||
return "", errors.New("捕获目标整库失败:" + err.Error())
|
||||
}
|
||||
snapshotID = backup.SnapshotID
|
||||
report.SnapshotID = snapshotID
|
||||
if err := window.SetBackupSnapshot(ctx, snapshotID); err != nil {
|
||||
return "", errors.New("记录预备份身份失败:" + err.Error())
|
||||
}
|
||||
log.Printf("drill: snapshot captured snapshot=%s bytes=%d in=%s", snapshotID, backup.Bytes, time.Since(start))
|
||||
|
||||
// 2) 基线校验:预备份必须与目标完全一致。
|
||||
baseline = drillVerification{SnapshotID: snapshotID, At: time.Now().UTC()}
|
||||
if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); err != nil {
|
||||
baseline.Error = err.Error()
|
||||
report.BaselineCheck = &baseline
|
||||
return "", errors.New("预备份与目标不一致(预备份本身不可信):" + err.Error())
|
||||
}
|
||||
baseline.Verified = true
|
||||
report.BaselineCheck = &baseline
|
||||
|
||||
// Canary is an isolated tag, including on an empty target. Never seed
|
||||
// a document or change an existing row merely to make a drill possible.
|
||||
var random [16]byte
|
||||
if _, err := rand.Read(random[:]); err != nil {
|
||||
return "", err
|
||||
}
|
||||
canary = fmt.Sprintf("%x", random[:])
|
||||
identityValue := target.Identity()
|
||||
var exists bool
|
||||
if err := target.Pool().QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM knowledge_tags WHERE id=$1)", canary).Scan(&exists); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if exists {
|
||||
return "", errors.New("canary identity collision; no write attempted")
|
||||
}
|
||||
ledger := libraryrelease.WriteSet{Version: libraryrelease.WriteSetVersion, JobID: jobID, Target: scope,
|
||||
TenantID: identityValue.TenantID, KnowledgeBaseID: identityValue.KnowledgeBaseID,
|
||||
BackupSnapshotID: snapshotID, TargetSnapshotID: snapshotID,
|
||||
CreatedAt: time.Now().UTC(), Universe: backup.Universe}
|
||||
report.Canary = canary
|
||||
if err := writeDrillCanary(ctx, config.LibraryArchiveRoot, &ledger, canary,
|
||||
func(ctx context.Context) error {
|
||||
if err := window.MarkContentRisk(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
contentTouched = true
|
||||
return nil
|
||||
}, func(ctx context.Context) error {
|
||||
_, err := target.Pool().Exec(ctx, `INSERT INTO knowledge_tags(id,tenant_id,knowledge_base_id,name) VALUES($1,$2,$3,$4)`, canary, identityValue.TenantID, identityValue.KnowledgeBaseID, "drill-canary-"+canary)
|
||||
return err
|
||||
}); err != nil {
|
||||
return "", fmt.Errorf("canary preparation/write failed: %w", err)
|
||||
}
|
||||
drift = drillVerification{SnapshotID: snapshotID, At: time.Now().UTC()}
|
||||
if verifyErr := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); verifyErr == nil {
|
||||
drift.Error = "写入 canary 后校验仍然通过:校验没有真正比对内容"
|
||||
report.DriftCheck = &drift
|
||||
return "", errors.New("写入 canary 后校验仍然通过:校验没有真正比对内容,拒绝把这次演练当成通过")
|
||||
} else {
|
||||
drift.Error = verifyErr.Error()
|
||||
}
|
||||
report.DriftCheck = &drift
|
||||
log.Printf("drill: drift detected as expected (%s)", drift.Error)
|
||||
|
||||
// 4) 按预备份恢复。
|
||||
// Restore removes only the exact new canary identity in the durable write set.
|
||||
if err := target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + snapshotID, SnapshotID: snapshotID,
|
||||
WriteSet: &ledger}); err != nil {
|
||||
return "", errors.New("按预备份恢复失败:" + err.Error())
|
||||
}
|
||||
// 5) 恢复后逐类复核:必须与预备份完全一致。
|
||||
restored = drillVerification{SnapshotID: snapshotID, At: time.Now().UTC()}
|
||||
if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); err != nil {
|
||||
restored.Error = err.Error()
|
||||
report.RestoreCheck = &restored
|
||||
return "", errors.New("恢复后内容仍与预备份不一致:" + err.Error())
|
||||
}
|
||||
restored.Verified = true
|
||||
report.RestoreCheck = &restored
|
||||
log.Printf("drill: restored and verified snapshot=%s", snapshotID)
|
||||
return snapshotID, nil
|
||||
}
|
||||
|
||||
window, windowErr := coordinator.RunWindow(ctx, jobID,
|
||||
func(ctx context.Context) error { return manager.VerifyRunning(ctx) }, work,
|
||||
// 没写过目标内容 ⇒ 按原状态恢复永远安全;写过 canary ⇒ 只有恢复并校验通过才开放。
|
||||
func(error) bool { return !contentTouched || restored.Verified })
|
||||
|
||||
report.Until = map[string]string{"job": jobID, "window_state": window.State}
|
||||
if windowErr != nil || window.State != "succeeded" {
|
||||
cause := windowErr
|
||||
if cause == nil {
|
||||
cause = fmt.Errorf("维护窗口状态 %s(未成功)", window.State)
|
||||
}
|
||||
if window.State == "held" {
|
||||
return fail("window", cause, true)
|
||||
}
|
||||
return fail("window", cause, false)
|
||||
}
|
||||
report.Status = "passed"
|
||||
report.FinishedAt = time.Now().UTC()
|
||||
return report, nil
|
||||
// 主机侧 CLI 不持有治理库连接:维护记录由调用它的作业/运维流程负责。
|
||||
// 页面发起的验证走 worker,那里 Options 同时接治理生命周期与写集合登记。
|
||||
return libraryverify.Run(context.TODO(), config, scope, jobID, libraryverify.Options{
|
||||
Progress: func(stage, state, detail string) { log.Printf("stage=%s state=%s %s", stage, state, detail) },
|
||||
})
|
||||
}
|
||||
|
||||
func writeReport(path string, report *drillReport) error {
|
||||
|
||||
@@ -87,7 +87,7 @@ func run(ctx context.Context, config appconfig.Config, scope, snapshotID, backup
|
||||
if strings.TrimSpace(expectSource) != want {
|
||||
return nil, fmt.Errorf("来源范围与 -expect-source 不一致:归档为 %s,参数为 %s", want, expectSource)
|
||||
}
|
||||
target, _, err := librarystack.OpenProductionStack(ctx, config, scope, backupSnapshotID, "")
|
||||
target, _, err := librarystack.OpenProductionStack(ctx, config, scope, backupSnapshotID, "", nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("装配目标失败: %w", err)
|
||||
}
|
||||
|
||||
@@ -2,8 +2,6 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"log"
|
||||
"net"
|
||||
@@ -35,10 +33,6 @@ func run() error {
|
||||
if len(data) < 32 || len(control) < 32 || data == control {
|
||||
return errors.New("gateway_distinct_tokens_required")
|
||||
}
|
||||
pub, e := base64.StdEncoding.DecodeString(os.Getenv("OBJECT_GATEWAY_AUDIT_PUBLIC_KEY"))
|
||||
if e != nil || len(pub) != ed25519.PublicKeySize {
|
||||
return errors.New("gateway_auditor_key_required")
|
||||
}
|
||||
provider, endpoint, bucket := os.Getenv("OBJECT_GATEWAY_PROVIDER"), os.Getenv("OBJECT_GATEWAY_ENDPOINT"), os.Getenv("OBJECT_GATEWAY_BUCKET")
|
||||
scope := objectgateway.Digest([]byte(provider + "\x00" + endpoint + "\x00" + bucket))
|
||||
pool, e := pgxpool.New(ctx, os.Getenv("OBJECT_GATEWAY_DATABASE_URL"))
|
||||
@@ -50,7 +44,10 @@ func run() error {
|
||||
if e != nil {
|
||||
return errors.New("gateway_backend_configuration_invalid")
|
||||
}
|
||||
g := &objectgateway.Gateway{Pool: pool, Scope: scope, Backend: backend, MaxBytes: 256 * 1024 * 1024, Isolation: objectgateway.AuditVerifier{Path: os.Getenv("OBJECT_GATEWAY_AUDIT_FILE"), PublicKey: pub, Scope: scope, GatewayID: os.Getenv("OBJECT_GATEWAY_ID")}}
|
||||
// The target bucket is shared with other writers. Admission is the journal
|
||||
// plus backend capability; retired attestation env vars are deliberately
|
||||
// ignored so an old deployment file cannot gate this path.
|
||||
g := &objectgateway.Gateway{Pool: pool, Scope: scope, Backend: backend, MaxBytes: 256 * 1024 * 1024}
|
||||
if governanceURL := os.Getenv("OBJECT_GATEWAY_GOVERNANCE_DATABASE_URL"); governanceURL != "" {
|
||||
cfg, e := pgxpool.ParseConfig(governanceURL)
|
||||
if e != nil {
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// startErrorMessage builds the addressed binary and runs it long enough to
|
||||
// observe why the real entry point refused to listen.
|
||||
func startErrorMessage(t *testing.T, extra ...string) (string, int) {
|
||||
t.Helper()
|
||||
bin := filepath.Join(t.TempDir(), "object-gateway")
|
||||
build := exec.Command("go", "build", "-o", bin, ".")
|
||||
if out, err := build.CombinedOutput(); err != nil {
|
||||
t.Fatalf("build gateway: %v\n%s", err, out)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, bin)
|
||||
cmd.Env = append([]string{
|
||||
"PATH=" + os.Getenv("PATH"),
|
||||
"HOME=" + os.Getenv("HOME"),
|
||||
"OBJECT_GATEWAY_LISTEN=127.0.0.1:0",
|
||||
"OBJECT_GATEWAY_DATABASE_URL=postgres://gateway@127.0.0.1:1/gateway?sslmode=disable&connect_timeout=2",
|
||||
"OBJECT_GATEWAY_PROVIDER=minio",
|
||||
"OBJECT_GATEWAY_ENDPOINT=http://127.0.0.1:19100",
|
||||
"OBJECT_GATEWAY_BUCKET=maintenance-fixture-bucket",
|
||||
"OBJECT_GATEWAY_REGION=us-east-1",
|
||||
"OBJECT_GATEWAY_ACCESS_KEY=fixture-access-key",
|
||||
"OBJECT_GATEWAY_SECRET_KEY=fixture-secret-key",
|
||||
"OBJECT_GATEWAY_DATA_TOKEN=" + strings.Repeat("d", 32),
|
||||
"OBJECT_GATEWAY_CONTROL_TOKEN=" + strings.Repeat("c", 32),
|
||||
}, extra...)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if ctx.Err() != nil {
|
||||
t.Fatalf("gateway did not terminate; output=%s", out)
|
||||
}
|
||||
code := 0
|
||||
if exit, ok := err.(*exec.ExitError); ok {
|
||||
code = exit.ExitCode()
|
||||
} else if err != nil {
|
||||
t.Fatalf("run gateway: %v", err)
|
||||
}
|
||||
return string(out), code
|
||||
}
|
||||
|
||||
// The default shared-bucket path must not require a whole-bucket exclusive
|
||||
// writer attestation. Only the journal/backend admission gate may refuse it,
|
||||
// and a stale attestation file left behind by an older deployment must not
|
||||
// gate the default path either.
|
||||
func TestGatewayEntrypointDoesNotRequireBucketAttestation(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
env []string
|
||||
}{
|
||||
{name: "no_audit_configured"},
|
||||
{name: "stale_audit_file_ignored", env: []string{
|
||||
"OBJECT_GATEWAY_ID=retired-gateway",
|
||||
"OBJECT_GATEWAY_AUDIT_FILE=/nonexistent/expired-attestation.json",
|
||||
"OBJECT_GATEWAY_AUDIT_PUBLIC_KEY=" + base64.StdEncoding.EncodeToString(make([]byte, 32)),
|
||||
}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
out, code := startErrorMessage(t, tc.env...)
|
||||
if code == 0 {
|
||||
t.Fatalf("gateway exited 0 instead of reporting its admission state: %s", out)
|
||||
}
|
||||
if strings.Contains(out, "gateway_auditor_key_required") || strings.Contains(out, "gateway_exclusive_writer_evidence_unverified") {
|
||||
t.Fatalf("shared-bucket admission still gated by whole-bucket attestation: %s", out)
|
||||
}
|
||||
if !strings.Contains(out, "gateway_persistence_unavailable") && !strings.Contains(out, "gateway_backend_not_ready") {
|
||||
t.Fatalf("gateway did not reach journal/backend admission: %s", out)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
type capturePersistenceFake struct {
|
||||
failAdvance error
|
||||
failStage error
|
||||
advanced []governance.LibraryJobState
|
||||
stages []string
|
||||
}
|
||||
|
||||
func (f *capturePersistenceFake) AdvanceLibraryJob(_ context.Context, _ string, state governance.LibraryJobState,
|
||||
_, _, _, _, _ string, _ time.Time) error {
|
||||
f.advanced = append(f.advanced, state)
|
||||
return f.failAdvance
|
||||
}
|
||||
|
||||
func (f *capturePersistenceFake) UpdateLibraryStage(_ context.Context, _, stage, state string,
|
||||
_, _, _, _ *int64, _ string, _ time.Time) error {
|
||||
f.stages = append(f.stages, stage+"/"+state)
|
||||
return f.failStage
|
||||
}
|
||||
|
||||
// The handler must stop before touching the source when the starting state
|
||||
// cannot be persisted. This is the failure injection that the old log-and-ignore
|
||||
// path hid behind a completed queue row.
|
||||
func TestBeginLibraryCaptureStopsOnAdvanceFailure(t *testing.T) {
|
||||
store := &capturePersistenceFake{failAdvance: errors.New("injected advance failure")}
|
||||
err := beginLibraryCaptureJob(context.Background(), store, "libseal-test", time.Now())
|
||||
if err == nil || !strings.Contains(err.Error(), "cannot be advanced") {
|
||||
t.Fatalf("err=%v", err)
|
||||
}
|
||||
if len(store.advanced) != 1 || store.advanced[0] != governance.LibraryJobRunning {
|
||||
t.Fatalf("advance calls=%v", store.advanced)
|
||||
}
|
||||
if len(store.stages) != 0 {
|
||||
t.Fatalf("stage was written before advance failure: %v", store.stages)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBeginLibraryCaptureStopsOnStageFailure(t *testing.T) {
|
||||
store := &capturePersistenceFake{failStage: errors.New("injected stage failure")}
|
||||
err := beginLibraryCaptureJob(context.Background(), store, "libseal-test", time.Now())
|
||||
if err == nil || !strings.Contains(err.Error(), "stage cannot be advanced") {
|
||||
t.Fatalf("err=%v", err)
|
||||
}
|
||||
if len(store.stages) != 1 || store.stages[0] != "export/running" {
|
||||
t.Fatalf("stage calls=%v", store.stages)
|
||||
}
|
||||
}
|
||||
|
||||
// Terminal persistence is two writes; either failure must be returned rather
|
||||
// than turning the queue row into completed while the business job is pending.
|
||||
func TestCompleteLibraryCaptureStopsOnStageFailure(t *testing.T) {
|
||||
store := &capturePersistenceFake{failStage: errors.New("injected success stage failure")}
|
||||
err := completeLibraryCaptureJob(context.Background(), store, "libseal-test", "snap-test", time.Now())
|
||||
if err == nil || !strings.Contains(err.Error(), "success stage cannot be persisted") {
|
||||
t.Fatalf("err=%v", err)
|
||||
}
|
||||
if len(store.advanced) != 0 {
|
||||
t.Fatalf("advance must not run after stage failure: %v", store.advanced)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompleteLibraryCaptureStopsOnAdvanceFailure(t *testing.T) {
|
||||
store := &capturePersistenceFake{failAdvance: errors.New("injected success advance failure")}
|
||||
err := completeLibraryCaptureJob(context.Background(), store, "libseal-test", "snap-test", time.Now())
|
||||
if err == nil || !strings.Contains(err.Error(), "success terminal cannot be persisted") {
|
||||
t.Fatalf("err=%v", err)
|
||||
}
|
||||
if len(store.stages) != 1 || store.stages[0] != "export/succeeded" {
|
||||
t.Fatalf("stage calls=%v", store.stages)
|
||||
}
|
||||
if len(store.advanced) != 1 || store.advanced[0] != governance.LibraryJobSucceeded {
|
||||
t.Fatalf("advance calls=%v", store.advanced)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompleteLibraryCapturePersistsBothFacts(t *testing.T) {
|
||||
store := &capturePersistenceFake{}
|
||||
if err := completeLibraryCaptureJob(context.Background(), store, "libseal-test", "snap-test", time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(store.stages) != 1 || store.stages[0] != "export/succeeded" ||
|
||||
len(store.advanced) != 1 || store.advanced[0] != governance.LibraryJobSucceeded {
|
||||
t.Fatalf("stages=%v advanced=%v", store.stages, store.advanced)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"knowledge-governance/internal/jobs"
|
||||
)
|
||||
|
||||
// R3g: only retained deterministic capture failures become permanent after the
|
||||
// agent has recovered the writer. Cancelled body-index coverage markers are
|
||||
// deliberately absent.
|
||||
func TestRetainedLibraryCaptureFailuresArePermanent(t *testing.T) {
|
||||
for _, marker := range []string{
|
||||
"missing library generated question index",
|
||||
"empty library index content",
|
||||
} {
|
||||
t.Run(marker, func(t *testing.T) {
|
||||
original := errors.New("LIBRARYCTL_FAILED: " + marker + ";已执行 recover 并确认写入者恢复")
|
||||
got := terminalLibraryCaptureError(original)
|
||||
terminal, code := jobs.Terminal(got)
|
||||
if !terminal || code != jobs.ErrCodeValidationFailed {
|
||||
t.Fatalf("retained validation failure must park immediately: terminal=%v code=%q err=%v", terminal, code, got)
|
||||
}
|
||||
if got.Error() != original.Error() {
|
||||
t.Fatalf("original message changed: got %q want %q", got.Error(), original.Error())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCancelledCoverageFailureStaysRetryable(t *testing.T) {
|
||||
for _, marker := range []string{
|
||||
"unverified legacy library chunk index coverage",
|
||||
"missing indexed library chunk",
|
||||
} {
|
||||
err := terminalLibraryCaptureError(errors.New(marker + ";已执行 recover 并确认写入者恢复"))
|
||||
if terminal, code := jobs.Terminal(err); terminal {
|
||||
t.Fatalf("cancelled marker %q must not be terminal; code=%q", marker, code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTransientLibraryCaptureFailureStaysRetryable(t *testing.T) {
|
||||
for _, message := range []string{
|
||||
"library agent is unreachable",
|
||||
"context deadline exceeded",
|
||||
`LIBRARYCTL_FAILED: job "lib-internal-x" already exists with state "failed";已执行 recover 并确认写入者恢复`,
|
||||
"missing library generated question index; writer restore not confirmed",
|
||||
} {
|
||||
got := terminalLibraryCaptureError(errors.New(message))
|
||||
if terminal, code := jobs.Terminal(got); terminal {
|
||||
t.Fatalf("%q must stay retryable, got terminal code=%q", message, code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetainedFailureDoesNotChangeMessage(t *testing.T) {
|
||||
original := errors.New("LIBRARYCTL_FAILED: missing library generated question index;已执行 recover 并确认写入者恢复")
|
||||
got := terminalLibraryCaptureError(original)
|
||||
if !strings.Contains(got.Error(), "missing library generated question index") {
|
||||
t.Fatalf("lost original cause: %v", got)
|
||||
}
|
||||
}
|
||||
@@ -233,8 +233,8 @@ func runLibraryReleaseAttempt(ctx context.Context, writer *libraryAttemptWriter,
|
||||
if !config.ProductionReleaseRequestsEnabled {
|
||||
gateReasons = append(gateReasons, "生产写入闸口关闭")
|
||||
}
|
||||
if !config.ProductionBackupRestoreVerified {
|
||||
gateReasons = append(gateReasons, "没有已核验的生产恢复演练证据")
|
||||
if ok, reason := libraryRestoreEvidenceSatisfied(ctx, store, config, payload.Scope); !ok {
|
||||
gateReasons = append(gateReasons, reason)
|
||||
}
|
||||
if len(gateReasons) > 0 {
|
||||
message := "生产" + kindLabel(kind) + "被闸口拒绝:" + strings.Join(gateReasons, ";")
|
||||
@@ -272,7 +272,7 @@ func runLibraryReleaseAttempt(ctx context.Context, writer *libraryAttemptWriter,
|
||||
// 生产索引映射基线:权威来源是当前生效版本(精确查),其次任何一次成功记录的映射。
|
||||
// 只认"生效版本"会在恢复把版本顶替掉之后误判"索引被改动"。
|
||||
knownMapping := knownIndexMapping(ctx, store, payload.Scope)
|
||||
target, backups, wireErr := buildLibraryProductionStack(ctx, config, payload.Scope, payload.TargetSnapshotID, knownMapping)
|
||||
target, backups, wireErr := buildLibraryProductionStack(ctx, config, restoreDispatchGate(config, store.Store), payload.Scope, payload.TargetSnapshotID, knownMapping)
|
||||
if wireErr == nil {
|
||||
if mapping, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil {
|
||||
recorder.mappingDigest = mapping
|
||||
@@ -628,6 +628,10 @@ func resumeInterruptedLibraryJobs(ctx context.Context, store libraryResumeStore,
|
||||
// 基线作业被杀时窗口可能还停着目标:用同一个作业 id 接管那个窗口,
|
||||
// 继续捕获/校验,而不是留下一个 held 目标等人工处理。
|
||||
queueType = libraryBaselineJobType
|
||||
case governance.LibraryJobVerify:
|
||||
// 恢复验证同样持有维护窗口:接管时必须用同一份作业 id 收掉那个窗口,
|
||||
// 否则目标会一直停在维护里而作业显示失败。
|
||||
queueType = libraryVerifyJobType
|
||||
}
|
||||
payload, _ := json.Marshal(map[string]string{"library_job_id": job.ID, "scope": job.Scope,
|
||||
"target_snapshot_id": job.TargetSnapshotID})
|
||||
|
||||
@@ -114,7 +114,7 @@ func runLibraryRecovery(ctx context.Context, store *postgresstore.Store, config
|
||||
claim.RecoveryJobID, claim.RecoveryOperationID = job.ID, recoveryOp.ID
|
||||
ctx = objectgateway.WithClaim(ctx, claim)
|
||||
knownMapping := knownIndexMapping(ctx, store, payload.Scope)
|
||||
target, _, wireErr := buildLibraryProductionStack(ctx, config, payload.Scope, prebackup, knownMapping)
|
||||
target, _, wireErr := buildLibraryProductionStack(ctx, config, restoreDispatchGate(config, store), payload.Scope, prebackup, knownMapping)
|
||||
if wireErr != nil {
|
||||
return fmt.Errorf("恢复目标装配失败:%w", wireErr)
|
||||
}
|
||||
@@ -127,7 +127,10 @@ func runLibraryRecovery(ctx context.Context, store *postgresstore.Store, config
|
||||
defer control.Close()
|
||||
defer coordinator.Close()
|
||||
|
||||
restorer := recoveryRestorer{target: target, expected: prebackup, writeSet: ledger}
|
||||
restorer := recoveryRestorer{target: target, expected: prebackup, writeSet: ledger,
|
||||
// 恢复验证写的是"隔离 canary",不是发布计划:同一 job 的收尾走受限的演练
|
||||
// 恢复入口(同样的身份合同 + 整段回放),不能拿发布计划合同去回放它。
|
||||
drill: windowJob.Kind == governance.LibraryJobVerify}
|
||||
verifier := recoveryVerifier{target: target, expected: prebackup}
|
||||
result, recoverErr := coordinator.RecoverWindow(ctx, payload.WindowJobID, restorer, verifier)
|
||||
now := time.Now().UTC()
|
||||
@@ -171,6 +174,8 @@ type recoveryRestorer struct {
|
||||
target *v071.ProductionLibraryTarget
|
||||
expected string
|
||||
writeSet *libraryrelease.WriteSet
|
||||
// drill 表示窗口属于恢复演练:它的写集合是 canary,不是发布执行计划。
|
||||
drill bool
|
||||
}
|
||||
|
||||
func (r recoveryRestorer) Restore(ctx context.Context, backupID string) error {
|
||||
@@ -183,8 +188,11 @@ func (r recoveryRestorer) Restore(ctx context.Context, backupID string) error {
|
||||
if r.writeSet.BackupSnapshotID != backupID {
|
||||
return fmt.Errorf("写集合记录绑定预备份 %s,不是 %s", r.writeSet.BackupSnapshotID, backupID)
|
||||
}
|
||||
return r.target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + backupID, SnapshotID: backupID,
|
||||
WriteSet: r.writeSet})
|
||||
ref := libraryrelease.BackupRef{ID: "backup-" + backupID, SnapshotID: backupID, WriteSet: r.writeSet}
|
||||
if r.drill {
|
||||
return r.target.RestoreDrill(ctx, ref)
|
||||
}
|
||||
return r.target.Restore(ctx, ref)
|
||||
}
|
||||
|
||||
// recoveryVerifier 做真实的内容校验,并把目标当前状态摘要作为"校验过什么"的凭证。
|
||||
|
||||
@@ -30,4 +30,9 @@ func registerLibraryReleaseHandlers(handlers map[string]jobs.Handler, store *pos
|
||||
handlers[libraryBaselineJobType] = admittanceGuarded(admissionErr, func(ctx context.Context, job jobs.Job) error {
|
||||
return runLibraryBaseline(ctx, store, config, job)
|
||||
})
|
||||
// 恢复验证:真实"备份→canary→恢复→复核",通过后写入与当前目标绑定的可恢复证据。
|
||||
// 它是同步闸口的证据来源,所以与上面三类共用同一把执行锁与准入。
|
||||
handlers[libraryVerifyJobType] = admittanceGuarded(admissionErr, func(ctx context.Context, job jobs.Job) error {
|
||||
return runLibraryVerify(ctx, store, config, job)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"knowledge-governance/internal/appconfig"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
)
|
||||
|
||||
// A12:能力默认关闭;只有"配置开启 + 目标身份存在 + 真实治理台账"同时成立才装配。
|
||||
// 配置本身不是独占证明,baseline 装配永远不传该台账。
|
||||
func TestRestoreDispatchGateRequiresExplicitConfiguration(t *testing.T) {
|
||||
store := &postgresstore.Store{}
|
||||
identity := appconfig.Config{LibraryProductionDatabaseURL: "postgres://local@127.0.0.1:5432/target_db",
|
||||
LibraryTargetInstance: "inst", LibraryProductionTenantID: 1, LibraryProductionInternalKB: "kb"}
|
||||
if gate := restoreDispatchGate(identity, store); gate != nil {
|
||||
t.Fatal("gate wired without the explicit switch")
|
||||
}
|
||||
enabled := identity
|
||||
enabled.LibraryRestoreDispatchEnabled = true
|
||||
if gate := restoreDispatchGate(enabled, store); gate == nil {
|
||||
t.Fatal("gate not wired when explicitly enabled")
|
||||
}
|
||||
if gate := restoreDispatchGate(enabled, nil); gate != nil {
|
||||
t.Fatal("gate wired without a governance store")
|
||||
}
|
||||
// 空目标配置仍会得到一个 gate:真实闸门是治理库激活登记(见 store 用例),
|
||||
// 用"配置看起来完整"冒充独占证明才是要避免的。
|
||||
}
|
||||
@@ -15,7 +15,8 @@ import (
|
||||
func TestRegisterLibraryReleaseHandlersCoversOnlyReleaseJobs(t *testing.T) {
|
||||
handlers := map[string]jobs.Handler{}
|
||||
registerLibraryReleaseHandlers(handlers, nil, appconfig.Config{}, nil)
|
||||
want := []string{librarySyncJobType, libraryRestoreJobType, libraryRecoverJobType, libraryBaselineJobType}
|
||||
want := []string{librarySyncJobType, libraryRestoreJobType, libraryRecoverJobType,
|
||||
libraryBaselineJobType, libraryVerifyJobType}
|
||||
if len(handlers) != len(want) {
|
||||
t.Fatalf("registered %d handlers, want %d: %#v", len(handlers), len(want), handlers)
|
||||
}
|
||||
@@ -30,7 +31,8 @@ func TestRegisterLibraryReleaseHandlersCoversOnlyReleaseJobs(t *testing.T) {
|
||||
func TestRegisterLibraryReleaseHandlersFailsClosedWhenNotAdmitted(t *testing.T) {
|
||||
handlers := map[string]jobs.Handler{}
|
||||
registerLibraryReleaseHandlers(handlers, nil, appconfig.Config{}, errNotAdmitted{})
|
||||
for _, jobType := range []string{librarySyncJobType, libraryRestoreJobType, libraryRecoverJobType, libraryBaselineJobType} {
|
||||
for _, jobType := range []string{librarySyncJobType, libraryRestoreJobType, libraryRecoverJobType,
|
||||
libraryBaselineJobType, libraryVerifyJobType} {
|
||||
err := handlers[jobType](context.Background(), jobs.Job{})
|
||||
if err == nil || !strings.Contains(err.Error(), "准入未通过") {
|
||||
t.Fatalf("handler %q error = %v, want admission refusal", jobType, err)
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
// fakeStageStore 记录封存失败分支对业务作业行的两次落库调用。
|
||||
type fakeStageStore struct {
|
||||
stageCalls []string
|
||||
jobCalls []string
|
||||
}
|
||||
|
||||
func (f *fakeStageStore) UpdateLibraryStage(_ context.Context, jobID, stage, state string, _, _, _, _ *int64, detail string, _ time.Time) error {
|
||||
f.stageCalls = append(f.stageCalls, jobID+"|"+stage+"|"+state+"|"+detail)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeStageStore) AdvanceLibraryJob(_ context.Context, jobID string, state governance.LibraryJobState, stage, message, recovery, _, _ string, _ time.Time) error {
|
||||
f.jobCalls = append(f.jobCalls, jobID+"|"+string(state)+"|"+stage+"|"+message+"|"+recovery)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeStageStore) LibraryVersionByRef(context.Context, string, governance.LibraryVersionKind, string, string) (governance.LibraryVersion, bool, error) {
|
||||
return governance.LibraryVersion{}, false, nil
|
||||
}
|
||||
|
||||
func (f *fakeStageStore) RecordLibraryVersion(context.Context, governance.LibraryVersion) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 归档根必须是非符号链接的 0700 目录;t.TempDir 的权限受 umask 影响。
|
||||
func privateDir(t *testing.T) string {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
if err := os.Chmod(root, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return root
|
||||
}
|
||||
|
||||
func sourceFor(scope, kb string) governance.LibrarySourceIdentity {
|
||||
return governance.LibrarySourceIdentity{Project: "kgmaint-weknora", App: "kgmaint-app", Scope: scope, TenantID: 1, KnowledgeBaseID: kb}
|
||||
}
|
||||
|
||||
// RED 用例:快照来源与配置不一致时,队列已 failed、页面却永远"导出中"(真实事故:
|
||||
// libseal-lib-internal-20261102T234053 running/export/error 空)。失败必须同时写进
|
||||
// stage 与 job,且原错原样出现在两处与返回值里 —— 不能被"已记录失败"遮住。
|
||||
func TestCapturedSourceMismatchPersistsFailedStageAndJob(t *testing.T) {
|
||||
root := privateDir(t)
|
||||
snapshotID := sealTinyBaseline(t, root, "internal", time.Date(2026, 11, 2, 23, 40, 53, 0, time.UTC))
|
||||
// 清单里是 kb-baseline;worker 配置漂移后拿的是另一个 KB(256 验收遗留正是这样)。
|
||||
source := sourceFor("internal", "0c84a098-d3ed-4b82-a165-e6de9691d6a8")
|
||||
store := &fakeStageStore{}
|
||||
|
||||
err := ensureCapturedSource(context.Background(), store, "libseal-lib-internal-20261102T234053",
|
||||
root, snapshotID, source, time.Now().UTC())
|
||||
|
||||
if err == nil || !strings.Contains(err.Error(), "captured manifest differs from configured source") {
|
||||
t.Fatalf("mismatch must keep the original error, got %v", err)
|
||||
}
|
||||
if len(store.stageCalls) != 1 {
|
||||
t.Fatalf("stage writes = %v, want exactly one failed stage", store.stageCalls)
|
||||
}
|
||||
if !strings.HasPrefix(store.stageCalls[0], "libseal-lib-internal-20261102T234053|export|failed|") ||
|
||||
!strings.Contains(store.stageCalls[0], "captured manifest differs from configured source") {
|
||||
t.Fatalf("stage write = %q, want failed+original error", store.stageCalls[0])
|
||||
}
|
||||
if len(store.jobCalls) != 1 {
|
||||
t.Fatalf("job writes = %v, want exactly one failed job", store.jobCalls)
|
||||
}
|
||||
if !strings.HasPrefix(store.jobCalls[0], "libseal-lib-internal-20261102T234053|failed|export|") ||
|
||||
!strings.Contains(store.jobCalls[0], "captured manifest differs from configured source") {
|
||||
t.Fatalf("job write = %q, want failed+original error", store.jobCalls[0])
|
||||
}
|
||||
// 恢复说明必须指向真实下一步,而不是暗示数据损坏。
|
||||
if !strings.Contains(store.jobCalls[0], "重新封存") {
|
||||
t.Fatalf("job recovery = %q, want a re-seal instruction", store.jobCalls[0])
|
||||
}
|
||||
}
|
||||
|
||||
// 正例对照:身份一致时不得写任何失败,也不得把成功路径变成失败。
|
||||
func TestCapturedSourceMatchStaysSuccessful(t *testing.T) {
|
||||
root := privateDir(t)
|
||||
snapshotID := sealTinyBaseline(t, root, "internal", time.Date(2026, 11, 2, 23, 40, 53, 0, time.UTC))
|
||||
store := &fakeStageStore{}
|
||||
|
||||
if err := ensureCapturedSource(context.Background(), store, "libseal-ok",
|
||||
root, snapshotID, sourceFor("internal", "kb-baseline"), time.Now().UTC()); err != nil {
|
||||
t.Fatalf("matching source rejected: %v", err)
|
||||
}
|
||||
if len(store.stageCalls) != 0 || len(store.jobCalls) != 0 {
|
||||
t.Fatalf("success path wrote failures: stage=%v job=%v", store.stageCalls, store.jobCalls)
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,12 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"knowledge-governance/internal/appconfig"
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/librarysnapshot"
|
||||
"time"
|
||||
)
|
||||
|
||||
func configuredCaptureSource(config appconfig.Config, scope string) governance.LibrarySourceIdentity {
|
||||
@@ -32,3 +34,20 @@ func verifyCapturedSource(root, snapshotID string, source governance.LibrarySour
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ensureCapturedSource 校验快照来源。校验失败必须把失败写进业务行:
|
||||
// 队列侧此时已 failed,只 return err 会让页面永远停在"导出中"
|
||||
// (实际事故:libseal-lib-internal-20261102T234053 running/export/error 空)。
|
||||
func ensureCapturedSource(ctx context.Context, store libraryStageStore, jobID, root, snapshotID string, source governance.LibrarySourceIdentity, now time.Time) error {
|
||||
err := verifyCapturedSource(root, snapshotID, source)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
// 快照本身已产生且字节完好,错的只是 worker 读到的来源身份;因此恢复路径是
|
||||
// "修配置后重新登记/重新封存",不是"重采数据"。原错原样写进 stage、job 与
|
||||
// 返回值:改写成更友好的文案会让真正的配置漂移查不出来。
|
||||
libraryStageRecorder{store: store, jobID: jobID}.Stage(ctx, "export", "failed", err.Error())
|
||||
_ = store.AdvanceLibraryJob(ctx, jobID, governance.LibraryJobFailed, "export", err.Error(),
|
||||
"快照已产生但来源身份与配置不一致,未登记为测试封存;修正来源配置后重新封存", "", "", now)
|
||||
return err
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,262 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/appconfig"
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/jobs"
|
||||
"knowledge-governance/internal/librarymaintenance"
|
||||
"knowledge-governance/internal/libraryverify"
|
||||
"knowledge-governance/internal/pgdsn"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
)
|
||||
|
||||
// libraryVerifyPayload 是"恢复验证"的队列载荷。
|
||||
//
|
||||
// 它只带作业号与 scope:目标(库/索引/桶/实例)来自本进程的部署配置,不接受请求方
|
||||
// 指定。否则一次页面点击就能把"验证"引到别的库上,证据也就证明不了当前目标。
|
||||
type libraryVerifyPayload struct {
|
||||
LibraryJobID string `json:"library_job_id"`
|
||||
Scope string `json:"scope"`
|
||||
}
|
||||
|
||||
// runLibraryVerify 在目标实例上真实跑一次恢复验证,并把结果登记成证据。
|
||||
//
|
||||
// 与 sync/restore/baseline 共用同一套准入与代次守卫:它会停写入者、在目标上写 canary
|
||||
// 再按预备份恢复,所以必须走 admitLibraryJob + runOwnedLibraryAttempt,不能绕开。
|
||||
// 执行体本身是 internal/libraryverify——与 cmd/librarydrill 同一份实现,不允许两份。
|
||||
func runLibraryVerify(ctx context.Context, base *postgresstore.Store, config appconfig.Config, job jobs.Job) error {
|
||||
var payload libraryVerifyPayload
|
||||
if err := json.Unmarshal(job.Payload, &payload); err != nil || payload.LibraryJobID == "" ||
|
||||
(payload.Scope != "internal" && payload.Scope != "external") {
|
||||
return errors.New("invalid library verify payload")
|
||||
}
|
||||
if _, admitErr := admitLibraryJob(ctx, base, job, payload.LibraryJobID, payload.Scope,
|
||||
governance.LibraryJobVerify, true); admitErr != nil {
|
||||
return admitErr
|
||||
}
|
||||
return runOwnedLibraryAttempt(ctx, base, config, job, payload.LibraryJobID,
|
||||
func(runCtx context.Context, writer *libraryAttemptWriter) error {
|
||||
return runLibraryVerifyAttempt(runCtx, writer, config, payload)
|
||||
})
|
||||
}
|
||||
|
||||
func runLibraryVerifyAttempt(ctx context.Context, writer *libraryAttemptWriter, config appconfig.Config,
|
||||
payload libraryVerifyPayload) error {
|
||||
store := writer
|
||||
now := time.Now().UTC()
|
||||
recorder := libraryStageRecorder{store: store, jobID: payload.LibraryJobID, scope: payload.Scope,
|
||||
operationID: writer.attempt.OperationID}
|
||||
|
||||
existing, jobErr := store.LibraryJob(ctx, payload.LibraryJobID)
|
||||
if jobErr != nil {
|
||||
// 作业行读不到就没有可登记的终态:直接失败,不落任何证据。
|
||||
return fmt.Errorf("读不到业务作业 %s:%w", payload.LibraryJobID, jobErr)
|
||||
}
|
||||
if existing.State == governance.LibraryJobSucceeded {
|
||||
// 幂等重放:已经成功过的验证不重复停目标、不重复写 canary。
|
||||
recorder.Stage(ctx, "finalize", "skipped", "该恢复验证已成功完成,重复投递不重复执行")
|
||||
return nil
|
||||
}
|
||||
if !config.ProductionReleaseRequestsEnabled {
|
||||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
|
||||
"生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false):恢复验证要停写入者并在目标上写 canary,闸口关闭时不执行",
|
||||
"capture")
|
||||
}
|
||||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobRunning, "capture",
|
||||
"", "", "", "", now)
|
||||
|
||||
report, runErr := libraryverify.Run(ctx, config, payload.Scope, payload.LibraryJobID,
|
||||
libraryverify.Options{
|
||||
Progress: func(stage, state, detail string) { recorder.Stage(ctx, stage, state, detail) },
|
||||
Governance: verifyGovernance{store: store, config: config},
|
||||
Ledger: recorder,
|
||||
})
|
||||
|
||||
// 证据只由真实执行产生:失败也照实登记 failure 与报告,绝不写成 passed。
|
||||
evidence := governance.LibraryRestoreVerification{
|
||||
ID: "libverify-" + payload.Scope + "-" + payload.LibraryJobID,
|
||||
Scope: payload.Scope,
|
||||
JobID: payload.LibraryJobID,
|
||||
Status: governance.LibraryVerificationFailed,
|
||||
StartedAt: now,
|
||||
}
|
||||
if report != nil {
|
||||
evidence.StartedAt = report.StartedAt
|
||||
evidence.SnapshotID = report.SnapshotID
|
||||
evidence.Report = verificationReportObject(report)
|
||||
}
|
||||
if runErr == nil && report != nil && report.Status == "passed" {
|
||||
evidence.Status = governance.LibraryVerificationPassed
|
||||
evidence.FinishedAt = report.FinishedAt
|
||||
} else {
|
||||
evidence.FinishedAt = time.Now().UTC()
|
||||
if report != nil && report.FinishedAt.After(evidence.StartedAt) {
|
||||
evidence.FinishedAt = report.FinishedAt
|
||||
}
|
||||
switch {
|
||||
case runErr != nil:
|
||||
evidence.Failure = runErr.Error()
|
||||
case report != nil && report.Status != "passed":
|
||||
evidence.Failure = "恢复验证未通过(状态 " + report.Status + ")"
|
||||
default:
|
||||
evidence.Failure = "恢复验证未产生报告"
|
||||
}
|
||||
}
|
||||
if saveErr := store.SaveLibraryRestoreVerification(ctx, evidence); saveErr != nil {
|
||||
// 证据落不了库就不能宣称验证通过:这会把闸口建立在一次没留痕的执行上。
|
||||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
|
||||
"恢复验证执行结束但证据登记失败:"+saveErr.Error(), "finalize")
|
||||
}
|
||||
|
||||
if runErr != nil || evidence.Status != governance.LibraryVerificationPassed {
|
||||
held := report != nil && report.Status == "maintenance_held"
|
||||
message := "恢复验证未通过:" + evidence.Failure
|
||||
if held {
|
||||
message += "(目标仍在维护窗口,入口关闭、写入者已停)"
|
||||
}
|
||||
// 失败出口只写真实结果:报告里的 restore_check 与窗口终态说明目标此刻是
|
||||
// 仍在维护,还是已经按原运行状态安全收尾。不能拿"有快照"冒充"已恢复并复核"。
|
||||
recovery := "未对目标执行任何写入,无需恢复"
|
||||
switch {
|
||||
case held:
|
||||
recovery = "目标仍在维护窗口:在同一作业上执行「按预备份恢复」(library.recover),恢复与逐类校验通过后才会开入口并释放占用"
|
||||
case report != nil && report.WindowState != "":
|
||||
recovery = "维护窗口已按原运行状态收尾(window=" + report.WindowState + "):目标内容未写入或已恢复并复核,无需再恢复"
|
||||
}
|
||||
snapshot := ""
|
||||
if report != nil {
|
||||
snapshot = report.SnapshotID
|
||||
}
|
||||
recorder.Stage(ctx, "finalize", "failed", message)
|
||||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "finalize",
|
||||
message, recovery, snapshot, snapshot, time.Now().UTC())
|
||||
return errors.New(message)
|
||||
}
|
||||
|
||||
finishedAt := evidence.FinishedAt
|
||||
if finishedAt.IsZero() {
|
||||
finishedAt = time.Now().UTC()
|
||||
}
|
||||
recorder.Stage(ctx, "finalize", "succeeded",
|
||||
"恢复验证通过:快照 "+evidence.SnapshotID+" 的预备份可真实恢复,证据已按当前目标登记")
|
||||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize",
|
||||
"", "", evidence.SnapshotID, evidence.SnapshotID, finishedAt)
|
||||
log.Printf("library verify %s passed target snapshot=%s", payload.LibraryJobID, evidence.SnapshotID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// verifyGovernance 把演练接进**既有**维护生命周期:实例行 + scope 行都由 worker 的
|
||||
// 真实 attempt 写入,入口状态与状态文字来自实际执行。页面读的维护接口只认这些行,
|
||||
// 所以只写磁盘 journal 的窗口在 UI 上等于不存在(审核 B2 的真实缺口)。
|
||||
type verifyGovernance struct {
|
||||
store *libraryAttemptWriter
|
||||
config appconfig.Config
|
||||
}
|
||||
|
||||
func (g verifyGovernance) targetIdentity(scope string) string {
|
||||
return fmt.Sprintf("scope=%s db=%s index=%s bucket=%s", scope,
|
||||
pgdsn.Endpoint(g.config.LibraryProductionDatabaseURL), g.config.LibraryProductionIndex, g.config.LibraryProductionS3Bucket)
|
||||
}
|
||||
|
||||
func (g verifyGovernance) Acquire(ctx context.Context, jobID, scope string, target libraryverify.TargetIdentity, originalServiceState string) error {
|
||||
now := time.Now().UTC()
|
||||
note := "实例级维护:关闭目标入口并停止全部写入者,恢复验证完成并复核后开放"
|
||||
if _, err := g.store.AcquireInstanceMaintenance(ctx, postgresstore.LibraryInstanceMaintenance{
|
||||
Instance: target.Instance, OwnerJobID: jobID, OwnerScope: scope,
|
||||
TargetFingerprint: target.Fingerprint, Level: librarymaintenance.LevelInstance,
|
||||
Services: target.Writers, IngressState: "unknown", State: "acquired",
|
||||
OriginalServiceState: originalServiceState, Note: note,
|
||||
}, now); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := g.store.AcquireLibraryMaintenance(ctx, postgresstore.LibraryMaintenance{
|
||||
Scope: scope, OwnerJobID: jobID, TargetIdentity: g.targetIdentity(scope),
|
||||
Level: librarymaintenance.LevelInstance, TargetFingerprint: target.Fingerprint,
|
||||
Services: target.Writers, IngressState: "unknown", State: "acquired",
|
||||
OriginalServiceState: originalServiceState, Note: note,
|
||||
}, now); err != nil {
|
||||
// scope 行没拿到时目标还没被停用:释放实例占用,避免留下一个空窗口。
|
||||
_ = g.store.ReleaseInstanceMaintenance(ctx, target.Instance, jobID, now)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g verifyGovernance) Ingress(ctx context.Context, jobID, scope, state, note string) error {
|
||||
now := time.Now().UTC()
|
||||
if err := g.store.SetLibraryMaintenanceIngress(ctx, scope, jobID, state, note, now); err != nil {
|
||||
return err
|
||||
}
|
||||
return g.store.SetInstanceMaintenanceIngress(ctx, g.config.LibraryTargetInstance, jobID, state, note, now)
|
||||
}
|
||||
|
||||
func (g verifyGovernance) State(ctx context.Context, jobID, scope, state, note string) error {
|
||||
now := time.Now().UTC()
|
||||
if err := g.store.UpdateLibraryMaintenanceState(ctx, scope, jobID, state, note, now); err != nil {
|
||||
return err
|
||||
}
|
||||
return g.store.UpdateInstanceMaintenanceState(ctx, g.config.LibraryTargetInstance, jobID, state, note, now)
|
||||
}
|
||||
|
||||
func (g verifyGovernance) Release(ctx context.Context, jobID, scope, note string) error {
|
||||
now := time.Now().UTC()
|
||||
if err := g.store.ReleaseLibraryMaintenance(ctx, scope, jobID, now); err != nil {
|
||||
return err
|
||||
}
|
||||
return g.store.ReleaseInstanceMaintenance(ctx, g.config.LibraryTargetInstance, jobID, now)
|
||||
}
|
||||
|
||||
// verificationReportObject 把报告转成可落库的 JSON 对象(保留 canary/漂移校验等证据字段)。
|
||||
func verificationReportObject(report *libraryverify.Report) map[string]any {
|
||||
if report == nil {
|
||||
return nil
|
||||
}
|
||||
raw, err := json.Marshal(report)
|
||||
if err != nil {
|
||||
return map[string]any{"encode_error": err.Error()}
|
||||
}
|
||||
var object map[string]any
|
||||
if err := json.Unmarshal(raw, &object); err != nil {
|
||||
return map[string]any{"encode_error": err.Error()}
|
||||
}
|
||||
return object
|
||||
}
|
||||
|
||||
// libraryRestoreEvidenceSatisfied 回答"当前目标 + scope 有没有可支撑写生产的恢复证据"。
|
||||
//
|
||||
// 证据是第一来源:library_restore_verifications 按当前 target_id 读取,换库/换索引/换桶
|
||||
// 之后旧证据自动失效,不需要谁记得关开关。历史部署里的 PRODUCTION_BACKUP_RESTORE_VERIFIED
|
||||
// 仍作为显式覆盖保留(不删除旧部署能力),但只允许在**明确没有证据行**时兜底;
|
||||
// 明确 failed/invalid 与读取错误一律拒绝(审核 B3:不能把"不知道"当"满足")。
|
||||
func libraryRestoreEvidenceSatisfied(ctx context.Context, store interface {
|
||||
LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error)
|
||||
}, config appconfig.Config, scope string) (bool, string) {
|
||||
// 存储没有证据能力(nil/迁移前形状)等同于"没有证据行":历史 env 覆盖可兜底;
|
||||
// 但只要读得出证据行,failed/invalid 与读取错误都优先拒绝。
|
||||
missing := true
|
||||
if store != nil {
|
||||
evidence, found, err := store.LatestLibraryRestoreVerification(ctx, scope)
|
||||
switch {
|
||||
case err != nil:
|
||||
return false, "读取恢复验证证据失败:" + err.Error()
|
||||
case found:
|
||||
missing = false
|
||||
if ok, reason := governance.LibraryVerificationSatisfied(evidence, scope); ok {
|
||||
return true, ""
|
||||
} else {
|
||||
return false, reason
|
||||
}
|
||||
}
|
||||
}
|
||||
if missing && config.ProductionBackupRestoreVerified {
|
||||
return true, ""
|
||||
}
|
||||
return false, "当前目标还没有恢复验证证据:先跑一次恢复验证(备份 → canary → 恢复 → 复核)"
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/appconfig"
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
// 同步闸口的"恢复可验证"判断必须以当前目标的证据为准:
|
||||
// 没有证据 ⇒ 拒绝并说清怎么补;有 passed 证据 ⇒ 放行;只有失败证据 ⇒ 拒绝并带失败原因。
|
||||
// 历史 env 覆盖只在证据缺失时兜底(页面不再写它)。
|
||||
type verifyEvidenceStore struct {
|
||||
evidence governance.LibraryRestoreVerification
|
||||
found bool
|
||||
}
|
||||
|
||||
func (s verifyEvidenceStore) LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error) {
|
||||
return s.evidence, s.found, nil
|
||||
}
|
||||
|
||||
// 审核 B3:读取失败不能被 env=true 洗成"满足"。
|
||||
type unreadableEvidenceStore struct{}
|
||||
|
||||
func (unreadableEvidenceStore) LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error) {
|
||||
return governance.LibraryRestoreVerification{}, false, errors.New("evidence lookup failed")
|
||||
}
|
||||
|
||||
func TestWorkerUnreadableEvidenceBeatsEnv(t *testing.T) {
|
||||
ok, reason := libraryRestoreEvidenceSatisfied(context.Background(), unreadableEvidenceStore{},
|
||||
appconfig.Config{ProductionBackupRestoreVerified: true}, "internal")
|
||||
if ok {
|
||||
t.Fatalf("unreadable evidence must block even with env=true; reason=%q", reason)
|
||||
}
|
||||
if !strings.Contains(reason, "读取恢复验证证据失败") {
|
||||
t.Fatalf("reason=%q", reason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryRestoreEvidenceSatisfied(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
passed := verifyEvidenceStore{found: true, evidence: governance.LibraryRestoreVerification{
|
||||
Scope: "internal", Status: governance.LibraryVerificationPassed, SnapshotID: "snap-1", FinishedAt: now}}
|
||||
if ok, reason := libraryRestoreEvidenceSatisfied(context.Background(), passed, appconfig.Config{}, "internal"); !ok {
|
||||
t.Fatalf("passed evidence must satisfy gate: %q", reason)
|
||||
}
|
||||
failed := verifyEvidenceStore{found: true, evidence: governance.LibraryRestoreVerification{
|
||||
Scope: "internal", Status: governance.LibraryVerificationFailed, SnapshotID: "snap-1",
|
||||
Failure: "恢复后内容仍与预备份不一致", FinishedAt: now}}
|
||||
if ok, reason := libraryRestoreEvidenceSatisfied(context.Background(), failed, appconfig.Config{}, "internal"); ok ||
|
||||
!strings.Contains(reason, "恢复后内容仍与预备份不一致") {
|
||||
t.Fatalf("failed evidence must block with its own failure: ok=%v reason=%q", ok, reason)
|
||||
}
|
||||
missing := verifyEvidenceStore{}
|
||||
if ok, reason := libraryRestoreEvidenceSatisfied(context.Background(), missing, appconfig.Config{}, "internal"); ok ||
|
||||
!strings.Contains(reason, "恢复验证") {
|
||||
t.Fatalf("missing evidence must block and point at the remedy: ok=%v reason=%q", ok, reason)
|
||||
}
|
||||
// 历史部署的 env 覆盖:证据缺失时才生效,证据存在时不再起作用。
|
||||
if ok, reason := libraryRestoreEvidenceSatisfied(context.Background(), missing,
|
||||
appconfig.Config{ProductionBackupRestoreVerified: true}, "internal"); !ok {
|
||||
t.Fatalf("legacy env override must still allow when no evidence: %q", reason)
|
||||
}
|
||||
if ok, _ := libraryRestoreEvidenceSatisfied(context.Background(), failed,
|
||||
appconfig.Config{ProductionBackupRestoreVerified: true}, "internal"); ok {
|
||||
t.Fatal("evidence must win over the legacy env override")
|
||||
}
|
||||
}
|
||||
@@ -5,7 +5,9 @@ import (
|
||||
|
||||
"knowledge-governance/internal/appconfig"
|
||||
"knowledge-governance/internal/librarymaintenance"
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
"knowledge-governance/internal/librarystack"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
v071 "knowledge-governance/internal/releaseadapter/v071"
|
||||
)
|
||||
|
||||
@@ -18,13 +20,27 @@ func openLibraryMaintenance(config appconfig.Config) (*librarymaintenance.Manage
|
||||
return librarystack.OpenMaintenance(config)
|
||||
}
|
||||
|
||||
// restoreDispatchGate 只在部署显式开启时才把治理台账交给目标。
|
||||
//
|
||||
// 为什么不能只信配置:配置为 true 只说明"允许装配",不证明旧 writer 已退出;
|
||||
// 真正的发放许可还要求治理库里对该目标存在未撤销的激活登记(见迁移 78)。
|
||||
func restoreDispatchGate(config appconfig.Config, store *postgresstore.Store) libraryrelease.RestoreMarkerGate {
|
||||
// 只判断"是否允许装配":目标身份与独占证明由治理台账的激活登记回答,
|
||||
// 这里再用一个空配置也能算出来的摘要去当门槛只会制造虚假的严格。
|
||||
if !config.LibraryRestoreDispatchEnabled || store == nil {
|
||||
return nil
|
||||
}
|
||||
return store
|
||||
}
|
||||
|
||||
// buildLibraryProductionStack 用"要同步的这份快照"做契约装配目标与预备份存储。
|
||||
func buildLibraryProductionStack(ctx context.Context, config appconfig.Config, scope, snapshotID, knownMapping string) (
|
||||
*v071.ProductionLibraryTarget, *v071.ProductionBackupStore, error) {
|
||||
return librarystack.OpenProductionStack(ctx, config, scope, snapshotID, knownMapping)
|
||||
func buildLibraryProductionStack(ctx context.Context, config appconfig.Config, gate libraryrelease.RestoreMarkerGate,
|
||||
scope, snapshotID, knownMapping string) (*v071.ProductionLibraryTarget, *v071.ProductionBackupStore, error) {
|
||||
return librarystack.OpenProductionStack(ctx, config, scope, snapshotID, knownMapping, gate)
|
||||
}
|
||||
|
||||
// buildLibraryBaselineStack 装配"给目标自己建基线"所需的目标与捕获器(契约取目标当前映射)。
|
||||
// 基线不参与发布/恢复派发,永远不传派发台账:只读/建基线不得获得恢复写许可。
|
||||
func buildLibraryBaselineStack(ctx context.Context, config appconfig.Config, scope string) (
|
||||
*v071.ProductionLibraryTarget, *v071.ProductionBackupStore, error) {
|
||||
return librarystack.OpenBaselineStack(ctx, config, scope)
|
||||
|
||||
@@ -135,9 +135,7 @@ func libraryQueueOwner(workerID string, job jobs.Job, libraryJobID string) strin
|
||||
if queueID == "" {
|
||||
queueID = libraryJobID
|
||||
}
|
||||
worker := workerID
|
||||
if worker == "" {
|
||||
worker = "worker"
|
||||
}
|
||||
return fmt.Sprintf("%s#%s#%d", worker, queueID, job.Attempt)
|
||||
// 合成规则只有一份(governance.LibraryDeliveryOwner):派发台账要按同一格式核
|
||||
// "这次许可属于哪次真实投递"。
|
||||
return governance.LibraryDeliveryOwner(workerID, queueID, job.Attempt)
|
||||
}
|
||||
|
||||
+90
-23
@@ -9,6 +9,7 @@ import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@@ -23,7 +24,9 @@ import (
|
||||
"knowledge-governance/internal/librarysnapshot"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
"knowledge-governance/internal/readiness"
|
||||
"knowledge-governance/internal/versionhistory"
|
||||
"knowledge-governance/internal/weknoraaccess"
|
||||
artifactfetch "knowledge-governance/internal/workflows/artifactfetch"
|
||||
catalogrefresh "knowledge-governance/internal/workflows/catalogrefresh"
|
||||
"knowledge-governance/internal/workflows/sourcescan"
|
||||
@@ -98,6 +101,27 @@ func stageSizesFrom(manifest librarysnapshot.Manifest) map[string]stageSize {
|
||||
// libraryCaptureJobType is shared with the API which enqueues it.
|
||||
const libraryCaptureJobType = "library.capture"
|
||||
|
||||
// Retained deterministic source-content failures: the agent has already
|
||||
// restored the writer, so retrying the same journal only replaces the first
|
||||
// cause with "already exists". Body-index coverage markers are intentionally
|
||||
// absent; that inferred rule was removed in R3f.
|
||||
var terminalLibraryCaptureMarkers = []string{
|
||||
"missing library generated question index",
|
||||
"empty library index content",
|
||||
}
|
||||
|
||||
func terminalLibraryCaptureError(err error) error {
|
||||
if err == nil || !strings.Contains(err.Error(), "已执行 recover 并确认写入者恢复") {
|
||||
return err
|
||||
}
|
||||
for _, marker := range terminalLibraryCaptureMarkers {
|
||||
if strings.Contains(err.Error(), marker) {
|
||||
return jobs.Permanent(jobs.ErrCodeValidationFailed, err)
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// librarySyncJobType 是生产同步作业的队列类型;与封存严格分开,状态不共用。
|
||||
const librarySyncJobType = "library.sync"
|
||||
|
||||
@@ -110,6 +134,10 @@ const libraryRestoreJobType = "library.restore"
|
||||
// libraryBaselineJobType 是"建立目标权威基线"的队列类型(与 API 侧常量同值)。
|
||||
const libraryBaselineJobType = "library.baseline"
|
||||
|
||||
// libraryVerifyJobType 是"恢复验证"的队列类型:真实备份→canary→恢复→复核,
|
||||
// 通过后才会产生同步闸口认的恢复证据。与 API 侧常量同值。
|
||||
const libraryVerifyJobType = "library.verify"
|
||||
|
||||
func main() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
@@ -142,7 +170,8 @@ func run(ctx context.Context, getenv func(string) string) error {
|
||||
} else if len(pending) > 0 {
|
||||
return fmt.Errorf("治理库 schema 落后(待应用迁移 %v):本角色不拥有 DDL,先由控制面进程完成迁移", pending)
|
||||
}
|
||||
store := postgresstore.NewStore(pool).WithLibraryTarget(config.LibraryTargetIdentity())
|
||||
store := postgresstore.NewStore(pool).WithLibraryTarget(config.LibraryTargetIdentity()).WithRoutingDefaults(config.InternalRoutingKnowledgeBaseID, config.ExternalRoutingKnowledgeBaseID).
|
||||
WithRestoreDispatchEvidence(config.LibraryTargetJournalRoot)
|
||||
log.Printf("library target evidence identity=%s", config.LibraryTargetIdentity())
|
||||
// 启动先接管被中断的写作业(复用已持久化的预备份),再把剩下的孤儿判失败:
|
||||
// 顺序反了会把可接管的作业先标失败,丢掉已完成的备份。
|
||||
@@ -259,6 +288,41 @@ func run(ctx context.Context, getenv func(string) string) error {
|
||||
return worker.Run(ctx, config.PollInterval)
|
||||
}
|
||||
|
||||
type libraryCapturePersistence interface {
|
||||
UpdateLibraryStage(ctx context.Context, jobID, stage, state string, total, completed, bytesTotal, bytesDone *int64, detail string, now time.Time) error
|
||||
AdvanceLibraryJob(ctx context.Context, jobID string, state governance.LibraryJobState, stage, message, recovery, snapshotID, targetSnapshotID string, now time.Time) error
|
||||
}
|
||||
|
||||
// beginLibraryCaptureJob persists the starting state before any source
|
||||
// operation. If either write fails the handler must return, so the queue cannot
|
||||
// be reported completed while the business job is still pending.
|
||||
func beginLibraryCaptureJob(ctx context.Context, store libraryCapturePersistence, jobID string, now time.Time) error {
|
||||
if err := store.AdvanceLibraryJob(ctx, jobID, governance.LibraryJobRunning, "export", "", "", "", "", now); err != nil {
|
||||
return fmt.Errorf("library seal job %s cannot be advanced: %w", jobID, err)
|
||||
}
|
||||
if err := store.UpdateLibraryStage(ctx, jobID, "export", "running", nil, nil, nil, nil,
|
||||
"主机侧正在暂停写入者、导出并恢复", now); err != nil {
|
||||
return fmt.Errorf("library seal job %s stage cannot be advanced: %w", jobID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// completeLibraryCaptureJob persists both terminal facts. A failed write must
|
||||
// be returned (queue failed), not logged and ignored; the capture journal keeps
|
||||
// replay idempotent, and the admin reconcile path can repair only after the
|
||||
// exact queue/version/archive evidence is present.
|
||||
func completeLibraryCaptureJob(ctx context.Context, store libraryCapturePersistence, jobID, snapshotID string, now time.Time) error {
|
||||
if err := store.UpdateLibraryStage(ctx, jobID, "export", "succeeded", nil, nil, nil, nil,
|
||||
"已封存快照 "+snapshotID, now); err != nil {
|
||||
return fmt.Errorf("library seal job %s success stage cannot be persisted: %w", jobID, err)
|
||||
}
|
||||
if err := store.AdvanceLibraryJob(ctx, jobID, governance.LibraryJobSucceeded, "export", "", "",
|
||||
snapshotID, "", now); err != nil {
|
||||
return fmt.Errorf("library seal job %s success terminal cannot be persisted: %w", jobID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func workerHandlers(store *postgresstore.Store, config appconfig.Config) (map[string]jobs.Handler, error) {
|
||||
handlers := map[string]jobs.Handler{
|
||||
"demo": func(_ context.Context, job jobs.Job) error {
|
||||
@@ -320,12 +384,8 @@ func workerHandlers(store *postgresstore.Store, config appconfig.Config) (map[st
|
||||
// 持久作业:页面读的是这里写的阶段与结果,worker 重启后仍能如实展示。
|
||||
libraryJobID := "libseal-" + payload.CaptureJob
|
||||
startedAt := time.Now().UTC()
|
||||
if advanceErr := store.AdvanceLibraryJob(ctx, libraryJobID, governance.LibraryJobRunning, "export", "", "", "", "", startedAt); advanceErr != nil {
|
||||
log.Printf("library seal job %s cannot be advanced: %v", libraryJobID, advanceErr)
|
||||
}
|
||||
if stageErr := store.UpdateLibraryStage(ctx, libraryJobID, "export", "running", nil, nil, nil, nil,
|
||||
"主机侧正在暂停写入者、导出并恢复", startedAt); stageErr != nil {
|
||||
log.Printf("library seal job %s stage update failed: %v", libraryJobID, stageErr)
|
||||
if beginErr := beginLibraryCaptureJob(ctx, store, libraryJobID, startedAt); beginErr != nil {
|
||||
return beginErr
|
||||
}
|
||||
// 平台审计过的精确死信解除指纹:读取失败必须阻断(fail closed),
|
||||
// 不能因为存储不可用就静默按"没有解除"或"全部解除"处理。
|
||||
@@ -357,7 +417,7 @@ func workerHandlers(store *postgresstore.Store, config appconfig.Config) (map[st
|
||||
_ = store.AdvanceLibraryJob(ctx, libraryJobID, governance.LibraryJobFailed, "export", err.Error(),
|
||||
"agent 已执行恢复检查;确认来源 healthy 后可重试同一作业号", "", "", finishedAt)
|
||||
log.Printf("library capture failed scope=%s capture_job=%s: %v", payload.Scope, payload.CaptureJob, err)
|
||||
return err
|
||||
return terminalLibraryCaptureError(err)
|
||||
}
|
||||
// 执行后复核:结果里 agent 自报的身份必须还是期望实例,否则这次封存
|
||||
// 可能来自别的栈,绝不能登记成"本实例的测试封存"。
|
||||
@@ -367,7 +427,7 @@ func workerHandlers(store *postgresstore.Store, config appconfig.Config) (map[st
|
||||
"快照已产生但来源身份不符,未登记为测试封存;核对执行器后重新封存", "", "", finishedAt)
|
||||
return identityErr
|
||||
}
|
||||
if err := verifyCapturedSource(config.LibraryArchiveRoot, result.SnapshotID, source); err != nil {
|
||||
if err := ensureCapturedSource(ctx, store, libraryJobID, config.LibraryArchiveRoot, result.SnapshotID, source, finishedAt); err != nil {
|
||||
return err
|
||||
}
|
||||
// 封存产物成版本记录:只登记"测试封存",绝不写成生产生效版本。
|
||||
@@ -383,10 +443,9 @@ func workerHandlers(store *postgresstore.Store, config appconfig.Config) (map[st
|
||||
if versionErr != nil {
|
||||
return fmt.Errorf("library seal job %s cannot record version: %w", libraryJobID, versionErr)
|
||||
}
|
||||
_ = store.UpdateLibraryStage(ctx, libraryJobID, "export", "succeeded", nil, nil, nil, nil,
|
||||
"已封存快照 "+result.SnapshotID, finishedAt)
|
||||
_ = store.AdvanceLibraryJob(ctx, libraryJobID, governance.LibraryJobSucceeded, "export", "", "",
|
||||
result.SnapshotID, "", finishedAt)
|
||||
if completeErr := completeLibraryCaptureJob(ctx, store, libraryJobID, result.SnapshotID, finishedAt); completeErr != nil {
|
||||
return completeErr
|
||||
}
|
||||
log.Printf("library capture completed scope=%s capture_job=%s snapshot=%s replay=%v requested_by=%s",
|
||||
payload.Scope, payload.CaptureJob, result.SnapshotID, result.Replay, payload.RequestedBy)
|
||||
return nil
|
||||
@@ -466,19 +525,27 @@ func workerHandlers(store *postgresstore.Store, config appconfig.Config) (map[st
|
||||
if !config.TestWeKnoraEnabled || !config.LibraryWorkerRole.ExecutesSourceWork() {
|
||||
return handlers, nil
|
||||
}
|
||||
apiKey, err := weknora.ReadAPIKeyFile(config.TestWeKnoraAPIKeyFile)
|
||||
if err != nil {
|
||||
// 启动只验证凭据“存在”(缺失像以前一样拒绝启动);具体值仍由 resolver 每次调用重读。
|
||||
if _, err := weknora.ReadAPIKeyFile(config.TestWeKnoraAPIKeyFile); err != nil {
|
||||
return nil, fmt.Errorf("read test WeKnora API key: %w", err)
|
||||
}
|
||||
client, err := weknora.NewClient(weknora.ClientConfig{
|
||||
BaseURL: config.TestWeKnoraBaseURL,
|
||||
APIKey: apiKey,
|
||||
Timeout: config.TestWeKnoraHTTPTimeout,
|
||||
AllowDockerHTTP: config.TestWeKnoraAllowDockerHTTP,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create test WeKnora client: %w", err)
|
||||
// 与 API 探测共用同一有效目标:网页保存的端点优先、env 只作未配置兜底;
|
||||
// 当前持久化绑定优先、env 兜底;凭据每次调用重读同一 key 文件。
|
||||
remoteResolver := readiness.Resolver{
|
||||
Endpoint: func(resolveCtx context.Context) string {
|
||||
if settings, settingsErr := store.GetActiveUISettings(resolveCtx); settingsErr == nil && strings.TrimSpace(settings.TestWeKnoraBaseURL) != "" {
|
||||
return settings.TestWeKnoraBaseURL
|
||||
}
|
||||
return config.TestWeKnoraBaseURL
|
||||
},
|
||||
LoadBinding: func(resolveCtx context.Context) (string, error) {
|
||||
internal, _, _, bindingErr := store.LoadRoutingKBBindings(resolveCtx)
|
||||
return internal, bindingErr
|
||||
},
|
||||
ReadKey: func() (string, error) { return weknora.ReadAPIKeyFile(config.TestWeKnoraAPIKeyFile) },
|
||||
DefaultKB: config.InternalRoutingKnowledgeBaseID,
|
||||
}
|
||||
client := &weknoraaccess.Client{Resolver: remoteResolver, Timeout: config.TestWeKnoraHTTPTimeout, AllowDockerHTTP: config.TestWeKnoraAllowDockerHTTP}
|
||||
upload, err := testweknora.NewUploadHandler(testweknora.UploadHandlerConfig{
|
||||
Store: store,
|
||||
Client: client,
|
||||
|
||||
+6
-1
@@ -18,4 +18,9 @@
|
||||
|
||||
1. 跑 `make verify`(本地全绿不等于远端跑通,需要一次真实 run 的证据)。
|
||||
2. 把**使用者可见**的变化写成 `CHANGELOG.md` 的 `[Unreleased]` 条目(按四类,别写过程)。
|
||||
3. 从本文件划掉该项;过程记录归档到 `docs/archive/`。
|
||||
3. 从本文件划掉该项;过程记录归档到 `docs/archive/`。
|
||||
## 资源首次复用的归属审计(后续)
|
||||
|
||||
- 当前首次复用只依赖“目标行逐列一致 + 对象字节一致”,不再要求本目标历史成功同步/预备份来源。
|
||||
- 后续可补:资源首次写入者/首次绑定 KB、跨 KB 引用计数、孤儿资源与同 tenant 共享资源的可审计归属视图;
|
||||
不以此重新阻断正常的同内容首次复用。
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
# DS01 激活证据的不变量(v2)
|
||||
|
||||
状态:源码与隔离测试已实现;**未在正式治理库执行激活**。
|
||||
相关实现:`internal/libraryrelease/restore_dispatch_activation.go`、
|
||||
`internal/platform/postgres/library_restore_dispatch_activation.go`、
|
||||
`internal/libraryfreeze/journal_evidence.go`。
|
||||
|
||||
## 1. 当前生产装配从哪里取得可核证据
|
||||
|
||||
```
|
||||
worker(LIBRARY_RESTORE_DISPATCH_V2=true)
|
||||
└─ librarymaintenance.Manager(LIBRARY_TARGET_*)
|
||||
└─ libraryfreeze.Coordinator
|
||||
├─ 关闭入口 → 读 gate drain.json(closed / drained / active_at_close)
|
||||
├─ 等后台静默 → QuiesceProof(depth/pending_ops/unsettled=0,稳定窗口)
|
||||
├─ 停写者 → Controller.Stop + Inventory 复核 + 退出码
|
||||
└─ 写 <LIBRARY_TARGET_JOURNAL_ROOT>/<job>.json(state=held)
|
||||
└─ library_instance_maintenance(治理库):instance / owner_job_id / target_fingerprint /
|
||||
level=instance / services / state / released_at
|
||||
```
|
||||
|
||||
激活是**部署动作**,不是业务请求:`Store.ActivateRestoreDispatch` 只在部署流程里调用。
|
||||
`Store.WithRestoreDispatchEvidence(LIBRARY_TARGET_JOURNAL_ROOT)` 是唯一允许读取 journal 的根;
|
||||
未配置该根 ⇒ `ErrRestoreDispatchActivationDisabled`(能力关闭),不退回"只看字符串"。
|
||||
|
||||
## 2. 证据文件只声明身份,不声明结论
|
||||
|
||||
v2 证据文档(`protocol_version=2`)字段。注意:台账行 `library_index_restore_dispatch_activation.protocol_version`
|
||||
仍是 1(迁移 78 的派发协议身份约束),证据文档版本与台账协议版本不是同一个数字:
|
||||
|
||||
| 字段 | 含义 |
|
||||
|---|---|
|
||||
| `target_identity` | 目标身份 |
|
||||
| `target_fingerprint` | 维护窗口/部署目标指纹(64 位 sha256 hex),必须等于 journal 与维护行 |
|
||||
| `restore_target_fingerprint` | 激活时对恢复目标(库/索引/桶)的实时观察指纹 |
|
||||
| `target_instance` | 治理库维护窗口的实例名,必须等于 journal 的 `source` |
|
||||
| `governance_database` | 当前治理库名,与 `current_database()` 比对 |
|
||||
| `protocol_build` | 必须是**运行中二进制**的 `version.DisplayVersion()` |
|
||||
| `maintenance_job` / `maintenance_journal` | 维护窗口作业;journal 名必须恰好是 `<job>.json` |
|
||||
| `maintenance_journal_sha256` | 磁盘 journal 字节的 sha256(不是文件内自报值) |
|
||||
| `writers[] {name, kind, instance_id}` | 旧 writer 集合:名字对治理库 `services`,实例 ID 对 journal `services[].id` |
|
||||
|
||||
**没有** `stopped` / `queue_drained` / `exited_at` / `drain_evidence` 这些调用方布尔字段。
|
||||
"已退出、已排空"只能从 journal 事实与治理库行推出来。
|
||||
|
||||
## 3. 激活门逐项不变量
|
||||
|
||||
- **A1 来源**:journal 由 `libraryfreeze` 写、位于配置的私有根(目录 0700、非符号链接);
|
||||
文件以 `O_NOFOLLOW` 打开、≤1MiB、严格 JSON(未知字段拒绝)。
|
||||
- **A2 摘要**:磁盘字节 sha256 == 证据声明值;写入台账的 `drain_evidence_digest` 是这个值。
|
||||
- **A3 窗口状态**:journal `state=held`、`level=instance`;已 `recovered`/`released` 的窗口
|
||||
不能证明"旧 writer 仍停着"。
|
||||
- **A4 实例绑定**:journal `source` == 证据 `target_instance`,且 == 治理库
|
||||
`library_instance_maintenance.instance`。
|
||||
- **A5 作业绑定**:journal `job` == 证据 `maintenance_job` == 维护行 `owner_job_id`。
|
||||
- **A6 目标绑定**:journal `target_fingerprint` == 证据 `target_fingerprint` == 维护行指纹;
|
||||
证据 `restore_target_fingerprint` == 调用方激活时对恢复目标的只读观察值。两者是
|
||||
不同层(实例/部署 vs 内容/索引/桶身份),不能用一个值互相顶替。
|
||||
- **A7 入口排空**:`ingress_closed=true`、`drain_verified=true`、gate `closed && drained &&
|
||||
active_at_close==0`。
|
||||
- **A8 后台静默**:`quiesce` 存在且 depth/pending_ops/unsettled 全 0、有合法 digest、
|
||||
停写后复核(`post_stop_at ≥ proven_at`、`post_stop_depth=0`、`post_stop_digest=digest`)。
|
||||
- **A9 writer 集合**:证据 writer 名字集合 == 维护行 `services`;实例 ID 集合 == journal
|
||||
`services[].id`;每个 writer `stop_requested=true` 且有 `exit_code`;数量必须一致。
|
||||
- **A10 有效性**:停写/排空观察(`quiesce.post_stop_at`)在 2 小时有效期内;
|
||||
证据 `attested_at` 不早于该观察、不晚于未来 5 分钟;窗口 `acquired_at` 早于观察。
|
||||
- **A11 现场身份**:`current_database()` 等于证据库名;运行构建必须已注入身份且等于
|
||||
`protocol_build`;调用方传入的目标指纹必须等于证据指纹。
|
||||
- **A12 失败关闭**:任一不满足 ⇒ 不写 activation 行;已撤销行的终局性仍由迁移 78 的
|
||||
trigger 保证(`revoked_at→NULL`、删行、状态倒退都拒绝)。
|
||||
|
||||
## 4. 它不证明什么
|
||||
|
||||
- 不独立观测"外部 writer 进程此刻不在运行";A7–A9 是维护链自己的受控观察
|
||||
(入口 drain、退出码、后台静默),不是通用进程扫描。
|
||||
- 不提供 ES 端 fence,也不保证已 `dispatching` 的旧 owner 不会迟到一次。
|
||||
- 不抵抗超级管理员禁用 trigger 或恢复旧 PG 快照。
|
||||
|
||||
## 5. 测试如何证明它不是"验字符串"
|
||||
|
||||
反例(真 PG,见 `library_restore_dispatch_linearization_test.go`):伪造 journal 摘要、
|
||||
摘要计算后替换真实文件、错 writer 名字/实例、无退出码、从未要求停、入口未排空
|
||||
(`drained=false` / `active_at_close=1` / `drain_verified=false`)、停写观察过期、
|
||||
证据过期、治理库身份不符、现场指纹不符、构建不符、维护窗口已释放、作业不属于窗口、
|
||||
journal 已 recovered、未配置受信根。全部要求零 activation 行;正例才产生 1 行,并把
|
||||
已核事实(journal 摘要、gate、quiesce、每个 writer 的退出码)写进 `evidence.verified`。
|
||||
@@ -0,0 +1,90 @@
|
||||
# DS01:删除型恢复的派发台账(实现说明)
|
||||
|
||||
状态:源码与隔离测试已实现;**未在 8088 / 正式治理库 / 业务 WeKnora 上执行 DDL 或激活**。
|
||||
设计正本:`astra-ds01-design.md`。有限验收表:握手目录 `ds01-acceptance-freeze.md`。
|
||||
|
||||
## 它证明什么、不证明什么
|
||||
|
||||
证明:对一个固定的删除型恢复 effect(`Before` 非空、`After` 空,恢复写入是独占 `_create`),
|
||||
台账里 `prepared` 意味着没有任何遵守本协议的派发者取得过发送许可;`dispatching` 之后永不回退,
|
||||
因此"曾经发送过"不可能再表现为 `prepared`。
|
||||
|
||||
不证明:ES 端 fencing;已 `dispatching` 的旧 owner 不会迟到一次;目标历史上没被外部写过;
|
||||
PG 快照回滚/手工删行之后的协议有效性(这类历史倒退必须 held 并人工处置)。
|
||||
|
||||
## 状态与唯一发送路径
|
||||
|
||||
```
|
||||
prepared --(CAS, 同一短事务内校验 activation/job/operation/attempt/lease/binding)--> dispatching
|
||||
dispatching --(真实 receipt + 当前 owner/lease)--> marker_confirmed
|
||||
```
|
||||
|
||||
- 发送许可只由 `AcquireRestoreMarker` 在 **COMMIT 明确返回后**给出,`RestoreSendPermit` 消费一次即作废。
|
||||
- CAS 事务按 `job → operation → activation → active attempt → effect` 取锁:
|
||||
- 激活撤销(`revoked_at`)与 CAS 在同一事务里线性化:撤销先完成则拒绝,CAS 先取锁则撤销排队;
|
||||
- lease 用 `clock_timestamp()` 在取锁之后判定,等锁跨过租约即拒绝;
|
||||
- attempt 必须属于原 window job;job 的 owner/epoch/lease、operation 的 scope/来源快照/
|
||||
已登记写集摘要、实例维护窗口(未释放且属于该作业)逐项核对;
|
||||
- 恢复路径另核 recovery queue job(运行中且有 lease)与 recovery operation(`kind=recover`、
|
||||
绑定同一 backup);
|
||||
- 同一 effect 换 plan 摘要会被台账拒绝(不能靠新 key 重开证明)。
|
||||
- 不可变绑定来自原持久锚点:`scope`、来源快照(发布基线)、写前镜像(backup)、目标实例/
|
||||
实际指纹/索引 UUID、变更/标记/请求摘要与 attempt 身份。来源与写前镜像不允许是同一份快照。
|
||||
目标指纹与索引 UUID 每次重建绑定都实时重读并与本次恢复冻结值比较:同名重建的目标/索引
|
||||
无法复用旧证明。
|
||||
- 恢复期 `_create` 只从 `sendRestoreMarkerOnce` 发出;preflight/故障钩子都在 CAS 之前。
|
||||
- 请求设置 `GetBody=nil`,Go 传输层不会在 keep-alive 重连时重放该 mutation。
|
||||
|
||||
## Restore 准入与收尾
|
||||
|
||||
`Restore` 在 `AdoptWriteSet` 之后、任何阶段写入之前调用纯读准入,并先对照原 window 的
|
||||
**权威台账全集**(跨 plan 摘要):出现另一份 plan、计划外 effect 或不可识别状态立即 held。
|
||||
|
||||
| 台账/证据 | 结果 |
|
||||
|---|---|
|
||||
| 无意图、无证据、无台账 | 首次恢复,可建立 `prepared` |
|
||||
| 旧意图、无台账 | held(旧协议未知历史,不补 prepared) |
|
||||
| `prepared` + 绑定冲突(`Held`) | held |
|
||||
| `prepared` + 已存在 claimed/finalize/目标精确 marker | held(协议证据矛盾) |
|
||||
| `prepared` + 无矛盾 | 可竞争一次许可 |
|
||||
| `prepared` + mark-intent 未落盘(文件失败/崩溃区段) | 核对绑定一致后补写**同内容**意图,再竞争 |
|
||||
| `dispatching` + 目标无精确 marker | held(零新发送) |
|
||||
| `dispatching` + 精确 marker/内容/世代 | 只走确认分支,不发 create |
|
||||
| `marker_confirmed` + 无 `restored` 证据 | held |
|
||||
| `restored` 证据与目标内容/世代一致 | 终态,允许重放 |
|
||||
|
||||
收尾前(整库指纹比对之后)以同一份权威全集再检查一次:任何 `prepared`/`dispatching` 记录、
|
||||
计划外记录、换 plan 摘要、或缺失 `restored` 文件证据的 effect 都拒绝返回成功,维护窗口不会
|
||||
被释放。
|
||||
|
||||
## 激活(部署动作,不属于本开发包)
|
||||
|
||||
新能力默认关闭。启用需要两步,缺一不可:
|
||||
|
||||
1. 部署配置 `LIBRARY_RESTORE_DISPATCH_V2=true`(只表示允许装配);
|
||||
2. 部署流程在**维护窗口 held 的当下**登记激活:`Store` 必须绑定
|
||||
`LIBRARY_TARGET_JOURNAL_ROOT`(`WithRestoreDispatchEvidence`),由
|
||||
`ActivateRestoreDispatch` 消费一份 v2 证据文件。
|
||||
|
||||
v2 证据只声明身份(目标/实例/治理库/构建、维护作业、journal 文件名与 sha256、
|
||||
旧 writer 集合);"已退出、已排空"由代码直接读维护 journal 与治理库
|
||||
`library_instance_maintenance` 行核出:窗口必须 `held`、入口 `closed/drained/
|
||||
active_at_close=0`、后台 quiesce 全零且有停写后复核、每个 writer 有
|
||||
`stop_requested` 与退出码、writer 集合与治理库一致、观察在 2 小时内。
|
||||
|
||||
没有配置受信 journal 根时返回 `ErrRestoreDispatchActivationDisabled`(能力关闭),
|
||||
不会退回"三个非空字符串就开门"。逐项不变量见
|
||||
`docs/architecture/ds01-activation-evidence.md`。
|
||||
|
||||
撤销是终局:`revoked_at` 一旦写入,SQL 不能改回 `NULL`、不能改时间,activation 行也不能删除;
|
||||
重新开放同一物理目标只能换新的目标身份。
|
||||
|
||||
边界:代码核验 journal 完整性、身份绑定与维护链自己的停写/排空观察;它不独立扫描
|
||||
外部进程,也不提供 ES 端 fence。
|
||||
|
||||
## 迁移与回滚
|
||||
|
||||
- `000078_library_restore_dispatch.up.sql` 只建治理库表与触发器,不碰 WeKnora Schema。
|
||||
- `000078_..._down.sql` 在 `library_index_restore_dispatch` 或 activation 表有**任意行**时拒绝执行:
|
||||
这些行是"是否发送过 / 是否曾被授权"的唯一证明,抹掉等于把 unknown 伪装成 prepared,或把
|
||||
已撤销的激活悄悄删掉。
|
||||
@@ -1,6 +1,6 @@
|
||||
# v0.7.1 整库发布的数据契约与覆盖边界
|
||||
|
||||
这是候选代码的契约,不是部署许可。真实隔离执行证据在交付目录 `data-contract-r10.md`、`test-summary-r10.json` 及其 raw 索引。PG/ES/MinIO 均为本机隔离真实服务;发布/恢复测试直接调用适配器;B22 另走实际 `librarystack.OpenProductionStack` 和 HTTP 网关。网关的独占审核是测试夹具,不代表生产已隔离;B22 控制读取角色是真实 SELECT-only PG 角色。
|
||||
这是候选代码的契约,不是部署许可。真实隔离执行证据在交付目录 `data-contract-r10.md`、`test-summary-r10.json` 及其 raw 索引。PG/ES/MinIO 均为本机隔离真实服务;发布/恢复测试直接调用适配器;B22 另走实际 `librarystack.OpenProductionStack` 和 HTTP 网关。网关按限定 key 工作,不要求整桶独占审核;B22 控制读取角色是真实 SELECT-only PG 角色。
|
||||
|
||||
## 1. 源 schema、历史身份与准入
|
||||
|
||||
@@ -55,7 +55,7 @@
|
||||
- 真实v071历史catalog镜像对照及UUID/自然键矩阵在r11执行,隔离结论与外部当前目录采样分开。A04已识别嵌套引用有有限支持,旧外部cursor仍不自动转换,不因内部结构正例而升级整项。
|
||||
- r8/r9进程恢复历史证据只在原发送协议、不可变计划/receipt及相应场景前提不变时复用。本轮对schema/引用准入有收紧,不能把历史执行日志称为当前源码正式进程全链重跑;完整当前隔离适配器回归单列。
|
||||
- clean-finalization unknown保持held是安全行为,但未满足全部自动收敛要求。若要求自动完成,需要能绑定原写入、完整内容和代次的充分服务端证据;相同干净正文、人工猜测归属或放宽共享规则均不可替代。
|
||||
- 最终提交/push、远端CI、父复核和本机部署为明确下一门;整体安全验收未齐不部署。生产独占/ACL网络凭据、网关OSS业务端到端与宿主重启/断电是另行授权的部署/破坏性验收,不能作为停止本机授权开发的借口。
|
||||
- 最终提交/push、远端CI、父复核和本机部署为明确下一门;整体安全验收未齐不部署。网关真实 OSS 业务端到端与宿主重启/断电是另行授权的部署/破坏性验收,不能作为停止本机授权开发的借口。
|
||||
|
||||
## 6. 目录与唯一键的有限支持矩阵
|
||||
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
# 目标库隔离 ID 映射
|
||||
|
||||
同一 WeKnora 实例里可以有多个知识库。同步把源快照写进目标库时,如果沿用源库的 UUID 主键,
|
||||
第二个目标库会与实例里已有库的行撞主键(真实证据:2 knowledges / 2 chunks / 23 wiki_pages /
|
||||
17 wiki_folders 已存在于旧库),写入前的跨范围检查会拒绝整次同步。关闭检查等于让两个库共享身份,
|
||||
删除/恢复/校验会互相误伤,因此身份必须在目标侧派生。
|
||||
|
||||
## 映射规则
|
||||
|
||||
```
|
||||
digest = sha256("knowledge-governance/target-identity/v1" || tenant || kb || table || source_id)
|
||||
target_id = 前 16 字节按 8-4-4-4-12 十六进制格式化(置 RFC 4122 variant/version 位)
|
||||
```
|
||||
|
||||
这是**本产品自定义的稳定 ID 算法**(`target_id_mode=scoped-sha256-v1`),只保证确定性与目标隔离;
|
||||
它不是标准 RFC 4122 UUIDv5(UUIDv5 用 SHA-1 + 标准 namespace),不要按 UUIDv5 互操作。
|
||||
|
||||
- 只绑定**稳定目标身份**(目标 tenant、目标 KB)与源身份,表名参与命名空间;
|
||||
- 不绑定 job、时间、快照版本等瞬态值:同一目标重复同步/重试/跨版本得到同一 ID;
|
||||
- 不同目标 KB、不同表得到不同 ID;
|
||||
- 源归档字节不变,映射只存在于目标侧派生视图(写集合、写入行、索引文档、校验预期);
|
||||
- 计划把策略持久化为 `target_id_mode=scoped-sha256-v1`:新进程按该字段重建映射,旧计划/写前镜像
|
||||
没有该字段则按原 ID 回放,绝不二次映射。
|
||||
|
||||
## 参与映射的表与字段
|
||||
|
||||
| 表 | 身份 | 引用字段 |
|
||||
|---|---|---|
|
||||
| knowledges | id | — |
|
||||
| chunks | id | knowledge_id, tag_id, parent_chunk_id, pre_chunk_id, next_chunk_id, relation_chunks, indirect_relation_chunks, metadata(仅精确匹配) |
|
||||
| knowledge_tags | id | — |
|
||||
| knowledge_tag_relations | (knowledge_id, tag_id) | 两个分量分别映射 |
|
||||
| wiki_folders | id | parent_id |
|
||||
| wiki_pages | id | folder_id, chunk_refs, source_refs(保留 `id|title`), page_metadata(仅精确匹配) |
|
||||
| wiki_page_issues | id | page_id, knowledge_id, chunk_id |
|
||||
| resource_bindings | id | owner_id(按 owner_type) |
|
||||
| resources | **不映射** | resource_id 保持共享身份 |
|
||||
| 索引文档 | 文档物理 ID | chunk_id, knowledge_id, source_id, tag_id |
|
||||
|
||||
正文、标题、摘要、向量、slug、in_links/out_links、pages_affected 不改写;本地自增列
|
||||
(如 `wiki_log_entries.id`)仍由既有 LocalIDs 分配。引用指向未登记身份时拒绝(悬挂引用不静默放行)。
|
||||
|
||||
## 一致性边界
|
||||
|
||||
- 写集合登记的是目标身份;verify 的身份摘要与内容摘要都按同一 `rowTransform` 计算;
|
||||
- 索引计划的目标文档 ID 与 `_source` 引用同步映射,before-image(目标写前镜像)保持原样;
|
||||
- 恢复重放写前镜像时映射关闭,避免把目标原始 ID 再映射一次;
|
||||
- 资源对象路径/location_hash 仍由既有资源改写逻辑处理,不因 ID 映射而变化。
|
||||
@@ -4,12 +4,14 @@
|
||||
|
||||
`cmd/object-gateway` 提供 `kg-object-gateway-v1`,只监听 loopback;外部接入需要另行审核的 TLS 终结与网络隔离。治理 PG 保存不可变计划、grant、intent、逐版本 receipt 和 attempt→grant delivery。部署迁移身份显式应用 `internal/objectgateway/schema.sql`;启动不自动迁移。运行身份只授必要 SELECT/INSERT 和 scope 行 UPDATE,不授 DDL/TRUNCATE/证据修改或业务库写权限。
|
||||
|
||||
scope 是 provider、原 endpoint、bucket 的 NUL 分隔 SHA256。同一物理桶只能有一个受审核 scope 和一套共享 journal,不能用 endpoint 别名或另一日志库建立平行写域。
|
||||
scope 是 provider、原 endpoint、bucket 的 NUL 分隔 SHA256。同一物理桶在本网关内只使用一个 scope 和一套共享 journal,不能用 endpoint 别名或另一日志库建立平行写域。这不要求整桶只有本网关一个写者:别的服务可以在同一桶的其它 key 上正常读写。
|
||||
|
||||
数据认证端点:`POST /v1/acquire`、`/v1/resolve`、`/v1/execute`、`/v1/reconcile`、`/v1/observe`,以及 `GET /v1/capabilities`。所有请求核验协议和 scope。`capabilities.automatic_grants` 必须为真,正式 stack 才接受自动授权能力;未接控制源、只读角色权限异常、协议不符或不可达均拒绝,无直写回退。
|
||||
|
||||
PG 行锁负责原子准入,不是远端存储 fence。intent 必须持久提交后才发送一次请求;连接死亡、租约过期不清除未知 intent。任何未终态 intent 均阻止 scope 换 epoch。重复 operation 只读回执或精确对账,不重发 PUT;暂时查不到版本仍 held。删除必须绑定已确认 create 的精确 VersionID,不删 latest、不生成删除标记。
|
||||
|
||||
同 key 竞争按 key 判定,不按桶判定:create 前已存在任何版本 → 回执 `rejected` 且不修改该对象;PUT 后除精确版本读回外还核对当前 latest,若并发写者已产生更新的当前版本,则只按精确 VersionID 撤回本次写入,保留对方版本并返回冲突;读不到当前版本时保持 held,不删除、不猜测。版本化存储的 PUT 不是原子 create-if-absent,因此“本 key 多数情况下归本计划”不是存储层保证。
|
||||
|
||||
不可变证据表拒绝 UPDATE/DELETE/TRUNCATE。Ready 核 fsync、synchronous_commit、full_page_writes、主库可写及必要表/触发器;管理员与迁移身份仍是可信边界,触发器不能抵御 DBA 的 DDL。
|
||||
|
||||
## 正式控制交付路径
|
||||
@@ -54,13 +56,13 @@ SELECT-only治理角色需要读取`library_jobs`、`library_operations`、`libr
|
||||
- `OBJECT_GATEWAY_ARCHIVE_ROOT`:原持久计划所在归档绝对路径,部署只读挂载。
|
||||
- `OBJECT_GATEWAY_TARGET_ID`:与正式 worker `LibraryTargetIdentity()` 完全相同的逻辑目标身份;不能用另一个环境的值。
|
||||
- `OBJECT_GATEWAY_PROVIDER`、`OBJECT_GATEWAY_ENDPOINT`、`OBJECT_GATEWAY_BUCKET`、`OBJECT_GATEWAY_REGION`:精确存储身份。
|
||||
- `OBJECT_GATEWAY_ACCESS_KEY`、`OBJECT_GATEWAY_SECRET_KEY`:仅网关持有的对象写凭据。
|
||||
- `OBJECT_GATEWAY_ACCESS_KEY`、`OBJECT_GATEWAY_SECRET_KEY`:网关持有的对象凭据,需在目标桶 PUT/GET/按版本 DELETE/列举版本。沿用现有 AK 即可,不要求另建独立只读或独占写凭据。
|
||||
- `OBJECT_GATEWAY_DATA_TOKEN`、`OBJECT_GATEWAY_CONTROL_TOKEN`:不同且至少 32 字节;普通 worker 仅持有 data token。
|
||||
- `OBJECT_GATEWAY_ID`、`OBJECT_GATEWAY_AUDIT_FILE`、`OBJECT_GATEWAY_AUDIT_PUBLIC_KEY`:独立审核者的 Ed25519 签名隔离证据,绑定协议、scope、gateway ID、时间窗及 writer/policy/拒写/排空摘要。审核私钥不能交给 worker 或网关。
|
||||
- `OBJECT_GATEWAY_ID`、`OBJECT_GATEWAY_AUDIT_FILE`、`OBJECT_GATEWAY_AUDIT_PUBLIC_KEY`:**已退役**。旧部署残留的值被忽略,不参与启动、Ready、每次 Execute 或能力判断;不需要管理员签发整桶独占证明,也不会因过期文件拒绝发布。
|
||||
|
||||
秘密只经服务环境/私有文件注入,不进 argv、URL 或日志。所有调用端错误都使用固定原因码。
|
||||
|
||||
正式 worker 使用 `LIBRARY_OBJECT_GATEWAY_URL` 和 `LIBRARY_OBJECT_GATEWAY_TOKEN`;不再依赖预先生成的 grant 文件。`LIBRARY_OBJECT_GATEWAY_GRANT_DIRECTORY` 保留旧离线兼容用途,不是自动模式的授权来源。发布 stack 对所有对象 provider 强制网关;S3 构造仍用于只读捕获/预备份,部署必须使用独立只读凭据。
|
||||
正式 worker 使用 `LIBRARY_OBJECT_GATEWAY_URL` 和 `LIBRARY_OBJECT_GATEWAY_TOKEN`;不再依赖预先生成的 grant 文件。`LIBRARY_OBJECT_GATEWAY_GRANT_DIRECTORY` 保留旧离线兼容用途,不是自动模式的授权来源。发布 stack 对所有对象 provider 强制网关;S3 构造仍用于只读捕获/预备份,可继续使用现有凭据,不再要求独立只读 OSS AK。
|
||||
|
||||
## 已关闭的本仓旁路
|
||||
|
||||
@@ -70,24 +72,34 @@ SELECT-only治理角色需要读取`library_jobs`、`library_operations`、`libr
|
||||
- `librarystack` 不再为缺网关的 MinIO/S3 返回直写 writer。
|
||||
- `libraryctl` capture 的 S3 构造与读取保留;未把读取当作写入口禁用。
|
||||
|
||||
这不等于已经撤销运行环境旧进程的凭据;源码变更尚未部署。`librarydrill` 等无 owned-attempt 的旧写型工具不能靠静态 grant 绕过自动模式,需补正式执行身份后才可用于写入。低层 S3 adapter 仍用于网关后端、读取及显式隔离夹具;部署权限才负责防止外部旧进程直接调用存储。
|
||||
这不等于已经撤销运行环境旧进程的凭据;源码变更尚未部署。`librarydrill` 等无 owned-attempt 的旧写型工具不能靠静态 grant 绕过自动模式,需补正式执行身份后才可用于写入。低层 S3 adapter 仍用于网关后端、读取及显式隔离夹具;共享桶不要求撤除其它写者,跨写者的同 key 竞争按冲突/held 处理。
|
||||
|
||||
## 实测与未完成边界
|
||||
|
||||
已实跑:真实治理 PG(独立 SELECT-only 登录角色)、原 archive、数据 HTTP、业务 PG 与 MinIO 的自动授权/跨 epoch 链;未锚定计划在第一笔 PG 前拒绝,接管保持原发送身份且不增加 PUT。另有真实网关二进制的两次中断验证。隔离审计 authority/故障注入仍是测试设施,不是生产权限证明。
|
||||
已实跑:真实治理 PG(独立 SELECT-only 登录角色)、原 archive、数据 HTTP、业务 PG 与 MinIO 的自动授权/跨 epoch 链;未锚定计划在第一笔 PG 前拒绝,接管保持原发送身份且不增加 PUT。另有真实网关二进制的两次中断验证,以及共享桶下“无整桶证明启动 + 其它 key 并发写入 + 同 key 既有对象拒绝”的真实 PG/MinIO 用例。历史隔离审计 authority/故障注入是当时的测试设施,不是当前部署前置。
|
||||
|
||||
正式 worker 隔离进程证据:真实 Docker 入口闸门/队列、只读 worker 对象身份、独立治理读取角色、网关与业务 PG/ES/MinIO;首次 PG 后和对象提交后两次 SIGKILL,同一原 job/operation/plan 的三代 attempt 完成发布,原对象发送 epoch 与回执不变。隔离构建标签 `kg_isolated_drill` 仅把真实租约缩短至4秒,并拒绝非 loopback 一次性 r8 库;正常构建仍为10分钟。维护上游是无业务凭据的独立 HTTP 夹具,不是运行中的 WeKnora;签名证据绑定本次真实拒写/排空材料,不代表生产独占审核。
|
||||
|
||||
已补独立 `library.recover` 正式worker链、恢复中再次SIGKILL、完整before-image/CAS/精确版本读回,以及来源/目标/原job/计划/operation/任意队列身份拒绝。ES故障代理只转发真实ES并丢弃已提交响应,未伪造服务响应。
|
||||
|
||||
尚未完成:clean-finalization unknown的充分归属与自动收敛;独立运行环境只读对象凭据、ACL/网络和全部writer排空;网关真实OSS端到端;原D0/F11/F12与55项其余缺口。测试/隔离验收与真实部署独占分列,不把生产变更授权当成本机开发授权的前置。
|
||||
尚未完成:clean-finalization unknown的充分归属与自动收敛;网关真实 OSS 端到端与真实部署验收(父执行);原D0/F11/F12与55项其余缺口。测试/隔离验收与真实部署分列。
|
||||
|
||||
维护隔离、独占审核与 grant 是独立前置。配置 true、健康 200、测试自签 authority 都不能代替已部署拒写/排空证据。目前未签发真实环境独占证明,未改生产权限或上游 WeKnora 源码/Schema。
|
||||
整桶独占审核、全部 writer 排空、独立只读对象凭据已不再是发布前置;维护窗口内停目标写入容器并读回确认即可更新。网关按 key 拒绝冲突、精确版本回滚,但不承诺无关写者同时修改同一 key 时仍能自动完成——那种情况会明确冲突或 held,需要人工按回执处理。未改生产权限或上游 WeKnora 源码/Schema。
|
||||
|
||||
## 维护式更新(已采用)
|
||||
|
||||
目标测试 KB 的更新走维护窗口,不再要求证明整桶只有一个写者:
|
||||
|
||||
1. 停目标测试 WeKnora 的写入应用容器(只停该服务,不停机器人、适配器、共享 PG/ES 或其它 KB 的服务),并由容器状态/健康读回确认已停止。这是本轮文档化的运维步骤;代码侧没有新增停容器实现。
|
||||
2. 用既有维护/预备份链记录本次目标 KB/对象范围:实例级维护窗口与写者停写证据由 `internal/libraryfreeze` + `internal/librarymaintenance` 采集,`libraryctl capture`(页面入口 `POST /api/v1/library/captures`)完成闸门排空→停写者→导出快照并保留 `post_stop_digest`;不把通用主机备份当作 KB 预备份的替代。
|
||||
3. 用原 plan、原 WriteSet 摘要和限定对象清单走既有 `ProductionLibraryTarget.Apply`(经正式 worker owned attempt 与网关 grant)更新资源;网关沿用现有 AK,不换凭据、不签独占证明。
|
||||
4. 启动原容器,健康检查与页面业务验收;失败走既有 `library.recover`/`RestoreDrill` 恢复链按原预备份回放并启动原服务,不无限留停机。
|
||||
5. 无关服务与无关 key 全程不需要停;同 key 若仍有外部写者,按上节的冲突/held 处理:双方版本都保留,由人按证据处置,不自动合并。
|
||||
|
||||
部署需变更的组件:`object-gateway`(本候选镜像)、`worker`(同提交镜像),以及两者 env 中移除/留空已退役的 `OBJECT_GATEWAY_ID`、`OBJECT_GATEWAY_AUDIT_FILE`、`OBJECT_GATEWAY_AUDIT_PUBLIC_KEY`。`api`、`web` 无协议变更,可不同步。journal、治理 PG 只读 authority、archive 与 target identity 配置保持不变。
|
||||
|
||||
## 部署与回退(未执行)
|
||||
|
||||
未来测试主机 `120.76.217.51`、生产 `120.76.218.137` 各自独立批准。每批先冻结主机、桶、全部角色/STS/预签名写入口、端口/网络规则、影响、维护窗口与回滚;当前没有执行这些外部变更。
|
||||
测试主机 `120.76.217.51`、生产 `120.76.218.137` 各自独立批准;本候选未在任一主机执行停机、备份、更新或启动。r5 的已完成 OSS 测试授权不能重复使用。
|
||||
|
||||
先关新写准入→排空并对账旧请求→撤旧直写权限/切只读身份并实测拒写→独立审核→批准的新限定对象验收→父复核,才可切业务客户端。r5 的已完成 OSS 测试授权不能重复使用。
|
||||
|
||||
回退先关准入并保留 journal;所有已发送请求有终态才可撤网关,unknown 继续 held。不得因网关故障自动恢复旧直写凭据。权限/网络回退须另批。当前整模块 INCOMPLETE,禁止部署部分候选。
|
||||
按“停目标写入容器→读回确认→备份目标 KB 范围→更新资源→启动→健康/页面验收”执行;失败按原备份恢复并启动原服务。回退先关新写准入并保留 journal;所有已发送请求有终态才可撤网关,unknown 继续 held。不得因网关故障自动恢复旧直写凭据。当前整模块仍未完成真实 OSS 端到端与 clean-finalization 收敛,禁止把本候选当作完整生产就绪。
|
||||
|
||||
@@ -23,8 +23,11 @@ type Config struct {
|
||||
ReleaseConfigFile string
|
||||
ProductionReleaseRequestsEnabled bool
|
||||
ProductionBackupRestoreVerified bool
|
||||
ProductionReleaseAuthorizerRef string
|
||||
ProductionReleaseAuthTokenFile string
|
||||
// LibraryRestoreDispatchEnabled 允许 worker 装配 DS01 派发台账;默认 false。
|
||||
// 它不能单独证明独占:治理库里的目标激活登记同样必需。
|
||||
LibraryRestoreDispatchEnabled bool
|
||||
ProductionReleaseAuthorizerRef string
|
||||
ProductionReleaseAuthTokenFile string
|
||||
// Production WeKnora endpoint: the release target itself stays declared in
|
||||
// the release runtime config (RELEASE_CONFIG_FILE); these settings only let
|
||||
// the API probe that endpoint so the console can show its real state.
|
||||
@@ -304,6 +307,14 @@ func Load(getenv func(string) string) (Config, error) {
|
||||
return Config{}, fmt.Errorf("PRODUCTION_BACKUP_RESTORE_VERIFIED: %w", err)
|
||||
}
|
||||
}
|
||||
// DS01 恢复标记派发协议:默认关闭。开关只是"允许装配"的意思,不是独占证明——
|
||||
// 真正发放许可还要求治理库里对该目标存在未撤销的激活登记。
|
||||
if raw := getenv("LIBRARY_RESTORE_DISPATCH_V2"); raw != "" {
|
||||
config.LibraryRestoreDispatchEnabled, err = strconv.ParseBool(raw)
|
||||
if err != nil {
|
||||
return Config{}, fmt.Errorf("LIBRARY_RESTORE_DISPATCH_V2: %w", err)
|
||||
}
|
||||
}
|
||||
if raw := getenv("PRODUCTION_WEKNORA_ALLOW_DOCKER_HTTP"); raw != "" {
|
||||
config.ProductionWeKnoraAllowDockerHTTP, err = strconv.ParseBool(raw)
|
||||
if err != nil {
|
||||
|
||||
@@ -73,12 +73,13 @@ func (r LibraryWorkerRole) LibraryJobKinds() []string {
|
||||
switch r {
|
||||
case LibraryWorkerRoleReleaseExecutor:
|
||||
return []string{string(governance.LibraryJobSync), string(governance.LibraryJobRestore),
|
||||
string(governance.LibraryJobRecover), string(governance.LibraryJobBaseline)}
|
||||
string(governance.LibraryJobRecover), string(governance.LibraryJobBaseline),
|
||||
string(governance.LibraryJobVerify)}
|
||||
case LibraryWorkerRoleCapture:
|
||||
return []string{string(governance.LibraryJobSeal)}
|
||||
default:
|
||||
return []string{string(governance.LibraryJobSeal), string(governance.LibraryJobSync),
|
||||
string(governance.LibraryJobRestore), string(governance.LibraryJobRecover),
|
||||
string(governance.LibraryJobBaseline)}
|
||||
string(governance.LibraryJobBaseline), string(governance.LibraryJobVerify)}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -63,13 +63,14 @@ func TestLoadRejectsUnknownLibraryWorkerRole(t *testing.T) {
|
||||
// 每个角色收拾哪些 kind、谁拥有 DDL,都必须互斥且覆盖完整。
|
||||
func TestLibraryWorkerRoleOwnershipMaps(t *testing.T) {
|
||||
release := LibraryWorkerRoleReleaseExecutor
|
||||
if got := release.LibraryJobKinds(); len(got) != 4 || got[0] != "sync" || got[1] != "restore" || got[2] != "recover" || got[3] != "baseline" {
|
||||
if got := release.LibraryJobKinds(); len(got) != 5 || got[0] != "sync" || got[1] != "restore" ||
|
||||
got[2] != "recover" || got[3] != "baseline" || got[4] != "verify" {
|
||||
t.Fatalf("release-executor kinds = %v", got)
|
||||
}
|
||||
if got := LibraryWorkerRoleCapture.LibraryJobKinds(); len(got) != 1 || got[0] != "seal" {
|
||||
t.Fatalf("capture kinds = %v", got)
|
||||
}
|
||||
if got := LibraryWorkerRoleFull.LibraryJobKinds(); len(got) != 5 {
|
||||
if got := LibraryWorkerRoleFull.LibraryJobKinds(); len(got) != 6 {
|
||||
t.Fatalf("full kinds = %v", got)
|
||||
}
|
||||
for _, role := range []LibraryWorkerRole{LibraryWorkerRoleFull, LibraryWorkerRoleCapture} {
|
||||
|
||||
@@ -10,6 +10,8 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strconv"
|
||||
@@ -127,12 +129,81 @@ func (r *Runner) runRemote(ctx context.Context, timeout time.Duration, remote st
|
||||
if ctx.Err() != nil {
|
||||
return text, fmt.Errorf("ssh %s 超时(%s)", r.config.Target, timeout)
|
||||
}
|
||||
// 主机侧脚本会用退出码表达"参数非法/有库失败",把输出带回去便于页面展示。
|
||||
return text, fmt.Errorf("ssh %s 执行失败: %s", r.config.Target, firstLines(text, 6))
|
||||
// 连接层失败与主机侧命令失败必须分开说:前者连脚本都没跑,
|
||||
// 任何"脚本缺失/计划未下发"都是把未观测当成事实。
|
||||
return text, hostFailure(r.config.Target, text)
|
||||
}
|
||||
return text, nil
|
||||
}
|
||||
|
||||
// hostFailure 把 ssh 的非零退出分成两类:连不上,和连上了但远端命令失败。
|
||||
func hostFailure(target, text string) error {
|
||||
if connectionFailurePattern.MatchString(text) {
|
||||
return fmt.Errorf("目标主机 %s 不可达(%s);未执行任何主机侧脚本,脚本与计划状态未知",
|
||||
target, firstLines(text, 3))
|
||||
}
|
||||
// 主机侧脚本会用退出码表达"参数非法/有库失败",把输出带回去便于页面展示。
|
||||
return fmt.Errorf("已连上 %s,但主机侧命令失败: %s", target, firstLines(text, 6))
|
||||
}
|
||||
|
||||
// connectionFailurePattern 是 OpenSSH 客户端在"根本没建立连接"时打的原文。
|
||||
// 只有命中它才允许说"不可达";其余非零退出属于远端命令自己的失败。
|
||||
var connectionFailurePattern = regexp.MustCompile(`(?i)ssh: connect to host|Operation timed out|Connection timed out|Connection refused|Connection reset by peer|No route to host|Network is unreachable|Could not resolve hostname|Host key verification failed|Permission denied|Connection closed by`)
|
||||
|
||||
// Endpoint 读回挂载的 ssh 配置里本别名真实拨号的 host:port。
|
||||
// 页面必须能说清"现在连的是哪个地址"——只显示别名时,改错文件也看不出来。
|
||||
func (r *Runner) Endpoint() string {
|
||||
if r == nil || r.config.SSHConfig == "" || r.config.Target == "" {
|
||||
return ""
|
||||
}
|
||||
raw, err := os.ReadFile(r.config.SSHConfig)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
host, port := sshConfigEndpoint(string(raw), r.config.Target)
|
||||
if host == "" {
|
||||
return ""
|
||||
}
|
||||
if port == "" {
|
||||
port = "22"
|
||||
}
|
||||
return net.JoinHostPort(host, port)
|
||||
}
|
||||
|
||||
// sshConfigEndpoint 解析 `Host <alias>` 段的 HostName/Port(首个匹配段生效)。
|
||||
// 只做"页面显示用"的最小解析,不替代 ssh 自己的配置语义。
|
||||
func sshConfigEndpoint(config, alias string) (string, string) {
|
||||
host, port := "", ""
|
||||
active := false
|
||||
for _, line := range strings.Split(config, "\n") {
|
||||
fields := strings.Fields(strings.TrimSpace(line))
|
||||
if len(fields) == 0 || strings.HasPrefix(fields[0], "#") {
|
||||
continue
|
||||
}
|
||||
key := strings.ToLower(fields[0])
|
||||
if key == "host" {
|
||||
active = false
|
||||
for _, pattern := range fields[1:] {
|
||||
if pattern == alias {
|
||||
active = true
|
||||
break
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !active || len(fields) < 2 {
|
||||
continue
|
||||
}
|
||||
switch key {
|
||||
case "hostname":
|
||||
host = strings.Trim(fields[1], `"`)
|
||||
case "port":
|
||||
port = strings.Trim(fields[1], `"`)
|
||||
}
|
||||
}
|
||||
return host, port
|
||||
}
|
||||
|
||||
var scheduleAppliedPattern = regexp.MustCompile(
|
||||
`SCHEDULE_APPLIED enabled=(\d)(?: time=(\S+))?(?: days=(\d+))?(?: months=(\d+))?(?: next=(\S+))?(?: cron=(\S+))?`)
|
||||
|
||||
|
||||
@@ -3,6 +3,8 @@ package backuphost
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -188,6 +190,41 @@ func TestRunnerSurfacesHostFailure(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 现场原错:ssh: connect to host 120.76.217.51 port 22: Operation timed out。
|
||||
// 连不上时脚本根本没跑,只能说"不可达";断言配对的反面:不能说脚本缺失。
|
||||
func TestHostFailureSeparatesUnreachableFromRemoteCommand(t *testing.T) {
|
||||
unreachable := hostFailure("weknora-test", "ssh: connect to host 120.76.217.51 port 22: Operation timed out")
|
||||
if !strings.Contains(unreachable.Error(), "不可达") || !strings.Contains(unreachable.Error(), "Operation timed out") {
|
||||
t.Fatalf("连接超时应报不可达并带原错: %v", unreachable)
|
||||
}
|
||||
if strings.Contains(unreachable.Error(), "主机侧命令失败") || strings.Contains(unreachable.Error(), "脚本缺失") {
|
||||
t.Fatalf("连接失败不能报成远端命令/脚本问题: %v", unreachable)
|
||||
}
|
||||
remote := hostFailure("weknora-test", "bash: /opt/weknora-test/logical-backup.sh: No such file or directory")
|
||||
if !strings.Contains(remote.Error(), "已连上") || !strings.Contains(remote.Error(), "No such file") {
|
||||
t.Fatalf("已连接但脚本失败时应原样带回原因: %v", remote)
|
||||
}
|
||||
if strings.Contains(remote.Error(), "不可达") {
|
||||
t.Fatalf("已连上就不能说不可达: %v", remote)
|
||||
}
|
||||
}
|
||||
|
||||
// 页面要能说清"实际拨号的地址",否则改错 ssh_config 也看不出来。
|
||||
func TestRunnerEndpointReadsDialedHostPort(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "ssh_config")
|
||||
config := "Host other\n HostName 10.0.0.9\n Port 2222\n\nHost weknora-test\n HostName 100.85.111.121\n HostKeyAlias 120.76.217.51\n"
|
||||
if err := os.WriteFile(path, []byte(config), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runner := New(Config{SSHConfig: path, Target: "weknora-test", ScriptDir: "/opt/weknora-test"})
|
||||
if got := runner.Endpoint(); got != "100.85.111.121:22" {
|
||||
t.Fatalf("应报真实拨号地址(默认 22),得到 %q", got)
|
||||
}
|
||||
if got := New(Config{SSHConfig: filepath.Join(t.TempDir(), "missing"), Target: "weknora-test"}).Endpoint(); got != "" {
|
||||
t.Fatalf("配置文件不可读时应返回空,不猜地址,得到 %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunNowReadsBackLastRun(t *testing.T) {
|
||||
calls := 0
|
||||
runner := testRunner(func(_ context.Context, _ time.Duration, remote string) (string, error) {
|
||||
|
||||
@@ -16,9 +16,8 @@ func developmentNotes(changelog, version, sha string) string {
|
||||
return ""
|
||||
}
|
||||
lines := strings.Split(changelog, "\n")
|
||||
prefix := "## 开发快照 snapshot/v" + version + " - "
|
||||
for i, line := range lines {
|
||||
if strings.HasPrefix(line, prefix) {
|
||||
if developmentHeadingMatches(line, version) {
|
||||
end := len(lines)
|
||||
for j := i + 1; j < len(lines); j++ {
|
||||
if strings.HasPrefix(lines[j], "## ") {
|
||||
@@ -39,3 +38,33 @@ func developmentNotes(changelog, version, sha string) string {
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// developmentHeadingMatches 接受两种真实标题:
|
||||
// - 旧格式:## 开发快照 snapshot/v0.1.1 - 2026-10-18
|
||||
// - 开发快照:## 开发快照 0.1.351(索引预检使用目标映射 ID)
|
||||
//
|
||||
// 版本号后必须是字符串结束或分隔符,避免 0.1.35 命中 0.1.351;
|
||||
// 只读取到下一个二级标题,不跨版本吞正文。
|
||||
func developmentHeadingMatches(line, version string) bool {
|
||||
prefixes := []string{
|
||||
"## 开发快照 snapshot/v" + version,
|
||||
"## 开发快照 " + version,
|
||||
}
|
||||
for _, prefix := range prefixes {
|
||||
if !strings.HasPrefix(line, prefix) {
|
||||
continue
|
||||
}
|
||||
rest := line[len(prefix):]
|
||||
if rest == "" {
|
||||
return true
|
||||
}
|
||||
if strings.HasPrefix(rest, "—") || strings.HasPrefix(rest, "(") {
|
||||
return true
|
||||
}
|
||||
switch rest[0] {
|
||||
case ' ', '\t', '-', '(':
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -44,6 +44,10 @@ const (
|
||||
// 读出来封存成快照并逐类校验,再登记成 production_effective。
|
||||
// 它不写目标内容、不清空目标、不猜版本——基线只能来自真实捕获与校验。
|
||||
LibraryJobBaseline LibraryJobKind = "baseline"
|
||||
// LibraryJobVerify 是"恢复验证":在维护窗口内捕获目标 → 校验基线一致 →
|
||||
// 写一个 canary 证明校验能发现差异 → 按预备份恢复 → 复核完全一致。
|
||||
// 它是唯一能产生"生产可恢复"证据的作业,页面不再接受手填开关。
|
||||
LibraryJobVerify LibraryJobKind = "verify"
|
||||
)
|
||||
|
||||
type LibraryJobState string
|
||||
@@ -173,6 +177,12 @@ func BaselineStages() []string {
|
||||
return []string{"capture", "verify", "finalize"}
|
||||
}
|
||||
|
||||
// VerifyStages 是恢复验证的阶段:捕获 → 基线校验 → 写 canary(证明校验有效)
|
||||
// → 按预备份恢复 → 恢复后复核。canary 与 restore 都是真实写入,失败必须可见。
|
||||
func VerifyStages() []string {
|
||||
return []string{"capture", "verify", "canary", "restore", "finalize"}
|
||||
}
|
||||
|
||||
// LibraryReleaseAuthorization 是生产发布授权:绑定目标、快照、参数摘要与有效期。
|
||||
type LibraryReleaseAuthorization struct {
|
||||
ID string `json:"id"`
|
||||
|
||||
@@ -147,7 +147,7 @@ func ValidateLibraryOperation(op LibraryOperation) error {
|
||||
return errors.New("library operation requires internal or external scope")
|
||||
}
|
||||
switch op.Kind {
|
||||
case LibraryJobSeal, LibraryJobSync, LibraryJobRestore, LibraryJobBaseline, LibraryJobRecover:
|
||||
case LibraryJobSeal, LibraryJobSync, LibraryJobRestore, LibraryJobBaseline, LibraryJobRecover, LibraryJobVerify:
|
||||
default:
|
||||
return fmt.Errorf("unsupported library operation kind %q", op.Kind)
|
||||
}
|
||||
|
||||
@@ -3,6 +3,8 @@ package governance
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The recovery request has its own queue identity and stable operation. It
|
||||
@@ -18,3 +20,33 @@ func LibraryRecoveryDigest(scope, queue, window, operation, target, source, back
|
||||
"target_id": target, "source_snapshot": source, "backup_snapshot": backup, "writeset_digest": plan,
|
||||
})
|
||||
}
|
||||
|
||||
// LibraryDeliveryOwner 是队列投递 + 业务 attempt 的合成 owner:
|
||||
//
|
||||
// <worker>#<queue job>#<queue attempt>
|
||||
//
|
||||
// 它就是 library_attempts.owner,也决定了"这次派发属于哪一次真实投递"。worker 与
|
||||
// 派发台账必须用同一份格式,否则"当前 owner"会变成两处各自解释的字符串。
|
||||
func LibraryDeliveryOwner(workerID, queueID string, attempt int) string {
|
||||
if queueID == "" {
|
||||
queueID = "-"
|
||||
}
|
||||
if workerID == "" {
|
||||
workerID = "worker"
|
||||
}
|
||||
return fmt.Sprintf("%s#%s#%d", workerID, queueID, attempt)
|
||||
}
|
||||
|
||||
// ParseLibraryDeliveryOwner 把 owner 拆回 (worker, ok);queue/attempt 必须逐位相同,
|
||||
// 不能靠后缀匹配把别的投递混进来。
|
||||
func ParseLibraryDeliveryOwner(owner, queueID string, attempt int) (string, bool) {
|
||||
suffix := fmt.Sprintf("#%s#%d", queueID, attempt)
|
||||
if queueID == "" || !strings.HasSuffix(owner, suffix) {
|
||||
return "", false
|
||||
}
|
||||
worker := strings.TrimSuffix(owner, suffix)
|
||||
if worker == "" || strings.Contains(worker, "#") {
|
||||
return "", false
|
||||
}
|
||||
return worker, true
|
||||
}
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
package governance
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// LibraryRestoreVerification 是一次**真实**恢复验证(备份 → canary → 恢复 → 复核)的登记。
|
||||
//
|
||||
// 为什么不能继续用环境变量 PRODUCTION_BACKUP_RESTORE_VERIFIED:那是一个可以手填的开关,
|
||||
// 它既证明不了"有人真跑过恢复",也证明不了"跑的是当前目标"。这里的每条记录由 worker
|
||||
// 在真实演练结束后写入,绑定 target_id(实例+库+索引+桶)与 scope,失败也照实留痕。
|
||||
type LibraryRestoreVerification struct {
|
||||
ID string `json:"id"`
|
||||
Scope string `json:"scope"`
|
||||
Status string `json:"status"` // passed | failed
|
||||
SnapshotID string `json:"snapshot_id"`
|
||||
JobID string `json:"job_id"`
|
||||
Failure string `json:"failure,omitempty"`
|
||||
Report map[string]any `json:"report,omitempty"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
FinishedAt time.Time `json:"finished_at"`
|
||||
}
|
||||
|
||||
const (
|
||||
LibraryVerificationPassed = "passed"
|
||||
LibraryVerificationFailed = "failed"
|
||||
)
|
||||
|
||||
// LibraryVerificationSatisfied 只回答一件事:这条证据能不能支撑"当前目标的恢复验证已通过"。
|
||||
// 通过的条件必须是全部成立——状态 passed、scope 相同、快照非空、完成时间有效。
|
||||
// target_id 的匹配由查询层(按当前 target_id 过滤)保证,这里不重复比较。
|
||||
func LibraryVerificationSatisfied(verification LibraryRestoreVerification, scope string) (bool, string) {
|
||||
if verification.Status != LibraryVerificationPassed {
|
||||
reason := "最近一次恢复验证没有通过"
|
||||
if detail := strings.TrimSpace(verification.Failure); detail != "" {
|
||||
reason += ":" + detail
|
||||
}
|
||||
return false, reason
|
||||
}
|
||||
if verification.Scope != scope {
|
||||
return false, "恢复验证证据属于另一个 scope(" + verification.Scope + ")"
|
||||
}
|
||||
if strings.TrimSpace(verification.SnapshotID) == "" {
|
||||
return false, "恢复验证证据没有绑定快照,不能作为备份可恢复的证明"
|
||||
}
|
||||
if verification.FinishedAt.IsZero() {
|
||||
return false, "恢复验证证据没有完成时间"
|
||||
}
|
||||
return true, ""
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package governance
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// 证据能不能支撑"当前目标可恢复"只由这四个条件回答:
|
||||
// 状态 passed、scope 相同、绑定了快照、有完成时间。少任何一条都不算。
|
||||
func TestLibraryVerificationSatisfied(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
cases := []struct {
|
||||
name string
|
||||
verification LibraryRestoreVerification
|
||||
scope string
|
||||
want bool
|
||||
reason string
|
||||
}{
|
||||
{name: "passed with snapshot", scope: "internal", want: true,
|
||||
verification: LibraryRestoreVerification{Status: LibraryVerificationPassed, Scope: "internal",
|
||||
SnapshotID: "snap-1", FinishedAt: now}},
|
||||
{name: "failed carries failure", scope: "internal", want: false, reason: "canary 后校验仍通过",
|
||||
verification: LibraryRestoreVerification{Status: LibraryVerificationFailed, Scope: "internal",
|
||||
SnapshotID: "snap-1", Failure: "canary 后校验仍通过", FinishedAt: now}},
|
||||
{name: "wrong scope", scope: "internal", want: false, reason: "另一个 scope",
|
||||
verification: LibraryRestoreVerification{Status: LibraryVerificationPassed, Scope: "external",
|
||||
SnapshotID: "snap-1", FinishedAt: now}},
|
||||
{name: "no snapshot", scope: "internal", want: false, reason: "没有绑定快照",
|
||||
verification: LibraryRestoreVerification{Status: LibraryVerificationPassed, Scope: "internal",
|
||||
FinishedAt: now}},
|
||||
{name: "no finish time", scope: "internal", want: false, reason: "没有完成时间",
|
||||
verification: LibraryRestoreVerification{Status: LibraryVerificationPassed, Scope: "internal",
|
||||
SnapshotID: "snap-1"}},
|
||||
}
|
||||
for _, testCase := range cases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
ok, reason := LibraryVerificationSatisfied(testCase.verification, testCase.scope)
|
||||
if ok != testCase.want {
|
||||
t.Fatalf("satisfied=%v want=%v reason=%q", ok, testCase.want, reason)
|
||||
}
|
||||
if !testCase.want && testCase.reason != "" && !containsText(reason, testCase.reason) {
|
||||
t.Fatalf("reason=%q want contains %q", reason, testCase.reason)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func containsText(haystack, needle string) bool {
|
||||
return len(haystack) >= len(needle) && (haystack == needle || indexOf(haystack, needle) >= 0)
|
||||
}
|
||||
|
||||
func indexOf(haystack, needle string) int {
|
||||
for i := 0; i+len(needle) <= len(haystack); i++ {
|
||||
if haystack[i:i+len(needle)] == needle {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
@@ -36,8 +36,11 @@ type ServiceHealth struct {
|
||||
ObservedAt time.Time `json:"observed_at"`
|
||||
LastSuccessAt *time.Time `json:"last_success_at"`
|
||||
ConsecutiveFailures int `json:"consecutive_failures"`
|
||||
Configured bool `json:"configured"`
|
||||
Applicable bool `json:"applicable"`
|
||||
// ProbeTarget names the object being probed (instance + short binding id),
|
||||
// never credentials or full UUIDs, so a red light can be attributed.
|
||||
ProbeTarget string `json:"probe_target"`
|
||||
Configured bool `json:"configured"`
|
||||
Applicable bool `json:"applicable"`
|
||||
}
|
||||
|
||||
type KnowledgeBaseProgress struct {
|
||||
|
||||
@@ -116,6 +116,7 @@ type CatalogRouteRequest struct {
|
||||
// fulfillment path needs to materialize and fetch it later.
|
||||
type CatalogRouteFile struct {
|
||||
NodeID string
|
||||
ParentNodeID string
|
||||
Name string
|
||||
RelativePath string
|
||||
SizeBytes *int64
|
||||
|
||||
@@ -148,6 +148,11 @@ func classifyRole(method, path string) string {
|
||||
if strings.HasPrefix(path, "/api/v1/backup-schedule") && method != http.MethodGet {
|
||||
return roleAdmin
|
||||
}
|
||||
// 核对封存结果会把一个 pending 业务作业修成 succeeded;只允许管理员,
|
||||
// 且 handler 仍要求 completed queue、精确版本和归档复核。
|
||||
if strings.HasPrefix(path, "/api/v1/library/jobs/") && strings.HasSuffix(path, "/reconcile-capture") {
|
||||
return roleAdmin
|
||||
}
|
||||
// 版本记录的同步会拉起一次对远端仓库的抓取(消耗网络与数据库写入),
|
||||
// 与"谁能改生产绑定"同级:读任何角色可看,同步只能管理员。
|
||||
if strings.HasPrefix(path, "/api/v1/version-history") && method != http.MethodGet {
|
||||
|
||||
@@ -24,6 +24,9 @@ type BackupScheduleStore interface {
|
||||
type BackupHost interface {
|
||||
Available() bool
|
||||
Target() string
|
||||
// Endpoint 是挂载的 ssh 配置里该别名真实拨号的 host:port;页面据此证明
|
||||
// "连的是哪条通道",而不是只显示一个别名。
|
||||
Endpoint() string
|
||||
Show(context.Context) (backuphost.ShowResult, error)
|
||||
Apply(context.Context, bool, string, int, int) (backuphost.ApplyResult, error)
|
||||
RunNow(context.Context) (backuphost.RunResult, error)
|
||||
@@ -82,8 +85,11 @@ type backupScheduleView struct {
|
||||
// HostConfigured/HostReachable 区分"没接线"与"接线了但连不上"。
|
||||
HostConfigured bool `json:"host_configured"`
|
||||
HostTarget string `json:"host_target"`
|
||||
HostReachable bool `json:"host_reachable"`
|
||||
HostError string `json:"host_error"`
|
||||
// HostEndpoint 是这条别名真实拨号的 host:port(来自挂载的 ssh 配置)。
|
||||
// 连接失败时它就是"到底往哪连"的唯一证据,不能只给一个别名让人猜。
|
||||
HostEndpoint string `json:"host_endpoint"`
|
||||
HostReachable bool `json:"host_reachable"`
|
||||
HostError string `json:"host_error"`
|
||||
|
||||
HostScheduleEnabled bool `json:"host_schedule_enabled"`
|
||||
HostScheduleTime string `json:"host_schedule_time"`
|
||||
@@ -92,6 +98,8 @@ type backupScheduleView struct {
|
||||
HostCronLines []string `json:"host_cron_lines"`
|
||||
HostCronService string `json:"host_cron_service"`
|
||||
HostScriptPresent bool `json:"host_script_present"`
|
||||
// HostCheckedAt 是本次真实主机探测时间;没有探测(只读库)时为空。
|
||||
HostCheckedAt string `json:"host_checked_at,omitempty"`
|
||||
|
||||
LastRunStamp string `json:"last_run_stamp"`
|
||||
LastRunCreatedAt string `json:"last_run_created_at"`
|
||||
@@ -281,6 +289,7 @@ func (config Config) backupSchedulePayload(ctx context.Context, probeHost bool)
|
||||
}
|
||||
view.HostConfigured = config.BackupHost.Available()
|
||||
view.HostTarget = config.BackupHost.Target()
|
||||
view.HostEndpoint = config.BackupHost.Endpoint()
|
||||
if !view.HostConfigured {
|
||||
view.InSync = false
|
||||
view.HostError = "目标主机凭据未挂载(BACKUP_HOST_SSH_CONFIG / BACKUP_HOST_TARGET)"
|
||||
@@ -289,10 +298,14 @@ func (config Config) backupSchedulePayload(ctx context.Context, probeHost bool)
|
||||
if !probeHost {
|
||||
return view, nil
|
||||
}
|
||||
view.HostCheckedAt = formatTime(config.Now().UTC())
|
||||
show, err := config.BackupHost.Show(ctx)
|
||||
if err != nil {
|
||||
view.HostReachable = false
|
||||
view.HostError = err.Error()
|
||||
if view.HostEndpoint != "" {
|
||||
view.HostError += "(拨号地址 " + view.HostEndpoint + ")"
|
||||
}
|
||||
view.InSync = false
|
||||
} else {
|
||||
view.HostReachable = true
|
||||
|
||||
@@ -46,6 +46,7 @@ func (store *fakeBackupScheduleStore) MarkBackupScheduleApplied(_ context.Contex
|
||||
type fakeBackupHost struct {
|
||||
available bool
|
||||
target string
|
||||
endpoint string
|
||||
show backuphost.ShowResult
|
||||
showErr error
|
||||
apply backuphost.ApplyResult
|
||||
@@ -59,6 +60,9 @@ type fakeBackupHost struct {
|
||||
|
||||
func (host *fakeBackupHost) Available() bool { return host.available }
|
||||
func (host *fakeBackupHost) Target() string { return host.target }
|
||||
func (host *fakeBackupHost) Endpoint() string {
|
||||
return host.endpoint
|
||||
}
|
||||
func (host *fakeBackupHost) Show(context.Context) (backuphost.ShowResult, error) {
|
||||
return host.show, host.showErr
|
||||
}
|
||||
@@ -166,6 +170,48 @@ func TestHandleBackupScheduleHostFlagsDrift(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleBackupScheduleHostUnreachableSeparatesConnectionFromHostState(t *testing.T) {
|
||||
// 现场原错:ssh connect to host 120.76.217.51 port 22 Operation timed out。
|
||||
// 连接没建立 ⇒ 既不能报"脚本缺失",也不能报"计划未下发",只能报不可达 + 拨号地址。
|
||||
host := &fakeBackupHost{
|
||||
available: true, target: "weknora-test", endpoint: "100.85.111.121:22",
|
||||
showErr: errors.New("目标主机 weknora-test 不可达(ssh: connect to host 120.76.217.51 port 22: Operation timed out);未执行任何主机侧脚本,脚本与计划状态未知"),
|
||||
}
|
||||
config := backupConfig(&fakeBackupScheduleStore{schedule: goldenSchedule}, host)
|
||||
recorder := httptest.NewRecorder()
|
||||
config.handleBackupScheduleHost(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/backup-schedule/host", nil))
|
||||
data := decodeData(t, recorder)
|
||||
if data["host_reachable"] != false || data["in_sync"] != false {
|
||||
t.Fatalf("连不上时不能报告可达或已同步: %+v", data)
|
||||
}
|
||||
message := data["host_error"].(string)
|
||||
if !strings.Contains(message, "不可达") || !strings.Contains(message, "100.85.111.121:22") {
|
||||
t.Fatalf("应报不可达并给出真实拨号地址: %q", message)
|
||||
}
|
||||
if strings.Contains(message, "脚本缺失") || strings.Contains(message, "未下发") {
|
||||
t.Fatalf("未观测到的脚本/计划状态不能当成结论: %q", message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleBackupScheduleHostReportsMissingScriptOnlyAfterConnecting(t *testing.T) {
|
||||
// 真正的脚本缺失:ssh 已连上,主机回的是脚本不存在 ⇒ 这时才允许说脚本缺失。
|
||||
host := &fakeBackupHost{
|
||||
available: true, target: "weknora-test", endpoint: "100.85.111.121:22",
|
||||
showErr: errors.New("已连上 weknora-test,但主机侧命令失败: bash: /opt/weknora-test/install_backup_schedule.sh: No such file or directory"),
|
||||
}
|
||||
config := backupConfig(&fakeBackupScheduleStore{schedule: goldenSchedule}, host)
|
||||
recorder := httptest.NewRecorder()
|
||||
config.handleBackupScheduleHost(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/backup-schedule/host", nil))
|
||||
data := decodeData(t, recorder)
|
||||
message := data["host_error"].(string)
|
||||
if !strings.Contains(message, "已连上") || !strings.Contains(message, "No such file") {
|
||||
t.Fatalf("已连接但脚本缺失时应原样带回主机原因: %q", message)
|
||||
}
|
||||
if strings.Contains(message, "不可达") {
|
||||
t.Fatalf("已连上就不能说不可达: %q", message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleBackupScheduleHostUnconfiguredDegrades(t *testing.T) {
|
||||
config := backupConfig(&fakeBackupScheduleStore{schedule: goldenSchedule}, &fakeBackupHost{available: false})
|
||||
recorder := httptest.NewRecorder()
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/jobs"
|
||||
"knowledge-governance/internal/librarysnapshot"
|
||||
)
|
||||
|
||||
// libraryCaptureReconciler is deliberately narrow: it repairs one business
|
||||
// seal job only after the completed capture queue, the successful export stage,
|
||||
// the exact test-sealed version and the archived bytes all agree. It never
|
||||
// captures, freezes writers, creates snapshots, writes source/target content or
|
||||
// changes historical failure evidence.
|
||||
type libraryCaptureReconciler interface {
|
||||
LibraryJob(ctx context.Context, id string) (governance.LibraryJob, error)
|
||||
LibraryCaptureExportEvidence(ctx context.Context, jobID string) (string, string, error)
|
||||
LibraryTestSealedVersionForJob(ctx context.Context, scope, jobID string) (governance.LibraryVersion, bool, error)
|
||||
EnsureLibraryOperation(ctx context.Context, op governance.LibraryOperation) (governance.LibraryOperation, bool, error)
|
||||
ReconcileLibraryCaptureTerminal(ctx context.Context, jobID, scope, queueJobID, snapshotID string, now time.Time) (bool, error)
|
||||
ReconcileMisclassifiedLibraryCapture(ctx context.Context, jobID, scope, queueJobID, snapshotID, expectedStage, expectedError, expectedRecovery string, expectedFinished *time.Time, auditID, actor, reason string, evidence map[string]any, now time.Time) (bool, error)
|
||||
}
|
||||
|
||||
const (
|
||||
legacyOrphanError = "队列中已无对应作业,业务作业未收到结果"
|
||||
legacyOrphanRecovery = "封存作业由 agent 恢复来源;确认来源 healthy 后可重发"
|
||||
)
|
||||
|
||||
type libraryCaptureQueueReader interface {
|
||||
QueueJob(ctx context.Context, id string) (jobs.Job, bool, error)
|
||||
}
|
||||
|
||||
func libraryCaptureReconcile(w http.ResponseWriter, r *http.Request, config Config) {
|
||||
if r.Method != http.MethodPost {
|
||||
writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"})
|
||||
return
|
||||
}
|
||||
store, ok := config.Store.(libraryCaptureReconciler)
|
||||
if !ok {
|
||||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"available": false,
|
||||
"reason": "封存作业核对存储不可用"})
|
||||
return
|
||||
}
|
||||
queueStore, ok := config.Store.(libraryCaptureQueueReader)
|
||||
if !ok {
|
||||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"available": false,
|
||||
"reason": "队列精确读取不可用"})
|
||||
return
|
||||
}
|
||||
jobID := strings.TrimSpace(r.PathValue("id"))
|
||||
if jobID == "" {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "job id required"})
|
||||
return
|
||||
}
|
||||
job, err := store.LibraryJob(r.Context(), jobID)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusNotFound, map[string]any{"available": false, "reason": "作业不存在"})
|
||||
return
|
||||
}
|
||||
if job.Kind != governance.LibraryJobSeal {
|
||||
conflict(w, "只有封存作业可以核对封存结果")
|
||||
return
|
||||
}
|
||||
systemMisclassified := job.State == governance.LibraryJobFailed && job.Stage == "" &&
|
||||
job.Error == legacyOrphanError && job.Recovery == legacyOrphanRecovery && job.SnapshotID == ""
|
||||
if job.State != governance.LibraryJobPending && job.State != governance.LibraryJobRunning &&
|
||||
job.State != governance.LibraryJobSucceeded && !systemMisclassified {
|
||||
conflict(w, "作业是真实失败/取消终态,不能改写成成功")
|
||||
return
|
||||
}
|
||||
if job.QueueJobID == "" {
|
||||
conflict(w, "作业没有关联队列号,不能证明封存完成")
|
||||
return
|
||||
}
|
||||
captureJob := strings.TrimPrefix(job.ID, "libseal-")
|
||||
if captureJob == "" || captureJob == job.ID {
|
||||
conflict(w, "作业号不是封存作业身份")
|
||||
return
|
||||
}
|
||||
queue, found, err := queueStore.QueueJob(r.Context(), job.QueueJobID)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot read capture queue"})
|
||||
return
|
||||
}
|
||||
if !found {
|
||||
conflict(w, "队列作业不存在,不能证明封存完成")
|
||||
return
|
||||
}
|
||||
if queue.Type != libraryCaptureJobType || queue.State != jobs.StateCompleted {
|
||||
conflict(w, "队列作业不是 completed 的 library.capture")
|
||||
return
|
||||
}
|
||||
var payload struct {
|
||||
Scope string `json:"scope"`
|
||||
CaptureJob string `json:"capture_job"`
|
||||
}
|
||||
if err := json.Unmarshal(queue.Payload, &payload); err != nil ||
|
||||
payload.Scope != job.Scope || payload.CaptureJob != captureJob ||
|
||||
queue.IdempotencyKey != "library.capture:"+captureJob {
|
||||
conflict(w, "队列作业的 scope/capture_job/幂等键与业务作业不一致")
|
||||
return
|
||||
}
|
||||
stageState, _, err := store.LibraryCaptureExportEvidence(r.Context(), job.ID)
|
||||
if err != nil || stageState != "succeeded" {
|
||||
conflict(w, "导出阶段没有成功证据,不能修复终态")
|
||||
return
|
||||
}
|
||||
version, found, err := store.LibraryTestSealedVersionForJob(r.Context(), job.Scope, job.ID)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot read exact test-sealed version"})
|
||||
return
|
||||
}
|
||||
if !found || version.Kind != governance.LibraryVersionTestSealed ||
|
||||
version.Scope != job.Scope || version.SourceJobID != job.ID || version.SnapshotID == "" {
|
||||
conflict(w, "没有与作业/scope 精确绑定的测试封存版本")
|
||||
return
|
||||
}
|
||||
source := configuredLibrarySource(config, job.Scope)
|
||||
sourceID, err := source.ID()
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"available": false,
|
||||
"reason": "来源身份配置不完整,无法核对封存来源"})
|
||||
return
|
||||
}
|
||||
evidenceSource, _ := version.Evidence["source_id"].(string)
|
||||
evidenceCapture, _ := version.Evidence["capture_job"].(string)
|
||||
if evidenceSource != sourceID || evidenceCapture != captureJob {
|
||||
conflict(w, "版本来源或 capture_job 与当前作业不一致")
|
||||
return
|
||||
}
|
||||
if config.LibraryArchiveRoot == "" {
|
||||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"available": false,
|
||||
"reason": "未接线归档根,无法复核归档完整性"})
|
||||
return
|
||||
}
|
||||
archive, err := librarysnapshot.Open(config.LibraryArchiveRoot, 256*1024*1024)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"available": false,
|
||||
"reason": "归档不可读,不能确认封存完整"})
|
||||
return
|
||||
}
|
||||
defer archive.Close()
|
||||
manifest, err := archive.Load(version.SnapshotID)
|
||||
if err != nil {
|
||||
conflict(w, "归档字节复核失败,不能修复作业")
|
||||
return
|
||||
}
|
||||
if manifest.Scope != source.Scope || manifest.TenantID != source.TenantID ||
|
||||
manifest.KnowledgeBaseID != source.KnowledgeBaseID {
|
||||
conflict(w, "归档清单来源身份与当前来源不一致")
|
||||
return
|
||||
}
|
||||
if job.SnapshotID != "" && job.SnapshotID != version.SnapshotID {
|
||||
conflict(w, "作业已有不同快照,不能覆盖")
|
||||
return
|
||||
}
|
||||
if job.State == governance.LibraryJobSucceeded {
|
||||
if job.SnapshotID != version.SnapshotID {
|
||||
conflict(w, "作业已成功但快照与精确版本不一致")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"repaired": false, "already_succeeded": true, "job_id": job.ID,
|
||||
"snapshot_id": version.SnapshotID, "queue_job_id": queue.ID,
|
||||
"version_id": version.ID,
|
||||
})
|
||||
return
|
||||
}
|
||||
actor := ""
|
||||
if identity, ok := sessionFrom(r.Context()); ok {
|
||||
actor = identity.Username
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
digest, err := governance.LibraryRequestDigest(job.Scope, governance.LibraryJobRecover, map[string]any{
|
||||
"kind": "capture_reconcile", "job_id": job.ID, "queue_job_id": queue.ID,
|
||||
"snapshot_id": version.SnapshotID, "capture_job": captureJob, "stage": "export",
|
||||
})
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot build reconcile audit digest"})
|
||||
return
|
||||
}
|
||||
if systemMisclassified {
|
||||
auditID := "librepair-" + digest[:16]
|
||||
updated, repairErr := store.ReconcileMisclassifiedLibraryCapture(r.Context(), job.ID, job.Scope,
|
||||
job.QueueJobID, version.SnapshotID, job.Stage, legacyOrphanError, legacyOrphanRecovery,
|
||||
job.FinishedAt, auditID, actor, "completed capture queue + export succeeded + exact version + archive verified; legacy orphan misclassification", map[string]any{
|
||||
"original_state": job.State, "original_error": job.Error, "original_recovery": job.Recovery,
|
||||
"original_finished_at": job.FinishedAt, "queue_job_id": queue.ID,
|
||||
"snapshot_id": version.SnapshotID, "version_id": version.ID, "archive_verified": true,
|
||||
}, now)
|
||||
if repairErr != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot persist misclassification repair"})
|
||||
return
|
||||
}
|
||||
if !updated {
|
||||
conflict(w, "原误判行状态/错误/完成时间已变化,拒绝覆盖")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"repaired": true, "already_succeeded": false, "misclassification_repaired": true,
|
||||
"job_id": job.ID, "snapshot_id": version.SnapshotID, "queue_job_id": queue.ID,
|
||||
"stage": "export", "version_id": version.ID, "audit_repair_id": auditID,
|
||||
})
|
||||
return
|
||||
}
|
||||
opID := "libop-capture-reconcile-" + digest[:16]
|
||||
op, _, err := store.EnsureLibraryOperation(r.Context(), governance.LibraryOperation{
|
||||
ID: opID, Scope: job.Scope, Kind: governance.LibraryJobRecover,
|
||||
TargetSnapshotID: version.SnapshotID, RequestDigest: digest,
|
||||
OperationKey: "capture-reconcile:" + job.ID, Actor: actor, CreatedAt: now,
|
||||
})
|
||||
if errors.Is(err, governance.ErrLibraryOperationConflict) {
|
||||
conflict(w, "同一核对键对应了不同证据,拒绝修复")
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot record reconcile audit"})
|
||||
return
|
||||
}
|
||||
updated, err := store.ReconcileLibraryCaptureTerminal(r.Context(), job.ID, job.Scope, job.QueueJobID, version.SnapshotID, now)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]any{
|
||||
"error": "cannot persist reconciled job terminal", "audit_operation_id": op.ID,
|
||||
})
|
||||
return
|
||||
}
|
||||
if !updated {
|
||||
conflict(w, "作业状态/身份在核对期间变化,拒绝覆盖")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"repaired": true, "already_succeeded": false, "job_id": job.ID,
|
||||
"snapshot_id": version.SnapshotID, "queue_job_id": queue.ID,
|
||||
"stage": "export", "version_id": version.ID, "audit_operation_id": op.ID,
|
||||
})
|
||||
}
|
||||
|
||||
func conflict(w http.ResponseWriter, reason string) {
|
||||
writeJSON(w, http.StatusConflict, map[string]any{"available": false, "repaired": false, "reason": reason})
|
||||
}
|
||||
@@ -0,0 +1,361 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/jobs"
|
||||
"knowledge-governance/internal/librarysnapshot"
|
||||
)
|
||||
|
||||
func (f *fakeLibraryStore) QueueJob(_ context.Context, id string) (jobs.Job, bool, error) {
|
||||
if f.queue == nil || f.queue.ID != id {
|
||||
return jobs.Job{}, false, nil
|
||||
}
|
||||
return *f.queue, true, nil
|
||||
}
|
||||
|
||||
func (f *fakeLibraryStore) LibraryCaptureExportEvidence(_ context.Context, _ string) (string, string, error) {
|
||||
return f.stageState, "", f.stageErr
|
||||
}
|
||||
|
||||
func (f *fakeLibraryStore) LibraryJobStage(_ context.Context, _, stage string) (string, string, error) {
|
||||
if f.stageErr != nil {
|
||||
return "", "", f.stageErr
|
||||
}
|
||||
if stage != "export" {
|
||||
return "", "", errors.New("unexpected stage")
|
||||
}
|
||||
return f.stageState, "", nil
|
||||
}
|
||||
|
||||
func (f *fakeLibraryStore) LibraryTestSealedVersionForJob(_ context.Context, scope, jobID string) (governance.LibraryVersion, bool, error) {
|
||||
if f.versionErr != nil {
|
||||
return governance.LibraryVersion{}, false, f.versionErr
|
||||
}
|
||||
if f.sealedVersion == nil || f.sealedVersion.Scope != scope || f.sealedVersion.SourceJobID != jobID {
|
||||
return governance.LibraryVersion{}, false, nil
|
||||
}
|
||||
return *f.sealedVersion, true, nil
|
||||
}
|
||||
|
||||
func reconcileFixture(t *testing.T) (*fakeLibraryStore, Config, governance.LibraryJob, []byte) {
|
||||
t.Helper()
|
||||
root := libraryFixtureRoot(t)
|
||||
snapshotID := sealSharedVersions(t, root, 1, 4)[0]
|
||||
archive, err := librarysnapshot.Open(root, 256*1024*1024)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
manifest, err := archive.Load(snapshotID)
|
||||
archive.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
config := Config{LibraryArchiveRoot: root,
|
||||
LibrarySourceProject: "fixture-project", LibrarySourceApp: "fixture-app",
|
||||
LibrarySourceTenantID: manifest.TenantID, LibrarySourceInternalKB: manifest.KnowledgeBaseID}
|
||||
source := configuredLibrarySource(config, "internal")
|
||||
sourceID, err := source.ID()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
job := governance.LibraryJob{ID: "libseal-cap-1", Scope: "internal", Kind: governance.LibraryJobSeal,
|
||||
State: governance.LibraryJobPending, QueueJobID: "libcap-1"}
|
||||
payload, _ := json.Marshal(map[string]string{"scope": "internal", "capture_job": "cap-1"})
|
||||
store := &fakeLibraryStore{
|
||||
jobs: []governance.LibraryJob{job},
|
||||
stageState: "succeeded",
|
||||
queue: &jobs.Job{ID: "libcap-1", Type: libraryCaptureJobType, State: jobs.StateCompleted,
|
||||
Payload: payload, IdempotencyKey: "library.capture:cap-1"},
|
||||
sealedVersion: &governance.LibraryVersion{ID: "libver-1", Scope: "internal",
|
||||
Kind: governance.LibraryVersionTestSealed, SnapshotID: snapshotID, SourceJobID: job.ID,
|
||||
Evidence: map[string]any{"source_id": sourceID, "capture_job": "cap-1"}},
|
||||
}
|
||||
config.Store = store
|
||||
return store, config, job, payload
|
||||
}
|
||||
|
||||
func (f *fakeLibraryStore) ReconcileLibraryCaptureTerminal(_ context.Context, jobID, scope, queueJobID, snapshotID string, _ time.Time) (bool, error) {
|
||||
if f.reconcileDenied {
|
||||
return false, nil
|
||||
}
|
||||
for index := range f.jobs {
|
||||
if f.jobs[index].ID != jobID || f.jobs[index].Scope != scope || f.jobs[index].Kind != governance.LibraryJobSeal ||
|
||||
f.jobs[index].QueueJobID != queueJobID || (f.jobs[index].State != governance.LibraryJobPending && f.jobs[index].State != governance.LibraryJobRunning) ||
|
||||
(f.jobs[index].SnapshotID != "" && f.jobs[index].SnapshotID != snapshotID) {
|
||||
return false, nil
|
||||
}
|
||||
f.jobs[index].State = governance.LibraryJobSucceeded
|
||||
f.jobs[index].SnapshotID = snapshotID
|
||||
f.lastState = governance.LibraryJobSucceeded
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (f *fakeLibraryStore) ReconcileMisclassifiedLibraryCapture(_ context.Context, jobID, scope, queueJobID, snapshotID, expectedStage, expectedError, expectedRecovery string, expectedFinished *time.Time, _ string, _ string, _ string, _ map[string]any, _ time.Time) (bool, error) {
|
||||
if f.reconcileDenied {
|
||||
return false, nil
|
||||
}
|
||||
for index := range f.jobs {
|
||||
j := &f.jobs[index]
|
||||
if j.ID != jobID || j.Scope != scope || j.Kind != governance.LibraryJobSeal || j.QueueJobID != queueJobID ||
|
||||
j.State != governance.LibraryJobFailed || j.Stage != expectedStage || j.Error != expectedError ||
|
||||
j.Recovery != expectedRecovery || j.SnapshotID != "" || (j.FinishedAt == nil) != (expectedFinished == nil) ||
|
||||
(j.FinishedAt != nil && expectedFinished != nil && !j.FinishedAt.Equal(*expectedFinished)) {
|
||||
return false, nil
|
||||
}
|
||||
j.State = governance.LibraryJobSucceeded
|
||||
j.SnapshotID = snapshotID
|
||||
j.Stage = "export"
|
||||
f.lastState = governance.LibraryJobSucceeded
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
var _ libraryCaptureReconciler = (*fakeLibraryStore)(nil)
|
||||
|
||||
func runReconcile(t *testing.T, config Config, jobID string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/library/jobs/"+jobID+"/reconcile-capture", nil)
|
||||
req.SetPathValue("id", jobID)
|
||||
rec := httptest.NewRecorder()
|
||||
libraryCaptureReconcile(rec, req, config)
|
||||
return rec
|
||||
}
|
||||
|
||||
func TestLibraryCaptureReconcileRepairsExactCompletedCapture(t *testing.T) {
|
||||
store, config, job, _ := reconcileFixture(t)
|
||||
rec := runReconcile(t, config, job.ID)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("code=%d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var body map[string]any
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if body["repaired"] != true || body["snapshot_id"] == "" || body["audit_operation_id"] == "" {
|
||||
t.Fatalf("body=%v", body)
|
||||
}
|
||||
if store.lastState != governance.LibraryJobSucceeded {
|
||||
t.Fatalf("advanced state=%s", store.lastState)
|
||||
}
|
||||
if len(store.operations) != 1 {
|
||||
t.Fatalf("audit operations=%v", store.operations)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryCaptureReconcileAlreadySucceededIsReadOnly(t *testing.T) {
|
||||
store, config, job, _ := reconcileFixture(t)
|
||||
job.State = governance.LibraryJobSucceeded
|
||||
job.SnapshotID = store.sealedVersion.SnapshotID
|
||||
store.jobs[0] = job
|
||||
rec := runReconcile(t, config, job.ID)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("code=%d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var body map[string]any
|
||||
_ = json.Unmarshal(rec.Body.Bytes(), &body)
|
||||
if body["already_succeeded"] != true || body["repaired"] != false || store.lastState != "" {
|
||||
t.Fatalf("body=%v advanced=%q", body, store.lastState)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryCaptureReconcileRejectsQueueNotCompleted(t *testing.T) {
|
||||
store, config, job, _ := reconcileFixture(t)
|
||||
store.queue.State = jobs.StateRunning
|
||||
rec := runReconcile(t, config, job.ID)
|
||||
if rec.Code != http.StatusConflict || store.lastState != "" {
|
||||
t.Fatalf("code=%d advanced=%q body=%s", rec.Code, store.lastState, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryCaptureReconcileRejectsStageNotSucceeded(t *testing.T) {
|
||||
store, config, job, _ := reconcileFixture(t)
|
||||
store.stageState = "failed"
|
||||
rec := runReconcile(t, config, job.ID)
|
||||
if rec.Code != http.StatusConflict || store.lastState != "" {
|
||||
t.Fatalf("code=%d advanced=%q", rec.Code, store.lastState)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryCaptureReconcileRejectsWrongVersionSource(t *testing.T) {
|
||||
store, config, job, _ := reconcileFixture(t)
|
||||
store.sealedVersion.Evidence["source_id"] = "other-source"
|
||||
rec := runReconcile(t, config, job.ID)
|
||||
if rec.Code != http.StatusConflict || store.lastState != "" {
|
||||
t.Fatalf("code=%d advanced=%q", rec.Code, store.lastState)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryCaptureReconcileRejectsWrongCaptureJob(t *testing.T) {
|
||||
store, config, job, _ := reconcileFixture(t)
|
||||
store.sealedVersion.Evidence["capture_job"] = "other-capture"
|
||||
rec := runReconcile(t, config, job.ID)
|
||||
if rec.Code != http.StatusConflict || store.lastState != "" {
|
||||
t.Fatalf("code=%d advanced=%q", rec.Code, store.lastState)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryCaptureReconcileRejectsBadArchive(t *testing.T) {
|
||||
store, config, job, _ := reconcileFixture(t)
|
||||
store.sealedVersion.SnapshotID = "missing-snapshot"
|
||||
rec := runReconcile(t, config, job.ID)
|
||||
if rec.Code != http.StatusConflict || store.lastState != "" {
|
||||
t.Fatalf("code=%d advanced=%q body=%s", rec.Code, store.lastState, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryCaptureReconcileRejectsFailedJob(t *testing.T) {
|
||||
store, config, job, _ := reconcileFixture(t)
|
||||
job.State = governance.LibraryJobFailed
|
||||
store.jobs[0] = job
|
||||
rec := runReconcile(t, config, job.ID)
|
||||
if rec.Code != http.StatusConflict || store.lastState != "" {
|
||||
t.Fatalf("code=%d advanced=%q", rec.Code, store.lastState)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryCaptureReconcileIsIdempotent(t *testing.T) {
|
||||
store, config, job, _ := reconcileFixture(t)
|
||||
first := runReconcile(t, config, job.ID)
|
||||
if first.Code != http.StatusOK {
|
||||
t.Fatalf("first code=%d body=%s", first.Code, first.Body.String())
|
||||
}
|
||||
second := runReconcile(t, config, job.ID)
|
||||
if second.Code != http.StatusOK {
|
||||
t.Fatalf("second code=%d body=%s", second.Code, second.Body.String())
|
||||
}
|
||||
var body map[string]any
|
||||
_ = json.Unmarshal(second.Body.Bytes(), &body)
|
||||
if body["already_succeeded"] != true || len(store.operations) != 1 {
|
||||
t.Fatalf("body=%v ops=%v", body, store.operations)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryCaptureReconcileRejectsMissingQueue(t *testing.T) {
|
||||
store, config, job, _ := reconcileFixture(t)
|
||||
store.queue = nil
|
||||
rec := runReconcile(t, config, job.ID)
|
||||
if rec.Code != http.StatusConflict || store.lastState != "" {
|
||||
t.Fatalf("code=%d advanced=%q", rec.Code, store.lastState)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryCaptureReconcileRejectsQueueScope(t *testing.T) {
|
||||
store, config, job, _ := reconcileFixture(t)
|
||||
store.queue.Payload, _ = json.Marshal(map[string]string{"scope": "external", "capture_job": "cap-1"})
|
||||
rec := runReconcile(t, config, job.ID)
|
||||
if rec.Code != http.StatusConflict || store.lastState != "" {
|
||||
t.Fatalf("code=%d advanced=%q", rec.Code, store.lastState)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryCaptureReconcileRejectsMissingVersion(t *testing.T) {
|
||||
store, config, job, _ := reconcileFixture(t)
|
||||
store.sealedVersion = nil
|
||||
rec := runReconcile(t, config, job.ID)
|
||||
if rec.Code != http.StatusConflict || store.lastState != "" {
|
||||
t.Fatalf("code=%d advanced=%q", rec.Code, store.lastState)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryCaptureReconcileRejectsWrongScopeVersion(t *testing.T) {
|
||||
store, config, job, _ := reconcileFixture(t)
|
||||
store.sealedVersion.Scope = "external"
|
||||
rec := runReconcile(t, config, job.ID)
|
||||
if rec.Code != http.StatusConflict || store.lastState != "" {
|
||||
t.Fatalf("code=%d advanced=%q", rec.Code, store.lastState)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryCaptureReconcileRejectsDifferentExistingSnapshot(t *testing.T) {
|
||||
store, config, job, _ := reconcileFixture(t)
|
||||
job.SnapshotID = "other-snapshot"
|
||||
store.jobs[0] = job
|
||||
rec := runReconcile(t, config, job.ID)
|
||||
if rec.Code != http.StatusConflict || store.lastState != "" {
|
||||
t.Fatalf("code=%d advanced=%q body=%s", rec.Code, store.lastState, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryCaptureReconcileConditionalWriteRejectsConcurrentState(t *testing.T) {
|
||||
store, config, job, _ := reconcileFixture(t)
|
||||
store.reconcileDenied = true
|
||||
rec := runReconcile(t, config, job.ID)
|
||||
if rec.Code != http.StatusConflict || store.lastState != "" {
|
||||
t.Fatalf("code=%d advanced=%q body=%s", rec.Code, store.lastState, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryCaptureReconcileRepairsExactLegacyMisclassification(t *testing.T) {
|
||||
store, config, job, _ := reconcileFixture(t)
|
||||
finished := time.Now().UTC()
|
||||
job.State = governance.LibraryJobFailed
|
||||
job.Stage = ""
|
||||
job.Error = legacyOrphanError
|
||||
job.Recovery = legacyOrphanRecovery
|
||||
job.FinishedAt = &finished
|
||||
store.jobs[0] = job
|
||||
rec := runReconcile(t, config, job.ID)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("code=%d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var body map[string]any
|
||||
_ = json.Unmarshal(rec.Body.Bytes(), &body)
|
||||
if body["misclassification_repaired"] != true || store.lastState != governance.LibraryJobSucceeded {
|
||||
t.Fatalf("body=%v advanced=%q", body, store.lastState)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryCaptureReconcileRejectsOtherFailedError(t *testing.T) {
|
||||
store, config, job, _ := reconcileFixture(t)
|
||||
finished := time.Now().UTC()
|
||||
job.State = governance.LibraryJobFailed
|
||||
job.Stage = ""
|
||||
job.Error = "capture export failed"
|
||||
job.Recovery = legacyOrphanRecovery
|
||||
job.FinishedAt = &finished
|
||||
store.jobs[0] = job
|
||||
rec := runReconcile(t, config, job.ID)
|
||||
if rec.Code != http.StatusConflict || store.lastState != "" {
|
||||
t.Fatalf("code=%d advanced=%q body=%s", rec.Code, store.lastState, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryCaptureReconcileRejectsRealFailureAndCancel(t *testing.T) {
|
||||
for _, state := range []governance.LibraryJobState{governance.LibraryJobFailed, governance.LibraryJobCancelled} {
|
||||
store, config, job, _ := reconcileFixture(t)
|
||||
job.State = state
|
||||
job.Error = "real failure"
|
||||
store.jobs[0] = job
|
||||
rec := runReconcile(t, config, job.ID)
|
||||
if rec.Code != http.StatusConflict || store.lastState != "" {
|
||||
t.Fatalf("state=%s code=%d advanced=%q", state, rec.Code, store.lastState)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryCaptureReconcileMisclassificationCASRejectsChange(t *testing.T) {
|
||||
store, config, job, _ := reconcileFixture(t)
|
||||
finished := time.Now().UTC()
|
||||
job.State = governance.LibraryJobFailed
|
||||
job.Stage = ""
|
||||
job.Error = legacyOrphanError
|
||||
job.Recovery = legacyOrphanRecovery
|
||||
job.FinishedAt = &finished
|
||||
store.jobs[0] = job
|
||||
store.reconcileDenied = true
|
||||
rec := runReconcile(t, config, job.ID)
|
||||
if rec.Code != http.StatusConflict || store.lastState != "" {
|
||||
t.Fatalf("code=%d advanced=%q", rec.Code, store.lastState)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/librarysnapshot"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
)
|
||||
|
||||
// Vertical PostgreSQL path: the real worker startup reconciliation must leave
|
||||
// a pending seal with a completed capture queue intact, then the actual HTTP
|
||||
// handler must verify archive/version/stage and repair it. This is not a fake
|
||||
// store or a direct helper test.
|
||||
func verticalReconcileFixture(t *testing.T) (*postgresstore.Store, Config, string, *pgxpool.Pool) {
|
||||
t.Helper()
|
||||
dsn := os.Getenv("KG_LIBRARY_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("requires explicitly isolated kg_library_test_ PostgreSQL database")
|
||||
}
|
||||
cfg, err := pgxpool.ParseConfig(dsn)
|
||||
if err != nil || !strings.HasPrefix(cfg.ConnConfig.Database, "kg_library_test_") || os.Getenv("KG_LIBRARY_TEST_ISOLATED") != "1" {
|
||||
t.Fatal("isolated database required")
|
||||
}
|
||||
cfg.MaxConns = 1
|
||||
cfg.ConnConfig.RuntimeParams["search_path"] = "pg_temp"
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.NewWithConfig(ctx, cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(pool.Close)
|
||||
if _, err := pool.Exec(ctx, `
|
||||
CREATE TEMP TABLE library_jobs(
|
||||
id varchar(64) PRIMARY KEY, scope varchar(16) NOT NULL, kind varchar(16) NOT NULL,
|
||||
state varchar(16) NOT NULL, stage varchar(32) NOT NULL DEFAULT '',
|
||||
idempotency_key varchar(255) NOT NULL DEFAULT '', request_digest varchar(64) NOT NULL DEFAULT '',
|
||||
queue_job_id varchar(64) NOT NULL DEFAULT '', snapshot_id varchar(64) NOT NULL DEFAULT '',
|
||||
target_snapshot_id varchar(64) NOT NULL DEFAULT '', actor varchar(128) NOT NULL DEFAULT '',
|
||||
error text NOT NULL DEFAULT '', recovery varchar(255) NOT NULL DEFAULT '',
|
||||
operation_id varchar(64) NOT NULL DEFAULT '', attempt_epoch integer NOT NULL DEFAULT 0,
|
||||
created_at timestamptz NOT NULL, updated_at timestamptz NOT NULL, finished_at timestamptz,
|
||||
target_id text NOT NULL DEFAULT '');
|
||||
CREATE TEMP TABLE library_job_stages(
|
||||
job_id varchar(64) NOT NULL, stage varchar(32) NOT NULL, state varchar(16) NOT NULL,
|
||||
total bigint, completed bigint, bytes_total bigint, bytes_done bigint,
|
||||
detail text NOT NULL DEFAULT '', updated_at timestamptz NOT NULL, PRIMARY KEY(job_id,stage));
|
||||
CREATE TEMP TABLE library_versions(
|
||||
id varchar(64) PRIMARY KEY, scope varchar(16) NOT NULL, kind varchar(24) NOT NULL,
|
||||
snapshot_id varchar(64) NOT NULL DEFAULT '', version_label varchar(64) NOT NULL DEFAULT '',
|
||||
source_job_id varchar(64) NOT NULL DEFAULT '', operation_id varchar(64) NOT NULL DEFAULT '',
|
||||
attempt_epoch integer NOT NULL DEFAULT 0, evidence jsonb NOT NULL DEFAULT '{}'::jsonb,
|
||||
created_by varchar(128) NOT NULL DEFAULT '', created_at timestamptz NOT NULL, superseded_at timestamptz);
|
||||
CREATE TEMP TABLE library_job_repairs(
|
||||
id varchar(64) PRIMARY KEY, job_id varchar(64) NOT NULL, scope varchar(16) NOT NULL,
|
||||
kind varchar(16) NOT NULL, queue_job_id varchar(64) NOT NULL, snapshot_id varchar(64) NOT NULL,
|
||||
original_state varchar(16) NOT NULL, original_stage varchar(32) NOT NULL DEFAULT '',
|
||||
original_error text NOT NULL DEFAULT '', original_recovery varchar(255) NOT NULL DEFAULT '',
|
||||
original_finished_at timestamptz, reason varchar(255) NOT NULL, actor varchar(128) NOT NULL DEFAULT '',
|
||||
evidence jsonb NOT NULL DEFAULT '{}'::jsonb, created_at timestamptz NOT NULL);
|
||||
CREATE TEMP TABLE library_operations(
|
||||
id varchar(64) PRIMARY KEY, scope varchar(16) NOT NULL, kind varchar(16) NOT NULL,
|
||||
target_snapshot_id varchar(64) NOT NULL DEFAULT '', request_digest varchar(64) NOT NULL,
|
||||
operation_key varchar(255) NOT NULL, actor varchar(128) NOT NULL DEFAULT '',
|
||||
created_at timestamptz NOT NULL, target_id text NOT NULL DEFAULT '',
|
||||
first_backup_snapshot_id varchar(64), first_backup_bytes bigint, first_backup_at timestamptz,
|
||||
writeset_digest varchar(64), writeset_rows bigint, target_fingerprint varchar(128), anchors_at timestamptz);
|
||||
CREATE UNIQUE INDEX library_operations_key_test ON library_operations(scope,kind,operation_key);
|
||||
CREATE TEMP TABLE jobs(
|
||||
id varchar(64) PRIMARY KEY, type varchar(64) NOT NULL, payload jsonb NOT NULL,
|
||||
state varchar(16) NOT NULL, priority integer NOT NULL DEFAULT 0, available_at timestamptz NOT NULL,
|
||||
lease_owner varchar(128), lease_until timestamptz, attempt integer NOT NULL DEFAULT 0,
|
||||
max_attempts integer NOT NULL DEFAULT 1, last_error text, idempotency_key varchar(255) NOT NULL,
|
||||
created_at timestamptz NOT NULL, updated_at timestamptz NOT NULL);`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
root := libraryFixtureRoot(t)
|
||||
snapshotID := sealSharedVersions(t, root, 1, 4)[0]
|
||||
archive, err := librarysnapshot.Open(root, 256*1024*1024)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
manifest, err := archive.Load(snapshotID)
|
||||
archive.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
config := Config{LibraryArchiveRoot: root,
|
||||
LibrarySourceProject: "fixture-project", LibrarySourceApp: "fixture-app",
|
||||
LibrarySourceTenantID: manifest.TenantID, LibrarySourceInternalKB: manifest.KnowledgeBaseID}
|
||||
source := configuredLibrarySource(config, "internal")
|
||||
sourceID, err := source.ID()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
payload, _ := json.Marshal(map[string]string{"scope": "internal", "capture_job": "vertical"})
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,available_at,idempotency_key,created_at,updated_at)
|
||||
VALUES('q-vertical','library.capture',$1,'completed',$2,'library.capture:vertical',$2,$2)`, payload, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO library_versions(id,scope,kind,snapshot_id,source_job_id,evidence,created_at)
|
||||
VALUES('ver-vertical','internal','test_sealed',$1,'libseal-vertical',$2,$3)`, snapshotID,
|
||||
map[string]any{"source_id": sourceID, "capture_job": "vertical"}, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO library_job_stages(job_id,stage,state,detail,updated_at)
|
||||
VALUES('libseal-vertical','export','succeeded','已封存快照', $1)`, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := postgresstore.NewStore(pool)
|
||||
config.Store = store
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO library_jobs(id,scope,kind,state,idempotency_key,request_digest,queue_job_id,created_at,updated_at)
|
||||
VALUES('libseal-vertical','internal','seal','pending','k','d','q-vertical',$1,$1)`, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return store, config, snapshotID, pool
|
||||
}
|
||||
|
||||
func TestStartupReconcileThenHandlerRepairsPendingCapture(t *testing.T) {
|
||||
store, config, _, _ := verticalReconcileFixture(t)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
if n, err := store.ReconcileLibraryJobs(ctx, now, []string{"seal"}); err != nil || n != 0 {
|
||||
t.Fatalf("startup reconcile n=%d err=%v", n, err)
|
||||
}
|
||||
job, err := store.LibraryJob(ctx, "libseal-vertical")
|
||||
if err != nil || job.State != governance.LibraryJobPending {
|
||||
t.Fatalf("startup changed job: %+v err=%v", job, err)
|
||||
}
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/library/jobs/libseal-vertical/reconcile-capture", nil)
|
||||
req.SetPathValue("id", "libseal-vertical")
|
||||
rec := httptest.NewRecorder()
|
||||
libraryCaptureReconcile(rec, req, config)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("handler code=%d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
job, err = store.LibraryJob(ctx, "libseal-vertical")
|
||||
if err != nil || job.State != governance.LibraryJobSucceeded || job.SnapshotID == "" {
|
||||
t.Fatalf("handler did not repair: %+v err=%v", job, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartupReconcilePreservesFailedCapturesForExactReconcile(t *testing.T) {
|
||||
store, config, snapshotID, pool := verticalReconcileFixture(t)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
source := configuredLibrarySource(config, "internal")
|
||||
sourceID, err := source.ID()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
payload, _ := json.Marshal(map[string]string{"scope": "internal", "capture_job": "vertical-failed"})
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO jobs(id,type,payload,state,available_at,idempotency_key,created_at,updated_at)
|
||||
VALUES('q-failed','library.capture',$1,'completed',$2,'library.capture:vertical-failed',$2,$2)`, payload, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO library_versions(id,scope,kind,snapshot_id,source_job_id,evidence,created_at)
|
||||
VALUES('ver-failed','internal','test_sealed',$1,'libseal-vertical-failed',$2,$3)`,
|
||||
snapshotID, map[string]any{"source_id": sourceID, "capture_job": "vertical-failed"}, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO library_job_stages(job_id,stage,state,detail,updated_at)
|
||||
VALUES('libseal-vertical-failed','export','succeeded','已封存快照', $1)`, now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
finished := now.Truncate(time.Microsecond)
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO library_jobs(id,scope,kind,state,idempotency_key,request_digest,queue_job_id,error,recovery,created_at,updated_at,finished_at)
|
||||
VALUES('libseal-vertical-failed','internal','seal','failed','kf','df','q-failed',$1,$2,$3,$3,$3)`,
|
||||
"队列中已无对应作业,业务作业未收到结果", "封存作业由 agent 恢复来源;确认来源 healthy 后可重发", finished); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n, err := store.ReconcileLibraryJobs(ctx, now, []string{"seal"}); err != nil || n != 0 {
|
||||
t.Fatalf("startup reconcile n=%d err=%v", n, err)
|
||||
}
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/v1/library/jobs/libseal-vertical-failed/reconcile-capture", nil)
|
||||
req.SetPathValue("id", "libseal-vertical-failed")
|
||||
rec := httptest.NewRecorder()
|
||||
libraryCaptureReconcile(rec, req, config)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("handler code=%d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
job, err := store.LibraryJob(ctx, "libseal-vertical-failed")
|
||||
if err != nil || job.State != governance.LibraryJobSucceeded || job.SnapshotID != snapshotID {
|
||||
t.Fatalf("failed misclassification not repaired: %+v err=%v", job, err)
|
||||
}
|
||||
var originalState, originalError string
|
||||
if err := pool.QueryRow(ctx, `SELECT original_state, original_error FROM library_job_repairs WHERE job_id='libseal-vertical-failed'`).
|
||||
Scan(&originalState, &originalError); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if originalState != "failed" || originalError == "" {
|
||||
t.Fatalf("audit missing original: state=%q error=%q", originalState, originalError)
|
||||
}
|
||||
}
|
||||
@@ -342,7 +342,11 @@ type libraryOverview struct {
|
||||
CaptureAvailable bool `json:"capture_available"`
|
||||
CaptureReason string `json:"capture_reason,omitempty"`
|
||||
Executor libraryExecutorView `json:"executor"`
|
||||
Now time.Time `json:"now"`
|
||||
// 实际目标身份:页面不能只显示"weknora-test"或旧文档级绑定名称。
|
||||
TargetInstance string `json:"target_instance,omitempty"`
|
||||
TargetProject string `json:"target_project,omitempty"`
|
||||
TargetKnowledgeBaseID string `json:"target_knowledge_base_id,omitempty"`
|
||||
Now time.Time `json:"now"`
|
||||
}
|
||||
|
||||
// libraryVersionRef 只回答"这个快照被登记成哪一类版本、由哪个作业登记"。
|
||||
@@ -642,9 +646,7 @@ func librarySyncGateWithGrant(config Config, production libraryKnownView, prebac
|
||||
if !config.ProductionReleaseRequestsEnabled {
|
||||
gate.Reasons = append(gate.Reasons, "生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false)")
|
||||
}
|
||||
if !config.ProductionBackupRestoreVerified {
|
||||
gate.Reasons = append(gate.Reasons, "没有已核验的生产恢复演练证据(PRODUCTION_BACKUP_RESTORE_VERIFIED=false)")
|
||||
}
|
||||
gate.Reasons = append(gate.Reasons, restoreGateReasons(config)...)
|
||||
// 生产端点以"整库目标是否配置"为准:整库同步直接写 PG/ES/对象,
|
||||
// 不依赖 WeKnora 的 HTTP 端,所以不能拿它是否配置当门禁。
|
||||
if strings.TrimSpace(config.LibraryProductionDatabaseURL) == "" {
|
||||
@@ -688,10 +690,16 @@ func libraryOverviewHandler(w http.ResponseWriter, r *http.Request, config Confi
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
overview := libraryOverview{Scope: scope, Jobs: []governance.LibraryJob{}, Now: now,
|
||||
TargetInstance: config.LibraryTargetInstance, TargetProject: config.LibraryTargetProject,
|
||||
Gate: libraryGateView{Reasons: []string{}},
|
||||
Diff: libraryDiffView{State: "unknown", Reason: "生产与测试的文档级差异读取未接线", Added: []any{}, Updated: []any{}, Removed: []any{}}}
|
||||
// 可发起封存 = worker 真实探测到执行器可用且指向本实例;原因逐条给全。
|
||||
overview.CaptureAvailable, overview.CaptureReason, overview.Executor = libraryExecutorAvailability(r.Context(), config)
|
||||
if scope == "internal" {
|
||||
overview.TargetKnowledgeBaseID = config.LibraryProductionInternalKB
|
||||
} else {
|
||||
overview.TargetKnowledgeBaseID = config.LibraryProductionExternalKB
|
||||
}
|
||||
|
||||
// 测试封存来自归档 + 版本登记:归档里**不止**测试封存——生产预备份与目标基线
|
||||
// 快照同样带 scope,如果只按"这个 scope 最新一份"挑,页面就会把目标自己的内容
|
||||
@@ -814,7 +822,8 @@ func libraryOverviewHandler(w http.ResponseWriter, r *http.Request, config Confi
|
||||
|
||||
grant, hasGrant := activeGrantFor(r.Context(), config, scope, target)
|
||||
overview.Authorization = authorizationViewOf(grant, hasGrant)
|
||||
overview.Gate = librarySyncGateWithGrant(config, overview.Production, overview.Prebackup, target, hasGrant)
|
||||
// 闸口按当前目标的真实恢复证据判断(没有证据时也接受历史 env 覆盖)。
|
||||
overview.Gate = librarySyncGateWithGrant(config.withRestoreEvidence(r.Context(), scope), overview.Production, overview.Prebackup, target, hasGrant)
|
||||
writeJSON(w, http.StatusOK, overview)
|
||||
}
|
||||
|
||||
@@ -963,7 +972,7 @@ func librarySyncEnqueue(w http.ResponseWriter, r *http.Request, config Config) {
|
||||
return
|
||||
}
|
||||
}
|
||||
gate := librarySyncGateWithGrant(config, overview.Production, overview.Prebackup, request.Target, hasGrant)
|
||||
gate := librarySyncGateWithGrant(config.withRestoreEvidence(r.Context(), request.Scope), overview.Production, overview.Prebackup, request.Target, hasGrant)
|
||||
if !gate.SyncAllowed {
|
||||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{
|
||||
"code": "library_sync_blocked", "reasons": gate.Reasons,
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/jobs"
|
||||
)
|
||||
|
||||
// 生产闸口与整库状态契约的针对性测试:不做数据库集成,只验证
|
||||
@@ -37,6 +38,13 @@ type fakeLibraryStore struct {
|
||||
resumeOK bool
|
||||
resumeErr error
|
||||
resumeCalls []string
|
||||
// capture reconciliation fixtures
|
||||
queue *jobs.Job
|
||||
stageState string
|
||||
stageErr error
|
||||
sealedVersion *governance.LibraryVersion
|
||||
versionErr error
|
||||
reconcileDenied bool
|
||||
}
|
||||
|
||||
func (f *fakeLibraryStore) CreateLibraryJob(_ context.Context, job governance.LibraryJob, _ []string, _ time.Time) (governance.LibraryJob, bool, error) {
|
||||
@@ -127,8 +135,16 @@ func (f *fakeLibraryStore) UpdateLibraryStage(_ context.Context, _, _, _ string,
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeLibraryStore) AdvanceLibraryJob(_ context.Context, _ string, state governance.LibraryJobState, _, _, _, _, _ string, _ time.Time) error {
|
||||
func (f *fakeLibraryStore) AdvanceLibraryJob(_ context.Context, jobID string, state governance.LibraryJobState, _, _, _, snapshotID, _ string, _ time.Time) error {
|
||||
f.lastState = state
|
||||
for index := range f.jobs {
|
||||
if f.jobs[index].ID == jobID {
|
||||
f.jobs[index].State = state
|
||||
if snapshotID != "" {
|
||||
f.jobs[index].SnapshotID = snapshotID
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -266,3 +266,25 @@ func TestLibraryReadPathsRejectTamperedManifest(t *testing.T) {
|
||||
t.Fatalf("storage must fail closed: status=%d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// F11: the HTTP projection must include binding and Wiki log counts; the
|
||||
// previous projection silently omitted them and the UI turned absence into 0.
|
||||
func TestLibrarySnapshotTotalsIncludeBindingsAndWikiLogs(t *testing.T) {
|
||||
view := librarySnapshotViewOfNoUsage("snap-totals", librarysnapshot.Manifest{
|
||||
Scope: "internal", KnowledgeBaseID: "kb", TenantID: 10000,
|
||||
Parts: []librarysnapshot.Part{
|
||||
{Kind: "database", Name: "knowledges", Count: 4},
|
||||
{Kind: "database", Name: "chunks", Count: 38},
|
||||
{Kind: "database", Name: "resources", Count: 21},
|
||||
{Kind: "database", Name: "resource_bindings", Count: 4},
|
||||
{Kind: "wiki", Name: "wiki_pages", Count: 31},
|
||||
{Kind: "wiki", Name: "wiki_folders", Count: 21},
|
||||
{Kind: "wiki", Name: "wiki_log_entries", Count: 11},
|
||||
},
|
||||
})
|
||||
for key, want := range map[string]int64{"documents": 4, "chunks": 38, "resources": 21, "bindings": 4, "wiki_pages": 31, "wiki_folders": 21, "wiki_log_entries": 11} {
|
||||
if view.Totals[key] != want {
|
||||
t.Fatalf("totals[%s]=%d want=%d (%v)", key, view.Totals[key], want, view.Totals)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -71,9 +71,7 @@ func libraryRestoreEnqueue(w http.ResponseWriter, r *http.Request, config Config
|
||||
if !config.ProductionReleaseRequestsEnabled {
|
||||
reasons = append(reasons, "生产写入闸口关闭")
|
||||
}
|
||||
if !config.ProductionBackupRestoreVerified {
|
||||
reasons = append(reasons, "没有已核验的生产恢复演练证据")
|
||||
}
|
||||
reasons = append(reasons, restoreGateReasons(config.withRestoreEvidence(r.Context(), request.Scope))...)
|
||||
authorizationOK := false
|
||||
if storeWithAuth, ok := config.Store.(interface {
|
||||
ActiveLibraryAuthorization(ctx context.Context, scope, snapshotID, digest string, now time.Time) (governance.LibraryReleaseAuthorization, bool, error)
|
||||
@@ -169,9 +167,7 @@ func libraryRestoreAvailability(w http.ResponseWriter, r *http.Request, config C
|
||||
if !config.ProductionReleaseRequestsEnabled {
|
||||
view.Reasons = append(view.Reasons, "生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false)")
|
||||
}
|
||||
if !config.ProductionBackupRestoreVerified {
|
||||
view.Reasons = append(view.Reasons, "没有已核验的生产恢复演练证据(PRODUCTION_BACKUP_RESTORE_VERIFIED=false)")
|
||||
}
|
||||
view.Reasons = append(view.Reasons, restoreGateReasons(config.withRestoreEvidence(r.Context(), scope))...)
|
||||
digest, err := libraryRestoreParamsDigest(scope, snapshotID)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot digest params"})
|
||||
@@ -228,9 +224,7 @@ func libraryJobResume(w http.ResponseWriter, r *http.Request, config Config) {
|
||||
if !config.ProductionReleaseRequestsEnabled {
|
||||
reasons = append(reasons, "生产写入闸口关闭")
|
||||
}
|
||||
if !config.ProductionBackupRestoreVerified {
|
||||
reasons = append(reasons, "没有已核验的生产恢复演练证据")
|
||||
}
|
||||
reasons = append(reasons, restoreGateReasons(config.withRestoreEvidence(r.Context(), source.Scope))...)
|
||||
var digest string
|
||||
switch source.Kind {
|
||||
case governance.LibraryJobSync:
|
||||
|
||||
@@ -261,10 +261,14 @@ func librarySnapshotViewOfNoUsage(id string, manifest librarysnapshot.Manifest)
|
||||
}
|
||||
case "resources":
|
||||
snapshot.Totals["resources"] = part.Count
|
||||
case "resource_bindings":
|
||||
snapshot.Totals["bindings"] = part.Count
|
||||
case "wiki_pages":
|
||||
snapshot.Totals["wiki_pages"] = part.Count
|
||||
case "wiki_folders":
|
||||
snapshot.Totals["wiki_folders"] = part.Count
|
||||
case "wiki_log_entries":
|
||||
snapshot.Totals["wiki_log_entries"] = part.Count
|
||||
}
|
||||
}
|
||||
sort.Slice(snapshot.Parts, func(i, j int) bool {
|
||||
|
||||
@@ -0,0 +1,328 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
// 恢复验证:页面点一次"跑恢复验证",后端建一个持久作业,worker 用与 CLI 同一份实现
|
||||
// (internal/libraryverify)真实执行:捕获目标 → 基线校验 → 写 canary 证明校验有效 →
|
||||
// 按预备份恢复 → 恢复后复核。通过与否只由这条证据表回答,页面不接受手填开关。
|
||||
const libraryVerifyJobType = "library.verify"
|
||||
|
||||
type libraryVerifyRequest struct {
|
||||
Scope string `json:"scope"`
|
||||
}
|
||||
|
||||
// libraryVerificationView 是页面需要的全部事实:当前目标是谁、最近一次验证结果、
|
||||
// 证据是否对当前目标成立、以及同步闸口能不能据此放行。
|
||||
type libraryVerificationView struct {
|
||||
Scope string `json:"scope"`
|
||||
Target libraryVerificationTarget `json:"target"`
|
||||
Verified bool `json:"verified"`
|
||||
Reason string `json:"reason"`
|
||||
Latest *governance.LibraryRestoreVerification `json:"latest,omitempty"`
|
||||
// SnapshotLatest 只在请求显式带 snapshot_id 时返回:快照报告不能拿 scope
|
||||
// 最近一条代替。
|
||||
SnapshotLatest *governance.LibraryRestoreVerification `json:"snapshot_latest,omitempty"`
|
||||
History []governance.LibraryRestoreVerification `json:"history"`
|
||||
Running bool `json:"running"`
|
||||
JobID string `json:"job_id,omitempty"`
|
||||
State string `json:"state,omitempty"`
|
||||
Stage string `json:"stage,omitempty"`
|
||||
}
|
||||
|
||||
type libraryVerificationTarget struct {
|
||||
Instance string `json:"instance"`
|
||||
Project string `json:"project"`
|
||||
InternalKB string `json:"internal_kb"`
|
||||
ExternalKB string `json:"external_kb"`
|
||||
Index string `json:"index"`
|
||||
S3Bucket string `json:"s3_bucket"`
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
DatabaseSet bool `json:"database_configured"`
|
||||
}
|
||||
|
||||
// libraryVerifyEnqueue 投递一次恢复验证作业(管理员)。
|
||||
//
|
||||
// 它不签发发布授权、不改版本:验证只回答"当前目标的预备份能不能恢复"。
|
||||
// 目标是部署配置,不接受请求体指定——否则一次点击就能验证别的库并把证据挂到当前目标上。
|
||||
func libraryVerifyEnqueue(w http.ResponseWriter, r *http.Request, config Config) {
|
||||
if r.Method != http.MethodPost {
|
||||
writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"})
|
||||
return
|
||||
}
|
||||
identity, ok := sessionFrom(r.Context())
|
||||
if !ok || roleRank(identity.Role) < roleRank(roleAdmin) {
|
||||
writeJSON(w, http.StatusForbidden, map[string]any{"error": "恢复验证仅管理员可发起"})
|
||||
return
|
||||
}
|
||||
var request libraryVerifyRequest
|
||||
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&request); err != nil {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"})
|
||||
return
|
||||
}
|
||||
if request.Scope != "internal" && request.Scope != "external" {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "scope must be internal or external"})
|
||||
return
|
||||
}
|
||||
store, ok := libraryStore(config)
|
||||
if !ok {
|
||||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"code": "library_verify_unavailable",
|
||||
"error": "恢复验证存储不可用"})
|
||||
return
|
||||
}
|
||||
// 验证要停写入者并在目标上写 canary:维护中或闸口关闭时直接拒绝,
|
||||
// 不让页面投一个 worker 必然失败的作业。
|
||||
if body, held := instanceMaintenanceGuard(r.Context(), config, request.Scope); held {
|
||||
writeJSON(w, http.StatusConflict, body)
|
||||
return
|
||||
}
|
||||
if !config.ProductionReleaseRequestsEnabled {
|
||||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{
|
||||
"code": "library_verify_blocked",
|
||||
"reasons": []string{
|
||||
"生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false):恢复验证要停写入者并在目标上写 canary",
|
||||
},
|
||||
})
|
||||
return
|
||||
}
|
||||
digest, err := governance.LibraryRequestDigest(request.Scope, governance.LibraryJobVerify,
|
||||
map[string]any{"scope": request.Scope})
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot digest request"})
|
||||
return
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
// 验证作业必须有和同步/恢复同一套不可变 operation:worker 的 owned attempt 从
|
||||
// job.operation_id 取身份,恢复派发闸口和对象网关都按它核验。没有 operation 的
|
||||
// verify 作业在真实恢复时会被 fail closed(审核 B2-R1)。
|
||||
intent, intentErr := ensureLibraryIntent(r.Context(), store, request.Scope, governance.LibraryJobVerify,
|
||||
"", digest, identity.Username, "", now)
|
||||
if intentErr != nil {
|
||||
if isIntentConflict(intentErr) {
|
||||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_verify_conflict",
|
||||
"error": "同一幂等键对应了不同的请求"})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot create library verify operation"})
|
||||
return
|
||||
}
|
||||
job, reused, err := store.CreateLibraryDispatch(r.Context(), governance.LibraryJob{
|
||||
ID: fmt.Sprintf("libverify-%s-%d", request.Scope, now.UnixNano()), Scope: request.Scope,
|
||||
Kind: governance.LibraryJobVerify,
|
||||
IdempotencyKey: intent.JobKey,
|
||||
RequestDigest: digest, OperationID: intent.Operation.ID, Actor: identity.Username,
|
||||
}, governance.VerifyStages(), false, now)
|
||||
if errors.Is(err, governance.ErrLibraryJobActive) {
|
||||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_write_active",
|
||||
"error": "该目标已有在飞的写作业(同步/恢复/基线/验证互斥),等它结束再发起"})
|
||||
return
|
||||
}
|
||||
if errors.Is(err, governance.ErrLibraryJobIdempotencyConflict) {
|
||||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_verify_conflict",
|
||||
"error": "同一幂等键对应了不同的请求"})
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot create library verify job"})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": reused})
|
||||
}
|
||||
|
||||
// libraryVerifications 读当前目标的恢复验证证据(页面轮询用)。
|
||||
func libraryVerifications(w http.ResponseWriter, r *http.Request, config Config) {
|
||||
scope, err := scopeFromQuery(r)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
view := libraryVerificationView{
|
||||
Scope: scope,
|
||||
History: []governance.LibraryRestoreVerification{},
|
||||
Target: libraryVerificationTarget{
|
||||
Instance: config.LibraryTargetInstance,
|
||||
Project: config.LibraryTargetProject,
|
||||
InternalKB: config.LibraryProductionInternalKB,
|
||||
ExternalKB: config.LibraryProductionExternalKB,
|
||||
Index: config.LibraryProductionIndex,
|
||||
S3Bucket: config.LibraryProductionS3Bucket,
|
||||
TenantID: config.LibraryProductionTenantID,
|
||||
DatabaseSet: config.LibraryProductionDatabaseURL != "",
|
||||
},
|
||||
}
|
||||
store, ok := config.Store.(interface {
|
||||
LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error)
|
||||
LibraryRestoreVerifications(context.Context, string, int) ([]governance.LibraryRestoreVerification, error)
|
||||
})
|
||||
if !ok {
|
||||
view.Reason = "当前部署没有恢复验证存储:无法读取证据"
|
||||
writeJSON(w, http.StatusServiceUnavailable, view)
|
||||
return
|
||||
}
|
||||
latest, found, err := store.LatestLibraryRestoreVerification(r.Context(), scope)
|
||||
if err != nil {
|
||||
view.Reason = "读取恢复验证证据失败:" + err.Error()
|
||||
writeJSON(w, http.StatusServiceUnavailable, view)
|
||||
return
|
||||
}
|
||||
if found {
|
||||
view.Latest = &latest
|
||||
view.JobID = latest.JobID
|
||||
}
|
||||
if history, err := store.LibraryRestoreVerifications(r.Context(), scope, 10); err == nil {
|
||||
view.History = history
|
||||
}
|
||||
if snapshotID := strings.TrimSpace(r.URL.Query().Get("snapshot_id")); snapshotID != "" {
|
||||
if exact, ok := config.Store.(interface {
|
||||
LibraryRestoreVerificationForSnapshot(context.Context, string, string) (governance.LibraryRestoreVerification, bool, error)
|
||||
}); ok {
|
||||
item, found, exactErr := exact.LibraryRestoreVerificationForSnapshot(r.Context(), scope, snapshotID)
|
||||
if exactErr != nil {
|
||||
view.Reason = "读取该快照的恢复验证记录失败"
|
||||
writeJSON(w, http.StatusServiceUnavailable, view)
|
||||
return
|
||||
}
|
||||
if found {
|
||||
view.SnapshotLatest = &item
|
||||
}
|
||||
}
|
||||
}
|
||||
view.Verified, view.Reason = governance.LibraryVerificationSatisfied(latest, scope)
|
||||
if !found {
|
||||
view.Verified = false
|
||||
view.Reason = "还没有对当前目标做过恢复验证:点「跑一次恢复验证」(真实备份 → canary → 恢复 → 复核)"
|
||||
}
|
||||
if snapshotID := strings.TrimSpace(r.URL.Query().Get("snapshot_id")); snapshotID != "" {
|
||||
// 显式按快照查询时,verified 只回答该快照;不能让 scope 最新通过记录
|
||||
// 把另一个未演练的快照染绿。
|
||||
view.Verified = false
|
||||
if view.SnapshotLatest == nil {
|
||||
view.Reason = "该快照没有真实恢复验证记录(未验证;不是读取失败)"
|
||||
} else if view.SnapshotLatest.Status == governance.LibraryVerificationPassed {
|
||||
view.Verified = true
|
||||
view.Reason = ""
|
||||
} else {
|
||||
view.Reason = "该快照最近一次恢复验证未通过"
|
||||
}
|
||||
}
|
||||
// 在飞作业必须从真实业务行读出来:旧实现只构造 running 字段却从不赋值,
|
||||
// 页面因此在验收里看不到"已接受/正在执行",按钮也不会进入执行态(审核 B4)。
|
||||
if jobs, ok := config.Store.(interface {
|
||||
ListLibraryJobs(context.Context, string, int) ([]governance.LibraryJob, error)
|
||||
}); ok {
|
||||
if items, err := jobs.ListLibraryJobs(r.Context(), scope, 20); err == nil {
|
||||
var active *governance.LibraryJob
|
||||
for index := range items {
|
||||
item := items[index]
|
||||
if item.Kind != governance.LibraryJobVerify || string(item.State) == "" {
|
||||
continue
|
||||
}
|
||||
if !libraryJobActiveState(item.State) {
|
||||
continue
|
||||
}
|
||||
if active == nil || item.CreatedAt.After(active.CreatedAt) {
|
||||
active = &item
|
||||
}
|
||||
}
|
||||
if active != nil {
|
||||
view.Running = true
|
||||
view.JobID = active.ID
|
||||
view.State = string(active.State)
|
||||
view.Stage = active.Stage
|
||||
view.Reason = "恢复验证作业 " + active.ID + " 正在执行(" + active.Stage + ");终态会在这里显示证据,页面会自动刷新"
|
||||
}
|
||||
}
|
||||
}
|
||||
writeJSON(w, http.StatusOK, view)
|
||||
}
|
||||
|
||||
// libraryJobActiveState 只认仍在执行的业务状态:终态不是"在飞"。
|
||||
func libraryJobActiveState(state governance.LibraryJobState) bool {
|
||||
switch state {
|
||||
case governance.LibraryJobPending, governance.LibraryJobRunning:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// withRestoreEvidence 把"当前目标是否有真实恢复证据"注入 Config 的副本,
|
||||
// 让同步/恢复闸口在同一个请求里按证据判断,而不是读一个可能被手填的 env。
|
||||
//
|
||||
// 证据优先级是不可交换的(审核 B3):明确 failed/invalid 或读取失败都必须拒绝,
|
||||
// 即使 env=true;历史 env 覆盖只在**明确没有证据行**(not-found)时兜底。
|
||||
// 把读取失败混进 not-found 等于把"不知道"说成"满足"。
|
||||
func (c Config) withRestoreEvidence(ctx context.Context, scope string) Config {
|
||||
satisfied, missing, err := c.restoreEvidence(ctx, scope)
|
||||
c.restoreEvidenceChecked = true
|
||||
switch {
|
||||
case err != nil:
|
||||
c.restoreVerified = false
|
||||
c.restoreEvidenceReason = err.Error()
|
||||
case satisfied:
|
||||
c.restoreVerified = true
|
||||
c.restoreEvidenceReason = ""
|
||||
case missing && c.ProductionBackupRestoreVerified:
|
||||
c.restoreVerified = true
|
||||
c.restoreEvidenceReason = ""
|
||||
default:
|
||||
c.restoreVerified = false
|
||||
if missing {
|
||||
c.restoreEvidenceReason = "当前目标还没有恢复演练证据:先在「生产保护与恢复」里跑一次恢复验证"
|
||||
} else {
|
||||
c.restoreEvidenceReason = "没有已核验的生产恢复演练证据"
|
||||
}
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// restoreEvidence 读当前目标的最近一条证据并给出它能不能支撑"可恢复"。
|
||||
// missing 只在"查询成功且确实没有这一行"时为 true;读取错误单独返回。
|
||||
func (c Config) restoreEvidence(ctx context.Context, scope string) (satisfied bool, missing bool, err error) {
|
||||
store, ok := c.Store.(interface {
|
||||
LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error)
|
||||
})
|
||||
if !ok {
|
||||
// 存储没有证据能力(迁移前的部署形状)等同于"没有证据行":
|
||||
// 历史 env 覆盖仍可兜底,但一旦读得出证据行,失败/无效/读取错误都优先拒绝。
|
||||
return false, true, nil
|
||||
}
|
||||
latest, found, err := store.LatestLibraryRestoreVerification(ctx, scope)
|
||||
if err != nil {
|
||||
return false, false, fmt.Errorf("读取恢复验证证据失败:%w", err)
|
||||
}
|
||||
if !found {
|
||||
return false, true, nil
|
||||
}
|
||||
ok, reason := governance.LibraryVerificationSatisfied(latest, scope)
|
||||
if !ok {
|
||||
return false, false, errors.New(reason)
|
||||
}
|
||||
return true, false, nil
|
||||
}
|
||||
|
||||
// restoreGateReasons 是闸口对"恢复可验证"的统一判断:只看 withRestoreEvidence 的结论。
|
||||
// 这里不再 OR env:env 兜底只在证据缺失时、且已经在 withRestoreEvidence 里判定过。
|
||||
func restoreGateReasons(config Config) []string {
|
||||
if config.restoreVerified {
|
||||
return nil
|
||||
}
|
||||
// 没有走过证据读取的调用点(历史路径/单元夹具)保持原有 env 语义;
|
||||
// 一旦读过证据,上面的 withRestoreEvidence 已经把优先级判定完毕。
|
||||
if !config.restoreEvidenceChecked && config.ProductionBackupRestoreVerified {
|
||||
return nil
|
||||
}
|
||||
reason := config.restoreEvidenceReason
|
||||
if reason == "" {
|
||||
reason = "没有已核验的生产恢复演练证据"
|
||||
}
|
||||
return []string{reason}
|
||||
}
|
||||
@@ -0,0 +1,309 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
// 恢复验证的页面契约:
|
||||
// - 只有管理员能发起(它会停写入者并在目标上写 canary);
|
||||
// - 闸口关闭 / 维护中时直接拒绝,不投一个 worker 必然失败的作业;
|
||||
// - 证据只由真实执行写入:没有证据 ⇒ verified=false,过期/换目标 ⇒ 失效。
|
||||
//
|
||||
// 这里不模拟"真跑一次恢复"(那需要真实目标);执行体本身在 internal/libraryverify,
|
||||
// 由库内 CLI 与 worker 共用,属于隔离 fixture 之外的集成范围。
|
||||
|
||||
type verificationStore struct {
|
||||
fakeLibraryStore
|
||||
verifications []governance.LibraryRestoreVerification
|
||||
}
|
||||
|
||||
func (s *verificationStore) LatestLibraryRestoreVerification(_ context.Context, scope string) (governance.LibraryRestoreVerification, bool, error) {
|
||||
for _, item := range s.verifications {
|
||||
if item.Scope == scope {
|
||||
return item, true, nil
|
||||
}
|
||||
}
|
||||
return governance.LibraryRestoreVerification{}, false, nil
|
||||
}
|
||||
|
||||
func (s *verificationStore) LibraryRestoreVerificationForSnapshot(_ context.Context, scope, snapshotID string) (governance.LibraryRestoreVerification, bool, error) {
|
||||
for _, item := range s.verifications {
|
||||
if item.Scope == scope && item.SnapshotID == snapshotID {
|
||||
return item, true, nil
|
||||
}
|
||||
}
|
||||
return governance.LibraryRestoreVerification{}, false, nil
|
||||
}
|
||||
|
||||
func (s *verificationStore) LibraryRestoreVerifications(_ context.Context, scope string, limit int) ([]governance.LibraryRestoreVerification, error) {
|
||||
items := make([]governance.LibraryRestoreVerification, 0, len(s.verifications))
|
||||
for _, item := range s.verifications {
|
||||
if item.Scope == scope {
|
||||
items = append(items, item)
|
||||
}
|
||||
}
|
||||
if limit > 0 && len(items) > limit {
|
||||
items = items[:limit]
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func verifyRequest(body, role, username string) *http.Request {
|
||||
request := httptest.NewRequest(http.MethodPost, "/api/v1/library/verifications", strings.NewReader(body))
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
ctx := context.WithValue(request.Context(), sessionKey{}, sessionIdentity{Username: username, Role: role})
|
||||
return request.WithContext(ctx)
|
||||
}
|
||||
|
||||
func TestLibraryVerifyRequiresAdmin(t *testing.T) {
|
||||
store := &verificationStore{}
|
||||
recorder := httptest.NewRecorder()
|
||||
libraryVerifyEnqueue(recorder, verifyRequest(`{"scope":"internal"}`, roleEngineer, "eng"),
|
||||
Config{Store: store, ProductionReleaseRequestsEnabled: true})
|
||||
if recorder.Code != http.StatusForbidden {
|
||||
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
if store.createN != 0 {
|
||||
t.Fatal("non-admin must not create a verify job")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryVerifyRefusesWhenWriteGateClosed(t *testing.T) {
|
||||
store := &verificationStore{}
|
||||
recorder := httptest.NewRecorder()
|
||||
libraryVerifyEnqueue(recorder, verifyRequest(`{"scope":"internal"}`, roleAdmin, "admin"),
|
||||
Config{Store: store, ProductionReleaseRequestsEnabled: false})
|
||||
if recorder.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
body := decodeBody(t, recorder)
|
||||
if body["code"] != "library_verify_blocked" {
|
||||
t.Fatalf("body=%v", body)
|
||||
}
|
||||
if store.createN != 0 {
|
||||
t.Fatal("closed gate must not create a verify job")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryVerifyEnqueuesVerifyJobWithCanaryStages(t *testing.T) {
|
||||
store := &verificationStore{}
|
||||
store.enqueueEchoRequestedID = true
|
||||
recorder := httptest.NewRecorder()
|
||||
libraryVerifyEnqueue(recorder, verifyRequest(`{"scope":"internal"}`, roleAdmin, "admin"),
|
||||
Config{Store: store, ProductionReleaseRequestsEnabled: true})
|
||||
if recorder.Code != http.StatusAccepted {
|
||||
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
if store.lastJob.Kind != governance.LibraryJobVerify || store.lastJob.Scope != "internal" {
|
||||
t.Fatalf("job=%#v", store.lastJob)
|
||||
}
|
||||
if !strings.HasPrefix(store.lastJob.ID, "libverify-internal-") {
|
||||
t.Fatalf("job id=%q", store.lastJob.ID)
|
||||
}
|
||||
// B2-R1:verify 作业必须带不可变 operation,worker 的 owned attempt / 恢复派发
|
||||
// 身份都从它取;没有 operation 的作业在真实恢复时会被 fail closed。
|
||||
if store.lastJob.OperationID == "" || store.lastJob.OperationID != store.lastOperation.ID {
|
||||
t.Fatalf("verify job must carry its operation: job=%#v op=%#v", store.lastJob, store.lastOperation)
|
||||
}
|
||||
if store.lastOperation.Kind != governance.LibraryJobVerify || store.lastOperation.Scope != "internal" {
|
||||
t.Fatalf("verify operation identity=%#v", store.lastOperation)
|
||||
}
|
||||
// 阶段必须包含 canary:没有"写 canary 证明校验有效"这一步,验证就只是自证。
|
||||
want := governance.VerifyStages()
|
||||
if len(want) != 5 || want[2] != "canary" || want[3] != "restore" || want[4] != "finalize" {
|
||||
t.Fatalf("verify stages=%v", want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryVerificationsWithoutEvidenceIsNotVerified(t *testing.T) {
|
||||
store := &verificationStore{}
|
||||
recorder := httptest.NewRecorder()
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/v1/library/verifications?scope=internal", nil)
|
||||
libraryVerifications(recorder, request, Config{Store: store})
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
body := decodeBody(t, recorder)
|
||||
if body["verified"] != false {
|
||||
t.Fatalf("verified must be false without evidence: %v", body)
|
||||
}
|
||||
if reason, _ := body["reason"].(string); !strings.Contains(reason, "还没有对当前目标做过恢复验证") {
|
||||
t.Fatalf("reason=%q", reason)
|
||||
}
|
||||
}
|
||||
|
||||
// F03: the selected snapshot must resolve its own evidence, never the scope's
|
||||
// newest passed record.
|
||||
func TestLibraryVerificationsExactSnapshotDoesNotBorrowLatest(t *testing.T) {
|
||||
store := &verificationStore{verifications: []governance.LibraryRestoreVerification{
|
||||
{ID: "verify-new", Scope: "internal", Status: governance.LibraryVerificationPassed,
|
||||
SnapshotID: "snap-new", JobID: "job-new", FinishedAt: time.Now().UTC()},
|
||||
{ID: "verify-old", Scope: "internal", Status: governance.LibraryVerificationFailed,
|
||||
SnapshotID: "snap-old", JobID: "job-old", Failure: "old restore mismatch",
|
||||
FinishedAt: time.Now().UTC().Add(-time.Hour)},
|
||||
}}
|
||||
recorder := httptest.NewRecorder()
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/v1/library/verifications?scope=internal&snapshot_id=snap-old", nil)
|
||||
libraryVerifications(recorder, request, Config{Store: store})
|
||||
body := decodeBody(t, recorder)
|
||||
exact, ok := body["snapshot_latest"].(map[string]any)
|
||||
if !ok || exact["snapshot_id"] != "snap-old" || exact["status"] != "failed" {
|
||||
t.Fatalf("snapshot_latest=%v body=%v", body["snapshot_latest"], body)
|
||||
}
|
||||
if body["verified"] != false {
|
||||
t.Fatalf("old failed snapshot must not inherit latest passed: %v", body["verified"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryVerificationsWithPassedEvidenceIsVerified(t *testing.T) {
|
||||
store := &verificationStore{verifications: []governance.LibraryRestoreVerification{{
|
||||
ID: "libverify-internal-job1", Scope: "internal", Status: governance.LibraryVerificationPassed,
|
||||
SnapshotID: "snap-1", JobID: "job1", StartedAt: time.Now().UTC().Add(-time.Minute),
|
||||
FinishedAt: time.Now().UTC(),
|
||||
}}}
|
||||
recorder := httptest.NewRecorder()
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/v1/library/verifications?scope=internal", nil)
|
||||
libraryVerifications(recorder, request, Config{Store: store})
|
||||
body := decodeBody(t, recorder)
|
||||
if body["verified"] != true {
|
||||
t.Fatalf("verified must be true with passed evidence: %v", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryVerificationsFailedEvidenceIsNotVerified(t *testing.T) {
|
||||
store := &verificationStore{verifications: []governance.LibraryRestoreVerification{{
|
||||
ID: "libverify-internal-job2", Scope: "internal", Status: governance.LibraryVerificationFailed,
|
||||
SnapshotID: "snap-2", JobID: "job2", Failure: "恢复后内容仍与预备份不一致",
|
||||
StartedAt: time.Now().UTC().Add(-time.Minute), FinishedAt: time.Now().UTC(),
|
||||
}}}
|
||||
recorder := httptest.NewRecorder()
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/v1/library/verifications?scope=internal", nil)
|
||||
libraryVerifications(recorder, request, Config{Store: store})
|
||||
body := decodeBody(t, recorder)
|
||||
if body["verified"] != false {
|
||||
t.Fatalf("failed evidence must not verify: %v", body)
|
||||
}
|
||||
if reason, _ := body["reason"].(string); !strings.Contains(reason, "恢复后内容仍与预备份不一致") {
|
||||
t.Fatalf("reason=%q", reason)
|
||||
}
|
||||
}
|
||||
|
||||
// 同步闸口必须消费同一 target 的证据:有 passed 证据即放行,
|
||||
// 只有失败证据时仍然拒绝并给出失败原因(不是笼统"没有证据")。
|
||||
// 审核 B3 反例(GPT-6 overlay)转成通过断言:env=true 不能覆盖明确失败或读取失败。
|
||||
type evidenceReadErrorStore struct{ verificationStore }
|
||||
|
||||
func (*evidenceReadErrorStore) LatestLibraryRestoreVerification(context.Context, string) (governance.LibraryRestoreVerification, bool, error) {
|
||||
return governance.LibraryRestoreVerification{}, false, errors.New("evidence lookup failed")
|
||||
}
|
||||
|
||||
func TestRestoreGateFailedEvidenceBeatsEnv(t *testing.T) {
|
||||
store := &verificationStore{verifications: []governance.LibraryRestoreVerification{{
|
||||
Scope: "internal", Status: governance.LibraryVerificationFailed, Failure: "restore mismatch",
|
||||
FinishedAt: time.Now().UTC(),
|
||||
}}}
|
||||
config := Config{Store: store, ProductionBackupRestoreVerified: true}.withRestoreEvidence(context.Background(), "internal")
|
||||
reasons := restoreGateReasons(config)
|
||||
if len(reasons) == 0 || !strings.Contains(strings.Join(reasons, ";"), "restore mismatch") {
|
||||
t.Fatalf("failed evidence must beat env=true: %v", reasons)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestoreGateUnreadableEvidenceBeatsEnv(t *testing.T) {
|
||||
config := Config{Store: &evidenceReadErrorStore{}, ProductionBackupRestoreVerified: true}.
|
||||
withRestoreEvidence(context.Background(), "internal")
|
||||
reasons := restoreGateReasons(config)
|
||||
if len(reasons) == 0 || !strings.Contains(strings.Join(reasons, ";"), "读取恢复验证证据失败") {
|
||||
t.Fatalf("unreadable evidence must beat env=true: %v", reasons)
|
||||
}
|
||||
}
|
||||
|
||||
// env 兜底只允许在"明确没有证据行"时生效。
|
||||
func TestRestoreGateEnvFallbackOnlyWhenEvidenceMissing(t *testing.T) {
|
||||
config := Config{Store: &verificationStore{}, ProductionBackupRestoreVerified: true}.
|
||||
withRestoreEvidence(context.Background(), "internal")
|
||||
if reasons := restoreGateReasons(config); len(reasons) != 0 {
|
||||
t.Fatalf("explicit not-found with legacy env must still allow: %v", reasons)
|
||||
}
|
||||
}
|
||||
|
||||
// 审核 B4:GET 必须从真实业务行投影在飞验证作业,而不是永远 running=false。
|
||||
func TestLibraryVerificationsProjectsActiveVerifyJob(t *testing.T) {
|
||||
store := &verificationStore{}
|
||||
store.jobs = []governance.LibraryJob{{
|
||||
ID: "verify-running", Scope: "internal", Kind: governance.LibraryJobVerify,
|
||||
State: governance.LibraryJobRunning, Stage: "restore", CreatedAt: time.Now().UTC(),
|
||||
}}
|
||||
recorder := httptest.NewRecorder()
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/v1/library/verifications?scope=internal", nil)
|
||||
libraryVerifications(recorder, request, Config{Store: store})
|
||||
body := decodeBody(t, recorder)
|
||||
if body["running"] != true {
|
||||
t.Fatalf("running verify job must be projected: %v", body)
|
||||
}
|
||||
if body["job_id"] != "verify-running" {
|
||||
t.Fatalf("job_id=%v", body["job_id"])
|
||||
}
|
||||
if body["stage"] != "restore" {
|
||||
t.Fatalf("stage=%v", body["stage"])
|
||||
}
|
||||
}
|
||||
|
||||
// 终态作业不是"在飞":页面不能一直显示执行中。
|
||||
func TestLibraryVerificationsIgnoresTerminalVerifyJob(t *testing.T) {
|
||||
store := &verificationStore{}
|
||||
store.jobs = []governance.LibraryJob{{
|
||||
ID: "verify-done", Scope: "internal", Kind: governance.LibraryJobVerify,
|
||||
State: governance.LibraryJobSucceeded, CreatedAt: time.Now().UTC(),
|
||||
}}
|
||||
recorder := httptest.NewRecorder()
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/v1/library/verifications?scope=internal", nil)
|
||||
libraryVerifications(recorder, request, Config{Store: store})
|
||||
body := decodeBody(t, recorder)
|
||||
if body["running"] != false {
|
||||
t.Fatalf("terminal verify job must not be projected as running: %v", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibrarySyncGateConsumesRestoreEvidence(t *testing.T) {
|
||||
base := Config{ProductionReleaseRequestsEnabled: true, ProductionBackupRestoreVerified: false,
|
||||
LibraryProductionDatabaseURL: "postgres://unused.invalid/db", LibraryCapacityBudgetBytes: 1 << 40}
|
||||
base.Store = &verificationStore{verifications: []governance.LibraryRestoreVerification{{
|
||||
Scope: "internal", Status: governance.LibraryVerificationPassed, SnapshotID: "snap-1",
|
||||
JobID: "job1", FinishedAt: time.Now().UTC(),
|
||||
}}}
|
||||
allowed := librarySyncGateWithGrant(base.withRestoreEvidence(context.Background(), "internal"),
|
||||
libraryKnownView{State: "known"}, libraryKnownView{State: "unknown"}, "snap-1", true)
|
||||
if !allowed.SyncAllowed {
|
||||
t.Fatalf("gate should allow with passed evidence: %v", allowed.Reasons)
|
||||
}
|
||||
noEvidence := Config{ProductionReleaseRequestsEnabled: true, ProductionBackupRestoreVerified: false,
|
||||
LibraryProductionDatabaseURL: "postgres://unused.invalid/db", LibraryCapacityBudgetBytes: 1 << 40,
|
||||
Store: &verificationStore{}}.withRestoreEvidence(context.Background(), "internal")
|
||||
blocked := librarySyncGateWithGrant(noEvidence,
|
||||
libraryKnownView{State: "known"}, libraryKnownView{State: "unknown"}, "snap-1", true)
|
||||
if blocked.SyncAllowed {
|
||||
t.Fatalf("gate without env and without evidence must block: %v", blocked.Reasons)
|
||||
}
|
||||
if len(blocked.Reasons) == 0 || !strings.Contains(strings.Join(blocked.Reasons, ";"), "恢复验证") {
|
||||
t.Fatalf("blocked reasons=%v", blocked.Reasons)
|
||||
}
|
||||
failedConfig := Config{ProductionReleaseRequestsEnabled: true, ProductionBackupRestoreVerified: false,
|
||||
LibraryProductionDatabaseURL: "postgres://unused.invalid/db", LibraryCapacityBudgetBytes: 1 << 40,
|
||||
Store: &verificationStore{verifications: []governance.LibraryRestoreVerification{{
|
||||
Scope: "internal", Status: governance.LibraryVerificationFailed, SnapshotID: "snap-2",
|
||||
JobID: "job2", Failure: "canary 后校验仍通过", FinishedAt: time.Now().UTC(),
|
||||
}}}}
|
||||
reason := strings.Join(restoreGateReasons(failedConfig.withRestoreEvidence(context.Background(), "internal")), ";")
|
||||
if !strings.Contains(reason, "canary 后校验仍通过") {
|
||||
t.Fatalf("失败证据的失败原因必须出现在闸口:%q", reason)
|
||||
}
|
||||
}
|
||||
@@ -97,6 +97,10 @@ func listMonitorTasks(w http.ResponseWriter, r *http.Request, config Config) {
|
||||
writeJSON(w, 500, map[string]string{"error": "无法读取文件处理状态"})
|
||||
return
|
||||
}
|
||||
// 目标同名不罕见(两侧都叫“对内知识库”):把当前生效绑定随页返回,
|
||||
// 前端才能在同一口径下标注“当前绑定/历史”,而不是拿本页条目反推全局。
|
||||
internal, external := config.effectiveRoutingTargets(r.Context())
|
||||
page.Bindings = postgresstore.MonitorBindings{Internal: internal, External: external}
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
writeJSON(w, 200, map[string]any{"data": page})
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
@@ -51,3 +52,39 @@ func TestMonitorTasksRoute(t *testing.T) {
|
||||
t.Fatalf("viewer removal %d", removal.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// F15:筛选里的“当前绑定”必须来自持久化绑定(与业务同一真源),不是 env 旧值,
|
||||
// 也不能让前端拿本页数据猜全局。
|
||||
func TestMonitorTasksReportsEffectiveTargetBindings(t *testing.T) {
|
||||
store := &monitorTasksTestStore{}
|
||||
handler, err := NewHandler(Config{
|
||||
Store: store,
|
||||
InternalRoutingKnowledgeBaseID: "env-old-internal",
|
||||
ExternalRoutingKnowledgeBaseID: "env-old-external",
|
||||
RoutingBindings: &bindingStoreFake{internal: "current-internal", external: "current-external"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req := httptest.NewRequest("GET", "/api/v1/monitor/tasks?limit=20", nil)
|
||||
req = req.WithContext(context.WithValue(req.Context(), sessionKey{}, sessionIdentity{Username: "tester", Role: roleViewer}))
|
||||
recorder := httptest.NewRecorder()
|
||||
handler.ServeHTTP(recorder, req)
|
||||
if recorder.Code != 200 {
|
||||
t.Fatalf("status %d: %s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
var payload struct {
|
||||
Data struct {
|
||||
Bindings struct {
|
||||
Internal string `json:"internal"`
|
||||
External string `json:"external"`
|
||||
} `json:"bindings"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if payload.Data.Bindings.Internal != "current-internal" || payload.Data.Bindings.External != "current-external" {
|
||||
t.Fatalf("bindings = %#v, want the persisted current bindings", payload.Data.Bindings)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/pgdsn"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
)
|
||||
|
||||
@@ -33,6 +34,55 @@ type productionBindingView struct {
|
||||
ProductionBaseURL string `json:"production_base_url"`
|
||||
ProductionConnected bool `json:"production_connected"`
|
||||
ProductionMessage string `json:"production_message"`
|
||||
// LegacyScope 说明这条记录属于哪条发布路径。整库发布(封存/同步/恢复)不读它,
|
||||
// 只读下面的 WholeTarget;页面必须把两者分开写,否则会把旧文档级目标当成整库目标。
|
||||
LegacyScope string `json:"legacy_scope"`
|
||||
// WholeTarget 是整库发布真正写入的目标(来自部署环境 LIBRARY_PRODUCTION_*),
|
||||
// 与上面可编辑的文档级绑定是两条独立路径。
|
||||
WholeTarget libraryPublishTargetView `json:"whole_target"`
|
||||
}
|
||||
|
||||
// libraryPublishTargetView 是"整库发布到底写哪儿"的只读投影。
|
||||
// 为什么由后端给出:页面上的 kb 名称/文档数来自生产端列表接口,而整库写入用的是
|
||||
// 另一组环境配置;让页面自己拼会把两条路径混成一个。
|
||||
type libraryPublishTargetView struct {
|
||||
Configured bool `json:"configured"`
|
||||
Source string `json:"source"`
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
InternalKBID string `json:"internal_kb_id"`
|
||||
ExternalKBID string `json:"external_kb_id"`
|
||||
Database string `json:"database"`
|
||||
ElasticIndex string `json:"elastic_index"`
|
||||
ObjectBucket string `json:"object_bucket"`
|
||||
MissingReason string `json:"missing_reason"`
|
||||
}
|
||||
|
||||
// libraryPublishTarget 从 config 派生整库目标;缺项照实说缺哪一项,不猜默认值。
|
||||
func (config Config) libraryPublishTarget() libraryPublishTargetView {
|
||||
view := libraryPublishTargetView{
|
||||
Source: "部署环境 LIBRARY_PRODUCTION_*",
|
||||
TenantID: config.LibraryProductionTenantID,
|
||||
InternalKBID: config.LibraryProductionInternalKB,
|
||||
ExternalKBID: config.LibraryProductionExternalKB,
|
||||
Database: pgdsn.Endpoint(config.LibraryProductionDatabaseURL),
|
||||
ElasticIndex: config.LibraryProductionIndex,
|
||||
ObjectBucket: config.LibraryProductionS3Bucket,
|
||||
}
|
||||
missing := make([]string, 0, 4)
|
||||
if config.LibraryProductionDatabaseURL == "" {
|
||||
missing = append(missing, "LIBRARY_PRODUCTION_DATABASE_URL")
|
||||
}
|
||||
if view.InternalKBID == "" {
|
||||
missing = append(missing, "LIBRARY_PRODUCTION_KB_INTERNAL")
|
||||
}
|
||||
if config.LibraryProductionIndex == "" {
|
||||
missing = append(missing, "LIBRARY_PRODUCTION_INDEX")
|
||||
}
|
||||
view.Configured = len(missing) == 0
|
||||
if len(missing) > 0 {
|
||||
view.MissingReason = "未配置:" + strings.Join(missing, "、")
|
||||
}
|
||||
return view
|
||||
}
|
||||
|
||||
// productionTargetBaseURL is the production endpoint the picker reads from: the
|
||||
@@ -79,6 +129,10 @@ func (config Config) productionBindingPayload(ctx context.Context) (productionBi
|
||||
InternalKBName: names[storedInternal], ExternalKBName: names[storedExternal],
|
||||
UpdatedBy: updatedBy, ProductionBaseURL: config.productionTargetBaseURL(ctx),
|
||||
ProductionConnected: message == "", ProductionMessage: message,
|
||||
// 这张表服务的是"文档级发布"(单篇文档写生产)。整库发布读的是部署环境
|
||||
// LIBRARY_PRODUCTION_*,两者不能互相顶替,所以页面必须分开显示。
|
||||
LegacyScope: "文档级发布(单篇文档写生产)· 旧路径",
|
||||
WholeTarget: config.libraryPublishTarget(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -178,3 +178,50 @@ func TestProductionBindingOptionsReportUnavailableReason(t *testing.T) {
|
||||
t.Fatalf("status = %d body %s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// 页面必须说清"整库发布真正写哪个 KB/端点"。这条投影只能来自部署配置,
|
||||
// 不能从可编辑的文档级绑定推出来,也不能把口令带进响应。
|
||||
func TestLibraryPublishTargetIsDerivedFromDeploymentConfig(t *testing.T) {
|
||||
config := Config{
|
||||
// 只放形状,不放任何可用凭据:口令段是占位符,主机是 .invalid。
|
||||
LibraryProductionDatabaseURL: "postgres://governance@db.invalid:5432/weknora_test?sslmode=disable",
|
||||
LibraryProductionInternalKB: "a8e9c8ad-411c-4c53-8421-599d29e9b179",
|
||||
LibraryProductionExternalKB: "2e980686-2233-4c4e-b5a6-8b0069477b8e",
|
||||
LibraryProductionTenantID: 10000,
|
||||
LibraryProductionIndex: "weknora_test_chunks",
|
||||
LibraryProductionS3Bucket: "dobot-weknora-test",
|
||||
}
|
||||
target := config.libraryPublishTarget()
|
||||
if !target.Configured || target.MissingReason != "" {
|
||||
t.Fatalf("完整配置应判为已配置: %+v", target)
|
||||
}
|
||||
if target.InternalKBID != "a8e9c8ad-411c-4c53-8421-599d29e9b179" || target.TenantID != 10000 {
|
||||
t.Fatalf("必须回显实际整库目标而不是占位: %+v", target)
|
||||
}
|
||||
if target.Database != "db.invalid:5432/weknora_test" {
|
||||
t.Fatalf("端点应只含 host:port/db,得到 %q", target.Database)
|
||||
}
|
||||
// 回显只能是端点:连接串里的用户信息(口令、账号)一个字符都不能进响应。
|
||||
if strings.Contains(target.Database, "@") || strings.Contains(target.Source, "@") ||
|
||||
strings.Contains(target.Source, "postgres://") {
|
||||
t.Fatalf("响应里不能出现连接串用户信息: %+v", target)
|
||||
}
|
||||
if target.ElasticIndex != "weknora_test_chunks" || target.ObjectBucket != "dobot-weknora-test" {
|
||||
t.Fatalf("索引与桶也要显示: %+v", target)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryPublishTargetNamesMissingConfigInsteadOfGuessing(t *testing.T) {
|
||||
target := Config{}.libraryPublishTarget()
|
||||
if target.Configured {
|
||||
t.Fatal("缺配置时不能报告已配置")
|
||||
}
|
||||
for _, key := range []string{"LIBRARY_PRODUCTION_DATABASE_URL", "LIBRARY_PRODUCTION_KB_INTERNAL"} {
|
||||
if !strings.Contains(target.MissingReason, key) {
|
||||
t.Fatalf("缺失说明应点名 %s,得到 %q", key, target.MissingReason)
|
||||
}
|
||||
}
|
||||
if target.InternalKBID != "" || target.Database != "(未配置)" {
|
||||
t.Fatalf("缺配置就不要编造目标: %+v", target)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"knowledge-governance/internal/dobotaiops"
|
||||
"knowledge-governance/internal/qawemail"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -20,6 +21,98 @@ type weeklyMailStore interface {
|
||||
TestSend(context.Context, string) (string, error)
|
||||
SaveDashboardImage(context.Context, string, []byte) error
|
||||
SendDue(context.Context, time.Time) (qawemail.DueResult, error)
|
||||
PreviewSend(context.Context, string, int64, string, time.Time) (qawemail.SendPreview, error)
|
||||
SendManual(context.Context, string, int64, string, string, time.Time) (qawemail.ManualResult, error)
|
||||
}
|
||||
|
||||
// weeklyMailSendError names the rule that refused a manual resend. Anything SMTP-related is
|
||||
// reported by the handler itself, so an unmet precondition never looks like a mail failure.
|
||||
func weeklyMailSendError(w http.ResponseWriter, err error) {
|
||||
switch {
|
||||
case errors.Is(err, qawemail.ErrNotCurrent):
|
||||
writeJSON(w, http.StatusConflict, map[string]string{"error": "该周周报未确认,或确认后正文/版本已变化;请先确认当前版本再补发"})
|
||||
case errors.Is(err, qawemail.ErrFutureWeek):
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "不能补发未来周"})
|
||||
case errors.Is(err, qawemail.ErrSlotPending):
|
||||
writeJSON(w, http.StatusConflict, map[string]string{"error": "本周尚未到达设定的发送时点,暂不能补发"})
|
||||
case errors.Is(err, qawemail.ErrNotConfigured):
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "未配置邮件服务器、发件人或收件人,无法补发"})
|
||||
case errors.Is(err, qawemail.ErrSendInFlight):
|
||||
writeJSON(w, http.StatusConflict, map[string]string{"error": "该周该版本有一次发送结果未知(记录停在发送中),为避免重复投递未重发;请先核对已发出的邮件"})
|
||||
case errors.Is(err, qawemail.ErrDeliveryUnknown):
|
||||
writeJSON(w, http.StatusConflict, map[string]string{"error": "SMTP 已提交但未收到最终确认,投递结果未知;为避免重复已停止重发,请先核对已发出的邮件"})
|
||||
case errors.Is(err, qawemail.ErrNoPassword):
|
||||
writeJSON(w, http.StatusConflict, map[string]string{"error": "SMTP 授权码缺失或无法解密,无法补发"})
|
||||
case errors.Is(err, qawemail.ErrInvalid):
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "补发参数无效:需要精确的周、已确认版本与正文 hash"})
|
||||
default:
|
||||
weeklyMailError(w, err)
|
||||
}
|
||||
}
|
||||
|
||||
// qaWeeklyMailSendPreview lets the confirmation dialog show the exact week, revision and
|
||||
// distribution this send would use — the same settings the send path reads.
|
||||
func qaWeeklyMailSendPreview(w http.ResponseWriter, r *http.Request, c Config) {
|
||||
if !authorizeQAOps(w, r) {
|
||||
return
|
||||
}
|
||||
store, ok := c.QAWeeklyMail.(weeklyMailStore)
|
||||
if !ok {
|
||||
weeklyMailError(w, qawemail.ErrUnavailable)
|
||||
return
|
||||
}
|
||||
query := r.URL.Query()
|
||||
revision, err := strconv.ParseInt(query.Get("revision"), 10, 64)
|
||||
if err != nil {
|
||||
weeklyMailSendError(w, qawemail.ErrInvalid)
|
||||
return
|
||||
}
|
||||
preview, err := store.PreviewSend(r.Context(), r.PathValue("week"), revision, query.Get("body_sha256"), time.Now())
|
||||
if err != nil {
|
||||
weeklyMailSendError(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"data": preview})
|
||||
}
|
||||
|
||||
// qaWeeklyMailSend is the formal resend: admin only, bound to the live confirmation, and the same
|
||||
// render/claim path as the automatic slot. It is not a test send and never targets the review box.
|
||||
func qaWeeklyMailSend(w http.ResponseWriter, r *http.Request, c Config) {
|
||||
if !authorizeQAOps(w, r) || !authorizeQAIntent(w, r, "qa-weekly-mail") {
|
||||
return
|
||||
}
|
||||
identity, ok := sessionFrom(r.Context())
|
||||
if !ok || identity.Role != "admin" {
|
||||
writeJSON(w, http.StatusForbidden, map[string]string{"error": "仅管理员可正式补发周报"})
|
||||
return
|
||||
}
|
||||
store, found := c.QAWeeklyMail.(weeklyMailStore)
|
||||
if !found {
|
||||
weeklyMailError(w, qawemail.ErrUnavailable)
|
||||
return
|
||||
}
|
||||
var request struct {
|
||||
Revision int64 `json:"revision"`
|
||||
BodySHA256 string `json:"body_sha256"`
|
||||
}
|
||||
if decodeQualityJSON(w, r, &request) != nil {
|
||||
weeklyMailError(w, qawemail.ErrInvalid)
|
||||
return
|
||||
}
|
||||
result, err := store.SendManual(r.Context(), r.PathValue("week"), request.Revision, request.BodySHA256, identity.Username, time.Now())
|
||||
if err != nil {
|
||||
weeklyMailSendError(w, err)
|
||||
return
|
||||
}
|
||||
if result.Status == "failed" {
|
||||
reason := result.Reason
|
||||
if len(reason) > 300 {
|
||||
reason = reason[:300]
|
||||
}
|
||||
writeJSON(w, http.StatusBadGateway, map[string]string{"error": "正式补发未送达:" + reason})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"data": result})
|
||||
}
|
||||
|
||||
func weeklyMailError(w http.ResponseWriter, err error) {
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/qawemail"
|
||||
)
|
||||
|
||||
// 页面 reload / “完成读真实发送记录”走的就是这个 handler。用真实 PG 的非空历史证明
|
||||
// 产品读回路径返回 200 且带 message_id,而不是 503——“历史行存在就整个列表不可用”
|
||||
// 正是上一版 Sends() 的确定性回归。
|
||||
func TestWeeklyMailSendsHandlerReadsRealHistoryWithMessageID(t *testing.T) {
|
||||
dsn := os.Getenv("KG_WEEKLY_MAIL_TEST_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("显式一次性周报邮件数据库未提供(KG_WEEKLY_MAIL_TEST_DSN):跳过不等于通过")
|
||||
}
|
||||
if !strings.Contains(dsn, "/kg_weekly_mail_test_") || strings.Contains(dsn, "/governance?") {
|
||||
t.Fatal("拒绝非一次性周报测试库")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, dsn)
|
||||
if err != nil {
|
||||
t.Fatal("测试库连接参数无效(详情省略)")
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := postgresstore.ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := qawemail.NewStore(pool, "fixture-http-history", os.Getenv("KG_WEEKLY_MAIL_TEST_KEY"), nil)
|
||||
for _, statement := range []string{
|
||||
`DELETE FROM qa_weekly_email_sends WHERE tenant_key=$1`,
|
||||
} {
|
||||
if _, err := pool.Exec(ctx, statement, "fixture-http-history"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO qa_weekly_email_sends(tenant_key,week,draft_revision,body_sha256,status,attempts,message_id,last_error,created_at,updated_at)
|
||||
VALUES($1,'2026-W45',16,$2,'sent',1,'<real@example.com>','',now(),now()),
|
||||
($1,'2026-W44',7,$2,'failed',2,'','smtp RCPT 被拒: 550',now(),now() - interval '1 hour')`,
|
||||
"fixture-http-history", strings.Repeat("a", 64)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/v1/qa-ops/master/weekly-mail/sends?week=2026-W45", nil)
|
||||
request = request.WithContext(context.WithValue(request.Context(), sessionKey{}, sessionIdentity{Username: "tester", Role: roleAdmin}))
|
||||
recorder := httptest.NewRecorder()
|
||||
qaWeeklyMailSends(recorder, request, Config{QAWeeklyMail: store})
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("非空历史必须 200,status=%d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
var payload struct {
|
||||
Data struct {
|
||||
Sends []qawemail.Send `json:"sends"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(payload.Data.Sends) != 1 || payload.Data.Sends[0].MessageID != "<real@example.com>" || payload.Data.Sends[0].Status != "sent" {
|
||||
t.Fatalf("发送记录必须带回真实 message_id:%+v", payload.Data.Sends)
|
||||
}
|
||||
|
||||
// 全量历史(不带 week)同样必须可读,且 failed 行不伪造 message_id。
|
||||
all := httptest.NewRequest(http.MethodGet, "/api/v1/qa-ops/master/weekly-mail/sends", nil)
|
||||
all = all.WithContext(context.WithValue(all.Context(), sessionKey{}, sessionIdentity{Username: "tester", Role: roleAdmin}))
|
||||
allRecorder := httptest.NewRecorder()
|
||||
qaWeeklyMailSends(allRecorder, all, Config{QAWeeklyMail: store})
|
||||
if allRecorder.Code != http.StatusOK {
|
||||
t.Fatalf("全量历史必须 200,status=%d body=%s", allRecorder.Code, allRecorder.Body.String())
|
||||
}
|
||||
if err := json.Unmarshal(allRecorder.Body.Bytes(), &payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(payload.Data.Sends) != 2 {
|
||||
t.Fatalf("应返回 2 条历史,got %+v", payload.Data.Sends)
|
||||
}
|
||||
for _, item := range payload.Data.Sends {
|
||||
if item.Status == "failed" && item.MessageID != "" {
|
||||
t.Fatalf("failed 行不得有 message_id:%+v", item)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/qawemail"
|
||||
)
|
||||
|
||||
type fakeWeeklyMailStore struct {
|
||||
preview qawemail.SendPreview
|
||||
previewErr error
|
||||
result qawemail.ManualResult
|
||||
sendErr error
|
||||
calls int
|
||||
lastWeek string
|
||||
lastRev int64
|
||||
lastHash string
|
||||
lastActor string
|
||||
}
|
||||
|
||||
func (f *fakeWeeklyMailStore) ReadSettings(context.Context) (qawemail.Settings, error) {
|
||||
return qawemail.Settings{}, nil
|
||||
}
|
||||
func (f *fakeWeeklyMailStore) SaveSettings(context.Context, string, qawemail.SettingsInput) (qawemail.Settings, error) {
|
||||
return qawemail.Settings{}, nil
|
||||
}
|
||||
func (f *fakeWeeklyMailStore) Confirm(context.Context, string, string) (qawemail.Confirmation, error) {
|
||||
return qawemail.Confirmation{}, nil
|
||||
}
|
||||
func (f *fakeWeeklyMailStore) Unconfirm(context.Context, string) error { return nil }
|
||||
func (f *fakeWeeklyMailStore) ReadConfirmation(context.Context, string) (*qawemail.Confirmation, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (f *fakeWeeklyMailStore) Sends(context.Context, string) ([]qawemail.Send, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (f *fakeWeeklyMailStore) TestSend(context.Context, string) (string, error) { return "", nil }
|
||||
func (f *fakeWeeklyMailStore) SaveDashboardImage(context.Context, string, []byte) error {
|
||||
return nil
|
||||
}
|
||||
func (f *fakeWeeklyMailStore) SendDue(context.Context, time.Time) (qawemail.DueResult, error) {
|
||||
return qawemail.DueResult{}, nil
|
||||
}
|
||||
func (f *fakeWeeklyMailStore) PreviewSend(_ context.Context, week string, revision int64, hash string, _ time.Time) (qawemail.SendPreview, error) {
|
||||
f.lastWeek, f.lastRev, f.lastHash = week, revision, hash
|
||||
if f.previewErr != nil {
|
||||
return qawemail.SendPreview{}, f.previewErr
|
||||
}
|
||||
out := f.preview
|
||||
out.Week, out.Revision, out.BodySHA256 = week, revision, hash
|
||||
return out, nil
|
||||
}
|
||||
func (f *fakeWeeklyMailStore) SendManual(_ context.Context, week string, revision int64, hash, actor string, _ time.Time) (qawemail.ManualResult, error) {
|
||||
f.calls++
|
||||
f.lastWeek, f.lastRev, f.lastHash, f.lastActor = week, revision, hash, actor
|
||||
if f.sendErr != nil {
|
||||
return qawemail.ManualResult{}, f.sendErr
|
||||
}
|
||||
out := f.result
|
||||
out.Week, out.Revision = week, revision
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func weeklyMailSendRequest(t *testing.T, role, body string, withIntent bool) *http.Request {
|
||||
t.Helper()
|
||||
request := httptest.NewRequest(http.MethodPost, "/api/v1/qa-ops/master/weekly-mail/send/2026-W45", strings.NewReader(body))
|
||||
request.SetPathValue("week", "2026-W45")
|
||||
request = request.WithContext(context.WithValue(request.Context(), sessionKey{}, sessionIdentity{Username: "tester", Role: role}))
|
||||
if withIntent {
|
||||
request.Header.Set("X-Governance-Action", "qa-weekly-mail")
|
||||
}
|
||||
return request
|
||||
}
|
||||
|
||||
// 正式补发是管理员动作:普通角色与缺少同源意图都必须被挡在 SMTP 之前。
|
||||
func TestWeeklyMailFormalSendRequiresAdminAndIntent(t *testing.T) {
|
||||
store := &fakeWeeklyMailStore{result: qawemail.ManualResult{Status: "sent", MessageID: "<m@example.com>"}}
|
||||
config := Config{QAWeeklyMail: store}
|
||||
body := `{"revision":16,"body_sha256":"` + strings.Repeat("a", 64) + `"}`
|
||||
|
||||
recorder := httptest.NewRecorder()
|
||||
qaWeeklyMailSend(recorder, weeklyMailSendRequest(t, "engineer", body, true), config)
|
||||
if recorder.Code != http.StatusForbidden {
|
||||
t.Fatalf("非管理员必须 403,status=%d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
recorder = httptest.NewRecorder()
|
||||
qaWeeklyMailSend(recorder, weeklyMailSendRequest(t, roleAdmin, body, false), config)
|
||||
if recorder.Code != http.StatusForbidden {
|
||||
t.Fatalf("缺少同源操作意图必须 403,status=%d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
if store.calls != 0 {
|
||||
t.Fatalf("被拒请求不得调用发送:calls=%d", store.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// 成功补发必须把 week/版本/hash/操作人原样交给 store,并回传真实 message_id 与收件人数。
|
||||
func TestWeeklyMailFormalSendPassesExactRevisionAndReturnsReceipt(t *testing.T) {
|
||||
store := &fakeWeeklyMailStore{result: qawemail.ManualResult{Status: "sent", MessageID: "<real@example.com>", Recipients: 119, Cc: 8}}
|
||||
config := Config{QAWeeklyMail: store}
|
||||
hash := strings.Repeat("b", 64)
|
||||
recorder := httptest.NewRecorder()
|
||||
qaWeeklyMailSend(recorder, weeklyMailSendRequest(t, roleAdmin, `{"revision":16,"body_sha256":"`+hash+`"}`, true), config)
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("管理员补发应成功,status=%d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
if store.lastWeek != "2026-W45" || store.lastRev != 16 || store.lastHash != hash || store.lastActor != "tester" {
|
||||
t.Fatalf("补发参数没有精确传递:week=%q rev=%d hash=%q actor=%q", store.lastWeek, store.lastRev, store.lastHash, store.lastActor)
|
||||
}
|
||||
var payload struct {
|
||||
Data qawemail.ManualResult `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if payload.Data.MessageID != "<real@example.com>" || payload.Data.Recipients != 119 || payload.Data.Cc != 8 {
|
||||
t.Fatalf("响应必须带回真实发送凭据与正式收件人数:%+v", payload.Data)
|
||||
}
|
||||
}
|
||||
|
||||
// 前置条件不满足必须给出具体原因,而不是冒充 SMTP 失败。
|
||||
func TestWeeklyMailFormalSendExplainsRefusalsInsteadOfSMTPFailure(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
err error
|
||||
status int
|
||||
contains string
|
||||
}{
|
||||
{"future", qawemail.ErrFutureWeek, http.StatusBadRequest, "未来周"},
|
||||
{"slot", qawemail.ErrSlotPending, http.StatusConflict, "尚未到达设定的发送时点"},
|
||||
{"stale", qawemail.ErrNotCurrent, http.StatusConflict, "正文/版本已变化"},
|
||||
{"inflight", qawemail.ErrSendInFlight, http.StatusConflict, "结果未知"},
|
||||
{"nopassword", qawemail.ErrNoPassword, http.StatusConflict, "授权码"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
store := &fakeWeeklyMailStore{sendErr: tc.err}
|
||||
recorder := httptest.NewRecorder()
|
||||
qaWeeklyMailSend(recorder, weeklyMailSendRequest(t, roleAdmin, `{"revision":16,"body_sha256":"`+strings.Repeat("c", 64)+`"}`, true), Config{QAWeeklyMail: store})
|
||||
if recorder.Code != tc.status || !strings.Contains(recorder.Body.String(), tc.contains) {
|
||||
t.Fatalf("%s 应 %d 且说明原因 %q,status=%d body=%s", tc.name, tc.status, tc.contains, recorder.Code, recorder.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// SMTP 真实失败:必须 502 且带上真实原因,绝不能回“已发送”。
|
||||
func TestWeeklyMailFormalSendReportsDeliveryFailure(t *testing.T) {
|
||||
store := &fakeWeeklyMailStore{result: qawemail.ManualResult{Status: "failed", Reason: "发送失败:smtp 登录失败: 535"}}
|
||||
recorder := httptest.NewRecorder()
|
||||
qaWeeklyMailSend(recorder, weeklyMailSendRequest(t, roleAdmin, `{"revision":16,"body_sha256":"`+strings.Repeat("d", 64)+`"}`, true), Config{QAWeeklyMail: store})
|
||||
if recorder.Code != http.StatusBadGateway || !strings.Contains(recorder.Body.String(), "535") {
|
||||
t.Fatalf("未送达必须 502 并带真实原因,status=%d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// 补发前预览必须给出真实 week/版本/收件人数,用于确认框;预览本身不投递。
|
||||
func TestWeeklyMailSendPreviewShowsExactDistribution(t *testing.T) {
|
||||
store := &fakeWeeklyMailStore{preview: qawemail.SendPreview{Status: "ready", Recipients: 119, Cc: 8}}
|
||||
hash := strings.Repeat("e", 64)
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/v1/qa-ops/master/weekly-mail/send/2026-W45?revision=16&body_sha256="+hash, nil)
|
||||
request.SetPathValue("week", "2026-W45")
|
||||
request = request.WithContext(context.WithValue(request.Context(), sessionKey{}, sessionIdentity{Username: "tester", Role: roleAdmin}))
|
||||
recorder := httptest.NewRecorder()
|
||||
qaWeeklyMailSendPreview(recorder, request, Config{QAWeeklyMail: store})
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("预览应成功,status=%d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
var payload struct {
|
||||
Data qawemail.SendPreview `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if payload.Data.Recipients != 119 || payload.Data.Cc != 8 || payload.Data.Revision != 16 || payload.Data.Status != "ready" {
|
||||
t.Fatalf("预览必须给出正式收件人数与版本:%+v", payload.Data)
|
||||
}
|
||||
if store.calls != 0 {
|
||||
t.Fatal("预览不得触发发送")
|
||||
}
|
||||
}
|
||||
@@ -181,6 +181,9 @@ type Config struct {
|
||||
LibraryProductionInternalKB string
|
||||
LibraryProductionExternalKB string
|
||||
LibraryProductionTenantID uint64
|
||||
// 整库发布还落在这些后端上;页面要显示"到底写哪儿",只给 KB ID 不够。
|
||||
LibraryProductionIndex string
|
||||
LibraryProductionS3Bucket string
|
||||
// LibraryScopeProbe 让测试注入"目标整库是否为空"的探测;留空即用真实连接。
|
||||
// 它不是运行时开关:生产部署不设置它,那次探测仍然直连目标库逐表计数。
|
||||
LibraryScopeProbe func(ctx context.Context, scope string) (bool, map[string]int64, error)
|
||||
@@ -198,8 +201,16 @@ type Config struct {
|
||||
RequireWeKnoraReady bool
|
||||
ProductionReleaseRequestsEnabled bool
|
||||
// TestAutoReviewPolicy 只用于展示:让运维在界面上看到"现在自动批准到什么程度"。
|
||||
TestAutoReviewPolicy string
|
||||
ProductionBackupRestoreVerified bool
|
||||
TestAutoReviewPolicy string
|
||||
ProductionBackupRestoreVerified bool
|
||||
// restoreVerified / restoreEvidenceReason 由请求期的证据读取填入(withRestoreEvidence):
|
||||
// 闸口按"当前目标是否有真实恢复验证证据"判断,而不是按可手填的 env。
|
||||
// 它们不可由配置注入,避免把证据变成另一个开关。
|
||||
restoreVerified bool
|
||||
restoreEvidenceReason string
|
||||
// restoreEvidenceChecked 表示这个请求真的读过证据行:只有读过之后才适用
|
||||
// "失败/读取错误优先于 env"。从未读过证据的旧调用点仍按历史 env 语义兜底。
|
||||
restoreEvidenceChecked bool
|
||||
ProductionReleaseAuthorizerRef string
|
||||
ProductionReleaseAuthorizationToken string
|
||||
// DingtalkSources 枚举「我账号下能看到」的钉钉知识库与钉盘空间(只读,dws)。
|
||||
@@ -271,6 +282,8 @@ func NewHandler(config Config) (http.Handler, error) {
|
||||
mux.HandleFunc("DELETE /api/v1/qa-ops/master/weekly-mail/confirmations/{week}", func(w http.ResponseWriter, r *http.Request) { qaWeeklyMailConfirm(w, r, config) })
|
||||
mux.HandleFunc("GET /api/v1/qa-ops/master/weekly-mail/sends", func(w http.ResponseWriter, r *http.Request) { qaWeeklyMailSends(w, r, config) })
|
||||
mux.HandleFunc("POST /api/v1/qa-ops/master/weekly-mail/test", func(w http.ResponseWriter, r *http.Request) { qaWeeklyMailTest(w, r, config) })
|
||||
mux.HandleFunc("GET /api/v1/qa-ops/master/weekly-mail/send/{week}", func(w http.ResponseWriter, r *http.Request) { qaWeeklyMailSendPreview(w, r, config) })
|
||||
mux.HandleFunc("POST /api/v1/qa-ops/master/weekly-mail/send/{week}", func(w http.ResponseWriter, r *http.Request) { qaWeeklyMailSend(w, r, config) })
|
||||
mux.HandleFunc("POST /api/v1/qa-ops/master/weekly-mail/confirmations/{week}/image", func(w http.ResponseWriter, r *http.Request) { qaWeeklyMailImage(w, r, config) })
|
||||
mux.HandleFunc("GET /api/v1/qa-ops/master/field-options", func(w http.ResponseWriter, r *http.Request) { qaMasterFieldOptions(w, r, config) })
|
||||
mux.HandleFunc("POST /api/v1/qa-ops/master/records/{id}/fields", func(w http.ResponseWriter, r *http.Request) { qaMasterFieldPatch(w, r, config) })
|
||||
@@ -548,6 +561,10 @@ func NewHandler(config Config) (http.Handler, error) {
|
||||
mux.HandleFunc("POST /api/v1/library/jobs/{id}/resume", func(w http.ResponseWriter, r *http.Request) {
|
||||
libraryJobResume(w, r, config)
|
||||
})
|
||||
// 封存完成但终态回写失败时,只按已完成队列+精确版本+归档复核修复该作业。
|
||||
mux.HandleFunc("POST /api/v1/library/jobs/{id}/reconcile-capture", func(w http.ResponseWriter, r *http.Request) {
|
||||
libraryCaptureReconcile(w, r, config)
|
||||
})
|
||||
// 生产同步唯一入口:闸口未齐不建作业,返回缺项而不是假 Accepted。
|
||||
mux.HandleFunc("POST /api/v1/library/syncs", func(w http.ResponseWriter, r *http.Request) {
|
||||
librarySyncEnqueue(w, r, config)
|
||||
@@ -556,6 +573,14 @@ func NewHandler(config Config) (http.Handler, error) {
|
||||
mux.HandleFunc("POST /api/v1/library/baselines", func(w http.ResponseWriter, r *http.Request) {
|
||||
libraryBaselineEnqueue(w, r, config)
|
||||
})
|
||||
// 恢复验证:真实备份 → canary → 恢复 → 复核,通过后证据按当前目标登记,
|
||||
// 同步闸口据此放行(不再需要页面/环境变量手填 verified)。
|
||||
mux.HandleFunc("POST /api/v1/library/verifications", func(w http.ResponseWriter, r *http.Request) {
|
||||
libraryVerifyEnqueue(w, r, config)
|
||||
})
|
||||
mux.HandleFunc("GET /api/v1/library/verifications", func(w http.ResponseWriter, r *http.Request) {
|
||||
libraryVerifications(w, r, config)
|
||||
})
|
||||
// 全部 main 提交时间线(读本库缓存,不直接打 Gitea)
|
||||
mux.HandleFunc("GET /api/v1/version-history", func(w http.ResponseWriter, r *http.Request) {
|
||||
versionHistoryList(w, r, config)
|
||||
|
||||
@@ -422,27 +422,28 @@ func (c *Coordinator) save(j journal) error {
|
||||
return dir.Sync()
|
||||
}
|
||||
func (c *Coordinator) load(job string) (journal, error) {
|
||||
f, err := c.root.OpenFile(job+".json", os.O_RDONLY|syscall.O_NOFOLLOW, 0)
|
||||
if err != nil {
|
||||
return journal{}, err
|
||||
}
|
||||
defer f.Close()
|
||||
info, err := f.Stat()
|
||||
if err != nil || !info.Mode().IsRegular() || info.Size() > 1024*1024 {
|
||||
return journal{}, errors.New("invalid freeze journal file")
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(f, 1024*1024+1))
|
||||
data, err := readJournalBytes(c.root, job)
|
||||
if err != nil {
|
||||
return journal{}, err
|
||||
}
|
||||
return parseJournal(data, job, c.source)
|
||||
}
|
||||
|
||||
// parseJournal 是 journal 结构校验的唯一实现:Coordinator.load 与
|
||||
// ReadJournal 必须用同一套判据,否则「台账认为合法、激活门认为不合法」迟早分叉。
|
||||
// source 为空表示调用方不预设来源(激活门另行把它绑到治理库实例上)。
|
||||
func parseJournal(data []byte, job, source string) (journal, error) {
|
||||
var j journal
|
||||
dec := json.NewDecoder(bytes.NewReader(data))
|
||||
dec.DisallowUnknownFields()
|
||||
if err = dec.Decode(&j); err != nil {
|
||||
if err := dec.Decode(&j); err != nil {
|
||||
return journal{}, errors.New("invalid freeze journal")
|
||||
}
|
||||
var extra any
|
||||
if dec.Decode(&extra) != io.EOF || j.Version != 1 || j.Source != c.source || j.Job != job || !validID(j.Job) {
|
||||
if dec.Decode(&extra) != io.EOF || j.Version != 1 || j.Job != job || !validID(j.Job) || !validID(j.Source) {
|
||||
return journal{}, errors.New("freeze journal identity mismatch")
|
||||
}
|
||||
if source != "" && j.Source != source {
|
||||
return journal{}, errors.New("freeze journal identity mismatch")
|
||||
}
|
||||
switch j.State {
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
package libraryfreeze
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// JournalEvidence 是维护窗口 journal 的**只读**事实视图。
|
||||
//
|
||||
// 恢复派发的激活门必须消费真实维护窗口写下的证据,而不是调用方自填的
|
||||
// "stopped=true"。这里复用 Coordinator.load 的同一套结构校验(parseJournal),
|
||||
// 只额外暴露事实字段与文件摘要;它不执行任何恢复动作,也不改 journal。
|
||||
//
|
||||
// 边界:本函数证明"这份 journal 字节是维护窗口写下的、结构与身份自洽",
|
||||
// 不独立证明外部 writer 此刻不在运行;后者由 journal 记录的入口排空/退出码
|
||||
// 与治理库维护行共同约束(见 docs/architecture/ds01-activation-evidence.md)。
|
||||
type JournalEvidence struct {
|
||||
Job string
|
||||
Source string
|
||||
Level string
|
||||
State string
|
||||
TargetFingerprint string
|
||||
Ingress string
|
||||
IngressClosed bool
|
||||
DrainVerified bool
|
||||
ContentRisk bool
|
||||
RestoreBlocked bool
|
||||
Gate *GateEvidence
|
||||
Quiesce *QuiesceProof
|
||||
Services []JournalService
|
||||
StartedAt time.Time
|
||||
Digest string
|
||||
}
|
||||
|
||||
// JournalService 是 journal 里登记的受控 writer 与它的退出观察。
|
||||
// Running 是"开窗口时的原始状态",不能拿它判断现在是否在跑;真正的停写证据是
|
||||
// StopRequested + ExitCode(以及 Accidental 的入口 drain),与 load 的校验口径一致。
|
||||
type JournalService struct {
|
||||
ID string
|
||||
Running bool
|
||||
Healthy bool
|
||||
ExitCode *int
|
||||
StopRequested bool
|
||||
StopVerified bool
|
||||
Restored bool
|
||||
}
|
||||
|
||||
// ReadJournal 从维护 journal 根目录读取 <job>.json,返回校验后的事实与 sha256。
|
||||
//
|
||||
// 目录必须真实且私有(0700、非符号链接),文件以 O_NOFOLLOW 打开、限长 1MiB;
|
||||
// 这些是维护链自己写 journal 时的同一组约束,放宽其中任何一条都会让"证据来源"
|
||||
// 变成随便一个可写目录。
|
||||
func ReadJournal(dir, job string) (JournalEvidence, error) {
|
||||
var evidence JournalEvidence
|
||||
if !validID(job) {
|
||||
return evidence, errors.New("invalid maintenance journal job id")
|
||||
}
|
||||
if !filepath.IsAbs(dir) {
|
||||
return evidence, errors.New("maintenance journal root must be absolute")
|
||||
}
|
||||
info, err := os.Lstat(dir)
|
||||
if err != nil || !info.IsDir() || info.Mode().Perm() != 0700 {
|
||||
return evidence, errors.New("maintenance journal root must be a private real directory")
|
||||
}
|
||||
resolved, err := filepath.EvalSymlinks(dir)
|
||||
if err != nil || resolved != filepath.Clean(dir) {
|
||||
return evidence, errors.New("maintenance journal root contains symlink")
|
||||
}
|
||||
root, err := os.OpenRoot(dir)
|
||||
if err != nil {
|
||||
return evidence, err
|
||||
}
|
||||
defer root.Close()
|
||||
data, err := readJournalBytes(root, job)
|
||||
if err != nil {
|
||||
return evidence, err
|
||||
}
|
||||
j, err := parseJournal(data, job, "")
|
||||
if err != nil {
|
||||
return evidence, err
|
||||
}
|
||||
sum := sha256.Sum256(data)
|
||||
services := make([]JournalService, len(j.Services))
|
||||
for i, s := range j.Services {
|
||||
services[i] = JournalService{ID: s.ID, Running: s.Running, Healthy: s.Healthy, ExitCode: s.ExitCode,
|
||||
StopRequested: s.StopRequested, StopVerified: s.StopVerified, Restored: s.Restored}
|
||||
}
|
||||
evidence = JournalEvidence{
|
||||
Job: job, Source: j.Source, Level: j.Level, State: j.State,
|
||||
TargetFingerprint: j.TargetFingerprint, Ingress: j.Ingress,
|
||||
IngressClosed: j.IngressClosed, DrainVerified: j.DrainVerified,
|
||||
ContentRisk: j.ContentRisk, RestoreBlocked: j.RestoreBlocked,
|
||||
Gate: j.Gate, Quiesce: j.Quiesce, Services: services,
|
||||
StartedAt: j.StartedAt.UTC(), Digest: hex.EncodeToString(sum[:]),
|
||||
}
|
||||
return evidence, nil
|
||||
}
|
||||
|
||||
func readJournalBytes(root *os.Root, job string) ([]byte, error) {
|
||||
if !validID(job) {
|
||||
return nil, errors.New("invalid freeze journal name")
|
||||
}
|
||||
f, err := root.OpenFile(job+".json", os.O_RDONLY|syscall.O_NOFOLLOW, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
info, err := f.Stat()
|
||||
if err != nil || !info.Mode().IsRegular() || info.Size() > 1024*1024 {
|
||||
return nil, errors.New("invalid freeze journal file")
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(f, 1024*1024+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(data) > 1024*1024 {
|
||||
return nil, errors.New("invalid freeze journal file")
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
package libraryrelease
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// 恢复标记派发协议(DS01 窄协议)。
|
||||
//
|
||||
// 这里定义的唯一持久事实是:某个固定的删除型恢复 effect 是否已经有人拿到过
|
||||
// "向索引发送一次 `_create`"的许可。它不是"ES 历史上从未有人写过这个 ID"的证明,
|
||||
// 也不是目标端 fence。判据是单调的持久状态,不是 missing、stats 或内存布尔值。
|
||||
const (
|
||||
// RestoreMarkerPhaseCreate 是恢复期独占 create(删除型文档的原文标记)的唯一相位。
|
||||
// 其它相位(apply create、compensation DELETE、clean CAS)不得借用它取得发送许可。
|
||||
RestoreMarkerPhaseCreate = "restore-marker-create"
|
||||
|
||||
RestoreDispatchPrepared = "prepared"
|
||||
RestoreDispatchDispatching = "dispatching"
|
||||
RestoreDispatchMarkerConfirmed = "marker_confirmed"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrRestoreDispatchHeld = errors.New("restore marker dispatch is held")
|
||||
ErrRestorePermitSpent = errors.New("restore send permit already used")
|
||||
ErrRestorePermitMismatch = errors.New("restore send permit bound to another effect")
|
||||
ErrRestoreDispatchUnavailable = errors.New("restore marker dispatch protocol is not activated for this target")
|
||||
)
|
||||
|
||||
// RestoreMarkerKey 是"同一个逻辑恢复 effect"的身份。epoch、恢复队列 ID、恢复作业 ID
|
||||
// 都不在 key 里:接管与重试必须落到同一行,不能靠换 key 绕开旧的 dispatching。
|
||||
type RestoreMarkerKey struct {
|
||||
TargetIdentity string `json:"target_identity"`
|
||||
OperationID string `json:"operation_id"`
|
||||
WindowJobID string `json:"window_job_id"`
|
||||
PlanDigest string `json:"plan_digest"`
|
||||
Index string `json:"index"`
|
||||
DocumentID string `json:"document_id"`
|
||||
Phase string `json:"phase"`
|
||||
}
|
||||
|
||||
func (k RestoreMarkerKey) Validate() error {
|
||||
for name, value := range map[string]string{
|
||||
"target": k.TargetIdentity, "operation": k.OperationID, "window job": k.WindowJobID,
|
||||
"plan": k.PlanDigest, "index": k.Index, "document": k.DocumentID, "phase": k.Phase,
|
||||
} {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return fmt.Errorf("restore marker key lacks %s", name)
|
||||
}
|
||||
}
|
||||
if k.Phase != RestoreMarkerPhaseCreate {
|
||||
return fmt.Errorf("restore marker key phase %q is not the single create phase", k.Phase)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RestoreMarkerBinding 是派发许可的不可变绑定。目标、快照、计划、变更体与标记体任一
|
||||
// 变化都必须 held,不能另起一行继续写。
|
||||
//
|
||||
// SourceSnapshot 是发布基线(原作业要同步到的那份来源),BackupSnapshot 是本次失败
|
||||
// 写集的写前镜像;两者必须来自原持久锚点且不能互相代替。TargetInstance /
|
||||
// TargetFingerprint / IndexUUID 冻结"实际目标",用于识别目标被同名字替换;它们不证明
|
||||
// 历史上没有发送过任何请求。
|
||||
type RestoreMarkerBinding struct {
|
||||
Key RestoreMarkerKey `json:"key"`
|
||||
Scope string `json:"scope"`
|
||||
SourceSnapshot string `json:"source_snapshot"`
|
||||
BackupSnapshot string `json:"backup_snapshot"`
|
||||
TargetInstance string `json:"target_instance"`
|
||||
TargetFingerprint string `json:"target_fingerprint"`
|
||||
IndexUUID string `json:"index_uuid"`
|
||||
RecoveryJobID string `json:"recovery_job_id,omitempty"`
|
||||
RecoveryOperationID string `json:"recovery_operation_id,omitempty"`
|
||||
ChangeDigest string `json:"change_digest"`
|
||||
MarkerDigest string `json:"marker_digest"`
|
||||
RequestDigest string `json:"request_digest"`
|
||||
Owner string `json:"owner"`
|
||||
Epoch int64 `json:"epoch"`
|
||||
}
|
||||
|
||||
func (b RestoreMarkerBinding) Validate() error {
|
||||
if err := b.Key.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
for name, value := range map[string]string{
|
||||
"scope": b.Scope, "source snapshot": b.SourceSnapshot, "backup snapshot": b.BackupSnapshot,
|
||||
"target instance": b.TargetInstance, "target fingerprint": b.TargetFingerprint,
|
||||
"index uuid": b.IndexUUID, "change digest": b.ChangeDigest, "marker digest": b.MarkerDigest,
|
||||
"request digest": b.RequestDigest, "owner": b.Owner,
|
||||
} {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return fmt.Errorf("restore marker binding lacks %s", name)
|
||||
}
|
||||
}
|
||||
if b.SourceSnapshot == b.BackupSnapshot {
|
||||
return errors.New("restore marker binding uses the source snapshot as the backup image")
|
||||
}
|
||||
if b.RecoveryJobID == "" != (b.RecoveryOperationID == "") {
|
||||
return errors.New("restore marker binding has a partial recovery delivery identity")
|
||||
}
|
||||
if b.Epoch <= 0 {
|
||||
return errors.New("restore marker binding lacks a positive attempt epoch")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Digest 覆盖**逻辑 effect 的不可变绑定**;台账用它拒绝"同 key 换绑定"。
|
||||
// owner/epoch 与 recovery queue/operation 是执行/投递锚点,不进摘要:自动回滚与
|
||||
// 随后的 recovery worker 必须落到同一个 effect 身份,但它们的投递身份不同。
|
||||
func (b RestoreMarkerBinding) Digest() (string, error) {
|
||||
if err := b.Validate(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
joined := strings.Join([]string{
|
||||
"v2", b.Key.TargetIdentity, b.Key.OperationID, b.Key.WindowJobID, b.Key.PlanDigest,
|
||||
b.Key.Index, b.Key.DocumentID, b.Key.Phase, b.Scope, b.SourceSnapshot, b.BackupSnapshot,
|
||||
b.TargetInstance, b.TargetFingerprint, b.IndexUUID,
|
||||
b.ChangeDigest, b.MarkerDigest, b.RequestDigest,
|
||||
}, "\x00")
|
||||
sum := sha256.Sum256([]byte(joined))
|
||||
return hex.EncodeToString(sum[:]), nil
|
||||
}
|
||||
|
||||
// RestoreDispatchStatus 是纯读分类结果。Held 为真时永远不能发送。
|
||||
//
|
||||
// BindingConflict 是与 Held 不同的**类型化**分类:Held 只说"这次不能再 create"
|
||||
// (prepared 冲突、dispatching 结果未知、marker 已确认都可能是合法的不可重发),
|
||||
// 而 BindingConflict 说明台账行的不可变绑定与本次绑定不是同一个 effect。绑定冲突
|
||||
// 在**任何状态**下都必须拒绝,包括已完成 clean 的 marker_confirmed;否则"只允许原
|
||||
// 确认分支继续"会变成放行一份不属于本计划的写入。
|
||||
type RestoreDispatchStatus struct {
|
||||
State string `json:"state"`
|
||||
Held bool `json:"held"`
|
||||
BindingConflict bool `json:"binding_conflict,omitempty"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
}
|
||||
|
||||
// RestoreSendPermit 是"这一次调用、这一个 effect、这一次发送"的许可。
|
||||
//
|
||||
// 它不是可复用凭证:Store 只在 dispatching 明确 commit 之后返回它;消费一次即作废。
|
||||
// 不能从 Load/Get/JSON/重启重建——那正是本协议要排除的"从外观推断未派发"。
|
||||
type RestoreSendPermit struct {
|
||||
key RestoreMarkerKey
|
||||
binding string
|
||||
nonce string
|
||||
used bool
|
||||
}
|
||||
|
||||
// IssueRestoreSendPermit 只允许实现 RestoreMarkerGate 的存储层调用。许可的权威性来自
|
||||
// 数据库 CAS 与提交,不来自这个构造函数;这里只保证调用栈内一次性。
|
||||
func IssueRestoreSendPermit(key RestoreMarkerKey, bindingDigest, nonce string) *RestoreSendPermit {
|
||||
return &RestoreSendPermit{key: key, binding: bindingDigest, nonce: nonce}
|
||||
}
|
||||
|
||||
func (p *RestoreSendPermit) Nonce() string {
|
||||
if p == nil {
|
||||
return ""
|
||||
}
|
||||
return p.nonce
|
||||
}
|
||||
|
||||
// Consume 校验 effect/绑定并作废许可。绑定不符时不消耗许可(避免被一次误用烧掉)。
|
||||
func (p *RestoreSendPermit) Consume(key RestoreMarkerKey, bindingDigest string) error {
|
||||
if p == nil {
|
||||
return ErrRestorePermitMismatch
|
||||
}
|
||||
if p.used {
|
||||
return ErrRestorePermitSpent
|
||||
}
|
||||
if p.key != key || p.binding != bindingDigest {
|
||||
return ErrRestorePermitMismatch
|
||||
}
|
||||
p.used = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// RestoreDispatchRecord 是权威台账里"这个原 window 还有哪些未决 effect"的一条。
|
||||
// 收尾与前置都查同一份记录集,不能只遍历当前 plan 的文档列表。
|
||||
type RestoreDispatchRecord struct {
|
||||
PlanDigest string `json:"plan_digest"`
|
||||
Index string `json:"index"`
|
||||
DocumentID string `json:"document_id"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
|
||||
// RestoreMarkerGate 是唯一能授予恢复标记 create 发送权的接口。
|
||||
//
|
||||
// Inspect 纯读、不返回许可;Prepare 只建立 prepared;Acquire 在独立短事务里做
|
||||
// prepared→dispatching 的 CAS,只有明确提交后才返回许可;Confirm 只记录真实回执;
|
||||
// UnresolvedRestoreDispatches 返回原 window 的权威未决全集(跨 plan,只读)。
|
||||
type RestoreMarkerGate interface {
|
||||
InspectRestoreMarker(ctx context.Context, b RestoreMarkerBinding) (RestoreDispatchStatus, bool, error)
|
||||
PrepareRestoreMarker(ctx context.Context, b RestoreMarkerBinding) (RestoreDispatchStatus, error)
|
||||
AcquireRestoreMarker(ctx context.Context, b RestoreMarkerBinding) (*RestoreSendPermit, RestoreDispatchStatus, error)
|
||||
ConfirmRestoreMarker(ctx context.Context, b RestoreMarkerBinding, receipt []byte) error
|
||||
UnresolvedRestoreDispatches(ctx context.Context, targetIdentity, operationID, windowJobID string) ([]RestoreDispatchRecord, error)
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package libraryrelease
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func restoreBinding() RestoreMarkerBinding {
|
||||
return RestoreMarkerBinding{
|
||||
Key: RestoreMarkerKey{TargetIdentity: "target-a", OperationID: "op-1", WindowJobID: "job-1",
|
||||
PlanDigest: "plan-1", Index: "kg-index", DocumentID: "doc-1", Phase: RestoreMarkerPhaseCreate},
|
||||
Scope: "internal", SourceSnapshot: "snap-src", BackupSnapshot: "snap-backup",
|
||||
TargetInstance: "inst-1", TargetFingerprint: "fingerprint-1", IndexUUID: "uuid-1",
|
||||
ChangeDigest: "change-1", MarkerDigest: "marker-1", RequestDigest: "request-1",
|
||||
Owner: "worker-a#queue#1", Epoch: 3,
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestoreMarkerBindingRejectsIncompleteIdentity(t *testing.T) {
|
||||
for name, mutate := range map[string]func(*RestoreMarkerBinding){
|
||||
"target": func(b *RestoreMarkerBinding) { b.Key.TargetIdentity = "" },
|
||||
"operation": func(b *RestoreMarkerBinding) { b.Key.OperationID = "" },
|
||||
"window": func(b *RestoreMarkerBinding) { b.Key.WindowJobID = "" },
|
||||
"plan": func(b *RestoreMarkerBinding) { b.Key.PlanDigest = "" },
|
||||
"index": func(b *RestoreMarkerBinding) { b.Key.Index = "" },
|
||||
"document": func(b *RestoreMarkerBinding) { b.Key.DocumentID = "" },
|
||||
"change": func(b *RestoreMarkerBinding) { b.ChangeDigest = "" },
|
||||
"marker": func(b *RestoreMarkerBinding) { b.MarkerDigest = "" },
|
||||
"request": func(b *RestoreMarkerBinding) { b.RequestDigest = "" },
|
||||
"owner": func(b *RestoreMarkerBinding) { b.Owner = "" },
|
||||
"epoch": func(b *RestoreMarkerBinding) { b.Epoch = 0 },
|
||||
"phase": func(b *RestoreMarkerBinding) { b.Key.Phase = "apply-create" },
|
||||
"instance": func(b *RestoreMarkerBinding) { b.TargetInstance = "" },
|
||||
"fingerprint": func(b *RestoreMarkerBinding) { b.TargetFingerprint = "" },
|
||||
"index uuid": func(b *RestoreMarkerBinding) { b.IndexUUID = "" },
|
||||
"source as backup": func(b *RestoreMarkerBinding) { b.BackupSnapshot = b.SourceSnapshot },
|
||||
} {
|
||||
binding := restoreBinding()
|
||||
mutate(&binding)
|
||||
if err := binding.Validate(); err == nil {
|
||||
t.Fatalf("%s: incomplete binding accepted", name)
|
||||
}
|
||||
}
|
||||
if err := restoreBinding().Validate(); err != nil {
|
||||
t.Fatalf("complete binding rejected: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestoreMarkerBindingDigestCoversEveryBoundValue(t *testing.T) {
|
||||
base, err := restoreBinding().Digest()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again, _ := restoreBinding().Digest(); again != base {
|
||||
t.Fatal("binding digest is not deterministic")
|
||||
}
|
||||
for name, mutate := range map[string]func(*RestoreMarkerBinding){
|
||||
"target": func(b *RestoreMarkerBinding) { b.Key.TargetIdentity = "target-b" },
|
||||
"index": func(b *RestoreMarkerBinding) { b.Key.Index = "other-index" },
|
||||
"doc": func(b *RestoreMarkerBinding) { b.Key.DocumentID = "doc-2" },
|
||||
"plan": func(b *RestoreMarkerBinding) { b.Key.PlanDigest = "plan-2" },
|
||||
"backup": func(b *RestoreMarkerBinding) { b.BackupSnapshot = "snap-2" },
|
||||
"change": func(b *RestoreMarkerBinding) { b.ChangeDigest = "change-2" },
|
||||
"marker": func(b *RestoreMarkerBinding) { b.MarkerDigest = "marker-2" },
|
||||
"request": func(b *RestoreMarkerBinding) { b.RequestDigest = "request-2" },
|
||||
"scope": func(b *RestoreMarkerBinding) { b.Scope = "external" },
|
||||
"source": func(b *RestoreMarkerBinding) { b.SourceSnapshot = "snap-9" },
|
||||
"instance": func(b *RestoreMarkerBinding) { b.TargetInstance = "inst-2" },
|
||||
"fingerprint": func(b *RestoreMarkerBinding) { b.TargetFingerprint = "fingerprint-2" },
|
||||
"index uuid": func(b *RestoreMarkerBinding) { b.IndexUUID = "uuid-2" },
|
||||
} {
|
||||
binding := restoreBinding()
|
||||
mutate(&binding)
|
||||
changed, err := binding.Digest()
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
if changed == base {
|
||||
t.Fatalf("%s: binding change not covered by digest", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 许可只能在本次调用栈用一次;换 key/换绑定都不能复用,也不能把已花的许可再花一次。
|
||||
// 恢复投递身份是执行锚点,不是 effect 身份:自动回滚与 recovery worker 的摘要必须一致。
|
||||
func TestRestoreBindingDigestIgnoresDeliveryIdentity(t *testing.T) {
|
||||
base, err := restoreBinding().Digest()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
delivered := restoreBinding()
|
||||
delivered.RecoveryJobID, delivered.RecoveryOperationID = "queue-1", "op-recovery"
|
||||
delivered.Owner, delivered.Epoch = "worker-b#queue#1", 7
|
||||
other, err := delivered.Digest()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if other != base {
|
||||
t.Fatal("delivery/attempt identity changed the logical effect digest")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestoreSendPermitIsSingleUseAndKeyBound(t *testing.T) {
|
||||
binding := restoreBinding()
|
||||
digest, err := binding.Digest()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
permit := IssueRestoreSendPermit(binding.Key, digest, "nonce-1")
|
||||
other := binding.Key
|
||||
other.DocumentID = "doc-2"
|
||||
if err := permit.Consume(other, digest); err == nil || !errors.Is(err, ErrRestorePermitMismatch) {
|
||||
t.Fatalf("permit usable for another effect: %v", err)
|
||||
}
|
||||
if err := permit.Consume(binding.Key, "other-binding"); err == nil || !errors.Is(err, ErrRestorePermitMismatch) {
|
||||
t.Fatalf("permit usable for another binding: %v", err)
|
||||
}
|
||||
if err := permit.Consume(binding.Key, digest); err != nil {
|
||||
t.Fatalf("valid permit rejected: %v", err)
|
||||
}
|
||||
if err := permit.Consume(binding.Key, digest); err == nil || !errors.Is(err, ErrRestorePermitSpent) {
|
||||
t.Fatalf("permit spent twice: %v", err)
|
||||
}
|
||||
if strings.Contains(permit.Nonce(), digest) {
|
||||
t.Fatal("nonce is not independent audit data")
|
||||
}
|
||||
var nilPermit *RestoreSendPermit
|
||||
if err := nilPermit.Consume(binding.Key, digest); err == nil {
|
||||
t.Fatal("nil permit accepted")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
package libraryrelease
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// 恢复派发激活证据(DS01-F3,协议版本 2)。
|
||||
//
|
||||
// 配置开关、"三个非空字符串"或调用方自填的 `stopped=true` 都不是独占证明。v2 证据
|
||||
// 只允许**指向**一份由维护窗口写下的 journal:调用方声明目标身份、维护作业、journal
|
||||
// 文件名/摘要与受控 writer 的不可变实例身份;真正的事实(入口关闭与排空、后台静默、
|
||||
// 每个 writer 的退出观察)由 Store 直接读 journal 字节并解析取得,再用治理库的
|
||||
// library_instance_maintenance 行把作业/实例/指纹/writer 集合钉死。
|
||||
//
|
||||
// 因此"把调用方 JSON 再 hash 一次"不再算核实:摘要只证明"读到的是这一份 journal",
|
||||
// 事实来自 journal 本体与治理库行,而不是调用方的布尔字段。
|
||||
//
|
||||
// 边界:代码能核验 journal 完整性、目标/实例/治理库/构建身份、writer 集合一致性与
|
||||
// 新鲜度;它不独立观测"外部 writer 此刻没在运行",但 journal 的入口 drain(
|
||||
// active_at_close=0)、quiesce 零点与退出码是维护链自己的受控观察,且窗口必须仍处于
|
||||
// held(已恢复/释放的窗口不构成"旧 writer 已停"的证据)。
|
||||
|
||||
const RestoreDispatchActivationMaxAge = 2 * time.Hour
|
||||
|
||||
// 证据文档 schema 版本与语义兼容批次是两件事:
|
||||
// - ProtocolVersion:这份 JSON 的字段形状(解析契约);
|
||||
// - ProtocolEpoch:派发协议语义批次;不同批次必须重新登记。它只约束派发语义,
|
||||
// 不能证明某个构建 SHA 属于该批次 —— 当前构建身份仍精确绑定运行 SHA,跨 SHA
|
||||
// 自动兼容在没有受信批次成员证明前保持关闭。
|
||||
//
|
||||
// 构建身份用完整 40 位 SHA,不用 display:开发号会随提交数变化,两个不同二进制
|
||||
// 完全可能都叫 0.1.322。
|
||||
const RestoreDispatchActivationProtocolVersion = 3
|
||||
|
||||
const RestoreDispatchProtocolEpoch = 3
|
||||
|
||||
// RestoreDispatchGovernanceAuthority 是治理库的**权威域**身份:系统标识(集群)
|
||||
// 加库 OID。只看 current_database() 无法区分"同名但另一个集群/另一份克隆"。
|
||||
// primary 由 Store 现场读 pg_is_in_recovery() 得出,不采信调用方声明。
|
||||
type RestoreDispatchGovernanceAuthority struct {
|
||||
SystemIdentifier string `json:"system_identifier"`
|
||||
DatabaseOID string `json:"database_oid"`
|
||||
}
|
||||
|
||||
// RestoreDispatchWriterDrain 是调用方对"旧 writer 集合"的声明:只给身份,不给结论。
|
||||
// InstanceID 必须等于 journal 里登记的不可变 writer 身份(容器/进程 ID)。
|
||||
type RestoreDispatchWriterDrain struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
InstanceID string `json:"instance_id"`
|
||||
}
|
||||
|
||||
type RestoreDispatchActivationEvidence struct {
|
||||
ProtocolVersion int `json:"protocol_version"`
|
||||
ProtocolEpoch int `json:"protocol_epoch"`
|
||||
TargetIdentity string `json:"target_identity"`
|
||||
// TargetFingerprint 是维护窗口/部署目标指纹(journal 与治理库维护行记录的那一个);
|
||||
// RestoreTargetFingerprint 是激活时对恢复目标内容/索引/桶的实时观察值。两者是不同层,
|
||||
// 不能相互替代:前者绑定"动的是哪个实例",后者绑定"恢复要写的那个目标身份"。
|
||||
TargetFingerprint string `json:"target_fingerprint"`
|
||||
RestoreTargetFingerprint string `json:"restore_target_fingerprint"`
|
||||
TargetInstance string `json:"target_instance"`
|
||||
GovernanceDatabase string `json:"governance_database"`
|
||||
GovernanceAuthority RestoreDispatchGovernanceAuthority `json:"governance_authority"`
|
||||
ProtocolBuild string `json:"protocol_build"`
|
||||
ProtocolBuildSHA string `json:"protocol_build_sha"`
|
||||
AttestedBy string `json:"attested_by"`
|
||||
AttestedAt time.Time `json:"attested_at"`
|
||||
MaintenanceJob string `json:"maintenance_job"`
|
||||
MaintenanceJournal string `json:"maintenance_journal"`
|
||||
MaintenanceJournalSHA256 string `json:"maintenance_journal_sha256"`
|
||||
Writers []RestoreDispatchWriterDrain `json:"writers"`
|
||||
}
|
||||
|
||||
func (e RestoreDispatchActivationEvidence) Validate(now time.Time, maxAge time.Duration) error {
|
||||
if e.ProtocolVersion != RestoreDispatchActivationProtocolVersion {
|
||||
return fmt.Errorf("restore dispatch activation protocol version %d is not supported", e.ProtocolVersion)
|
||||
}
|
||||
// 兼容批次不同 = 派发语义不兼容:即使 display 相同也不能共用激活证据。
|
||||
if e.ProtocolEpoch != RestoreDispatchProtocolEpoch {
|
||||
return fmt.Errorf("restore dispatch activation protocol epoch %d is not compatible with %d",
|
||||
e.ProtocolEpoch, RestoreDispatchProtocolEpoch)
|
||||
}
|
||||
for name, value := range map[string]string{
|
||||
"target identity": e.TargetIdentity, "target fingerprint": e.TargetFingerprint,
|
||||
"restore target fingerprint": e.RestoreTargetFingerprint,
|
||||
"target instance": e.TargetInstance, "governance database": e.GovernanceDatabase,
|
||||
"protocol build": e.ProtocolBuild, "attested by": e.AttestedBy,
|
||||
"maintenance job": e.MaintenanceJob, "maintenance journal": e.MaintenanceJournal,
|
||||
"maintenance journal digest": e.MaintenanceJournalSHA256,
|
||||
} {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return fmt.Errorf("restore dispatch activation evidence lacks %s", name)
|
||||
}
|
||||
}
|
||||
// 构建身份必须是完整 SHA;display 可以随提交数变化,不能作为身份。
|
||||
if !isFullSHA(e.ProtocolBuildSHA) {
|
||||
return errors.New("restore dispatch activation build identity is not a full 40-character commit sha")
|
||||
}
|
||||
// 治理权威域必须是可读回的 PG 事实形状:集群系统标识 + 库 OID,两个都是十进制数字。
|
||||
if !isDecimal(e.GovernanceAuthority.SystemIdentifier) || !isDecimal(e.GovernanceAuthority.DatabaseOID) {
|
||||
return errors.New("restore dispatch activation governance authority is not a cluster/database identity")
|
||||
}
|
||||
if !isHex64(e.TargetFingerprint) || !isHex64(e.RestoreTargetFingerprint) {
|
||||
return errors.New("restore dispatch activation target fingerprints are not sha256 hex strings")
|
||||
}
|
||||
if !isHex64(e.MaintenanceJournalSHA256) {
|
||||
return errors.New("restore dispatch activation maintenance journal digest is not a sha256 hex string")
|
||||
}
|
||||
if !validActivationID(e.MaintenanceJob) {
|
||||
return errors.New("restore dispatch activation maintenance job id is invalid")
|
||||
}
|
||||
// journal 只能按作业名定位在同一受信根目录里;带路径分隔符/相对引用会绕开
|
||||
// Store 配置的证据根,让"来源"变成调用方随手给的任意文件。
|
||||
if e.MaintenanceJournal != e.MaintenanceJob+".json" {
|
||||
return errors.New("restore dispatch activation maintenance journal is not the job journal")
|
||||
}
|
||||
if len(e.Writers) == 0 || len(e.Writers) > 256 {
|
||||
return errors.New("restore dispatch activation lists no controlled writers")
|
||||
}
|
||||
names := map[string]bool{}
|
||||
instances := map[string]bool{}
|
||||
for _, writer := range e.Writers {
|
||||
if strings.TrimSpace(writer.Name) == "" || strings.TrimSpace(writer.Kind) == "" ||
|
||||
strings.TrimSpace(writer.InstanceID) == "" {
|
||||
return errors.New("restore dispatch activation writer lacks name, kind or instance id")
|
||||
}
|
||||
if !validActivationID(writer.InstanceID) {
|
||||
return errors.New("restore dispatch activation writer instance id is invalid")
|
||||
}
|
||||
if names[writer.Name] {
|
||||
return fmt.Errorf("restore dispatch activation writer %s is listed twice", writer.Name)
|
||||
}
|
||||
names[writer.Name] = true
|
||||
if instances[writer.InstanceID] {
|
||||
return fmt.Errorf("restore dispatch activation writer instance %s is listed twice", writer.InstanceID)
|
||||
}
|
||||
instances[writer.InstanceID] = true
|
||||
}
|
||||
if e.AttestedAt.IsZero() || e.AttestedAt.After(now.Add(5*time.Minute)) {
|
||||
return errors.New("restore dispatch activation attestation time is not credible")
|
||||
}
|
||||
if maxAge <= 0 {
|
||||
maxAge = RestoreDispatchActivationMaxAge
|
||||
}
|
||||
if now.Sub(e.AttestedAt) > maxAge {
|
||||
return fmt.Errorf("restore dispatch activation evidence is older than %s", maxAge)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// LoadRestoreDispatchActivationEvidence 读取并核验证据文件字节摘要。expectedDigest 必须
|
||||
// 与文件实际 sha256 相同;不接受"文件里写的摘要"。
|
||||
func LoadRestoreDispatchActivationEvidence(path, expectedDigest string, now time.Time, maxAge time.Duration) (RestoreDispatchActivationEvidence, error) {
|
||||
var evidence RestoreDispatchActivationEvidence
|
||||
if strings.TrimSpace(path) == "" || strings.TrimSpace(expectedDigest) == "" {
|
||||
return evidence, errors.New("restore dispatch activation requires an evidence file and its digest")
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return evidence, fmt.Errorf("read restore dispatch activation evidence: %w", err)
|
||||
}
|
||||
sum := sha256.Sum256(raw)
|
||||
if got := hex.EncodeToString(sum[:]); got != strings.ToLower(strings.TrimSpace(expectedDigest)) {
|
||||
return evidence, errors.New("restore dispatch activation evidence digest does not match the file")
|
||||
}
|
||||
decoder := json.NewDecoder(strings.NewReader(string(raw)))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&evidence); err != nil {
|
||||
return RestoreDispatchActivationEvidence{}, fmt.Errorf("parse restore dispatch activation evidence: %w", err)
|
||||
}
|
||||
if err := evidence.Validate(now, maxAge); err != nil {
|
||||
return RestoreDispatchActivationEvidence{}, err
|
||||
}
|
||||
return evidence, nil
|
||||
}
|
||||
|
||||
// CanonicalBytes 是写入台账的规范字节(同一份证据只有一种序列化)。
|
||||
func (e RestoreDispatchActivationEvidence) CanonicalBytes() ([]byte, error) {
|
||||
return json.Marshal(e)
|
||||
}
|
||||
|
||||
func isHex64(value string) bool {
|
||||
if len(value) != 64 || strings.ToLower(value) != value {
|
||||
return false
|
||||
}
|
||||
_, err := hex.DecodeString(value)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func isFullSHA(value string) bool {
|
||||
if len(value) != 40 || strings.ToLower(value) != value {
|
||||
return false
|
||||
}
|
||||
_, err := hex.DecodeString(value)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func isDecimal(value string) bool {
|
||||
if value == "" || len(value) > 20 {
|
||||
return false
|
||||
}
|
||||
for _, r := range value {
|
||||
if r < '0' || r > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// validActivationID 与维护 journal 的作业/服务 ID 同一字符集:证据里出现的身份
|
||||
// 必须能原样出现在 journal 中,不能靠空格/路径字符制造"看起来不同"的别名。
|
||||
func validActivationID(value string) bool {
|
||||
if len(value) == 0 || len(value) > 128 {
|
||||
return false
|
||||
}
|
||||
for _, r := range value {
|
||||
if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_') {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -19,6 +19,12 @@ const WriteSetVersion = 1
|
||||
// MigrationWriteSetVersion rejects reinterpretation by old V1 executors.
|
||||
const MigrationWriteSetVersion = 2
|
||||
|
||||
// WriteSetKindRestoreDrill marks a write set that authorizes only a recovery
|
||||
// drill: canary rows plus a zero-object/zero-index execution plan. It shares
|
||||
// the V2 envelope so the object gateway controller can verify it, but it is
|
||||
// never a publish plan and must not be adopted as one.
|
||||
const WriteSetKindRestoreDrill = "restore-drill"
|
||||
|
||||
// WriteSetRow 是台账里的一行:本次发布明确要写的精确身份 + 写前状态。
|
||||
//
|
||||
// 为什么需要"写前状态":恢复不能在目标里按"没人引用"推断残渣。历史孤儿资源、
|
||||
@@ -89,6 +95,13 @@ type LocalIDMapping struct {
|
||||
}
|
||||
|
||||
type WriteSet struct {
|
||||
// Kind 区分发布执行计划与受限演练恢复计划。空值 = 历史发布计划。
|
||||
// 演练计划必须显式为 restore-drill:它只授权 canary 回放,不含对象/索引
|
||||
// 写项,读取方不能把它当发布计划接管。
|
||||
Kind string `json:"kind,omitempty"`
|
||||
// TargetIDMode 记录这份计划是否把源身份映射成目标隔离身份,以及用哪套算法。
|
||||
// 空值 = 历史计划/写前镜像:按原 ID 回放,绝不能在恢复时二次映射。
|
||||
TargetIDMode string `json:"target_id_mode,omitempty"`
|
||||
Index []IndexWrite `json:"index,omitempty"`
|
||||
Objects []ObjectWrite `json:"objects,omitempty"`
|
||||
SchemaDigest string `json:"schema_digest,omitempty"`
|
||||
@@ -111,6 +124,32 @@ type WriteSet struct {
|
||||
Digest string `json:"digest"`
|
||||
}
|
||||
|
||||
// ValidateRestoreBinding 是恢复身份合同(DS01):写集合必须绑定**本次**预备份、
|
||||
// 同一目标 tenant/KB,且来源快照(要回放到目标的内容身份)与写前镜像不能是同一份。
|
||||
// 它是 ProductionLibraryTarget.Restore 在建任何派发绑定之前调用的同一个判定:
|
||||
// 演练的写前/来源身份也必须过这一关,不能靠调用方约定。
|
||||
func (w WriteSet) ValidateRestoreBinding(snapshotID string, tenantID uint64, knowledgeBaseID string) error {
|
||||
if snapshotID == "" {
|
||||
return errors.New("恢复需要绑定的预备份快照")
|
||||
}
|
||||
if w.BackupSnapshotID != snapshotID {
|
||||
return fmt.Errorf("写集合记录绑定的是预备份 %s,不是 %s", w.BackupSnapshotID, snapshotID)
|
||||
}
|
||||
if w.TenantID != tenantID || w.KnowledgeBaseID != knowledgeBaseID {
|
||||
return fmt.Errorf("写集合记录绑定的是 %d/%s,与目标 %d/%s 不一致",
|
||||
w.TenantID, w.KnowledgeBaseID, tenantID, knowledgeBaseID)
|
||||
}
|
||||
// 派发绑定只能来自原锚点:来源快照是发布基线,backup 是这份写集的写前镜像。
|
||||
// 二者缺失或相同都说明装配没有拿到真实身份,必须在任何写之前失败关闭。
|
||||
if w.TargetSnapshotID == "" || snapshotID == "" {
|
||||
return errors.New("恢复缺少来源快照或写前镜像,拒绝建立派发绑定")
|
||||
}
|
||||
if w.TargetSnapshotID == snapshotID {
|
||||
return errors.New("来源快照与写前镜像不能是同一份快照,拒绝建立派发绑定")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Seal 把行序规范化为可复现的顺序并写入摘要。调用方在落盘与登记之前必须先 Seal。
|
||||
// 排序在这里落地(而不是只在算摘要时排),否则落盘字节与摘要覆盖的内容不一致。
|
||||
func (w *WriteSet) Seal() error {
|
||||
@@ -167,7 +206,20 @@ func (w *WriteSet) computeDigest() (string, error) {
|
||||
|
||||
// Verify 校验摘要与必备身份。缺任一身份都必须失败:没有绑定就没有"这份记录属于
|
||||
// 这个作业/目标/预备份"的证明,拿它去删目标数据无法追责。
|
||||
const (
|
||||
// TargetIDModeScoped 是目标隔离身份的持久契约标识。
|
||||
// 算法是自定义的 sha256(namespace|tenant|kb|table|source) 16 字节派生,
|
||||
// 只保证确定性与隔离,不自称标准 UUIDv5。
|
||||
TargetIDModeScoped = "scoped-sha256-v1"
|
||||
)
|
||||
|
||||
func (w *WriteSet) Verify() error {
|
||||
if w.TargetIDMode != "" && w.TargetIDMode != TargetIDModeScoped {
|
||||
return fmt.Errorf("unsupported target id mode %q", w.TargetIDMode)
|
||||
}
|
||||
if w.TargetIDMode == TargetIDModeScoped && (w.TenantID == 0 || w.KnowledgeBaseID == "") {
|
||||
return errors.New("target id mode lacks target identity")
|
||||
}
|
||||
if w.ContractVersion != 0 && w.ContractVersion != 1 {
|
||||
return errors.New("unsupported migration contract version")
|
||||
}
|
||||
@@ -192,6 +244,22 @@ func (w *WriteSet) Verify() error {
|
||||
if w.Version != WriteSetVersion && w.Version != MigrationWriteSetVersion {
|
||||
return fmt.Errorf("写集合记录版本 %d 不受支持(当前 %d)", w.Version, WriteSetVersion)
|
||||
}
|
||||
if w.Kind != "" && w.Kind != WriteSetKindRestoreDrill {
|
||||
return fmt.Errorf("写集合记录种类 %q 不受支持", w.Kind)
|
||||
}
|
||||
if w.Kind == WriteSetKindRestoreDrill {
|
||||
// 演练恢复计划只回放写前镜像:不新建对象、不按发布计划改索引、不做本地 ID 映射。
|
||||
// 任何一项非空都说明它被当成发布计划构造/改写过,拒绝。
|
||||
if w.Version != MigrationWriteSetVersion || w.ContractVersion != 1 || len(w.SchemaDigest) != 64 {
|
||||
return errors.New("演练恢复计划缺少可执行契约(V2 + contract 1 + schema 摘要)")
|
||||
}
|
||||
if len(w.Objects) != 0 || len(w.Index) != 0 || len(w.LocalIDs) != 0 {
|
||||
return errors.New("演练恢复计划不得包含对象/索引/本地映射写项")
|
||||
}
|
||||
if w.SourceTenantID == 0 || w.SourceKnowledgeBaseID == "" {
|
||||
return errors.New("演练恢复计划缺少来源身份")
|
||||
}
|
||||
}
|
||||
if w.JobID == "" || w.BackupSnapshotID == "" || w.TargetSnapshotID == "" {
|
||||
return errors.New("写集合记录缺少作业/预备份/快照绑定")
|
||||
}
|
||||
@@ -222,6 +290,9 @@ func (w *WriteSet) Verify() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// IsRestoreDrill 表示这是一份受限演练恢复计划,而不是发布执行计划。
|
||||
func (w WriteSet) IsRestoreDrill() bool { return w.Kind == WriteSetKindRestoreDrill }
|
||||
|
||||
// CreatedIDs 返回"本次新建"的精确身份(写前不存在的行)。恢复只允许删这些。
|
||||
func (w WriteSet) CreatedIDs(table string) []string {
|
||||
ids := make([]string, 0, 8)
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
|
||||
"knowledge-governance/internal/appconfig"
|
||||
"knowledge-governance/internal/librarymaintenance"
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
"knowledge-governance/internal/librarysnapshot"
|
||||
"knowledge-governance/internal/objectgateway"
|
||||
v071 "knowledge-governance/internal/releaseadapter/v071"
|
||||
@@ -89,7 +90,50 @@ func scopeConfigOf(config appconfig.Config, scope string) (scopeConfig, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
func newTarget(ctx context.Context, config appconfig.Config, scope scopeConfig) (*v071.ProductionLibraryTarget, error) {
|
||||
// RestoreDispatchWiring 是恢复派发身份的唯一装配来源:真实 attempt(objectgateway
|
||||
// claim)、原 window 作业、部署目标实例与 scope。adapter 不接受调用方另填身份。
|
||||
type RestoreDispatchWiring struct {
|
||||
TargetIdentity string
|
||||
Scope string
|
||||
Instance string
|
||||
OperationID string
|
||||
WindowJobID string
|
||||
AttemptOwner string
|
||||
AttemptEpoch int64
|
||||
RecoveryJobID string
|
||||
RecoveryOperationID string
|
||||
}
|
||||
|
||||
// RestoreDispatchWiringFor 把本次真实执行的 claim 映射成派发身份,并在字段缺失时
|
||||
// 失败关闭。它不读取快照:来源/写前镜像只能从原 WriteSet 与 backup 引用取得。
|
||||
func RestoreDispatchWiringFor(config appconfig.Config, scope string, claim objectgateway.Claim) (RestoreDispatchWiring, error) {
|
||||
if scope != "internal" && scope != "external" {
|
||||
return RestoreDispatchWiring{}, fmt.Errorf("未知 scope %q", scope)
|
||||
}
|
||||
identity := config.LibraryTargetIdentity()
|
||||
if identity == "" || config.LibraryTargetInstance == "" || config.LibraryProductionDatabaseURL == "" {
|
||||
return RestoreDispatchWiring{}, errors.New("恢复派发缺少目标身份/实例")
|
||||
}
|
||||
if (scope == "internal" && config.LibraryProductionInternalKB == "") ||
|
||||
(scope == "external" && config.LibraryProductionExternalKB == "") {
|
||||
return RestoreDispatchWiring{}, errors.New("恢复派发缺少 scope 对应的知识库")
|
||||
}
|
||||
if claim.JobID == "" || claim.OperationID == "" || claim.Owner == "" || claim.AttemptEpoch <= 0 {
|
||||
return RestoreDispatchWiring{}, errors.New("恢复派发缺少本次真实 attempt 身份")
|
||||
}
|
||||
if (claim.RecoveryJobID == "") != (claim.RecoveryOperationID == "") {
|
||||
return RestoreDispatchWiring{}, errors.New("恢复投递身份不完整")
|
||||
}
|
||||
return RestoreDispatchWiring{
|
||||
TargetIdentity: identity, Scope: scope, Instance: config.LibraryTargetInstance,
|
||||
OperationID: claim.OperationID, WindowJobID: claim.JobID,
|
||||
AttemptOwner: claim.Owner, AttemptEpoch: claim.AttemptEpoch,
|
||||
RecoveryJobID: claim.RecoveryJobID, RecoveryOperationID: claim.RecoveryOperationID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func newTarget(ctx context.Context, config appconfig.Config, scope scopeConfig, scopeName string,
|
||||
gate libraryrelease.RestoreMarkerGate) (*v071.ProductionLibraryTarget, error) {
|
||||
objects, err := v071.NewS3ObjectStore(scope.s3)
|
||||
if err != nil {
|
||||
return nil, errors.New("生产对象存储配置无效")
|
||||
@@ -110,13 +154,33 @@ func newTarget(ctx context.Context, config appconfig.Config, scope scopeConfig)
|
||||
}
|
||||
writer = gateway
|
||||
}
|
||||
return v071.NewProductionLibraryTarget(ctx, v071.ProductionLibraryConfig{
|
||||
// DS01:派发身份只能来自本次真实 attempt(objectgateway claim)与原窗口作业。
|
||||
// gate 为 nil 时该能力整体关闭,adapter 对新续作 fail closed;baseline 永远传 nil。
|
||||
claim := objectgateway.ClaimFromContext(ctx)
|
||||
targetConfig := v071.ProductionLibraryConfig{
|
||||
Postgres: config.LibraryProductionDatabaseURL, SearchURL: config.LibraryProductionElasticURL,
|
||||
SearchIndex: config.LibraryProductionIndex, Objects: writer, ObjectsReader: objects,
|
||||
ArchiveRoot: config.LibraryArchiveRoot,
|
||||
ObjectsBucket: config.LibraryProductionS3Bucket,
|
||||
MaxDocuments: 200000,
|
||||
}, scope.target)
|
||||
ArchiveRoot: config.LibraryArchiveRoot,
|
||||
ObjectsBucket: config.LibraryProductionS3Bucket,
|
||||
MaxDocuments: 200000,
|
||||
RestoreDispatch: gate,
|
||||
}
|
||||
if gate != nil {
|
||||
wiring, err := RestoreDispatchWiringFor(config, scopeName, claim)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
targetConfig.RestoreTargetIdentity = wiring.TargetIdentity
|
||||
targetConfig.RestoreScope = wiring.Scope
|
||||
targetConfig.RestoreInstance = wiring.Instance
|
||||
targetConfig.RestoreOperationID = wiring.OperationID
|
||||
targetConfig.RestoreWindowJobID = wiring.WindowJobID
|
||||
targetConfig.RestoreAttemptOwner = wiring.AttemptOwner
|
||||
targetConfig.RestoreAttemptEpoch = wiring.AttemptEpoch
|
||||
targetConfig.RestoreRecoveryJobID = wiring.RecoveryJobID
|
||||
targetConfig.RestoreRecoveryOperationID = wiring.RecoveryOperationID
|
||||
}
|
||||
return v071.NewProductionLibraryTarget(ctx, targetConfig, scope.target)
|
||||
}
|
||||
|
||||
func newBackups(config appconfig.Config, scope string, target *v071.ProductionLibraryTarget,
|
||||
@@ -130,13 +194,13 @@ func newBackups(config appconfig.Config, scope string, target *v071.ProductionLi
|
||||
|
||||
// OpenProductionStack 用部署注入的配置装配目标与独立预备份存储。
|
||||
// 契约以"要同步的这份快照"为准:生产必须向它看齐,而不是反过来。
|
||||
func OpenProductionStack(ctx context.Context, config appconfig.Config, scope, snapshotID, knownMapping string) (
|
||||
*v071.ProductionLibraryTarget, *v071.ProductionBackupStore, error) {
|
||||
func OpenProductionStack(ctx context.Context, config appconfig.Config, scope, snapshotID, knownMapping string,
|
||||
gate libraryrelease.RestoreMarkerGate) (*v071.ProductionLibraryTarget, *v071.ProductionBackupStore, error) {
|
||||
scopeCfg, err := scopeConfigOf(config, scope)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
target, err := newTarget(ctx, config, scopeCfg)
|
||||
target, err := newTarget(ctx, config, scopeCfg, scope, gate)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
@@ -218,7 +282,7 @@ func OpenBaselineStack(ctx context.Context, config appconfig.Config, scope strin
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
target, err := newTarget(ctx, config, scopeCfg)
|
||||
target, err := newTarget(ctx, config, scopeCfg, scope, nil)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
@@ -2,9 +2,80 @@ package librarystack
|
||||
|
||||
import (
|
||||
"knowledge-governance/internal/appconfig"
|
||||
"knowledge-governance/internal/objectgateway"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// DS01-F1:正式装配必须把 scope/instance/真实 attempt 身份传到 adapter;快照绑定
|
||||
// 不再来自这里(来源/写前镜像只从原 WriteSet 与 backup 引用取得)。
|
||||
func TestRestoreDispatchWiringUsesRealClaim(t *testing.T) {
|
||||
config := appconfig.Config{
|
||||
LibraryTargetInstance: "inst-prod",
|
||||
LibraryProductionDatabaseURL: "postgres://user:pass@127.0.0.1:5432/kg_prod?sslmode=disable",
|
||||
LibraryProductionInternalKB: "kb-internal",
|
||||
LibraryProductionExternalKB: "kb-external",
|
||||
}
|
||||
claim := objectgateway.Claim{
|
||||
JobID: "job-window", OperationID: "op-original", Owner: "worker-1#queue#job-window",
|
||||
AttemptEpoch: 3, RecoveryJobID: "queue-recovery", RecoveryOperationID: "op-recovery",
|
||||
}
|
||||
wiring, err := RestoreDispatchWiringFor(config, "internal", claim)
|
||||
if err != nil {
|
||||
t.Fatalf("real wiring: %v", err)
|
||||
}
|
||||
if wiring.Scope != "internal" || wiring.Instance != "inst-prod" {
|
||||
t.Fatalf("scope/instance = %q/%q", wiring.Scope, wiring.Instance)
|
||||
}
|
||||
if wiring.TargetIdentity == "" || wiring.TargetIdentity == "unconfigured" {
|
||||
t.Fatalf("target identity = %q", wiring.TargetIdentity)
|
||||
}
|
||||
if wiring.OperationID != claim.OperationID || wiring.WindowJobID != claim.JobID ||
|
||||
wiring.AttemptOwner != claim.Owner || wiring.AttemptEpoch != claim.AttemptEpoch {
|
||||
t.Fatalf("attempt identity not propagated: %+v", wiring)
|
||||
}
|
||||
if wiring.RecoveryJobID != claim.RecoveryJobID || wiring.RecoveryOperationID != claim.RecoveryOperationID {
|
||||
t.Fatalf("recovery delivery identity not propagated: %+v", wiring)
|
||||
}
|
||||
// 自动回滚(非恢复 worker)没有 recovery 投递身份,必须保持为空。
|
||||
rollback, err := RestoreDispatchWiringFor(config, "internal", objectgateway.Claim{
|
||||
JobID: "job-window", OperationID: "op-original", Owner: "worker-1#queue#job-window", AttemptEpoch: 3})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rollback.RecoveryJobID != "" || rollback.RecoveryOperationID != "" {
|
||||
t.Fatalf("rollback wiring invented a recovery identity: %+v", rollback)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestoreDispatchWiringFailsClosed(t *testing.T) {
|
||||
config := appconfig.Config{
|
||||
LibraryTargetInstance: "inst-prod",
|
||||
LibraryProductionDatabaseURL: "postgres://user:pass@127.0.0.1:5432/kg_prod?sslmode=disable",
|
||||
LibraryProductionInternalKB: "kb-internal",
|
||||
}
|
||||
if _, err := RestoreDispatchWiringFor(config, "internal", objectgateway.Claim{}); err == nil {
|
||||
t.Fatal("missing attempt identity was accepted")
|
||||
}
|
||||
if _, err := RestoreDispatchWiringFor(config, "tenant-wide", objectgateway.Claim{
|
||||
JobID: "j", OperationID: "o", Owner: "w", AttemptEpoch: 1}); err == nil {
|
||||
t.Fatal("unknown scope was accepted")
|
||||
}
|
||||
if _, err := RestoreDispatchWiringFor(config, "external", objectgateway.Claim{
|
||||
JobID: "j", OperationID: "o", Owner: "w", AttemptEpoch: 1}); err == nil {
|
||||
t.Fatal("external scope without its knowledge base was accepted")
|
||||
}
|
||||
if _, err := RestoreDispatchWiringFor(config, "internal", objectgateway.Claim{
|
||||
JobID: "j", OperationID: "o", Owner: "w", AttemptEpoch: 1, RecoveryJobID: "q"}); err == nil {
|
||||
t.Fatal("partial recovery delivery identity was accepted")
|
||||
}
|
||||
unconfigured := config
|
||||
unconfigured.LibraryProductionDatabaseURL = ""
|
||||
if _, err := RestoreDispatchWiringFor(unconfigured, "internal", objectgateway.Claim{
|
||||
JobID: "j", OperationID: "o", Owner: "w", AttemptEpoch: 1}); err == nil {
|
||||
t.Fatal("unconfigured target database was accepted as a dispatch target")
|
||||
}
|
||||
}
|
||||
|
||||
func TestObjectAddressingMatchesProvider(t *testing.T) {
|
||||
for _, provider := range []string{"oss", "s3", "minio"} {
|
||||
t.Run(provider, func(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
package libraryverify
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
)
|
||||
|
||||
func drillLedger() libraryrelease.WriteSet {
|
||||
return libraryrelease.WriteSet{Version: 1, JobID: "drill", Target: "internal", TenantID: 10000, KnowledgeBaseID: "kb",
|
||||
BackupSnapshotID: "backup", TargetSnapshotID: "source", CreatedAt: time.Now().UTC(),
|
||||
Universe: map[string]libraryrelease.TableFingerprint{"knowledge_tags": {Count: 0, Digest: "empty"}}}
|
||||
}
|
||||
|
||||
// 写集合必须先落盘并登记,再标记内容风险,最后才写目标:
|
||||
// 顺序颠倒会让恢复入口在"已经写过"时读不到 ledger。
|
||||
func TestCanaryPersistsAndRegistersEvidenceBeforeWrite(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
ledger := drillLedger()
|
||||
registered, marked, written := false, false, false
|
||||
err := writeDrillCanary(context.Background(), root, &ledger, "canary-id", nil,
|
||||
func(context.Context) error {
|
||||
if !registered {
|
||||
t.Fatal("risk marker preceded durable registration")
|
||||
}
|
||||
marked = true
|
||||
return nil
|
||||
},
|
||||
func(_ context.Context) error {
|
||||
if len(ledger.CreatedIDs("knowledge_tags")) != 1 {
|
||||
t.Fatal("canary absent from write set")
|
||||
}
|
||||
if ledger.Digest == "" {
|
||||
t.Fatal("write set was not sealed before registration")
|
||||
}
|
||||
matches, e := filepath.Glob(filepath.Join(root, "writesets", "*.json"))
|
||||
if e != nil || len(matches) != 1 {
|
||||
t.Fatalf("durable ledger missing: %v %v", matches, e)
|
||||
}
|
||||
if _, e := os.Stat(matches[0]); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
registered = true
|
||||
return nil
|
||||
},
|
||||
func(context.Context) error {
|
||||
if !registered || !marked {
|
||||
t.Fatal("write preceded registration/risk marker")
|
||||
}
|
||||
written = true
|
||||
return nil
|
||||
})
|
||||
if err != nil || !written || !registered {
|
||||
t.Fatalf("err=%v written=%v registered=%v", err, written, registered)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanaryNeverWritesWithoutDurableLedger(t *testing.T) {
|
||||
root := filepath.Join(t.TempDir(), "blocked")
|
||||
if err := os.WriteFile(root, []byte("not a directory"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ledger := drillLedger()
|
||||
marked, written := false, false
|
||||
err := writeDrillCanary(context.Background(), root, &ledger, "canary-id", nil,
|
||||
func(context.Context) error { marked = true; return nil },
|
||||
func(context.Context) error { marked = true; return nil },
|
||||
func(context.Context) error { written = true; return nil })
|
||||
if err == nil || marked || written {
|
||||
t.Fatalf("err=%v marked=%v written=%v", err, marked, written)
|
||||
}
|
||||
}
|
||||
|
||||
// 登记失败 = 恢复入口读不到写集合:此时一笔都不许写。
|
||||
func TestCanaryNeverWritesWhenRegistrationFails(t *testing.T) {
|
||||
ledger := drillLedger()
|
||||
marked, written := false, false
|
||||
err := writeDrillCanary(context.Background(), t.TempDir(), &ledger, "canary-id", nil,
|
||||
func(context.Context) error { marked = true; return nil },
|
||||
func(context.Context) error { return errors.New("governance registration unavailable") },
|
||||
func(context.Context) error { written = true; return nil })
|
||||
if err == nil || marked || written {
|
||||
t.Fatalf("err=%v marked=%v written=%v", err, marked, written)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanaryNeverWritesWhenRiskMarkerFails(t *testing.T) {
|
||||
ledger := drillLedger()
|
||||
written := false
|
||||
err := writeDrillCanary(context.Background(), t.TempDir(), &ledger, "canary-id", nil,
|
||||
func(context.Context) error { return errors.New("journal unavailable") },
|
||||
func(context.Context) error { return nil },
|
||||
func(context.Context) error { written = true; return nil })
|
||||
if err == nil || written {
|
||||
t.Fatalf("err=%v written=%v", err, written)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package libraryverify
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
v071 "knowledge-governance/internal/releaseadapter/v071"
|
||||
)
|
||||
|
||||
// B1 的定向回归:共享 Run 用 PlanWrite 产出的写前/来源身份与写集合,
|
||||
// 必须通过真实恢复入口调用的**同一份**身份合同;用同一份快照冒充两个身份
|
||||
// 的旧形状必须被拒绝。这里不连接任何业务目标(隔离 fixture)。
|
||||
func TestSharedDrillPlanPassesRealRestoreIdentityContract(t *testing.T) {
|
||||
preimage := libraryrelease.BackupRef{SnapshotID: strings.Repeat("a", 64),
|
||||
Universe: map[string]libraryrelease.TableFingerprint{"knowledge_tags": {Count: 0, Digest: "empty"}}}
|
||||
source := libraryrelease.BackupRef{SnapshotID: strings.Repeat("b", 64),
|
||||
Universe: map[string]libraryrelease.TableFingerprint{"knowledge_tags": {Count: 0, Digest: "empty"}}}
|
||||
|
||||
set, err := PlanWrite("libverify-internal-fixture", "internal", 10000, "fixture-kb", preimage, source)
|
||||
if err != nil {
|
||||
t.Fatalf("shared plan rejected: %v", err)
|
||||
}
|
||||
if err := set.Seal(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// ProductionLibraryTarget.Restore 与 RestoreDrill 都在写之前调用这一判定。
|
||||
if err := set.ValidateRestoreBinding(preimage.SnapshotID, 10000, "fixture-kb"); err != nil {
|
||||
t.Fatalf("real restore identity contract rejected the shared plan: %v", err)
|
||||
}
|
||||
// 演练恢复入口的写集合守卫(只允许可归因的 canary 行)也必须接受它。
|
||||
canary := set
|
||||
canary.Rows = append(canary.Rows, libraryrelease.WriteSetRow{Table: "knowledge_tags", ID: "canary-1"})
|
||||
if err := canary.Seal(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := v071.ValidateDrillWriteSet(canary); err != nil {
|
||||
t.Fatalf("drill write set guard rejected the shared plan: %v", err)
|
||||
}
|
||||
|
||||
// 旧缺陷形状:来源快照 = 写前镜像。同一判定必须拒绝。
|
||||
legacy := set
|
||||
legacy.TargetSnapshotID = preimage.SnapshotID
|
||||
legacy.Digest = ""
|
||||
if err := legacy.Seal(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = legacy.ValidateRestoreBinding(preimage.SnapshotID, 10000, "fixture-kb")
|
||||
if err == nil || !strings.Contains(err.Error(), "同一份快照") {
|
||||
t.Fatalf("legacy same-snapshot plan must be rejected, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSharedPlanRejectsSameSnapshotIdentity(t *testing.T) {
|
||||
preimage := libraryrelease.BackupRef{SnapshotID: strings.Repeat("c", 64),
|
||||
Universe: map[string]libraryrelease.TableFingerprint{"knowledge_tags": {Count: 0, Digest: "empty"}}}
|
||||
if _, err := PlanWrite("job", "internal", 10000, "kb", preimage, preimage); err == nil {
|
||||
t.Fatal("plan must refuse to use one snapshot as both source and pre-image")
|
||||
}
|
||||
if _, err := PlanWrite("job", "internal", 10000, "kb", preimage, libraryrelease.BackupRef{}); err == nil {
|
||||
t.Fatal("plan must refuse a missing source identity")
|
||||
}
|
||||
}
|
||||
|
||||
// 演练恢复入口只回放"本次写的隔离 canary 行";任何其它行都不是演练产物。
|
||||
func TestDrillWriteSetGuardRejectsForeignRows(t *testing.T) {
|
||||
set := libraryrelease.WriteSet{Version: libraryrelease.WriteSetVersion, Rows: []libraryrelease.WriteSetRow{
|
||||
{Table: "knowledges", ID: "doc-1"},
|
||||
}}
|
||||
if err := v071.ValidateDrillWriteSet(set); err == nil {
|
||||
t.Fatal("drill guard must reject non-canary rows")
|
||||
}
|
||||
set.Rows = []libraryrelease.WriteSetRow{{Table: "knowledge_tags", ID: "tag-1", Existing: true}}
|
||||
if err := v071.ValidateDrillWriteSet(set); err == nil {
|
||||
t.Fatal("drill guard must reject rows that existed before the write")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,445 @@
|
||||
// Package libraryverify 在目标实例上做一次**真实**的恢复验证(原 cmd/librarydrill 的实现):
|
||||
//
|
||||
// 维护隔离(关入口/停写入者/证明排空)
|
||||
// → 捕获目标整库(真预备份,落归档)
|
||||
// → 校验目标 == 预备份(基线一致)
|
||||
// → 写入一个 canary(证明"校验确实能发现内容变化")
|
||||
// → 按预备份恢复
|
||||
// → 逐表/索引/资源/Wiki 再校验一次,必须完全一致
|
||||
// → 关窗口(按原运行状态恢复服务、开入口、验证健康)
|
||||
//
|
||||
// 为什么搬进 internal:页面要能点一次"恢复验证",worker 必须在进程内执行同一份实现,
|
||||
// 不能再要求运维 SSH 上去跑 CLI。命令行使同一个 Run,不允许两份实现。
|
||||
// 它只读/写**被测目标自己**,绝不触碰封存源;所有凭据来自进程环境,不进命令行、不打印。
|
||||
// 任一步失败都不伪装成功:窗口保持维护、报告写明失败阶段。
|
||||
package libraryverify
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/appconfig"
|
||||
"knowledge-governance/internal/libraryfreeze"
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
"knowledge-governance/internal/librarystack"
|
||||
v071 "knowledge-governance/internal/releaseadapter/v071"
|
||||
)
|
||||
|
||||
type Verification struct {
|
||||
SnapshotID string `json:"snapshot_id"`
|
||||
Verified bool `json:"verified"`
|
||||
Error string `json:"error,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
type Report struct {
|
||||
Status string `json:"status"` // passed | failed | maintenance_held
|
||||
Scope string `json:"scope"`
|
||||
Target string `json:"target"`
|
||||
SnapshotID string `json:"snapshot_id"` // 写前镜像:恢复实际回放的那份预备份
|
||||
SourceSnapshot string `json:"source_snapshot_id,omitempty"`
|
||||
WriteSetDigest string `json:"writeset_digest,omitempty"`
|
||||
WindowState string `json:"window_state,omitempty"`
|
||||
Project string `json:"project"`
|
||||
Counts map[string]int64 `json:"counts"`
|
||||
BaselineCheck *Verification `json:"baseline_check"`
|
||||
DriftCheck *Verification `json:"drift_check"` // 必须失败:证明确实能发现差异
|
||||
RestoreCheck *Verification `json:"restore_check"`
|
||||
Canary string `json:"canary"`
|
||||
Failure string `json:"failure,omitempty"`
|
||||
Stage string `json:"stage,omitempty"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
FinishedAt time.Time `json:"finished_at"`
|
||||
Until map[string]string `json:"until,omitempty"`
|
||||
}
|
||||
|
||||
// Progress 把演练的阶段推进交给调用方(worker 写 library_job_stages,CLI 打日志)。
|
||||
type Progress func(stage, state, detail string)
|
||||
|
||||
func (p Progress) report(stage, state, detail string) {
|
||||
if p != nil {
|
||||
p(stage, state, detail)
|
||||
}
|
||||
}
|
||||
|
||||
// TargetIdentity 是治理侧维护记录必须钉住的目标事实(与 librarymaintenance.Target 同源)。
|
||||
type TargetIdentity struct {
|
||||
Instance string
|
||||
Fingerprint string
|
||||
Writers []string
|
||||
}
|
||||
|
||||
// Governance 是演练必须持有的**治理侧**维护生命周期:实例/scope 占用、入口真实状态、
|
||||
// 释放。为什么不能让调用方只写磁盘 journal:页面与恢复入口读的是治理行,只写 journal
|
||||
// 时"正在维护"不会出现在 UI,失败后也没有同 job 的恢复锚点(2026-11 审核 B2)。
|
||||
type Governance interface {
|
||||
Acquire(ctx context.Context, jobID, scope string, target TargetIdentity, originalServiceState string) error
|
||||
Ingress(ctx context.Context, jobID, scope, state, note string) error
|
||||
State(ctx context.Context, jobID, scope, state, note string) error
|
||||
Release(ctx context.Context, jobID, scope, note string) error
|
||||
}
|
||||
|
||||
// Ledger 把预备份身份与写集合摘要持久化到**同一个作业**(worker 直接传
|
||||
// libraryStageRecorder:它已经是 BackupObserver + WriteSetObserver)。
|
||||
// 恢复入口按 (scope, job, 写前镜像) 精确读这条记录,所以两件事必须发生在写第一笔之前。
|
||||
type Ledger interface {
|
||||
BackupPrepared(ctx context.Context, backup libraryrelease.BackupRef) error
|
||||
WriteSetPlanned(ctx context.Context, set libraryrelease.WriteSet) error
|
||||
}
|
||||
|
||||
// Options 是演练的可选接线。Governance/Ledger 为 nil 只允许主机侧 CLI 使用:
|
||||
// 页面发起的作业必须两者都在,否则演练结论不会出现在治理记录里。
|
||||
type Options struct {
|
||||
Progress Progress
|
||||
Governance Governance
|
||||
Ledger Ledger
|
||||
}
|
||||
|
||||
// Run 执行一次演练。jobID 同时是维护窗口的持有者标识,必须与业务作业 id 一致,
|
||||
// 否则崩溃恢复时窗口与作业对不上。
|
||||
func Run(ctx context.Context, config appconfig.Config, scope, jobID string, opts Options) (*Report, error) {
|
||||
if scope != "internal" && scope != "external" {
|
||||
return nil, errors.New("scope must be internal or external")
|
||||
}
|
||||
progress := opts.Progress
|
||||
report := &Report{Status: "failed", Scope: scope, Project: config.LibraryTargetProject,
|
||||
Target: config.LibraryTargetInstance, StartedAt: time.Now().UTC()}
|
||||
fail := func(stage string, cause error, held bool) (*Report, error) {
|
||||
report.Stage = stage
|
||||
report.Failure = cause.Error()
|
||||
report.FinishedAt = time.Now().UTC()
|
||||
if held {
|
||||
report.Status = "maintenance_held"
|
||||
}
|
||||
progress.report(stage, "failed", cause.Error())
|
||||
return report, cause
|
||||
}
|
||||
|
||||
progress.report("capture", "running", "打开目标栈与维护管理器")
|
||||
target, backups, err := librarystack.OpenBaselineStack(ctx, config, scope)
|
||||
if err != nil {
|
||||
return fail("capture", err, false)
|
||||
}
|
||||
defer target.Close()
|
||||
manager, err := librarystack.OpenMaintenance(config)
|
||||
if err != nil {
|
||||
return fail("capture", err, false)
|
||||
}
|
||||
defer manager.Close()
|
||||
if err := manager.VerifyTopology(ctx); err != nil {
|
||||
return fail("capture", err, false)
|
||||
}
|
||||
originalState, err := manager.OriginalState(ctx)
|
||||
if err != nil {
|
||||
return fail("capture", errors.New("读不到目标写入者的原运行状态,不能开维护窗口:"+err.Error()), false)
|
||||
}
|
||||
identity := manager.Target()
|
||||
// 治理占用必须在任何目标动作之前:拿不到就没有"入口已关/写入者已停"的权威记录,
|
||||
// 失败后也没有可恢复的同 job 锚点。
|
||||
if opts.Governance != nil {
|
||||
targetIdentity := TargetIdentity{Instance: identity.Instance, Fingerprint: identity.Fingerprint, Writers: identity.Writers}
|
||||
if err := opts.Governance.Acquire(ctx, jobID, scope, targetIdentity, originalState); err != nil {
|
||||
return fail("capture", errors.New("目标实例正在维护中或无法登记维护占用:"+err.Error()), false)
|
||||
}
|
||||
}
|
||||
coordinator, control, err := manager.Open(config.LibraryTargetJournalRoot)
|
||||
if err != nil {
|
||||
if opts.Governance != nil {
|
||||
// 还没进窗口、目标未被停用:释放占用是诚实的;释放失败就报告出来。
|
||||
_ = opts.Governance.Release(ctx, jobID, scope, "维护协调器不可用,未进入窗口")
|
||||
}
|
||||
return fail("capture", err, false)
|
||||
}
|
||||
defer control.Close()
|
||||
defer coordinator.Close()
|
||||
|
||||
var (
|
||||
snapshotID string
|
||||
sourceID string
|
||||
writesetDigest string
|
||||
counts map[string]int64
|
||||
canary string
|
||||
contentTouched bool
|
||||
baseline Verification
|
||||
drift Verification
|
||||
restored Verification
|
||||
)
|
||||
work := func(ctx context.Context, window *libraryfreeze.Window) (string, error) {
|
||||
start := time.Now()
|
||||
// 窗口已关入口、已停写入者并证明排空:此时才敢把 UI 的入口状态写成 closed。
|
||||
if opts.Governance != nil {
|
||||
if err := opts.Governance.Ingress(ctx, jobID, scope, "closed",
|
||||
"恢复验证已关闭目标入口并停止写入者"); err != nil {
|
||||
return "", errors.New("登记入口关闭失败:" + err.Error())
|
||||
}
|
||||
}
|
||||
current, err := target.Current(ctx)
|
||||
if err != nil {
|
||||
return "", errors.New("读取目标当前整库状态失败:" + err.Error())
|
||||
}
|
||||
counts = current.Counts
|
||||
report.Counts = counts
|
||||
// 1) 写前镜像:真预备份(落归档、内容寻址、导出后复核内容未变)。
|
||||
// 它是恢复阶段实际回放的唯一内容来源。
|
||||
backup, err := backups.Prepare(ctx, "", current)
|
||||
if err != nil {
|
||||
return "", errors.New("捕获目标整库失败:" + err.Error())
|
||||
}
|
||||
snapshotID = backup.SnapshotID
|
||||
report.SnapshotID = snapshotID
|
||||
if err := window.SetBackupSnapshot(ctx, snapshotID); err != nil {
|
||||
return "", errors.New("记录预备份身份失败:" + err.Error())
|
||||
}
|
||||
// 2) 来源身份:DS01 合同要求"来源快照 ≠ 写前镜像",演练不能用同一份快照冒充
|
||||
// 两个身份(真实 Restore 会在任何写之前拒绝)。这里再做一次**真实的**独立捕获,
|
||||
// 只作为这次写入的来源身份;两次捕获都在同一维护切点内、由同一备份存储密封,
|
||||
// 不伪造快照、也不放宽 Restore 的校验。
|
||||
source, err := backups.Prepare(ctx, "", current)
|
||||
if err != nil {
|
||||
return "", errors.New("捕获来源身份失败:" + err.Error())
|
||||
}
|
||||
if source.SnapshotID == "" || source.SnapshotID == snapshotID {
|
||||
return "", errors.New("来源身份与写前镜像相同:拒绝用一份快照冒充两个恢复身份")
|
||||
}
|
||||
sourceID = source.SnapshotID
|
||||
report.SourceSnapshot = sourceID
|
||||
// 预备份引用必须在任何写入之前落进同一个作业:恢复入口按它取写前镜像。
|
||||
if opts.Ledger != nil {
|
||||
if err := opts.Ledger.BackupPrepared(ctx, backup); err != nil {
|
||||
return "", errors.New("预备份身份无法持久化,未对目标执行任何写入:" + err.Error())
|
||||
}
|
||||
}
|
||||
progress.report("capture", "succeeded",
|
||||
fmt.Sprintf("已捕获写前镜像 %s 与来源身份 %s(%d 字节,用时 %s)",
|
||||
short(snapshotID), short(sourceID), backup.Bytes, time.Since(start).Round(time.Second)))
|
||||
log.Printf("verify: snapshot captured preimage=%s source=%s bytes=%d in=%s",
|
||||
snapshotID, sourceID, backup.Bytes, time.Since(start))
|
||||
|
||||
// 3) 基线校验:预备份必须与目标完全一致。
|
||||
progress.report("verify", "running", "逐表/索引/资源比对目标与预备份")
|
||||
baseline = Verification{SnapshotID: snapshotID, At: time.Now().UTC()}
|
||||
if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); err != nil {
|
||||
baseline.Error = err.Error()
|
||||
report.BaselineCheck = &baseline
|
||||
return "", errors.New("预备份与目标不一致(预备份本身不可信):" + err.Error())
|
||||
}
|
||||
baseline.Verified = true
|
||||
report.BaselineCheck = &baseline
|
||||
progress.report("verify", "succeeded", "预备份与目标逐类一致")
|
||||
|
||||
// Canary 是隔离的标签行,空目标上也一样:绝不为了"让演练能跑"去种文档或改既有行。
|
||||
var random [16]byte
|
||||
if _, err := rand.Read(random[:]); err != nil {
|
||||
return "", err
|
||||
}
|
||||
canary = fmt.Sprintf("%x", random[:])
|
||||
identityValue := target.Identity()
|
||||
var exists bool
|
||||
if err := target.Pool().QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM knowledge_tags WHERE id=$1)", canary).Scan(&exists); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if exists {
|
||||
return "", errors.New("canary identity collision; no write attempted")
|
||||
}
|
||||
ledger, planErr := PlanWrite(jobID, scope, identityValue.TenantID, identityValue.KnowledgeBaseID, backup, source)
|
||||
if planErr != nil {
|
||||
return "", planErr
|
||||
}
|
||||
report.Canary = canary
|
||||
progress.report("canary", "running", "写入一个隔离 canary,验证校验能发现差异")
|
||||
if err := writeDrillCanary(ctx, config.LibraryArchiveRoot, &ledger, canary,
|
||||
func(set *libraryrelease.WriteSet) error {
|
||||
// 计划必须在 Seal/落盘/登记之前定型:对象网关控制器按登记摘要加载,
|
||||
// 这里补 V2 信封、目标 schema、来源身份与当前 owned attempt 绑定。
|
||||
return target.PrepareDrillPlan(ctx, set)
|
||||
},
|
||||
func(ctx context.Context) error {
|
||||
if err := window.MarkContentRisk(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
contentTouched = true
|
||||
return nil
|
||||
}, func(ctx context.Context) error {
|
||||
if opts.Ledger == nil {
|
||||
return nil
|
||||
}
|
||||
// 写集合摘要必须在第一笔写入之前登记到同一个作业的预备份证据里:
|
||||
// 恢复入口按 (scope, job, 写前镜像) 精确读它,读不到就保持维护。
|
||||
return opts.Ledger.WriteSetPlanned(ctx, *ledgerClone(&ledger))
|
||||
}, func(ctx context.Context) error {
|
||||
_, err := target.Pool().Exec(ctx, `INSERT INTO knowledge_tags(id,tenant_id,knowledge_base_id,name) VALUES($1,$2,$3,$4)`, canary, identityValue.TenantID, identityValue.KnowledgeBaseID, "drill-canary-"+canary)
|
||||
return err
|
||||
}); err != nil {
|
||||
return "", fmt.Errorf("canary preparation/write failed: %w", err)
|
||||
}
|
||||
writesetDigest = ledger.Digest
|
||||
report.WriteSetDigest = writesetDigest
|
||||
drift = Verification{SnapshotID: snapshotID, At: time.Now().UTC()}
|
||||
if verifyErr := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); verifyErr == nil {
|
||||
drift.Error = "写入 canary 后校验仍然通过:校验没有真正比对内容"
|
||||
report.DriftCheck = &drift
|
||||
return "", errors.New("写入 canary 后校验仍然通过:校验没有真正比对内容,拒绝把这次演练当成通过")
|
||||
} else {
|
||||
drift.Error = verifyErr.Error()
|
||||
}
|
||||
report.DriftCheck = &drift
|
||||
progress.report("canary", "succeeded", "校验按预期发现了 canary 差异")
|
||||
log.Printf("verify: drift detected as expected (%s)", drift.Error)
|
||||
|
||||
// 4) 按写前镜像恢复:四个阶段整段回放(与真实恢复同一批 Apply 原语),
|
||||
// 而不是只删 canary。发布计划形态的 Restore 不能接受"只写过一行 canary"的
|
||||
// 写集合(v2 执行计划合同),所以演练走限定的安全恢复入口;它复用同一份
|
||||
// DS01 身份合同与整实例指纹校验。
|
||||
progress.report("restore", "running", "按预备份恢复目标整库")
|
||||
if err := target.RestoreDrill(ctx, libraryrelease.BackupRef{ID: "backup-" + snapshotID, SnapshotID: snapshotID,
|
||||
WriteSet: &ledger}); err != nil {
|
||||
return "", errors.New("按预备份恢复失败:" + err.Error())
|
||||
}
|
||||
// 5) 恢复后逐类复核:必须与预备份完全一致。
|
||||
restored = Verification{SnapshotID: snapshotID, At: time.Now().UTC()}
|
||||
if err := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); err != nil {
|
||||
restored.Error = err.Error()
|
||||
report.RestoreCheck = &restored
|
||||
return "", errors.New("恢复后内容仍与预备份不一致:" + err.Error())
|
||||
}
|
||||
restored.Verified = true
|
||||
report.RestoreCheck = &restored
|
||||
progress.report("restore", "succeeded", "恢复后与预备份逐类一致")
|
||||
log.Printf("verify: restored and verified snapshot=%s", snapshotID)
|
||||
return snapshotID, nil
|
||||
}
|
||||
|
||||
progress.report("finalize", "running", "关闭维护窗口并按原状态恢复目标")
|
||||
window, windowErr := coordinator.RunWindow(ctx, jobID,
|
||||
func(ctx context.Context) error { return manager.VerifyRunning(ctx) }, work,
|
||||
// 没写过目标内容 ⇒ 按原状态恢复永远安全;写过 canary ⇒ 只有恢复并校验通过才开放。
|
||||
func(error) bool { return !contentTouched || restored.Verified })
|
||||
|
||||
report.Until = map[string]string{"job": jobID, "window_state": window.State}
|
||||
report.WindowState = window.State
|
||||
// 服务真的回来、入口验证过(succeeded / 未写内容的安全失败)才释放治理占用;
|
||||
// held 时保留行,页面才能显示真实入口状态并从同一 job 走恢复入口。
|
||||
released := false
|
||||
if opts.Governance != nil {
|
||||
switch {
|
||||
case window.State == "succeeded" || (window.State == "failed" && !contentTouched):
|
||||
if err := opts.Governance.Ingress(ctx, jobID, scope, "open",
|
||||
"维护窗口已按原运行状态恢复,入口已验证可用"); err != nil {
|
||||
return fail("finalize", errors.New("恢复后登记入口状态失败:"+err.Error()), false)
|
||||
}
|
||||
if err := opts.Governance.Release(ctx, jobID, scope,
|
||||
"恢复验证结束,服务与入口已按原状态恢复"); err != nil {
|
||||
return fail("finalize", errors.New("恢复验证结束但维护占用释放失败:"+err.Error()), false)
|
||||
}
|
||||
released = true
|
||||
default:
|
||||
cause := "维护窗口保持关闭"
|
||||
if windowErr != nil {
|
||||
cause = windowErr.Error()
|
||||
}
|
||||
_ = opts.Governance.Ingress(ctx, jobID, scope, "closed", "目标内容状态未验证,入口保持关闭")
|
||||
_ = opts.Governance.State(ctx, jobID, scope, "needs_restore",
|
||||
"恢复验证失败且目标内容未验证:"+cause)
|
||||
}
|
||||
}
|
||||
if windowErr != nil || window.State != "succeeded" {
|
||||
cause := windowErr
|
||||
if cause == nil {
|
||||
cause = fmt.Errorf("维护窗口状态 %s(未成功)", window.State)
|
||||
}
|
||||
held := window.State == "held"
|
||||
if held && released {
|
||||
held = false
|
||||
}
|
||||
return fail("finalize", cause, held)
|
||||
}
|
||||
report.Status = "passed"
|
||||
report.FinishedAt = time.Now().UTC()
|
||||
progress.report("finalize", "succeeded", "维护窗口已关闭,目标恢复原运行状态")
|
||||
return report, nil
|
||||
}
|
||||
|
||||
// PlanWrite 构造演练的写前镜像/来源身份与写集合。Run 与定向回归共用这一份:
|
||||
// 只有来源身份 ≠ 写前镜像、且两者都是同一次维护切点里密封的真实捕获时,
|
||||
// 恢复入口(同一份 DS01 身份合同)才接受这次演练。
|
||||
func PlanWrite(jobID, scope string, tenantID uint64, knowledgeBaseID string,
|
||||
preimage, source libraryrelease.BackupRef) (libraryrelease.WriteSet, error) {
|
||||
if preimage.SnapshotID == "" || source.SnapshotID == "" {
|
||||
return libraryrelease.WriteSet{}, errors.New("演练缺少写前镜像或来源身份快照")
|
||||
}
|
||||
if preimage.SnapshotID == source.SnapshotID {
|
||||
return libraryrelease.WriteSet{}, errors.New("来源身份与写前镜像相同:拒绝用一份快照冒充两个恢复身份")
|
||||
}
|
||||
set := libraryrelease.WriteSet{Version: libraryrelease.WriteSetVersion, JobID: jobID, Target: scope,
|
||||
TenantID: tenantID, KnowledgeBaseID: knowledgeBaseID,
|
||||
BackupSnapshotID: preimage.SnapshotID, TargetSnapshotID: source.SnapshotID,
|
||||
CreatedAt: time.Now().UTC(), Universe: preimage.Universe}
|
||||
if err := set.ValidateRestoreBinding(preimage.SnapshotID, tenantID, knowledgeBaseID); err != nil {
|
||||
return libraryrelease.WriteSet{}, err
|
||||
}
|
||||
return set, nil
|
||||
}
|
||||
|
||||
func short(id string) string {
|
||||
if len(id) <= 12 {
|
||||
return id
|
||||
}
|
||||
return id[:12]
|
||||
}
|
||||
|
||||
// ledgerClone 只用于把台账交给登记方:Seal 会重排 Rows,调用方随后还要用同一份
|
||||
// 内容去恢复,不能让登记方持有可变的同一指针。
|
||||
func ledgerClone(set *libraryrelease.WriteSet) *libraryrelease.WriteSet {
|
||||
copy := *set
|
||||
copy.Rows = append([]libraryrelease.WriteSetRow(nil), set.Rows...)
|
||||
return ©
|
||||
}
|
||||
|
||||
// writeDrillCanary 是"写入前先留痕"的唯一实现:seal → 落盘 → 登记 → 标风险 → 写。
|
||||
// 任何一步失败都不写目标;登记在标风险之前,恢复入口才拿得到这份写集合。
|
||||
func writeDrillCanary(ctx context.Context, root string, ledger *libraryrelease.WriteSet, id string,
|
||||
prepare func(*libraryrelease.WriteSet) error,
|
||||
mark func(context.Context) error, record func(context.Context) error, write func(context.Context) error) error {
|
||||
ledger.Rows = append(ledger.Rows, libraryrelease.WriteSetRow{Table: "knowledge_tags", ID: id, Existing: false})
|
||||
if prepare != nil {
|
||||
if err := prepare(ledger); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := ledger.Seal(); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := v071.PersistWriteSet(root, ledger); err != nil {
|
||||
return err
|
||||
}
|
||||
if record != nil {
|
||||
if err := record(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := mark(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return write(ctx)
|
||||
}
|
||||
|
||||
// RecoverWindows 把非终态的维护窗口按原运行状态恢复(服务起来、入口验证)。
|
||||
// 演练失败但还没写目标内容时,用它把目标从"停住"状态救回来;治理记录由作业/接口那侧负责。
|
||||
func RecoverWindows(config appconfig.Config) error {
|
||||
manager, err := librarystack.OpenMaintenance(config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer manager.Close()
|
||||
coordinator, control, err := manager.Open(config.LibraryTargetJournalRoot)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer control.Close()
|
||||
defer coordinator.Close()
|
||||
return coordinator.Recover(context.Background())
|
||||
}
|
||||
@@ -177,10 +177,13 @@ func (provider *RealProvider) weKnoraProbe(ctx context.Context, now time.Time) g
|
||||
return provider.track(classify("weknora", "WeKnora API", false, true, ProbeOutcome{ObservedAt: now}))
|
||||
}
|
||||
outcome := provider.httpProbeOutcome(ctx, baseURL, now)
|
||||
probeTarget := ""
|
||||
if outcome.Reachability == nil && provider.weknoraReadiness != nil {
|
||||
outcome.WorkReadiness = provider.weknoraReadiness(ctx)
|
||||
probeTarget, outcome.WorkReadiness = provider.weknoraReadiness(ctx)
|
||||
}
|
||||
return provider.track(classify("weknora", "WeKnora API", true, true, outcome))
|
||||
health := provider.track(classify("weknora", "WeKnora API", true, true, outcome))
|
||||
health.ProbeTarget = probeTarget
|
||||
return health
|
||||
}
|
||||
|
||||
// productionWeKnoraProbe reports the real state of the production WeKnora
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/readiness"
|
||||
)
|
||||
|
||||
func testNow() time.Time { return time.Date(2026, 9, 12, 9, 0, 0, 0, time.UTC) }
|
||||
@@ -141,3 +142,31 @@ func TestOverallReadinessTreatsProductionEndpointAsAdvisory(t *testing.T) {
|
||||
t.Fatalf("real dependency failure must still block: ok=%v blocked=%v", ok, blocked)
|
||||
}
|
||||
}
|
||||
|
||||
// F13: 旧 env 绑定 404 是「业务就绪被阻断」,不是「服务不可达」;
|
||||
// 同时必须把脱敏后的探测对象带出来,否则红灯无法归因。
|
||||
func TestWeKnoraReadinessReportsBindingBlockWithoutLosingLiveness(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
writer.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
provider := &RealProvider{
|
||||
httpClient: server.Client(),
|
||||
weknoraBaseURL: server.URL,
|
||||
health: newHealthTracker(),
|
||||
weknoraReadiness: func(context.Context) (string, error) {
|
||||
target := "实例 weknora.test · 知识库 1b52c140"
|
||||
return target, fmt.Errorf("%w(%s)", readiness.ErrKnowledgeBaseMissing, target)
|
||||
},
|
||||
}
|
||||
health := provider.weKnoraProbe(context.Background(), testNow())
|
||||
if health.Liveness != LiveUp || health.Readiness != ReadBlocked {
|
||||
t.Fatalf("404 binding = %+v, want up/blocked", health)
|
||||
}
|
||||
if health.ProbeTarget != "实例 weknora.test · 知识库 1b52c140" {
|
||||
t.Fatalf("probe target = %q", health.ProbeTarget)
|
||||
}
|
||||
if !strings.Contains(health.Reason, "1b52c140") || strings.Contains(health.Reason, "1b52c140-2e18-4676-afc5-4c7d8d190a8c") {
|
||||
t.Fatalf("reason = %q, want short target only", health.Reason)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,9 +44,9 @@ type RealProviderConfig struct {
|
||||
// place operators configure it — and the env value above stays the
|
||||
// bootstrap fallback for containers that were wired before the UI existed.
|
||||
ProductionWeKnoraResolver func(ctx context.Context) string
|
||||
// WeKnoraReadiness optionally exercises a real read against a routing
|
||||
// knowledge base (nil = API key not configured; liveness-only).
|
||||
WeKnoraReadiness func(ctx context.Context) error
|
||||
// WeKnoraReadiness optionally exercises a real read against the CURRENT
|
||||
// routing binding and reports the sanitized probe target alongside the error.
|
||||
WeKnoraReadiness func(ctx context.Context) (target string, err error)
|
||||
// WorkerStaleAfter bounds how long a missing worker heartbeat still counts
|
||||
// as alive (P10).
|
||||
WorkerStaleAfter time.Duration
|
||||
@@ -61,7 +61,7 @@ type RealProvider struct {
|
||||
testWeKnoraResolver func(ctx context.Context) string
|
||||
productionWeKnoraBaseURL string
|
||||
productionWeKnoraResolver func(ctx context.Context) string
|
||||
weknoraReadiness func(ctx context.Context) error
|
||||
weknoraReadiness func(ctx context.Context) (string, error)
|
||||
workerStaleAfter time.Duration
|
||||
health *healthTracker
|
||||
}
|
||||
|
||||
@@ -24,7 +24,7 @@ func TestGatewayAcquisitionPersistsAttemptDelivery(t *testing.T) {
|
||||
expected := g.Grant{Epoch: 1, Owner: g.Digest([]byte(claim.JobID)), Plan: claim.PlanDigest}
|
||||
req := request("acquired-object", expected)
|
||||
authority := ¤tPlanAuthority{claim, planFor(expected, req)}
|
||||
gateway := &g.Gateway{Pool: pool, Scope: backend.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20, Authority: authority}
|
||||
gateway := &g.Gateway{Pool: pool, Scope: backend.Bucket, Backend: backend, MaxBytes: 1 << 20, Authority: authority}
|
||||
grant, e := gateway.Acquire(ctx, claim)
|
||||
if e != nil || grant != expected {
|
||||
t.Fatal("anchored active attempt grant unavailable", e)
|
||||
@@ -79,7 +79,7 @@ func TestGatewayAcquisitionCannotHandOverUnknownSend(t *testing.T) {
|
||||
req := request("acquire-held", expected)
|
||||
body := []byte("gateway fixture acquire-held")
|
||||
authority := ¤tPlanAuthority{claim, planFor(expected, req)}
|
||||
gateway := &g.Gateway{Pool: pool, Scope: real.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20, Authority: authority}
|
||||
gateway := &g.Gateway{Pool: pool, Scope: real.Bucket, Backend: backend, MaxBytes: 1 << 20, Authority: authority}
|
||||
if _, e := gateway.Acquire(ctx, claim); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
package objectgateway
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Attestation is the retired whole-bucket exclusive-writer evidence format.
|
||||
// It is kept only so historical audit artifacts remain readable; no runtime
|
||||
// path loads, verifies or requires it. Admission is the journal, the backend
|
||||
// capability and the frozen plan, all of which work on a shared bucket.
|
||||
type Attestation struct {
|
||||
Protocol string `json:"protocol"`
|
||||
Scope string `json:"scope"`
|
||||
GatewayID string `json:"gateway_id"`
|
||||
IssuedAt time.Time `json:"issued_at"`
|
||||
ExpiresAt time.Time `json:"expires_at"`
|
||||
WriterInventory string `json:"writer_inventory_sha256"`
|
||||
PolicyEvidence string `json:"policy_evidence_sha256"`
|
||||
DenialEvidence string `json:"denial_evidence_sha256"`
|
||||
DrainEvidence string `json:"drain_evidence_sha256"`
|
||||
}
|
||||
type SignedAttestation struct {
|
||||
Payload json.RawMessage `json:"payload"`
|
||||
Signature []byte `json:"signature"`
|
||||
}
|
||||
@@ -1,57 +0,0 @@
|
||||
package objectgateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Attestation is issued by an independent deployment auditor, never by a worker.
|
||||
// Its signature authenticates an audit, not the truth of self-reported config.
|
||||
// Runtime cutover must invalidate/replace the audit whenever writer rights change.
|
||||
type Attestation struct {
|
||||
Protocol string `json:"protocol"`
|
||||
Scope string `json:"scope"`
|
||||
GatewayID string `json:"gateway_id"`
|
||||
IssuedAt time.Time `json:"issued_at"`
|
||||
ExpiresAt time.Time `json:"expires_at"`
|
||||
WriterInventory string `json:"writer_inventory_sha256"`
|
||||
PolicyEvidence string `json:"policy_evidence_sha256"`
|
||||
DenialEvidence string `json:"denial_evidence_sha256"`
|
||||
DrainEvidence string `json:"drain_evidence_sha256"`
|
||||
}
|
||||
type SignedAttestation struct {
|
||||
Payload json.RawMessage `json:"payload"`
|
||||
Signature []byte `json:"signature"`
|
||||
}
|
||||
type AuditVerifier struct {
|
||||
Path string
|
||||
PublicKey ed25519.PublicKey
|
||||
Scope, GatewayID string
|
||||
}
|
||||
|
||||
func (v AuditVerifier) Verify(ctx context.Context) error {
|
||||
if ctx.Err() != nil {
|
||||
return ErrIsolation
|
||||
}
|
||||
st, e := os.Lstat(v.Path)
|
||||
if e != nil || !st.Mode().IsRegular() || st.Size() > 16384 || st.Mode().Perm()&0022 != 0 {
|
||||
return ErrIsolation
|
||||
}
|
||||
raw, e := os.ReadFile(v.Path)
|
||||
if e != nil {
|
||||
return ErrIsolation
|
||||
}
|
||||
var signed SignedAttestation
|
||||
var a Attestation
|
||||
if json.Unmarshal(raw, &signed) != nil || len(v.PublicKey) != ed25519.PublicKeySize || !ed25519.Verify(v.PublicKey, signed.Payload, signed.Signature) || json.Unmarshal(signed.Payload, &a) != nil {
|
||||
return ErrIsolation
|
||||
}
|
||||
now := time.Now()
|
||||
if a.Protocol != Protocol || a.Scope != v.Scope || a.GatewayID != v.GatewayID || a.GatewayID == "" || a.IssuedAt.After(now) || !a.ExpiresAt.After(now) || a.ExpiresAt.Sub(a.IssuedAt) > time.Hour || !validHash(a.WriterInventory) || !validHash(a.PolicyEvidence) || !validHash(a.DenialEvidence) || !validHash(a.DrainEvidence) {
|
||||
return ErrIsolation
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,59 +0,0 @@
|
||||
package objectgateway_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
g "knowledge-governance/internal/objectgateway"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestGatewayAuditCannotBeConfigurationBoolean(t *testing.T) {
|
||||
pub, priv, e := ed25519.GenerateKey(rand.Reader)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "audit.json")
|
||||
verifier := g.AuditVerifier{Path: path, PublicKey: pub, Scope: "target", GatewayID: "gateway"}
|
||||
if verifier.Verify(context.Background()) == nil {
|
||||
t.Fatal("missing audit accepted")
|
||||
}
|
||||
if os.WriteFile(path, []byte(`{"isolated":true,"ready":true}`), 0600) != nil {
|
||||
t.Fatal("fixture")
|
||||
}
|
||||
if verifier.Verify(context.Background()) == nil {
|
||||
t.Fatal("boolean accepted as isolation proof")
|
||||
}
|
||||
a := g.Attestation{Protocol: g.Protocol, Scope: "target", GatewayID: "gateway", IssuedAt: time.Now().Add(-time.Minute), ExpiresAt: time.Now().Add(30 * time.Minute), WriterInventory: g.Digest([]byte("inventory")), PolicyEvidence: g.Digest([]byte("policy")), DenialEvidence: g.Digest([]byte("denials")), DrainEvidence: g.Digest([]byte("drain"))}
|
||||
write := func(v g.Attestation) {
|
||||
t.Helper()
|
||||
raw, _ := json.Marshal(v)
|
||||
signed, _ := json.Marshal(g.SignedAttestation{Payload: raw, Signature: ed25519.Sign(priv, raw)})
|
||||
if os.WriteFile(path, signed, 0600) != nil {
|
||||
t.Fatal("fixture")
|
||||
}
|
||||
}
|
||||
write(a)
|
||||
if verifier.Verify(context.Background()) != nil {
|
||||
t.Fatal("valid test auditor signature rejected")
|
||||
}
|
||||
for _, mutate := range []func(*g.Attestation){func(v *g.Attestation) { v.Scope = "other" }, func(v *g.Attestation) { v.GatewayID = "other" }, func(v *g.Attestation) { v.ExpiresAt = time.Now().Add(-time.Second) }, func(v *g.Attestation) { v.DenialEvidence = "" }, func(v *g.Attestation) { v.IssuedAt = time.Now().Add(time.Hour) }, func(v *g.Attestation) { v.ExpiresAt = time.Now().Add(24 * time.Hour) }} {
|
||||
bad := a
|
||||
mutate(&bad)
|
||||
write(bad)
|
||||
if verifier.Verify(context.Background()) == nil {
|
||||
t.Fatal("invalid isolation audit accepted")
|
||||
}
|
||||
}
|
||||
write(a)
|
||||
other, _, _ := ed25519.GenerateKey(rand.Reader)
|
||||
verifier.PublicKey = other
|
||||
if verifier.Verify(context.Background()) == nil {
|
||||
t.Fatal("untrusted signer accepted")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package objectgateway
|
||||
|
||||
import "context"
|
||||
|
||||
// sendCreate performs exactly one non-retried versioned PUT and then confirms
|
||||
// the result. Versioned object stores do not provide atomic create-if-absent,
|
||||
// so the confirmation re-reads the current version to make sure this request's
|
||||
// version is still the one a reader would observe.
|
||||
func sendCreate(ctx context.Context, b Backend, r Request, body []byte) (Version, error) {
|
||||
v, err := b.Put(ctx, r.Key, body, r.token())
|
||||
if err != nil {
|
||||
// Unknown outcome: the intent stays unresolved, nothing is retried and
|
||||
// nothing is deleted because no version id is confirmed.
|
||||
return Version{}, ErrHeld
|
||||
}
|
||||
return confirmCreate(ctx, b, r, v)
|
||||
}
|
||||
|
||||
// confirmCreate binds the receipt to a version that is both the exact version
|
||||
// this request wrote and the current version of the key. It is read-only: the
|
||||
// create intent is already committed, so withdrawing a version here would be an
|
||||
// unrecorded mutation with no delete intent or receipt and would leave the
|
||||
// create intent permanently held. A concurrent writer that produced a newer
|
||||
// current version therefore wins by explicit conflict; both versions stay in
|
||||
// the store and an operator resolves the key with evidence.
|
||||
func confirmCreate(ctx context.Context, b Backend, r Request, v Version) (Version, error) {
|
||||
exact, exists, err := b.Observe(ctx, r.Key, v.ID)
|
||||
if err != nil || !exists || !r.matches(exact) || v != exact {
|
||||
return Version{}, ErrHeld
|
||||
}
|
||||
latest, current, err := b.Observe(ctx, r.Key, "")
|
||||
if err != nil || !current {
|
||||
// Cannot tell whether our write is still current: keep the intent held
|
||||
// rather than mutating anything on a guess.
|
||||
return Version{}, ErrHeld
|
||||
}
|
||||
if latest.ID != v.ID {
|
||||
// Another writer owns the current version. Report the conflict and leave
|
||||
// both versions untouched; this path never merges them automatically.
|
||||
return Version{}, ErrConflict
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
package objectgateway
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type fakeBackend struct {
|
||||
byID map[string]Version
|
||||
latest Version
|
||||
latestExists bool
|
||||
latestErr error
|
||||
putVersion Version
|
||||
putErr error
|
||||
putCalls int
|
||||
deletes []string
|
||||
deleteErr error
|
||||
}
|
||||
|
||||
func (f *fakeBackend) Ready(context.Context) error { return nil }
|
||||
func (f *fakeBackend) Observe(_ context.Context, _ string, id string) (Version, bool, error) {
|
||||
if id == "" {
|
||||
if f.latestErr != nil {
|
||||
return Version{}, false, f.latestErr
|
||||
}
|
||||
return f.latest, f.latestExists, nil
|
||||
}
|
||||
v, ok := f.byID[id]
|
||||
return v, ok, nil
|
||||
}
|
||||
func (f *fakeBackend) List(context.Context, string) ([]Version, error) { return nil, nil }
|
||||
func (f *fakeBackend) Put(context.Context, string, []byte, string) (Version, error) {
|
||||
f.putCalls++
|
||||
if f.putErr != nil {
|
||||
return Version{}, f.putErr
|
||||
}
|
||||
return f.putVersion, nil
|
||||
}
|
||||
func (f *fakeBackend) Delete(_ context.Context, _ string, v Version) error {
|
||||
if f.deleteErr != nil {
|
||||
return f.deleteErr
|
||||
}
|
||||
f.deletes = append(f.deletes, v.ID)
|
||||
delete(f.byID, v.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func createFixture(t *testing.T) (Request, Version) {
|
||||
t.Helper()
|
||||
body := []byte("shared bucket object")
|
||||
grant := Grant{Epoch: 1, Owner: Digest([]byte("owner")), Plan: Digest([]byte("plan"))}
|
||||
r := Request{Grant: grant, Operation: Digest([]byte("operation")), Key: "kb/bound/object-1", Action: "create", SHA256: Digest(body), Bytes: int64(len(body))}
|
||||
v := Version{ID: "version-own", Owner: r.token(), SHA256: r.SHA256, Bytes: r.Bytes, ETag: `"etag-own"`}
|
||||
return r, v
|
||||
}
|
||||
|
||||
func TestCreateConfirmationDetectsNewerForeignVersionWithoutSideEffects(t *testing.T) {
|
||||
r, own := createFixture(t)
|
||||
foreign := Version{ID: "version-foreign", Owner: Digest([]byte("foreign-owner")), SHA256: Digest([]byte("foreign")), Bytes: 7, ETag: `"etag-foreign"`}
|
||||
f := &fakeBackend{byID: map[string]Version{own.ID: own, foreign.ID: foreign}, latest: foreign, latestExists: true, putVersion: own}
|
||||
if _, err := confirmCreate(context.Background(), f, r, own); !errors.Is(err, ErrConflict) {
|
||||
t.Fatalf("concurrent newer version accepted as committed: %v", err)
|
||||
}
|
||||
// Confirmation runs after the journal intent exists. Deleting here would be
|
||||
// an unrecorded mutation with no delete intent/receipt and would strand the
|
||||
// create intent in held forever, so confirmation must be read-only.
|
||||
if len(f.deletes) != 0 {
|
||||
t.Fatalf("confirmation deleted without a delete intent/receipt: %v", f.deletes)
|
||||
}
|
||||
if _, ok := f.byID[own.ID]; !ok {
|
||||
t.Fatal("own version removed without a recorded delete")
|
||||
}
|
||||
if _, ok := f.byID[foreign.ID]; !ok {
|
||||
t.Fatal("foreign version was deleted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateConfirmationCommitsOnlyWhenOwnVersionIsCurrent(t *testing.T) {
|
||||
r, own := createFixture(t)
|
||||
f := &fakeBackend{byID: map[string]Version{own.ID: own}, latest: own, latestExists: true, putVersion: own}
|
||||
got, err := confirmCreate(context.Background(), f, r, own)
|
||||
if err != nil || got != own {
|
||||
t.Fatalf("own current version not committed: %v %v", got, err)
|
||||
}
|
||||
if len(f.deletes) != 0 {
|
||||
t.Fatalf("unexpected rollback %v", f.deletes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateConfirmationHoldsWithoutDeletingWhenCurrentVersionUnreadable(t *testing.T) {
|
||||
r, own := createFixture(t)
|
||||
f := &fakeBackend{byID: map[string]Version{own.ID: own}, putVersion: own, latestErr: errors.New("read failed")}
|
||||
if _, err := confirmCreate(context.Background(), f, r, own); !errors.Is(err, ErrHeld) {
|
||||
t.Fatalf("unreadable current version treated as terminal: %v", err)
|
||||
}
|
||||
if len(f.deletes) != 0 {
|
||||
t.Fatalf("deleted on an unknown observation: %v", f.deletes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateConfirmationHoldsOnExactReadbackMismatch(t *testing.T) {
|
||||
r, own := createFixture(t)
|
||||
other := own
|
||||
other.ETag = `"different"`
|
||||
f := &fakeBackend{byID: map[string]Version{own.ID: other}, latest: other, latestExists: true, putVersion: own}
|
||||
if _, err := confirmCreate(context.Background(), f, r, own); !errors.Is(err, ErrHeld) {
|
||||
t.Fatalf("mismatched readback treated as committed: %v", err)
|
||||
}
|
||||
if len(f.deletes) != 0 {
|
||||
t.Fatalf("deleted on mismatch: %v", f.deletes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateSendIsSingleAttemptAndNeverDeletesUnknownPut(t *testing.T) {
|
||||
r, _ := createFixture(t)
|
||||
f := &fakeBackend{putErr: errors.New("transport lost")}
|
||||
if _, err := sendCreate(context.Background(), f, r, []byte("shared bucket object")); !errors.Is(err, ErrHeld) {
|
||||
t.Fatalf("unknown PUT treated as terminal: %v", err)
|
||||
}
|
||||
if f.putCalls != 1 {
|
||||
t.Fatalf("PUT retried: calls=%d", f.putCalls)
|
||||
}
|
||||
if len(f.deletes) != 0 {
|
||||
t.Fatalf("deleted without a confirmed version: %v", f.deletes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanAuthorizesOnlyFrozenObjectList(t *testing.T) {
|
||||
body := []byte("planned")
|
||||
owner, digest := Digest([]byte("owner")), Digest([]byte("plan-digest"))
|
||||
p := Plan{Digest: digest, Owner: owner, Objects: []ObjectBinding{{Operation: Digest([]byte("op-1")), Key: "kb/allowed/1", SHA256: Digest(body), Bytes: int64(len(body))}}}
|
||||
allowed := Request{Grant: Grant{1, owner, digest}, Operation: Digest([]byte("op-1")), Key: "kb/allowed/1", Action: "create", SHA256: Digest(body), Bytes: int64(len(body))}
|
||||
if !p.allows(allowed) {
|
||||
t.Fatal("frozen object rejected")
|
||||
}
|
||||
for name, mutate := range map[string]func(*Request){
|
||||
"other_kb_key": func(r *Request) { r.Key = "kb/other-kb/1" },
|
||||
"other_object": func(r *Request) { r.Key = "kb/allowed/2" },
|
||||
"other_operation": func(r *Request) { r.Operation = Digest([]byte("op-2")) },
|
||||
} {
|
||||
got := allowed
|
||||
mutate(&got)
|
||||
if p.allows(got) {
|
||||
t.Fatalf("%s authorized outside the frozen write set", name)
|
||||
}
|
||||
}
|
||||
dup := p
|
||||
dup.Objects = append(dup.Objects, p.Objects[0])
|
||||
if dup.valid() {
|
||||
t.Fatal("duplicate key accepted in plan")
|
||||
}
|
||||
}
|
||||
@@ -20,17 +20,18 @@ type Gateway struct {
|
||||
Pool *pgxpool.Pool
|
||||
Scope string
|
||||
Backend Backend
|
||||
Isolation IsolationVerifier
|
||||
MaxBytes int64
|
||||
}
|
||||
|
||||
// Ready gates admission on the local journal and the object backend only. A
|
||||
// shared bucket is the supported deployment shape: unrelated writers on other
|
||||
// keys do not need to be drained, and no whole-bucket exclusivity evidence is
|
||||
// required. Per-key competition is handled at execute time, not by a global
|
||||
// "only writer" attestation.
|
||||
func (g *Gateway) Ready(ctx context.Context) error {
|
||||
if g.Pool == nil || g.Scope == "" || g.Backend == nil || g.MaxBytes < 1 {
|
||||
return ErrPersistence
|
||||
}
|
||||
if g.Isolation == nil || g.Isolation.Verify(ctx) != nil {
|
||||
return ErrIsolation
|
||||
}
|
||||
var sync, fsync, full string
|
||||
if g.Pool.QueryRow(ctx, "SELECT current_setting('synchronous_commit'), current_setting('fsync'), current_setting('full_page_writes')").Scan(&sync, &fsync, &full) != nil || sync != "on" || fsync != "on" || full != "on" {
|
||||
return ErrPersistence
|
||||
@@ -249,13 +250,9 @@ func (g *Gateway) execute(ctx context.Context, r Request, body []byte, authoriza
|
||||
if exists {
|
||||
return g.finish(ctx, r, "rejected", Version{})
|
||||
}
|
||||
v, err := g.Backend.Put(ctx, r.Key, body, r.token())
|
||||
v, err := sendCreate(ctx, g.Backend, r, body)
|
||||
if err != nil {
|
||||
return Receipt{}, ErrHeld
|
||||
}
|
||||
exact, exists, err := g.Backend.Observe(ctx, r.Key, v.ID)
|
||||
if err != nil || !exists || !r.matches(exact) || v != exact {
|
||||
return Receipt{}, ErrHeld
|
||||
return Receipt{}, err
|
||||
}
|
||||
return g.finish(ctx, r, "committed", v)
|
||||
}
|
||||
@@ -322,5 +319,15 @@ func (g *Gateway) Reconcile(ctx context.Context, r Request) (Receipt, error) {
|
||||
if err != nil || !exists || exact != found[0] {
|
||||
return Receipt{}, ErrHeld
|
||||
}
|
||||
latest, current, err := g.Backend.Observe(ctx, r.Key, "")
|
||||
if err != nil || !current {
|
||||
return Receipt{}, ErrHeld
|
||||
}
|
||||
if latest.ID != exact.ID {
|
||||
// Another writer owns the current version of this key. Do not delete
|
||||
// anything during reconcile and do not report a committed create for a
|
||||
// key this plan no longer controls.
|
||||
return Receipt{}, ErrConflict
|
||||
}
|
||||
return g.finish(ctx, r, "committed", exact)
|
||||
}
|
||||
|
||||
@@ -30,9 +30,6 @@ import (
|
||||
g "knowledge-governance/internal/objectgateway"
|
||||
)
|
||||
|
||||
type verifiedFixture struct{}
|
||||
|
||||
func (verifiedFixture) Verify(context.Context) error { return nil }
|
||||
func fixture(t *testing.T) (context.Context, *pgxpool.Pool, *g.S3Backend) {
|
||||
t.Helper()
|
||||
dsn := os.Getenv("KG_GATEWAY_TEST_DATABASE_URL")
|
||||
@@ -118,7 +115,7 @@ func TestGatewayReadinessRequiresJournal(t *testing.T) {
|
||||
t.Fatal(e)
|
||||
}
|
||||
defer other.Close()
|
||||
gw := &g.Gateway{Pool: other, Scope: "missing-journal", Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20}
|
||||
gw := &g.Gateway{Pool: other, Scope: "missing-journal", Backend: backend, MaxBytes: 1 << 20}
|
||||
if e := gw.Ready(ctx); !errors.Is(e, g.ErrPersistence) {
|
||||
t.Fatalf("missing durable journal accepted: %v", e)
|
||||
}
|
||||
@@ -126,7 +123,7 @@ func TestGatewayReadinessRequiresJournal(t *testing.T) {
|
||||
func TestGatewayRealReplayAndImmutableLedger(t *testing.T) {
|
||||
ctx, pool, backend := fixture(t)
|
||||
grant := g.Grant{Epoch: 1, Owner: g.Digest([]byte(t.Name())), Plan: g.Digest([]byte("fixed-plan"))}
|
||||
gw := &g.Gateway{Pool: pool, Scope: backend.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20}
|
||||
gw := &g.Gateway{Pool: pool, Scope: backend.Bucket, Backend: backend, MaxBytes: 1 << 20}
|
||||
if _, exists, err := backend.Observe(ctx, "missing-key-probe", ""); err != nil || exists {
|
||||
t.Fatalf("initial authoritative absence unavailable: %v exists=%v", err, exists)
|
||||
}
|
||||
|
||||
@@ -239,8 +239,6 @@ func (c *Client) call(ctx context.Context, path string, in, out any) error {
|
||||
return ErrHeld
|
||||
case ErrPersistence.Error():
|
||||
return ErrPersistence
|
||||
case ErrIsolation.Error():
|
||||
return ErrIsolation
|
||||
}
|
||||
return errors.New("gateway_request_refused")
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
func TestGatewayTargetIdentityOnEveryCall(t *testing.T) {
|
||||
ctx, pool, backend := fixture(t)
|
||||
grant := g.Grant{Epoch: 1, Owner: g.Digest([]byte(t.Name())), Plan: g.Digest([]byte("fixed-plan"))}
|
||||
gw := &g.Gateway{Pool: pool, Scope: backend.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20}
|
||||
gw := &g.Gateway{Pool: pool, Scope: backend.Bucket, Backend: backend, MaxBytes: 1 << 20}
|
||||
if e := gw.InstallPlan(ctx, planFor(grant, request(t.Name(), grant))); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@ func TestGatewayIntentCommitFailureNeverSends(t *testing.T) {
|
||||
ctx, pool, real := fixture(t)
|
||||
backend := &stoppedBeforeSend{Backend: real}
|
||||
grant := g.Grant{Epoch: 1, Owner: g.Digest([]byte(t.Name())), Plan: g.Digest([]byte("plan"))}
|
||||
gateway := &g.Gateway{Pool: pool, Scope: real.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20}
|
||||
gateway := &g.Gateway{Pool: pool, Scope: real.Bucket, Backend: backend, MaxBytes: 1 << 20}
|
||||
if e := gateway.InstallPlan(ctx, planFor(grant, request(t.Name(), grant))); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
func TestGatewayGrantCannotAuthorizeUnplannedKey(t *testing.T) {
|
||||
ctx, pool, backend := fixture(t)
|
||||
grant := g.Grant{Epoch: 1, Owner: g.Digest([]byte(t.Name())), Plan: g.Digest([]byte("fixed-plan"))}
|
||||
gateway := &g.Gateway{Pool: pool, Scope: backend.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20}
|
||||
gateway := &g.Gateway{Pool: pool, Scope: backend.Bucket, Backend: backend, MaxBytes: 1 << 20}
|
||||
if e := gateway.Advance(ctx, 0, grant); !errors.Is(e, g.ErrConflict) {
|
||||
t.Fatal("unregistered plan grant", e)
|
||||
}
|
||||
|
||||
@@ -19,7 +19,6 @@ var (
|
||||
ErrPersistence = errors.New("gateway_persistence_unavailable")
|
||||
ErrAuthority = errors.New("gateway_control_authority_unavailable")
|
||||
ErrAuthorityPrivileges = errors.New("gateway_control_reader_privileges_invalid")
|
||||
ErrIsolation = errors.New("gateway_exclusive_writer_evidence_unverified")
|
||||
ErrRejected = errors.New("gateway_create_rejected_existing_object")
|
||||
)
|
||||
|
||||
@@ -60,7 +59,6 @@ type Backend interface {
|
||||
Put(context.Context, string, []byte, string) (Version, error)
|
||||
Delete(context.Context, string, Version) error
|
||||
}
|
||||
type IsolationVerifier interface{ Verify(context.Context) error }
|
||||
|
||||
func Digest(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) }
|
||||
func validHash(s string) bool {
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
package objectgateway_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
g "knowledge-governance/internal/objectgateway"
|
||||
)
|
||||
|
||||
// A maintenance update of the target KB runs while the same bucket still
|
||||
// serves other writers. Publishing planned objects must neither require a
|
||||
// whole-bucket exclusivity attestation nor disturb an unrelated key or a
|
||||
// pre-existing object at a planned key. External writes here are made before
|
||||
// execute (coexistence), not interleaved concurrently with the PUT.
|
||||
func TestGatewaySharedBucketCoexistenceWithOtherWriters(t *testing.T) {
|
||||
ctx, pool, backend := fixture(t)
|
||||
endpoint := os.Getenv("KG_GATEWAY_TEST_ENDPOINT")
|
||||
scope := g.Digest([]byte("minio\x00" + endpoint + "\x00" + backend.Bucket))
|
||||
listener, e := net.Listen("tcp", "127.0.0.1:0")
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
addr := listener.Addr().String()
|
||||
listener.Close()
|
||||
data, control := g.Digest([]byte("data "+scope)), g.Digest([]byte("control "+scope))
|
||||
client, e := g.NewClient("http://"+addr, data, scope)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
admin, _ := g.NewClient("http://"+addr, control, scope)
|
||||
// Deliberately no OBJECT_GATEWAY_AUDIT_FILE / OBJECT_GATEWAY_AUDIT_PUBLIC_KEY:
|
||||
// the default shared-bucket path must start without whole-bucket evidence.
|
||||
env := []string{
|
||||
"PATH=" + os.Getenv("PATH"),
|
||||
"HOME=" + os.Getenv("HOME"),
|
||||
"OBJECT_GATEWAY_LISTEN=" + addr,
|
||||
"OBJECT_GATEWAY_DATABASE_URL=" + os.Getenv("KG_GATEWAY_TEST_DATABASE_URL"),
|
||||
"OBJECT_GATEWAY_PROVIDER=minio",
|
||||
"OBJECT_GATEWAY_ENDPOINT=" + endpoint,
|
||||
"OBJECT_GATEWAY_BUCKET=" + backend.Bucket,
|
||||
"OBJECT_GATEWAY_REGION=us-east-1",
|
||||
"OBJECT_GATEWAY_ACCESS_KEY=" + os.Getenv("KG_GATEWAY_TEST_ACCESS_KEY"),
|
||||
"OBJECT_GATEWAY_SECRET_KEY=" + os.Getenv("KG_GATEWAY_TEST_SECRET_KEY"),
|
||||
"OBJECT_GATEWAY_DATA_TOKEN=" + data,
|
||||
"OBJECT_GATEWAY_CONTROL_TOKEN=" + control,
|
||||
}
|
||||
startGateway(t, env, client)
|
||||
if e = client.ReadyForAcquisition(ctx); e == nil {
|
||||
t.Fatal("gateway without control authority advertised automatic grants")
|
||||
}
|
||||
|
||||
// Another service keeps writing unrelated keys during the maintenance window.
|
||||
externalOwner := g.Digest([]byte("other-service"))
|
||||
unrelatedKey := "other-service/unrelated-" + g.Digest([]byte(backend.Bucket))[:8]
|
||||
unrelatedBody := []byte("unrelated service body")
|
||||
if _, e = backend.Client.PutObject(ctx, &s3.PutObjectInput{Bucket: aws.String(backend.Bucket), Key: aws.String(unrelatedKey), Body: bytes.NewReader(unrelatedBody), Metadata: map[string]string{"kg-release-owner": externalOwner}}); e != nil {
|
||||
t.Fatalf("external unrelated write failed: %v", e)
|
||||
}
|
||||
unrelated, _, e := backend.Observe(ctx, unrelatedKey, "")
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
|
||||
// A planned key already holds a foreign object: the plan must refuse it and
|
||||
// leave that object byte-identical.
|
||||
existingKey := "unchanged/existing-object"
|
||||
existingBody := []byte("pre-existing production content")
|
||||
if _, e = backend.Client.PutObject(ctx, &s3.PutObjectInput{Bucket: aws.String(backend.Bucket), Key: aws.String(existingKey), Body: bytes.NewReader(existingBody), Metadata: map[string]string{"kg-release-owner": externalOwner}}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
existing, _, e := backend.Observe(ctx, existingKey, "")
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
|
||||
grant := g.Grant{Epoch: 1, Owner: g.Digest([]byte("maintenance-owner")), Plan: g.Digest([]byte("maintenance-plan"))}
|
||||
fresh := request("shared-fresh", grant)
|
||||
clash := request("shared-clash", grant)
|
||||
clash.Key = existingKey
|
||||
if e = admin.InstallPlan(ctx, planFor(grant, fresh, clash)); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e = admin.Advance(ctx, 0, grant); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
receipt, e := client.Execute(ctx, fresh, []byte("gateway fixture shared-fresh"))
|
||||
if e != nil || receipt.State != "committed" {
|
||||
t.Fatalf("planned create blocked by unrelated writers: %v %s", e, receipt.State)
|
||||
}
|
||||
rejected, e := client.Execute(ctx, clash, []byte("gateway fixture shared-clash"))
|
||||
if e != nil || rejected.State != "rejected" {
|
||||
t.Fatalf("pre-existing planned key not rejected: %v %s", e, rejected.State)
|
||||
}
|
||||
after, exists, e := backend.Observe(ctx, existingKey, "")
|
||||
if e != nil || !exists || after.ID != existing.ID || after.SHA256 != g.Digest(existingBody) {
|
||||
t.Fatalf("pre-existing object modified: %v %+v", e, after)
|
||||
}
|
||||
still, exists, e := backend.Observe(ctx, unrelatedKey, "")
|
||||
if e != nil || !exists || still.ID != unrelated.ID {
|
||||
t.Fatalf("unrelated object disturbed: %v %+v", e, still)
|
||||
}
|
||||
|
||||
// Precise rollback removes only the version this gateway created.
|
||||
cleanup(t, ctx, &g.Gateway{Pool: pool, Scope: scope, Backend: backend, MaxBytes: 1 << 20}, receipt)
|
||||
for key, keep := range map[string]g.Version{existingKey: existing, unrelatedKey: unrelated} {
|
||||
if e = backend.Delete(ctx, key, keep); e != nil {
|
||||
t.Fatalf("external fixture cleanup %s: %v", key, e)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
|
||||
func TestGatewayNewEpochPreservesOriginalSendIdentity(t *testing.T) {
|
||||
ctx, pool, backend := fixture(t)
|
||||
gateway := &g.Gateway{Pool: pool, Scope: backend.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20}
|
||||
gateway := &g.Gateway{Pool: pool, Scope: backend.Bucket, Backend: backend, MaxBytes: 1 << 20}
|
||||
first := g.Grant{Epoch: 1, Owner: g.Digest([]byte(t.Name())), Plan: g.Digest([]byte("takeover-plan"))}
|
||||
sent, unsent := request("takeover-sent", first), request("takeover-unsent", first)
|
||||
if e := gateway.InstallPlan(ctx, planFor(first, sent, unsent)); e != nil {
|
||||
|
||||
@@ -22,7 +22,7 @@ func TestGatewayAbsenceNeverSettlesUnknown(t *testing.T) {
|
||||
ctx, pool, real := fixture(t)
|
||||
backend := &stoppedBeforeSend{Backend: real}
|
||||
grant := g.Grant{Epoch: 1, Owner: g.Digest([]byte(t.Name())), Plan: g.Digest([]byte("plan"))}
|
||||
gateway := &g.Gateway{Pool: pool, Scope: real.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20}
|
||||
gateway := &g.Gateway{Pool: pool, Scope: real.Bucket, Backend: backend, MaxBytes: 1 << 20}
|
||||
if e := gateway.InstallPlan(ctx, planFor(grant, request(t.Name(), grant))); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
@@ -62,7 +62,7 @@ func TestGatewayReadOnlyPersistenceBlocksSend(t *testing.T) {
|
||||
t.Fatal(e)
|
||||
}
|
||||
defer other.Close()
|
||||
gateway := &g.Gateway{Pool: other, Scope: backend.Bucket, Backend: backend, Isolation: verifiedFixture{}, MaxBytes: 1 << 20}
|
||||
gateway := &g.Gateway{Pool: other, Scope: backend.Bucket, Backend: backend, MaxBytes: 1 << 20}
|
||||
if e = gateway.Ready(ctx); !errors.Is(e, g.ErrPersistence) {
|
||||
t.Fatal("read-only persistence accepted", e)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,236 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"knowledge-governance/internal/governance"
|
||||
)
|
||||
|
||||
// F17:取件 verified 后必须沿同一策略/当前绑定自动接续唯一 test.upload。
|
||||
// 这里用真实隔离 PG 跑 RouteCatalogFiles + MarkArtifactFetched 的真实 SQL 路径。
|
||||
type autoIngestFixture struct {
|
||||
t *testing.T
|
||||
ctx context.Context
|
||||
pool *pgxpool.Pool
|
||||
store *Store
|
||||
now time.Time
|
||||
connID string
|
||||
kbID string
|
||||
}
|
||||
|
||||
func newAutoIngestFixture(t *testing.T) *autoIngestFixture {
|
||||
t.Helper()
|
||||
url := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
|
||||
if url == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(pool.Close)
|
||||
if err = ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Date(2026, 11, 5, 4, 0, 0, 0, time.UTC)
|
||||
exec := func(q string, args ...any) {
|
||||
t.Helper()
|
||||
if _, err := pool.Exec(ctx, q, args...); err != nil {
|
||||
t.Fatalf("%v: %s", err, q)
|
||||
}
|
||||
}
|
||||
exec(`INSERT INTO source_connections (id,source_type,name,enabled) VALUES ('conn-auto','dingtalk','ATOM自动衔接',TRUE)`)
|
||||
exec(`INSERT INTO source_catalog_snapshots (id,source_connection_id,snapshot_sha256,source_label,node_count,folder_count,file_count,observed_at)
|
||||
VALUES ('snap-auto','conn-auto',repeat('a',64),'fixture',2,1,1,$1)`, now)
|
||||
exec(`INSERT INTO source_catalog_nodes (snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash)
|
||||
VALUES ('snap-auto','node-dir',NULL,'folder','常见问题汇总','常见问题汇总',repeat('b',64))`)
|
||||
exec(`INSERT INTO source_catalog_nodes (snapshot_id,node_id,parent_node_id,node_type,name,relative_path,source_locator_hash)
|
||||
VALUES ('snap-auto','node-file','node-dir','file','人形常见问题.adoc','常见问题汇总/人形常见问题.adoc',repeat('c',64))`)
|
||||
exec(`INSERT INTO source_catalog_heads (source_connection_id,snapshot_id,activated_at) VALUES ('conn-auto','snap-auto',$1)`, now)
|
||||
exec(`INSERT INTO routing_kb_bindings (id,internal_kb_id,external_kb_id,updated_by) VALUES ('active','kb-current','kb-ext', 'fixture')
|
||||
ON CONFLICT (id) DO UPDATE SET internal_kb_id='kb-current',external_kb_id='kb-ext'`)
|
||||
return &autoIngestFixture{t: t, ctx: ctx, pool: pool, store: NewStore(pool).WithRoutingDefaults("kb-env", "kb-ext"), now: now, connID: "conn-auto", kbID: "kb-current"}
|
||||
}
|
||||
|
||||
func (f *autoIngestFixture) route() (versionID, publicationID string) {
|
||||
f.t.Helper()
|
||||
result, err := f.store.RouteCatalogFiles(f.ctx, governance.CatalogRouteRequest{
|
||||
ConnectionID: f.connID,
|
||||
NodeIDs: []string{"node-file"},
|
||||
Visibility: governance.VisibilityInternal,
|
||||
TargetKnowledgeBaseID: f.kbID,
|
||||
OperatorRef: "fixture",
|
||||
Reason: "fixture",
|
||||
Now: f.now,
|
||||
})
|
||||
if err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
if result.SelectedFiles != 1 || result.PlannedPublications != 1 {
|
||||
f.t.Fatalf("route result = %#v", result)
|
||||
}
|
||||
if err := f.pool.QueryRow(f.ctx, `SELECT publication.source_item_version_id,publication.id
|
||||
FROM test_publications publication
|
||||
JOIN catalog_import_intents intent ON intent.source_item_id=(SELECT source_item_id FROM source_item_versions WHERE id=publication.source_item_version_id)
|
||||
WHERE intent.node_id='node-file' AND publication.test_kb_id=$1`, f.kbID).Scan(&versionID, &publicationID); err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
return versionID, publicationID
|
||||
}
|
||||
|
||||
func (f *autoIngestFixture) markFetched(versionID string) {
|
||||
f.t.Helper()
|
||||
if err := f.store.MarkArtifactFetched(f.ctx, versionID, "32222102cafd6e5cdaeee415aad51e7df52629136d5ef2c3eae6e0e30ff7153d", 6991, "spool/auto.docx", "docx", true, f.now.Add(time.Minute)); err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (f *autoIngestFixture) uploadJobs(publicationID string) int {
|
||||
f.t.Helper()
|
||||
var count int
|
||||
if err := f.pool.QueryRow(f.ctx, `SELECT COUNT(*) FROM jobs WHERE type='test.upload' AND payload->>'publication_id'=$1`, publicationID).Scan(&count); err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func (f *autoIngestFixture) publicationState(publicationID string) string {
|
||||
f.t.Helper()
|
||||
var state string
|
||||
if err := f.pool.QueryRow(f.ctx, `SELECT processing_state FROM test_publications WHERE id=$1`, publicationID).Scan(&state); err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
func TestFetchedVerifiedArtifactAutoStartsExactlyOneUpload(t *testing.T) {
|
||||
f := newAutoIngestFixture(t)
|
||||
versionID, publicationID := f.route()
|
||||
if jobs := f.uploadJobs(publicationID); jobs != 0 {
|
||||
t.Fatalf("upload job before verification = %d", jobs)
|
||||
}
|
||||
f.markFetched(versionID)
|
||||
if jobs := f.uploadJobs(publicationID); jobs != 1 {
|
||||
t.Fatalf("upload jobs after verified fetch = %d, want 1", jobs)
|
||||
}
|
||||
if state := f.publicationState(publicationID); state != "pending" {
|
||||
t.Fatalf("publication state = %q, want pending", state)
|
||||
}
|
||||
// 幂等:重复完成不能造第二个上传任务。
|
||||
f.markFetched(versionID)
|
||||
if jobs := f.uploadJobs(publicationID); jobs != 1 {
|
||||
t.Fatalf("duplicate verified fetch created %d upload jobs", jobs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRouteReusingVerifiedContentAlsoAutoStarts(t *testing.T) {
|
||||
f := newAutoIngestFixture(t)
|
||||
versionID, publicationID := f.route()
|
||||
// 模拟"本地已有先比对":内容已 verified,删除尚未消费的取件作业后再路由。
|
||||
if _, err := f.pool.Exec(f.ctx, `UPDATE source_item_versions SET content_object_id=$2,content_state='verified'
|
||||
WHERE id=$1`, versionID, f.insertVerifiedContent()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := f.pool.Exec(f.ctx, `DELETE FROM jobs WHERE type='artifact.fetch' AND payload->>'version_id'=$1`, versionID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := f.store.RouteCatalogFiles(f.ctx, governance.CatalogRouteRequest{
|
||||
ConnectionID: f.connID, NodeIDs: []string{"node-file"}, Visibility: governance.VisibilityInternal,
|
||||
TargetKnowledgeBaseID: f.kbID, OperatorRef: "fixture", Reason: "reuse", Now: f.now.Add(time.Minute),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if jobs := f.uploadJobs(publicationID); jobs != 1 {
|
||||
t.Fatalf("reused verified content upload jobs = %d, want 1", jobs)
|
||||
}
|
||||
}
|
||||
|
||||
func (f *autoIngestFixture) insertVerifiedContent() string {
|
||||
f.t.Helper()
|
||||
if _, err := f.pool.Exec(f.ctx, `INSERT INTO content_objects(id,sha256,size_bytes,mime,validated_format,local_spool_ref,content_state)
|
||||
VALUES('co-auto',repeat('d',64),6991,'','docx','spool/auto.docx','verified')
|
||||
ON CONFLICT(sha256) DO UPDATE SET local_spool_ref=EXCLUDED.local_spool_ref,content_state='verified'`); err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
var id string
|
||||
if err := f.pool.QueryRow(f.ctx, `SELECT id FROM content_objects WHERE sha256=repeat('d',64)`).Scan(&id); err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func TestStuckPlannedVerifiedPublicationRecoversViaReRoute(t *testing.T) {
|
||||
f := newAutoIngestFixture(t)
|
||||
versionID, publicationID := f.route()
|
||||
f.markFetched(versionID) // 新代码本会入队;先清掉,复现 356 已卡记录
|
||||
if _, err := f.pool.Exec(f.ctx, `DELETE FROM jobs WHERE type='test.upload' AND payload->>'publication_id'=$1`, publicationID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := f.pool.Exec(f.ctx, `UPDATE test_publications SET processing_state='planned' WHERE id=$1`, publicationID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// 父在 UI 对同一文件再次保存当前策略:只这一条、只当前绑定,幂等接续。
|
||||
if _, err := f.store.RouteCatalogFiles(f.ctx, governance.CatalogRouteRequest{
|
||||
ConnectionID: f.connID, NodeIDs: []string{"node-file"}, Visibility: governance.VisibilityInternal,
|
||||
TargetKnowledgeBaseID: f.kbID, OperatorRef: "fixture", Reason: "resume", Now: f.now.Add(2 * time.Minute),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if jobs := f.uploadJobs(publicationID); jobs != 1 {
|
||||
t.Fatalf("stuck record recovered upload jobs = %d, want 1", jobs)
|
||||
}
|
||||
if state := f.publicationState(publicationID); state != "pending" {
|
||||
t.Fatalf("stuck record state = %q", state)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAutoStartRefusesStaleOrIneligiblePolicies(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
mutate func(f *autoIngestFixture, publicationID string)
|
||||
}{
|
||||
{"disabled source", func(f *autoIngestFixture, _ string) {
|
||||
if _, err := f.pool.Exec(f.ctx, `UPDATE source_connections SET enabled=FALSE WHERE id='conn-auto'`); err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
}},
|
||||
{"blocked policy", func(f *autoIngestFixture, _ string) {
|
||||
if _, err := f.pool.Exec(f.ctx, `UPDATE source_items SET policy_state='blocked' WHERE source_connection_id='conn-auto'`); err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
}},
|
||||
{"filtered filename", func(f *autoIngestFixture, _ string) {
|
||||
if _, err := f.pool.Exec(f.ctx, `INSERT INTO filename_filter_rules(source_connection_id,directory_path,excluded_extensions,updated_by,mode)
|
||||
VALUES('conn-auto','',ARRAY['adoc'],'fixture','exclude')`); err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
}},
|
||||
{"superseded publication", func(f *autoIngestFixture, publicationID string) {
|
||||
if _, err := f.pool.Exec(f.ctx, `UPDATE test_publications SET superseded_at=$2 WHERE id=$1`, publicationID, f.now); err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
}},
|
||||
{"binding changed", func(f *autoIngestFixture, _ string) {
|
||||
if _, err := f.pool.Exec(f.ctx, `UPDATE routing_kb_bindings SET internal_kb_id='kb-other' WHERE id='active'`); err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
f := newAutoIngestFixture(t)
|
||||
versionID, publicationID := f.route()
|
||||
tc.mutate(f, publicationID)
|
||||
f.markFetched(versionID)
|
||||
if jobs := f.uploadJobs(publicationID); jobs != 0 {
|
||||
t.Fatalf("%s auto-started %d upload jobs, want 0", tc.name, jobs)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -27,7 +27,7 @@ func (s *Store) RouteCatalogFiles(ctx context.Context, request governance.Catalo
|
||||
return governance.CatalogRouteResult{}, fmt.Errorf("begin catalog route: %w", err)
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
result, err := routeCatalogFilesTx(ctx, tx, request)
|
||||
result, err := routeCatalogFilesTx(ctx, tx, s, request)
|
||||
if err != nil {
|
||||
return governance.CatalogRouteResult{}, err
|
||||
}
|
||||
@@ -57,7 +57,7 @@ func validateCatalogRouteRequest(request governance.CatalogRouteRequest) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func routeCatalogFilesTx(ctx context.Context, tx pgx.Tx, request governance.CatalogRouteRequest) (governance.CatalogRouteResult, error) {
|
||||
func routeCatalogFilesTx(ctx context.Context, tx pgx.Tx, store *Store, request governance.CatalogRouteRequest) (governance.CatalogRouteResult, error) {
|
||||
targetKnowledgeBaseIDs := request.TargetKnowledgeBaseIDs
|
||||
if len(targetKnowledgeBaseIDs) == 0 && request.TargetKnowledgeBaseID != "" {
|
||||
targetKnowledgeBaseIDs = []string{request.TargetKnowledgeBaseID}
|
||||
@@ -114,7 +114,7 @@ FOR UPDATE OF node`, request.ConnectionID, nodeIDs)
|
||||
JOIN source_catalog_nodes child ON child.snapshot_id=parent.snapshot_id AND child.parent_node_id=parent.node_id
|
||||
WHERE parent.node_type='folder'
|
||||
)
|
||||
SELECT DISTINCT node.node_id,node.name,node.relative_path,node.size_bytes,node.modified_at,node.content_sha256,
|
||||
SELECT DISTINCT node.node_id,node.parent_node_id,node.name,node.relative_path,node.size_bytes,node.modified_at,node.content_sha256,
|
||||
node.source_locator_hash,(node.node_id=ANY($2::text[])) AS directly_selected
|
||||
FROM descendants
|
||||
JOIN source_catalog_nodes node ON node.snapshot_id=descendants.snapshot_id AND node.node_id=descendants.node_id
|
||||
@@ -130,7 +130,7 @@ ORDER BY node.node_id`, request.ConnectionID, nodeIDs)
|
||||
files := make([]governance.CatalogRouteFile, 0, len(nodeIDs))
|
||||
for rows.Next() {
|
||||
var file governance.CatalogRouteFile
|
||||
if err := rows.Scan(&file.NodeID, &file.Name, &file.RelativePath, &file.SizeBytes, &file.ModifiedAt,
|
||||
if err := rows.Scan(&file.NodeID, &file.ParentNodeID, &file.Name, &file.RelativePath, &file.SizeBytes, &file.ModifiedAt,
|
||||
&file.ContentSHA256, &file.LocatorHash, &file.Direct); err != nil {
|
||||
rows.Close()
|
||||
return governance.CatalogRouteResult{}, fmt.Errorf("scan catalog route file: %w", err)
|
||||
@@ -142,7 +142,7 @@ ORDER BY node.node_id`, request.ConnectionID, nodeIDs)
|
||||
return governance.CatalogRouteResult{}, fmt.Errorf("iterate catalog route files: %w", err)
|
||||
}
|
||||
rows.Close()
|
||||
return fulfillCatalogIntentsTx(ctx, tx, request, files, request.Now)
|
||||
return fulfillCatalogIntentsTx(ctx, tx, store, request, files, request.Now)
|
||||
}
|
||||
|
||||
func setCatalogDirectoryPolicy(ctx context.Context, tx pgx.Tx, request governance.CatalogRouteRequest, directoryPath string) error {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user