姚章浩
c27a0c14cc
build / build (push) Successful in 1m22s
真实演练暴露两个只有跑起来才看得见的问题： 1. 归档要求完整兼容性契约（v0.7.1 / schema 74 / elasticsearch_v8 / 1024 维 / 映射摘要），而基线装配只填了 embedding 与映射，捕获被 "unsupported snapshot compatibility contract" 拒绝。 现在 schema 版本读目标库 schema_migrations（dirty 或版本不符直接拒）， embedding 优先读目标库模型绑定（本机目标 models 表为空、引用悬空， 此时回落部署声明并说清来源），版本/引擎用受支持栈常量，映射取目标当前值。 2. 演练在"还没写目标内容"时就允许 restore_blocked=false 的窗口以 blocked 收尾， 写出 content_risk=false + restore_blocked=true 的非法 journal（恢复器拒绝加载， 目标停在停机态）。现在写 canary 时先 MarkContentRisk，allowRestore 按 "没碰内容 ⇒ 安全恢复；碰过 ⇒ 必须恢复并校验通过"。 另外给演练工具加 -recover：只把非终态窗口按原运行状态恢复，用于演练失败后 把目标救回来（不改治理记录）。 验证：库内模拟真实目标不可得，契约分支用真实 kgseal 演练覆盖： snapshot 5cb945e6… 捕获 143ms → 基线一致 → canary 后校验如预期失败 （knowledges 1 行摘要不一致）→ 恢复后校验通过 → 窗口 succeeded、服务与入口恢复。
252 lines
11 KiB
Go
252 lines
11 KiB
Go
// Package librarystack 是"目标整库读写栈"的唯一装配点：同步、恢复、基线，
|
||
// 以及主机侧的真实恢复演练都从这里拿同一个目标/预备份存储/维护协调器。
|
||
//
|
||
// 为什么单独成包：装配一旦被复制两份，"目标是谁、用哪份契约"就会随调用路径漂移。
|
||
// 这个包只做装配与配置校验，不包含任何业务编排。
|
||
package librarystack
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"fmt"
|
||
"strings"
|
||
|
||
"github.com/jackc/pgx/v5"
|
||
|
||
"knowledge-governance/internal/appconfig"
|
||
"knowledge-governance/internal/librarymaintenance"
|
||
"knowledge-governance/internal/librarysnapshot"
|
||
v071 "knowledge-governance/internal/releaseadapter/v071"
|
||
"knowledge-governance/internal/releasebundle"
|
||
)
|
||
|
||
// OpenMaintenance 按部署配置装配实例级维护目标；未接线时返回明确原因。
|
||
func OpenMaintenance(config appconfig.Config) (*librarymaintenance.Manager, error) {
|
||
database, err := librarymaintenance.DatabaseIdentity(config.LibraryProductionDatabaseURL)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
return librarymaintenance.New(librarymaintenance.Config{
|
||
Instance: config.LibraryTargetInstance,
|
||
Project: config.LibraryTargetProject,
|
||
Writers: config.LibraryTargetWriters,
|
||
NonWriters: config.LibraryTargetNonWriters,
|
||
DockerSocket: config.LibraryTargetDockerSocket,
|
||
IngressURL: config.LibraryTargetIngressURL,
|
||
IngressService: config.LibraryTargetIngressService,
|
||
IngressDrainFile: config.LibraryTargetIngressDrainFile,
|
||
Database: database,
|
||
Index: config.LibraryProductionIndex,
|
||
Bucket: config.LibraryProductionS3Bucket,
|
||
DatabaseURL: config.LibraryProductionDatabaseURL,
|
||
RedisAddr: config.LibraryTargetRedisAddr,
|
||
RedisPassword: config.LibraryTargetRedisPassword,
|
||
RedisDB: config.LibraryTargetRedisDB,
|
||
})
|
||
}
|
||
|
||
// scopeConfig 是"某个 scope 的生产整库目标"的部署注入配置。
|
||
type scopeConfig struct {
|
||
target v071.ProductionScope
|
||
s3 v071.S3ObjectStoreConfig
|
||
}
|
||
|
||
func scopeConfigOf(config appconfig.Config, scope string) (scopeConfig, error) {
|
||
missing := make([]string, 0, 5)
|
||
if config.LibraryProductionDatabaseURL == "" {
|
||
missing = append(missing, "LIBRARY_PRODUCTION_DATABASE_URL")
|
||
}
|
||
if config.LibraryProductionElasticURL == "" {
|
||
missing = append(missing, "LIBRARY_PRODUCTION_ELASTIC_URL")
|
||
}
|
||
if config.LibraryProductionTenantID == 0 {
|
||
missing = append(missing, "LIBRARY_PRODUCTION_TENANT_ID")
|
||
}
|
||
kb := config.LibraryProductionInternalKB
|
||
if scope == "external" {
|
||
kb = config.LibraryProductionExternalKB
|
||
}
|
||
if kb == "" {
|
||
missing = append(missing, "LIBRARY_PRODUCTION_KB_"+strings.ToUpper(scope))
|
||
}
|
||
if config.LibraryProductionS3Endpoint == "" || config.LibraryProductionS3Bucket == "" ||
|
||
config.LibraryProductionS3AccessKey == "" || config.LibraryProductionS3SecretKey == "" {
|
||
missing = append(missing, "LIBRARY_PRODUCTION_S3_ENDPOINT/BUCKET/ACCESS_KEY/SECRET_KEY")
|
||
}
|
||
if len(missing) > 0 {
|
||
return scopeConfig{}, fmt.Errorf("生产整库目标未配置：缺少 %s", strings.Join(missing, "、"))
|
||
}
|
||
return scopeConfig{
|
||
target: v071.ProductionScope{TenantID: config.LibraryProductionTenantID, KnowledgeBaseID: kb},
|
||
s3: v071.S3ObjectStoreConfig{
|
||
Provider: config.LibraryProductionS3Provider, Endpoint: config.LibraryProductionS3Endpoint,
|
||
Region: config.LibraryProductionS3Region, Bucket: config.LibraryProductionS3Bucket,
|
||
AccessKeyID: config.LibraryProductionS3AccessKey, SecretAccessKey: config.LibraryProductionS3SecretKey,
|
||
Secure: strings.HasPrefix(config.LibraryProductionS3Endpoint, "https://"),
|
||
ForcePathStyle: true, MaxObjectBytes: 256 * 1024 * 1024,
|
||
},
|
||
}, nil
|
||
}
|
||
|
||
func newTarget(ctx context.Context, config appconfig.Config, scope scopeConfig) (*v071.ProductionLibraryTarget, error) {
|
||
objects, err := v071.NewS3ObjectStore(scope.s3)
|
||
if err != nil {
|
||
return nil, errors.New("生产对象存储配置无效")
|
||
}
|
||
return v071.NewProductionLibraryTarget(ctx, v071.ProductionLibraryConfig{
|
||
Postgres: config.LibraryProductionDatabaseURL, SearchURL: config.LibraryProductionElasticURL,
|
||
SearchIndex: config.LibraryProductionIndex, Objects: objects, ObjectsReader: objects,
|
||
ArchiveRoot: config.LibraryArchiveRoot,
|
||
ObjectsBucket: config.LibraryProductionS3Bucket,
|
||
MaxDocuments: 200000,
|
||
}, scope.target)
|
||
}
|
||
|
||
func newBackups(config appconfig.Config, scope string, target *v071.ProductionLibraryTarget,
|
||
contract librarysnapshot.Compatibility) (*v071.ProductionBackupStore, error) {
|
||
return v071.NewProductionBackupStore(v071.ProductionBackupConfig{
|
||
Pool: target.Pool(), Search: target.Search(), Objects: target.Objects(),
|
||
ArchiveRoot: config.LibraryArchiveRoot, Scope: scope, Identity: target.Identity(),
|
||
Compatibility: contract, MaxDocuments: 200000, MaxObjectByte: 256 * 1024 * 1024,
|
||
})
|
||
}
|
||
|
||
// OpenProductionStack 用部署注入的配置装配目标与独立预备份存储。
|
||
// 契约以"要同步的这份快照"为准：生产必须向它看齐，而不是反过来。
|
||
func OpenProductionStack(ctx context.Context, config appconfig.Config, scope, snapshotID, knownMapping string) (
|
||
*v071.ProductionLibraryTarget, *v071.ProductionBackupStore, error) {
|
||
scopeCfg, err := scopeConfigOf(config, scope)
|
||
if err != nil {
|
||
return nil, nil, err
|
||
}
|
||
target, err := newTarget(ctx, config, scopeCfg)
|
||
if err != nil {
|
||
return nil, nil, err
|
||
}
|
||
archive, err := librarysnapshot.Open(config.LibraryArchiveRoot, 256*1024*1024)
|
||
if err != nil {
|
||
target.Close()
|
||
return nil, nil, fmt.Errorf("归档不可用，无法读取快照契约: %w", err)
|
||
}
|
||
manifest, err := archive.Load(snapshotID)
|
||
archive.Close()
|
||
if err != nil {
|
||
target.Close()
|
||
return nil, nil, fmt.Errorf("读取目标快照失败: %w", err)
|
||
}
|
||
contract := manifest.Compatibility
|
||
if contract.EmbeddingDimension == 0 {
|
||
contract.EmbeddingModel = config.LibraryEmbeddingModel
|
||
contract.EmbeddingDimension = config.LibraryEmbeddingDimension
|
||
}
|
||
// 生产索引存在时必须与快照契约一致；不存在或为空基线时跳过比对
|
||
// （空索引的映射没有可比性，非空之后仍然严格比对）。
|
||
if mapping, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil {
|
||
// 映射基线：优先用"上一次生产生效版本登记的实际映射"，
|
||
// 它才是这份生产索引的既有事实；快照契约只管"来自哪份数据"。
|
||
baseline := knownMapping
|
||
if baseline == "" {
|
||
baseline = mapping
|
||
}
|
||
if baseline != "" && mapping != baseline {
|
||
current, currentErr := target.Current(ctx)
|
||
emptyBaseline := currentErr == nil && current.Counts["index"] == 0 && current.Counts["chunks"] == 0
|
||
if !emptyBaseline {
|
||
target.Close()
|
||
return nil, nil, errors.New("生产索引映射与上次生效版本不一致（索引被改动过），先对齐索引映射")
|
||
}
|
||
}
|
||
}
|
||
backupContract := contract
|
||
if knownMapping != "" {
|
||
backupContract.IndexMappingSHA256 = knownMapping
|
||
}
|
||
backups, err := newBackups(config, scope, target, backupContract)
|
||
if err != nil {
|
||
target.Close()
|
||
return nil, nil, err
|
||
}
|
||
return target, backups, nil
|
||
}
|
||
|
||
// OpenBaselineStack 装配"给目标自己建基线"所需的目标与捕获器。
|
||
//
|
||
// 契约来源按"能验证的优先级"取，绝不去借别人的快照：
|
||
// - WeKnora 版本 / 检索引擎：受支持栈的常量（v0.7.1 / elasticsearch_v8），
|
||
// 目标镜像与 ES 实测同属这一代；
|
||
// - schema 版本：**读目标数据库**的 schema_migrations，必须等于受支持版本且不 dirty；
|
||
// - embedding 模型与维度：优先读目标库的模型绑定，读不到（这台测试目标的
|
||
// models 表是空的、KB 里的模型引用悬空）才回落到部署声明，并在错误信息里说清楚；
|
||
// - 索引映射：目标此刻的真实映射。
|
||
func OpenBaselineStack(ctx context.Context, config appconfig.Config, scope string) (
|
||
*v071.ProductionLibraryTarget, *v071.ProductionBackupStore, error) {
|
||
scopeCfg, err := scopeConfigOf(config, scope)
|
||
if err != nil {
|
||
return nil, nil, err
|
||
}
|
||
target, err := newTarget(ctx, config, scopeCfg)
|
||
if err != nil {
|
||
return nil, nil, err
|
||
}
|
||
mapping, mapErr := target.Search().MappingSHA256(ctx)
|
||
if mapErr != nil {
|
||
target.Close()
|
||
return nil, nil, fmt.Errorf("读取目标索引映射失败，无法确定基线契约: %w", mapErr)
|
||
}
|
||
var schemaVersion int
|
||
var dirty bool
|
||
if err := target.Pool().QueryRow(ctx,
|
||
`SELECT version, dirty FROM schema_migrations ORDER BY version DESC LIMIT 1`).Scan(&schemaVersion, &dirty); err != nil {
|
||
target.Close()
|
||
return nil, nil, fmt.Errorf("读取目标 WeKnora schema 版本失败: %w", err)
|
||
}
|
||
if dirty || schemaVersion != releasebundle.WeKnoraMigrationV071 {
|
||
target.Close()
|
||
return nil, nil, fmt.Errorf("目标 schema 版本 %d（dirty=%v）与受支持的 %d 不一致，拒绝建基线",
|
||
schemaVersion, dirty, releasebundle.WeKnoraMigrationV071)
|
||
}
|
||
embeddingModel := config.LibraryEmbeddingModel
|
||
embeddingDimension := config.LibraryEmbeddingDimension
|
||
var dbModel string
|
||
var dbDimension int
|
||
err = target.Pool().QueryRow(ctx, `
|
||
SELECT model.name, COALESCE(NULLIF(model.parameters->'embedding_parameters'->>'dimension', '')::int, 0)
|
||
FROM knowledge_bases AS kb
|
||
JOIN models AS model ON model.id = kb.embedding_model_id AND model.deleted_at IS NULL
|
||
WHERE kb.id = $1 AND kb.deleted_at IS NULL`, scopeCfg.target.KnowledgeBaseID).Scan(&dbModel, &dbDimension)
|
||
switch {
|
||
case err == nil && dbModel != "" && dbDimension > 0:
|
||
// 目标库自己说得清楚时以它为准；与部署声明不一致就不建（否则基线记录的是另一套栈）。
|
||
if config.LibraryEmbeddingDimension != 0 && dbDimension != config.LibraryEmbeddingDimension {
|
||
target.Close()
|
||
return nil, nil, fmt.Errorf("目标库声明的 embedding 维度 %d 与部署声明 %d 不一致，拒绝建基线",
|
||
dbDimension, config.LibraryEmbeddingDimension)
|
||
}
|
||
embeddingModel, embeddingDimension = dbModel, dbDimension
|
||
case errors.Is(err, pgx.ErrNoRows):
|
||
// 这台目标没有模型绑定（models 表为空 / KB 引用悬空）：用部署声明的值，
|
||
// 并在基线的阶段说明里标注来源，别把它说成"从目标读到的"。
|
||
if embeddingModel == "" || embeddingDimension <= 0 {
|
||
target.Close()
|
||
return nil, nil, errors.New("目标库没有模型绑定，且部署没有声明 LIBRARY_EMBEDDING_MODEL/DIMENSION")
|
||
}
|
||
default:
|
||
target.Close()
|
||
return nil, nil, fmt.Errorf("读取目标 embedding 模型绑定失败: %w", err)
|
||
}
|
||
contract := librarysnapshot.Compatibility{
|
||
WeKnoraVersion: releasebundle.WeKnoraVersionV071,
|
||
SchemaVersion: schemaVersion,
|
||
RetrieverEngine: "elasticsearch_v8",
|
||
EmbeddingModel: embeddingModel,
|
||
EmbeddingDimension: embeddingDimension,
|
||
IndexMappingSHA256: mapping,
|
||
}
|
||
backups, err := newBackups(config, scope, target, contract)
|
||
if err != nil {
|
||
target.Close()
|
||
return nil, nil, err
|
||
}
|
||
return target, backups, nil
|
||
}
|