姚章浩
98294b9583
build / build (push) Failing after 24s
why：审计 19 项（12 P1 / 7 P2）来自同一批根因，逐点试错会把协议撕成十几份：
· 意图与执行混为一谈（F01/F15）：`library.sync:<scope>:<快照>` 是终身幂等键，
A→B→A 的第三次发布拿到第一次的历史成功作业，什么都不做；
· 版本行既是实体又是事件（F02/F09）：主键只由 (kind,scope,快照) 决定、写入是
upsert，第二个作业覆盖 `source_job_id`/`evidence`/`created_at`；
· 接管没有代次（F13）：两个 worker 都能认领同一 pending 作业，失租的旧执行还能
写业务状态、释放别人的维护锁；
· 早退与终态没有统一政策（F03/F04/F05/F06）：半写窗口的前置检查失败会重新开门、
服务起不来也照样开入口、journal 终态后收尾不可重建、恢复成功不回写 current；
· 数据身份与准入各写一套（F07/F08/F14/F17）：接管丢了复用语义、全库搬运不改
location_hash、通用入口可造 library.* 作业、目标模型从没被问过；
· 存储/容量/UI/CI 判据各自为政（F10/F11/F12/F16/F18/F19）。
what：按批准的一套协议集中实施（详见 handoffs/release-consolidated-implementation/
current-contract.md）：
· operation（一次明确意图）/ attempt（epoch 代次）/ 不可变版本事件 / current 指针；
治理库迁移 76，只动治理库，不碰 WeKnora Schema/源码/镜像；
· 所有业务状态、阶段、版本事件、维护锁写入带 epoch 守卫（失租代次写不进去）；
心跳失租立刻取消 handler；
· 半写窗口早退保持关闭；服务未全部健康绝不开入口；journal 终态可从持久事实重建
收尾；恢复成功登记 recovery 事件并推进 current；
· worker 在动维护前核对业务行/kind/scope/队列关联；通用入口拒绝 library.*；
· 全库搬运重算 location_hash 并与文档级导入共用实现；接管消费写集合的复用决定；
发布前核目标实时模型契约、容量门禁前置；
· 存储保护闭包改为全量依赖、类型只认显式登记、保护未知则不出可删候选；
· 页面核 HTTP 状态与结构、恢复纳入轮询、Wiki 差异覆盖标题/摘要/父级。
how：`make verify` rc=0（16 项必需检查全绿，含 go test 全量与前端 485 例）；
`go vet ./...` rc=0；`npx tsc -p tsconfig.json --noEmit` rc=0；逐项负例见各包测试
（F03/F04/F08/F12/F13/F14/F16/F18/F19 都有先红后绿记录）。
未执行：治理库迁移到任何已有库、真实 PG/ES/S3/WeKnora 集成、故障注入、部署。
263 lines
11 KiB
Go
263 lines
11 KiB
Go
// Package librarystack 是"目标整库读写栈"的唯一装配点：同步、恢复、基线，
|
||
// 以及主机侧的真实恢复演练都从这里拿同一个目标/预备份存储/维护协调器。
|
||
//
|
||
// 为什么单独成包：装配一旦被复制两份，"目标是谁、用哪份契约"就会随调用路径漂移。
|
||
// 这个包只做装配与配置校验，不包含任何业务编排。
|
||
package librarystack
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"fmt"
|
||
"strings"
|
||
|
||
"github.com/jackc/pgx/v5"
|
||
|
||
"knowledge-governance/internal/appconfig"
|
||
"knowledge-governance/internal/librarymaintenance"
|
||
"knowledge-governance/internal/librarysnapshot"
|
||
v071 "knowledge-governance/internal/releaseadapter/v071"
|
||
"knowledge-governance/internal/releasebundle"
|
||
)
|
||
|
||
// OpenMaintenance 按部署配置装配实例级维护目标；未接线时返回明确原因。
|
||
func OpenMaintenance(config appconfig.Config) (*librarymaintenance.Manager, error) {
|
||
database, err := librarymaintenance.DatabaseIdentity(config.LibraryProductionDatabaseURL)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
return librarymaintenance.New(librarymaintenance.Config{
|
||
Instance: config.LibraryTargetInstance,
|
||
Project: config.LibraryTargetProject,
|
||
Writers: config.LibraryTargetWriters,
|
||
NonWriters: config.LibraryTargetNonWriters,
|
||
DockerSocket: config.LibraryTargetDockerSocket,
|
||
IngressURL: config.LibraryTargetIngressURL,
|
||
IngressService: config.LibraryTargetIngressService,
|
||
IngressDrainFile: config.LibraryTargetIngressDrainFile,
|
||
Database: database,
|
||
Index: config.LibraryProductionIndex,
|
||
Bucket: config.LibraryProductionS3Bucket,
|
||
DatabaseURL: config.LibraryProductionDatabaseURL,
|
||
RedisAddr: config.LibraryTargetRedisAddr,
|
||
RedisPassword: config.LibraryTargetRedisPassword,
|
||
RedisDB: config.LibraryTargetRedisDB,
|
||
})
|
||
}
|
||
|
||
// scopeConfig 是"某个 scope 的生产整库目标"的部署注入配置。
|
||
type scopeConfig struct {
|
||
target v071.ProductionScope
|
||
s3 v071.S3ObjectStoreConfig
|
||
}
|
||
|
||
func scopeConfigOf(config appconfig.Config, scope string) (scopeConfig, error) {
|
||
missing := make([]string, 0, 5)
|
||
if config.LibraryProductionDatabaseURL == "" {
|
||
missing = append(missing, "LIBRARY_PRODUCTION_DATABASE_URL")
|
||
}
|
||
if config.LibraryProductionElasticURL == "" {
|
||
missing = append(missing, "LIBRARY_PRODUCTION_ELASTIC_URL")
|
||
}
|
||
if config.LibraryProductionTenantID == 0 {
|
||
missing = append(missing, "LIBRARY_PRODUCTION_TENANT_ID")
|
||
}
|
||
kb := config.LibraryProductionInternalKB
|
||
if scope == "external" {
|
||
kb = config.LibraryProductionExternalKB
|
||
}
|
||
if kb == "" {
|
||
missing = append(missing, "LIBRARY_PRODUCTION_KB_"+strings.ToUpper(scope))
|
||
}
|
||
if config.LibraryProductionS3Endpoint == "" || config.LibraryProductionS3Bucket == "" ||
|
||
config.LibraryProductionS3AccessKey == "" || config.LibraryProductionS3SecretKey == "" {
|
||
missing = append(missing, "LIBRARY_PRODUCTION_S3_ENDPOINT/BUCKET/ACCESS_KEY/SECRET_KEY")
|
||
}
|
||
if len(missing) > 0 {
|
||
return scopeConfig{}, fmt.Errorf("生产整库目标未配置：缺少 %s", strings.Join(missing, "、"))
|
||
}
|
||
return scopeConfig{
|
||
target: v071.ProductionScope{TenantID: config.LibraryProductionTenantID, KnowledgeBaseID: kb},
|
||
s3: v071.S3ObjectStoreConfig{
|
||
Provider: config.LibraryProductionS3Provider, Endpoint: config.LibraryProductionS3Endpoint,
|
||
Region: config.LibraryProductionS3Region, Bucket: config.LibraryProductionS3Bucket,
|
||
AccessKeyID: config.LibraryProductionS3AccessKey, SecretAccessKey: config.LibraryProductionS3SecretKey,
|
||
Secure: strings.HasPrefix(config.LibraryProductionS3Endpoint, "https://"),
|
||
ForcePathStyle: true, MaxObjectBytes: 256 * 1024 * 1024,
|
||
},
|
||
}, nil
|
||
}
|
||
|
||
func newTarget(ctx context.Context, config appconfig.Config, scope scopeConfig) (*v071.ProductionLibraryTarget, error) {
|
||
objects, err := v071.NewS3ObjectStore(scope.s3)
|
||
if err != nil {
|
||
return nil, errors.New("生产对象存储配置无效")
|
||
}
|
||
return v071.NewProductionLibraryTarget(ctx, v071.ProductionLibraryConfig{
|
||
Postgres: config.LibraryProductionDatabaseURL, SearchURL: config.LibraryProductionElasticURL,
|
||
SearchIndex: config.LibraryProductionIndex, Objects: objects, ObjectsReader: objects,
|
||
ArchiveRoot: config.LibraryArchiveRoot,
|
||
ObjectsBucket: config.LibraryProductionS3Bucket,
|
||
MaxDocuments: 200000,
|
||
}, scope.target)
|
||
}
|
||
|
||
func newBackups(config appconfig.Config, scope string, target *v071.ProductionLibraryTarget,
|
||
contract librarysnapshot.Compatibility) (*v071.ProductionBackupStore, error) {
|
||
return v071.NewProductionBackupStore(v071.ProductionBackupConfig{
|
||
Pool: target.Pool(), Search: target.Search(), Objects: target.Objects(),
|
||
ArchiveRoot: config.LibraryArchiveRoot, Scope: scope, Identity: target.Identity(),
|
||
Compatibility: contract, MaxDocuments: 200000, MaxObjectByte: 256 * 1024 * 1024,
|
||
})
|
||
}
|
||
|
||
// OpenProductionStack 用部署注入的配置装配目标与独立预备份存储。
|
||
// 契约以"要同步的这份快照"为准：生产必须向它看齐，而不是反过来。
|
||
func OpenProductionStack(ctx context.Context, config appconfig.Config, scope, snapshotID, knownMapping string) (
|
||
*v071.ProductionLibraryTarget, *v071.ProductionBackupStore, error) {
|
||
scopeCfg, err := scopeConfigOf(config, scope)
|
||
if err != nil {
|
||
return nil, nil, err
|
||
}
|
||
target, err := newTarget(ctx, config, scopeCfg)
|
||
if err != nil {
|
||
return nil, nil, err
|
||
}
|
||
archive, err := librarysnapshot.Open(config.LibraryArchiveRoot, 256*1024*1024)
|
||
if err != nil {
|
||
target.Close()
|
||
return nil, nil, fmt.Errorf("归档不可用，无法读取快照契约: %w", err)
|
||
}
|
||
manifest, err := archive.Load(snapshotID)
|
||
archive.Close()
|
||
if err != nil {
|
||
target.Close()
|
||
return nil, nil, fmt.Errorf("读取目标快照失败: %w", err)
|
||
}
|
||
contract := manifest.Compatibility
|
||
if contract.EmbeddingDimension == 0 {
|
||
contract.EmbeddingModel = config.LibraryEmbeddingModel
|
||
contract.EmbeddingDimension = config.LibraryEmbeddingDimension
|
||
}
|
||
// 目标实时契约：来源快照的声明只说明"数据来自哪"，不能当目标合同（F17）。
|
||
// 读不到就不许发布——猜一个目标模型正是"知识行声明 B、向量却是 A"的来源。
|
||
live, liveErr := target.LiveTargetContract(ctx)
|
||
if liveErr != nil {
|
||
target.Close()
|
||
return nil, nil, fmt.Errorf("读不到目标实时模型契约，拒绝发布：%w", liveErr)
|
||
}
|
||
if verifyErr := v071.VerifySourceAgainstTarget(contract, live); verifyErr != nil {
|
||
target.Close()
|
||
return nil, nil, verifyErr
|
||
}
|
||
// 生产索引存在时必须与快照契约一致；不存在或为空基线时跳过比对
|
||
// （空索引的映射没有可比性，非空之后仍然严格比对）。
|
||
if mapping, mapErr := target.Search().MappingSHA256(ctx); mapErr == nil {
|
||
// 映射基线：优先用"上一次生产生效版本登记的实际映射"，
|
||
// 它才是这份生产索引的既有事实；快照契约只管"来自哪份数据"。
|
||
baseline := knownMapping
|
||
if baseline == "" {
|
||
baseline = mapping
|
||
}
|
||
if baseline != "" && mapping != baseline {
|
||
current, currentErr := target.Current(ctx)
|
||
emptyBaseline := currentErr == nil && current.Counts["index"] == 0 && current.Counts["chunks"] == 0
|
||
if !emptyBaseline {
|
||
target.Close()
|
||
return nil, nil, errors.New("生产索引映射与上次生效版本不一致（索引被改动过），先对齐索引映射")
|
||
}
|
||
}
|
||
}
|
||
backupContract := v071.ContractFromLive(contract, live)
|
||
if knownMapping != "" {
|
||
backupContract.IndexMappingSHA256 = knownMapping
|
||
}
|
||
backups, err := newBackups(config, scope, target, backupContract)
|
||
if err != nil {
|
||
target.Close()
|
||
return nil, nil, err
|
||
}
|
||
return target, backups, nil
|
||
}
|
||
|
||
// OpenBaselineStack 装配"给目标自己建基线"所需的目标与捕获器。
|
||
//
|
||
// 契约来源按"能验证的优先级"取，绝不去借别人的快照：
|
||
// - WeKnora 版本 / 检索引擎：受支持栈的常量（v0.7.1 / elasticsearch_v8），
|
||
// 目标镜像与 ES 实测同属这一代；
|
||
// - schema 版本：**读目标数据库**的 schema_migrations，必须等于受支持版本且不 dirty；
|
||
// - embedding 模型与维度：优先读目标库的模型绑定，读不到（这台测试目标的
|
||
// models 表是空的、KB 里的模型引用悬空）才回落到部署声明，并在错误信息里说清楚；
|
||
// - 索引映射：目标此刻的真实映射。
|
||
func OpenBaselineStack(ctx context.Context, config appconfig.Config, scope string) (
|
||
*v071.ProductionLibraryTarget, *v071.ProductionBackupStore, error) {
|
||
scopeCfg, err := scopeConfigOf(config, scope)
|
||
if err != nil {
|
||
return nil, nil, err
|
||
}
|
||
target, err := newTarget(ctx, config, scopeCfg)
|
||
if err != nil {
|
||
return nil, nil, err
|
||
}
|
||
mapping, mapErr := target.Search().MappingSHA256(ctx)
|
||
if mapErr != nil {
|
||
target.Close()
|
||
return nil, nil, fmt.Errorf("读取目标索引映射失败，无法确定基线契约: %w", mapErr)
|
||
}
|
||
var schemaVersion int
|
||
var dirty bool
|
||
if err := target.Pool().QueryRow(ctx,
|
||
`SELECT version, dirty FROM schema_migrations ORDER BY version DESC LIMIT 1`).Scan(&schemaVersion, &dirty); err != nil {
|
||
target.Close()
|
||
return nil, nil, fmt.Errorf("读取目标 WeKnora schema 版本失败: %w", err)
|
||
}
|
||
if dirty || schemaVersion != releasebundle.WeKnoraMigrationV071 {
|
||
target.Close()
|
||
return nil, nil, fmt.Errorf("目标 schema 版本 %d（dirty=%v）与受支持的 %d 不一致，拒绝建基线",
|
||
schemaVersion, dirty, releasebundle.WeKnoraMigrationV071)
|
||
}
|
||
embeddingModel := config.LibraryEmbeddingModel
|
||
embeddingDimension := config.LibraryEmbeddingDimension
|
||
var dbModel string
|
||
var dbDimension int
|
||
err = target.Pool().QueryRow(ctx, `
|
||
SELECT model.name, COALESCE(NULLIF(model.parameters->'embedding_parameters'->>'dimension', '')::int, 0)
|
||
FROM knowledge_bases AS kb
|
||
JOIN models AS model ON model.id = kb.embedding_model_id AND model.deleted_at IS NULL
|
||
WHERE kb.id = $1 AND kb.deleted_at IS NULL`, scopeCfg.target.KnowledgeBaseID).Scan(&dbModel, &dbDimension)
|
||
switch {
|
||
case err == nil && dbModel != "" && dbDimension > 0:
|
||
// 目标库自己说得清楚时以它为准；与部署声明不一致就不建（否则基线记录的是另一套栈）。
|
||
if config.LibraryEmbeddingDimension != 0 && dbDimension != config.LibraryEmbeddingDimension {
|
||
target.Close()
|
||
return nil, nil, fmt.Errorf("目标库声明的 embedding 维度 %d 与部署声明 %d 不一致，拒绝建基线",
|
||
dbDimension, config.LibraryEmbeddingDimension)
|
||
}
|
||
embeddingModel, embeddingDimension = dbModel, dbDimension
|
||
case errors.Is(err, pgx.ErrNoRows):
|
||
// 这台目标没有模型绑定（models 表为空 / KB 引用悬空）：用部署声明的值，
|
||
// 并在基线的阶段说明里标注来源，别把它说成"从目标读到的"。
|
||
if embeddingModel == "" || embeddingDimension <= 0 {
|
||
target.Close()
|
||
return nil, nil, errors.New("目标库没有模型绑定，且部署没有声明 LIBRARY_EMBEDDING_MODEL/DIMENSION")
|
||
}
|
||
default:
|
||
target.Close()
|
||
return nil, nil, fmt.Errorf("读取目标 embedding 模型绑定失败: %w", err)
|
||
}
|
||
contract := librarysnapshot.Compatibility{
|
||
WeKnoraVersion: releasebundle.WeKnoraVersionV071,
|
||
SchemaVersion: schemaVersion,
|
||
RetrieverEngine: "elasticsearch_v8",
|
||
EmbeddingModel: embeddingModel,
|
||
EmbeddingDimension: embeddingDimension,
|
||
IndexMappingSHA256: mapping,
|
||
}
|
||
backups, err := newBackups(config, scope, target, contract)
|
||
if err != nil {
|
||
target.Close()
|
||
return nil, nil, err
|
||
}
|
||
return target, backups, nil
|
||
}
|