281699a431
build / build (push) Successful in 1m21s
为什么:生产目标此前是"无条件失败"的占位,编排再完整也不能落地;本轮把它做成真实实现, 并在本机 WeKnora 测试栈的真实 PG/ES 上跑通导入、撤销、预备份与故障恢复。 - ProductionLibraryTarget:Current 读行数+身份摘要;Apply 按阶段真实写入(database/wiki 同事务先删本库再插、resources 传对象、index 搬运原始文档与向量);Verify 比对身份摘要; Restore 按预备份快照回放。 - 撤销真正生效:索引阶段写入后 PruneScope 删除快照里没有的文档。 - ProductionBackupStore:导出前独立完整预备份,导出前后行数不一致即失败。 - worker 用注入配置装配目标与预备份;缺项把变量名写进阶段并失败,不再有占位对象。 - capacity 安全加法防溢出;保留策略组合语义明确(各维度都超限才候选)。 - 新增本机真实服务集成测试(一次性 kg_library_test_* 库 + 随机 ES 索引), 覆盖导入/校验/撤销/预备份/故障注入/恢复。
85 lines
3.5 KiB
Go
85 lines
3.5 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"knowledge-governance/internal/appconfig"
|
|
"knowledge-governance/internal/librarysnapshot"
|
|
v071 "knowledge-governance/internal/releaseadapter/v071"
|
|
)
|
|
|
|
// buildLibraryProductionStack 用部署注入的配置装配生产整库目标与独立预备份存储。
|
|
// 任何一项缺失都返回明确原因,调用方据此把作业失败并把缺项写进阶段,
|
|
// 不用"未配置"的占位对象假装能跑。
|
|
func buildLibraryProductionStack(ctx context.Context, config appconfig.Config, scope string) (
|
|
*v071.ProductionLibraryTarget, *v071.ProductionBackupStore, error) {
|
|
missing := make([]string, 0, 4)
|
|
if config.LibraryProductionDatabaseURL == "" {
|
|
missing = append(missing, "LIBRARY_PRODUCTION_DATABASE_URL")
|
|
}
|
|
if config.LibraryProductionElasticURL == "" {
|
|
missing = append(missing, "LIBRARY_PRODUCTION_ELASTIC_URL")
|
|
}
|
|
if config.LibraryProductionTenantID == 0 {
|
|
missing = append(missing, "LIBRARY_PRODUCTION_TENANT_ID")
|
|
}
|
|
kb := config.LibraryProductionInternalKB
|
|
if scope == "external" {
|
|
kb = config.LibraryProductionExternalKB
|
|
}
|
|
if kb == "" {
|
|
missing = append(missing, "LIBRARY_PRODUCTION_KB_"+strings.ToUpper(scope))
|
|
}
|
|
if config.LibraryProductionS3Endpoint == "" || config.LibraryProductionS3Bucket == "" ||
|
|
config.LibraryProductionS3AccessKey == "" || config.LibraryProductionS3SecretKey == "" {
|
|
missing = append(missing, "LIBRARY_PRODUCTION_S3_ENDPOINT/BUCKET/ACCESS_KEY/SECRET_KEY")
|
|
}
|
|
if len(missing) > 0 {
|
|
return nil, nil, fmt.Errorf("生产整库目标未配置:缺少 %s", strings.Join(missing, "、"))
|
|
}
|
|
secure := strings.HasPrefix(config.LibraryProductionS3Endpoint, "https://")
|
|
objects, err := v071.NewS3ObjectStore(v071.S3ObjectStoreConfig{
|
|
Provider: "s3", Endpoint: config.LibraryProductionS3Endpoint,
|
|
Region: config.LibraryProductionS3Region, Bucket: config.LibraryProductionS3Bucket,
|
|
AccessKeyID: config.LibraryProductionS3AccessKey, SecretAccessKey: config.LibraryProductionS3SecretKey,
|
|
Secure: secure, ForcePathStyle: true, MaxObjectBytes: 256 * 1024 * 1024,
|
|
})
|
|
if err != nil {
|
|
return nil, nil, errors.New("生产对象存储配置无效")
|
|
}
|
|
identity := v071.ProductionScope{TenantID: config.LibraryProductionTenantID, KnowledgeBaseID: kb}
|
|
target, err := v071.NewProductionLibraryTarget(ctx, v071.ProductionLibraryConfig{
|
|
Postgres: config.LibraryProductionDatabaseURL, SearchURL: config.LibraryProductionElasticURL,
|
|
SearchIndex: config.LibraryProductionIndex, Objects: objects, ObjectsReader: objects,
|
|
ArchiveRoot: config.LibraryArchiveRoot,
|
|
MaxDocuments: 200000,
|
|
}, identity)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
// 契约以生产**当前**索引映射为准:这样预备份的兼容性与恢复目标是同一份事实。
|
|
mapping, err := target.Search().MappingSHA256(ctx)
|
|
if err != nil {
|
|
target.Close()
|
|
return nil, nil, fmt.Errorf("读取生产索引映射失败: %w", err)
|
|
}
|
|
contract := librarysnapshot.Compatibility{
|
|
WeKnoraVersion: "v0.7.1", SchemaVersion: 74, RetrieverEngine: "elasticsearch_v8",
|
|
EmbeddingModel: config.LibraryEmbeddingModel, EmbeddingDimension: config.LibraryEmbeddingDimension,
|
|
IndexMappingSHA256: mapping,
|
|
}
|
|
backups, err := v071.NewProductionBackupStore(v071.ProductionBackupConfig{
|
|
Pool: target.Pool(), Search: target.Search(), Objects: target.Objects(),
|
|
ArchiveRoot: config.LibraryArchiveRoot, Scope: scope, Identity: target.Identity(),
|
|
Compatibility: contract, MaxDocuments: 200000, MaxObjectByte: 256 * 1024 * 1024,
|
|
})
|
|
if err != nil {
|
|
target.Close()
|
|
return nil, nil, err
|
|
}
|
|
return target, backups, nil
|
|
}
|