feat(release): 发布前快照硬门禁 + 云内快照 + ES/OSS 保护
按生产级备份组合落地:
- internal/releasesnapshot:新增门禁包,要求 root 下存在覆盖目标库、未过期
(默认 24h)、含 wiki_pages.jsonl 与 sha256 的快照;本地文件再校验哈希。
7 条单测(过期/换库/篡改/缺页/未接线/半成品)。
- 发布器:写目标库之前(Publish/Delete/Rollback 三条路)先过门禁;映射目标
(源库→目标库,共享真实库)或 policy=always 时必须有快照;门禁未接线
同样拒绝(fail-closed)。新错误码 release_target_snapshot_missing(不可重试)
+ 页面中文提示。6 条发布器单测。
- worker 配置新增 target_snapshot{root,max_age_hours,policy},校验 policy/
max_age/绝对路径;compose.release-sim.yaml 只读挂载快照目录。
- scripts/target_kb_snapshot.sh:默认云内执行(与 DB 同 VPC,凭据取自 WeKnora
容器)+ 默认不导 chunks(44s 出 8 张表);修复 \copy 不能跨行/不能带 WITH、
ES 计数改用 knowledge_base_id.keyword。
- ES:注册 kb_backup 仓库(path.repo),首个快照 36s/4.7GB/0 分片失败,
SLM kb-hourly(3d)+kb-daily(7d) 已建并执行成功。
- OSS:目标桶开启版本控制(此前未开启)。
- 文档 P6 增补硬门禁与云内执行;真实快照目录上门禁实测:两个绑定库通过、
未覆盖库拒绝、未接线拒绝。
This commit is contained in:
@@ -15,8 +15,19 @@ import (
|
||||
const maxReleaseConfigBytes = 1024 * 1024
|
||||
|
||||
type releaseRuntimeConfig struct {
|
||||
ArchiveRoot string `json:"archive_root"`
|
||||
Routes []releaseRouteConfig `json:"routes"`
|
||||
ArchiveRoot string `json:"archive_root"`
|
||||
// TargetSnapshot 是"写目标库前必须留档"的门禁配置。
|
||||
TargetSnapshot releaseTargetSnapshotConfig `json:"target_snapshot"`
|
||||
Routes []releaseRouteConfig `json:"routes"`
|
||||
}
|
||||
|
||||
// releaseTargetSnapshotConfig 指向快照根目录(scripts/target_kb_snapshot.sh 的输出
|
||||
// 目录),并声明门禁对哪些项生效。
|
||||
type releaseTargetSnapshotConfig struct {
|
||||
Root string `json:"root"`
|
||||
MaxAgeHours int `json:"max_age_hours"`
|
||||
// Policy:mapped(默认,只对 源库→目标库 映射的共享库强制)| always(一律强制)。
|
||||
Policy string `json:"policy"`
|
||||
}
|
||||
|
||||
type releaseRouteConfig struct {
|
||||
@@ -98,6 +109,9 @@ func validateReleaseRuntimeConfig(config releaseRuntimeConfig) error {
|
||||
if !filepath.IsAbs(config.ArchiveRoot) || len(config.Routes) == 0 {
|
||||
return errors.New("release archive root and routes are required")
|
||||
}
|
||||
if err := validateTargetSnapshotConfig(config.TargetSnapshot); err != nil {
|
||||
return err
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, route := range config.Routes {
|
||||
if route.TenantID == 0 || !validConfigIdentifier(route.KnowledgeBaseID) || !validConfigIdentifier(route.EmbeddingModel) || seen[route.KnowledgeBaseID] {
|
||||
@@ -135,6 +149,27 @@ func validObjectEndpoint(object releaseObjectConfig) bool {
|
||||
|
||||
func absoluteFile(path string) bool { return filepath.IsAbs(path) && filepath.Clean(path) == path }
|
||||
|
||||
// validateTargetSnapshotConfig 校验快照门禁配置。留档目录缺省即视为"未接线",
|
||||
// 但一旦声明要留档(policy=always)就必须给出目录——否则是假门禁。
|
||||
func validateTargetSnapshotConfig(config releaseTargetSnapshotConfig) error {
|
||||
if config.Policy != "" && config.Policy != "mapped" && config.Policy != "always" {
|
||||
return errors.New("release target snapshot policy must be mapped or always")
|
||||
}
|
||||
if config.MaxAgeHours < 0 || config.MaxAgeHours > 720 {
|
||||
return errors.New("release target snapshot max_age_hours must be between 0 and 720")
|
||||
}
|
||||
if config.Root == "" {
|
||||
if config.Policy == "always" {
|
||||
return errors.New("release target snapshot root is required when policy is always")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !filepath.IsAbs(config.Root) || filepath.Clean(config.Root) != config.Root {
|
||||
return errors.New("release target snapshot root must be an absolute clean path")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validHTTPURL(value string) bool {
|
||||
parsed, err := url.Parse(value)
|
||||
return err == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") && parsed.Host != "" && parsed.User == nil && parsed.RawQuery == "" && parsed.Fragment == ""
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
v071 "knowledge-governance/internal/releaseadapter/v071"
|
||||
"knowledge-governance/internal/releasesnapshot"
|
||||
releaseworkflow "knowledge-governance/internal/workflows/release"
|
||||
)
|
||||
|
||||
@@ -31,7 +32,7 @@ func buildReleaseRuntime(ctx context.Context, config releaseRuntimeConfig) (*rel
|
||||
runtime := &releaseRuntime{}
|
||||
publishers := make(map[string]releaseworkflow.Publisher, len(config.Routes))
|
||||
for _, route := range config.Routes {
|
||||
publisher, pools, err := buildReleaseRoute(ctx, config.ArchiveRoot, route)
|
||||
publisher, pools, err := buildReleaseRoute(ctx, config.ArchiveRoot, config.TargetSnapshot, route)
|
||||
if err != nil {
|
||||
runtime.Close()
|
||||
return nil, err
|
||||
@@ -48,7 +49,7 @@ func buildReleaseRuntime(ctx context.Context, config releaseRuntimeConfig) (*rel
|
||||
return runtime, nil
|
||||
}
|
||||
|
||||
func buildReleaseRoute(ctx context.Context, archiveRoot string, route releaseRouteConfig) (releaseworkflow.Publisher, []*pgxpool.Pool, error) {
|
||||
func buildReleaseRoute(ctx context.Context, archiveRoot string, snapshot releaseTargetSnapshotConfig, route releaseRouteConfig) (releaseworkflow.Publisher, []*pgxpool.Pool, error) {
|
||||
sourceDatabaseURL, err := readRestrictedValue(route.Source.DatabaseURLFile)
|
||||
if err != nil {
|
||||
return nil, nil, errors.New("read source database credential")
|
||||
@@ -158,7 +159,21 @@ func buildReleaseRoute(ctx context.Context, archiveRoot string, route releaseRou
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create v0.7.1 document target"))
|
||||
}
|
||||
publisher, err := releaseworkflow.NewSidecarPublisher(documentExporter, documentTarget)
|
||||
var publisherOptions []releaseworkflow.PublisherOption
|
||||
if snapshot.Root != "" {
|
||||
// 写目标库前必须有覆盖该库的、未过期的快照留档(见 releasesnapshot.Guard)。
|
||||
maxAge := time.Duration(snapshot.MaxAgeHours) * time.Hour
|
||||
if snapshot.MaxAgeHours == 0 {
|
||||
maxAge = 24 * time.Hour
|
||||
}
|
||||
policy := releaseworkflow.SnapshotPolicyMapped
|
||||
if snapshot.Policy == "always" {
|
||||
policy = releaseworkflow.SnapshotPolicyAlways
|
||||
}
|
||||
publisherOptions = append(publisherOptions, releaseworkflow.WithTargetSnapshots(
|
||||
releasesnapshot.Guard{Root: snapshot.Root, MaxAge: maxAge, VerifyLocalFiles: true}, policy))
|
||||
}
|
||||
publisher, err := releaseworkflow.NewSidecarPublisher(documentExporter, documentTarget, publisherOptions...)
|
||||
if err != nil {
|
||||
return cleanup(errors.New("create v0.7.1 release publisher"))
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user