feat(release): 发布前快照硬门禁 + 云内快照 + ES/OSS 保护

按生产级备份组合落地:

- internal/releasesnapshot:新增门禁包,要求 root 下存在覆盖目标库、未过期
  (默认 24h)、含 wiki_pages.jsonl 与 sha256 的快照;本地文件再校验哈希。
  7 条单测(过期/换库/篡改/缺页/未接线/半成品)。
- 发布器:写目标库之前(Publish/Delete/Rollback 三条路)先过门禁;映射目标
  (源库→目标库,共享真实库)或 policy=always 时必须有快照;门禁未接线
  同样拒绝(fail-closed)。新错误码 release_target_snapshot_missing(不可重试)
  + 页面中文提示。6 条发布器单测。
- worker 配置新增 target_snapshot{root,max_age_hours,policy},校验 policy/
  max_age/绝对路径;compose.release-sim.yaml 只读挂载快照目录。
- scripts/target_kb_snapshot.sh:默认云内执行(与 DB 同 VPC,凭据取自 WeKnora
  容器)+ 默认不导 chunks(44s 出 8 张表);修复 \copy 不能跨行/不能带 WITH、
  ES 计数改用 knowledge_base_id.keyword。
- ES:注册 kb_backup 仓库(path.repo),首个快照 36s/4.7GB/0 分片失败,
  SLM kb-hourly(3d)+kb-daily(7d) 已建并执行成功。
- OSS:目标桶开启版本控制(此前未开启)。
- 文档 P6 增补硬门禁与云内执行;真实快照目录上门禁实测:两个绑定库通过、
  未覆盖库拒绝、未接线拒绝。
This commit is contained in:
姚章浩
2026-10-09 10:59:55 +08:00
parent f4e1ecc5f1
commit 1fa35e9870
14 changed files with 836 additions and 70 deletions
+37 -2
View File
@@ -15,8 +15,19 @@ import (
const maxReleaseConfigBytes = 1024 * 1024
type releaseRuntimeConfig struct {
ArchiveRoot string `json:"archive_root"`
Routes []releaseRouteConfig `json:"routes"`
ArchiveRoot string `json:"archive_root"`
// TargetSnapshot 是"写目标库前必须留档"的门禁配置。
TargetSnapshot releaseTargetSnapshotConfig `json:"target_snapshot"`
Routes []releaseRouteConfig `json:"routes"`
}
// releaseTargetSnapshotConfig 指向快照根目录(scripts/target_kb_snapshot.sh 的输出
// 目录),并声明门禁对哪些项生效。
type releaseTargetSnapshotConfig struct {
Root string `json:"root"`
MaxAgeHours int `json:"max_age_hours"`
// Policy:mapped(默认,只对 源库→目标库 映射的共享库强制)| always(一律强制)。
Policy string `json:"policy"`
}
type releaseRouteConfig struct {
@@ -98,6 +109,9 @@ func validateReleaseRuntimeConfig(config releaseRuntimeConfig) error {
if !filepath.IsAbs(config.ArchiveRoot) || len(config.Routes) == 0 {
return errors.New("release archive root and routes are required")
}
if err := validateTargetSnapshotConfig(config.TargetSnapshot); err != nil {
return err
}
seen := map[string]bool{}
for _, route := range config.Routes {
if route.TenantID == 0 || !validConfigIdentifier(route.KnowledgeBaseID) || !validConfigIdentifier(route.EmbeddingModel) || seen[route.KnowledgeBaseID] {
@@ -135,6 +149,27 @@ func validObjectEndpoint(object releaseObjectConfig) bool {
func absoluteFile(path string) bool { return filepath.IsAbs(path) && filepath.Clean(path) == path }
// validateTargetSnapshotConfig 校验快照门禁配置。留档目录缺省即视为"未接线",
// 但一旦声明要留档(policy=always)就必须给出目录——否则是假门禁。
func validateTargetSnapshotConfig(config releaseTargetSnapshotConfig) error {
if config.Policy != "" && config.Policy != "mapped" && config.Policy != "always" {
return errors.New("release target snapshot policy must be mapped or always")
}
if config.MaxAgeHours < 0 || config.MaxAgeHours > 720 {
return errors.New("release target snapshot max_age_hours must be between 0 and 720")
}
if config.Root == "" {
if config.Policy == "always" {
return errors.New("release target snapshot root is required when policy is always")
}
return nil
}
if !filepath.IsAbs(config.Root) || filepath.Clean(config.Root) != config.Root {
return errors.New("release target snapshot root must be an absolute clean path")
}
return nil
}
func validHTTPURL(value string) bool {
parsed, err := url.Parse(value)
return err == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") && parsed.Host != "" && parsed.User == nil && parsed.RawQuery == "" && parsed.Fragment == ""
+18 -3
View File
@@ -13,6 +13,7 @@ import (
"knowledge-governance/internal/platform/weknora"
v071 "knowledge-governance/internal/releaseadapter/v071"
"knowledge-governance/internal/releasesnapshot"
releaseworkflow "knowledge-governance/internal/workflows/release"
)
@@ -31,7 +32,7 @@ func buildReleaseRuntime(ctx context.Context, config releaseRuntimeConfig) (*rel
runtime := &releaseRuntime{}
publishers := make(map[string]releaseworkflow.Publisher, len(config.Routes))
for _, route := range config.Routes {
publisher, pools, err := buildReleaseRoute(ctx, config.ArchiveRoot, route)
publisher, pools, err := buildReleaseRoute(ctx, config.ArchiveRoot, config.TargetSnapshot, route)
if err != nil {
runtime.Close()
return nil, err
@@ -48,7 +49,7 @@ func buildReleaseRuntime(ctx context.Context, config releaseRuntimeConfig) (*rel
return runtime, nil
}
func buildReleaseRoute(ctx context.Context, archiveRoot string, route releaseRouteConfig) (releaseworkflow.Publisher, []*pgxpool.Pool, error) {
func buildReleaseRoute(ctx context.Context, archiveRoot string, snapshot releaseTargetSnapshotConfig, route releaseRouteConfig) (releaseworkflow.Publisher, []*pgxpool.Pool, error) {
sourceDatabaseURL, err := readRestrictedValue(route.Source.DatabaseURLFile)
if err != nil {
return nil, nil, errors.New("read source database credential")
@@ -158,7 +159,21 @@ func buildReleaseRoute(ctx context.Context, archiveRoot string, route releaseRou
if err != nil {
return cleanup(errors.New("create v0.7.1 document target"))
}
publisher, err := releaseworkflow.NewSidecarPublisher(documentExporter, documentTarget)
var publisherOptions []releaseworkflow.PublisherOption
if snapshot.Root != "" {
// 写目标库前必须有覆盖该库的、未过期的快照留档(见 releasesnapshot.Guard)。
maxAge := time.Duration(snapshot.MaxAgeHours) * time.Hour
if snapshot.MaxAgeHours == 0 {
maxAge = 24 * time.Hour
}
policy := releaseworkflow.SnapshotPolicyMapped
if snapshot.Policy == "always" {
policy = releaseworkflow.SnapshotPolicyAlways
}
publisherOptions = append(publisherOptions, releaseworkflow.WithTargetSnapshots(
releasesnapshot.Guard{Root: snapshot.Root, MaxAge: maxAge, VerifyLocalFiles: true}, policy))
}
publisher, err := releaseworkflow.NewSidecarPublisher(documentExporter, documentTarget, publisherOptions...)
if err != nil {
return cleanup(errors.New("create v0.7.1 release publisher"))
}