feat(release): 发布前快照硬门禁 + 云内快照 + ES/OSS 保护

按生产级备份组合落地:

- internal/releasesnapshot:新增门禁包,要求 root 下存在覆盖目标库、未过期
  (默认 24h)、含 wiki_pages.jsonl 与 sha256 的快照;本地文件再校验哈希。
  7 条单测(过期/换库/篡改/缺页/未接线/半成品)。
- 发布器:写目标库之前(Publish/Delete/Rollback 三条路)先过门禁;映射目标
  (源库→目标库,共享真实库)或 policy=always 时必须有快照;门禁未接线
  同样拒绝(fail-closed)。新错误码 release_target_snapshot_missing(不可重试)
  + 页面中文提示。6 条发布器单测。
- worker 配置新增 target_snapshot{root,max_age_hours,policy},校验 policy/
  max_age/绝对路径;compose.release-sim.yaml 只读挂载快照目录。
- scripts/target_kb_snapshot.sh:默认云内执行(与 DB 同 VPC,凭据取自 WeKnora
  容器)+ 默认不导 chunks(44s 出 8 张表);修复 \copy 不能跨行/不能带 WITH、
  ES 计数改用 knowledge_base_id.keyword。
- ES:注册 kb_backup 仓库(path.repo),首个快照 36s/4.7GB/0 分片失败,
  SLM kb-hourly(3d)+kb-daily(7d) 已建并执行成功。
- OSS:目标桶开启版本控制(此前未开启)。
- 文档 P6 增补硬门禁与云内执行;真实快照目录上门禁实测:两个绑定库通过、
  未覆盖库拒绝、未接线拒绝。
This commit is contained in:
姚章浩
2026-10-09 10:59:55 +08:00
parent f4e1ecc5f1
commit 1fa35e9870
14 changed files with 836 additions and 70 deletions
+6 -2
View File
@@ -27,7 +27,10 @@ const (
ErrorCodeRollbackArtifactMissing = "release_rollback_artifact_missing"
ErrorCodeRollbackFailed = "release_rollback_failed"
ErrorCodeStoppedAfterFailure = "release_stopped_after_failure"
ErrorCodeInternal = "release_internal_error"
// ErrorCodeTargetSnapshotMissing:写目标库之前必须存在覆盖该库的、未过期的
// 快照留档(scripts/target_kb_snapshot.sh 产出),否则拒绝发布。
ErrorCodeTargetSnapshotMissing = "release_target_snapshot_missing"
ErrorCodeInternal = "release_internal_error"
)
var (
@@ -305,7 +308,8 @@ func ValidErrorCode(code string) bool {
switch code {
case ErrorCodeContractMismatch, ErrorCodeBundleInvalid, ErrorCodeTargetUnavailable,
ErrorCodeTargetWriteFailed, ErrorCodeVerificationFailed, ErrorCodeRollbackArtifactMissing,
ErrorCodeRollbackFailed, ErrorCodeStoppedAfterFailure, ErrorCodeInternal:
ErrorCodeRollbackFailed, ErrorCodeStoppedAfterFailure, ErrorCodeTargetSnapshotMissing,
ErrorCodeInternal:
return true
default:
return false
@@ -34,13 +34,73 @@ type SidecarTarget interface {
type SidecarPublisher struct {
exporter BundleExporter
target SidecarTarget
// snapshots 是"写目标库前必须存在目标库快照"的门禁;nil 表示未接线。
snapshots TargetSnapshotGuard
snapshotPolicy SnapshotPolicy
}
func NewSidecarPublisher(exporter BundleExporter, target SidecarTarget) (*SidecarPublisher, error) {
// TargetSnapshotGuard 由 internal/releasesnapshot.Guard 实现。
type TargetSnapshotGuard interface {
Check(knowledgeBaseID string) error
}
// SnapshotPolicy 决定门禁对哪些项生效。
type SnapshotPolicy int
const (
// SnapshotPolicyMapped:只对"源库 → 目标库"映射的项强制(共享真实库)。
SnapshotPolicyMapped SnapshotPolicy = iota
// SnapshotPolicyAlways:所有写入目标库的项都要有快照。
SnapshotPolicyAlways
)
// PublisherOption 用于给发布器加装配件(保持既有两参构造的兼容)。
type PublisherOption func(*SidecarPublisher) error
// WithTargetSnapshots 装上快照门禁:写入目标库之前先证明"现场已留档"。
func WithTargetSnapshots(guard TargetSnapshotGuard, policy SnapshotPolicy) PublisherOption {
return func(publisher *SidecarPublisher) error {
if guard == nil {
return errors.New("target snapshot guard is required")
}
publisher.snapshots = guard
publisher.snapshotPolicy = policy
return nil
}
}
func NewSidecarPublisher(exporter BundleExporter, target SidecarTarget, options ...PublisherOption) (*SidecarPublisher, error) {
if exporter == nil || target == nil {
return nil, errors.New("release sidecar dependencies are required")
}
return &SidecarPublisher{exporter: exporter, target: target}, nil
publisher := &SidecarPublisher{exporter: exporter, target: target}
for _, option := range options {
if option == nil {
continue
}
if err := option(publisher); err != nil {
return nil, err
}
}
return publisher, nil
}
// ensureTargetSnapshot 在任何写目标库的动作之前调用:目标库是共享真实库
// (源库 → 目标库 映射)或策略要求"一律"时,必须存在覆盖该库的新快照。
// 门禁未接线(root 未配置)时对这类项**拒绝放行**,而不是跳过校验。
func (publisher *SidecarPublisher) ensureTargetSnapshot(item PlanItem) error {
required := publisher.snapshotPolicy == SnapshotPolicyAlways || item.TargetBase() != item.TestKnowledgeBaseID
if !required {
return nil
}
if publisher.snapshots == nil {
return newSidecarError(ErrorCodeTargetSnapshotMissing, false,
errors.New("发布前快照门禁未配置(release config target_snapshot.root)"))
}
if err := publisher.snapshots.Check(item.TargetBase()); err != nil {
return newSidecarError(ErrorCodeTargetSnapshotMissing, false, err)
}
return nil
}
// releaseItemMapping declares which production base this item must land in: the
@@ -53,6 +113,9 @@ func releaseItemMapping(item PlanItem) releasebundle.KnowledgeBaseMapping {
}
func (publisher *SidecarPublisher) Publish(ctx context.Context, item PlanItem) (ItemResult, error) {
if err := publisher.ensureTargetSnapshot(item); err != nil {
return ItemResult{}, err
}
bundle, err := publisher.exporter.Export(ctx, item)
if err != nil {
return ItemResult{}, newSidecarError(ErrorCodeTargetUnavailable, true, err)
@@ -95,6 +158,9 @@ func (publisher *SidecarPublisher) Publish(ctx context.Context, item PlanItem) (
}
func (publisher *SidecarPublisher) Rollback(ctx context.Context, item PlanItem) (ItemResult, error) {
if err := publisher.ensureTargetSnapshot(item); err != nil {
return ItemResult{}, err
}
if !validRevision(item.PreviousProductionRevision) {
return ItemResult{}, newSidecarError(ErrorCodeRollbackArtifactMissing, false, errors.New("missing rollback revision"))
}
@@ -112,6 +178,9 @@ func (publisher *SidecarPublisher) Rollback(ctx context.Context, item PlanItem)
}
func (publisher *SidecarPublisher) Delete(ctx context.Context, item PlanItem) (ItemResult, error) {
if err := publisher.ensureTargetSnapshot(item); err != nil {
return ItemResult{}, err
}
bundle, err := publisher.exporter.Export(ctx, item)
if err != nil {
return ItemResult{}, newSidecarError(ErrorCodeTargetUnavailable, true, err)
@@ -215,3 +215,148 @@ func testManifest() releasebundle.Manifest {
func testTargetContract() releasebundle.TargetContract {
return releasebundle.TargetContract{WeKnoraVersion: releasebundle.WeKnoraVersionV071, MigrationVersion: releasebundle.WeKnoraMigrationV071, TenantID: 10000, KnowledgeBaseID: "kb-safe", RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "bge-m3", EmbeddingDimension: releasebundle.EmbeddingDimensionV071, ElasticsearchMappingSHA256: strings.Repeat("a", 64)}
}
// --- 发布前快照门禁 ---------------------------------------------------------
type stubSnapshotGuard struct {
checked []string
err error
}
func (guard *stubSnapshotGuard) Check(knowledgeBaseID string) error {
guard.checked = append(guard.checked, knowledgeBaseID)
return guard.err
}
func mappedPlanItem() PlanItem {
item := testPlanItem()
item.TargetKnowledgeBaseID = "kb-production"
return item
}
// 共享真实库(源库 → 目标库 映射)没有快照时必须拒绝,且不得碰目标库。
func TestSidecarPublisherRefusesMappedItemWithoutSnapshot(t *testing.T) {
calls := []string{}
manifest, err := releasebundle.Seal(testManifest())
if err != nil {
t.Fatal(err)
}
exporter := &fakeExporter{manifest: manifest, calls: &calls}
target := &fakeSidecarTarget{contract: testTargetContract(), calls: &calls, prepared: PreparedChange{PreviousRevision: "prod-v0", ProductionRevision: "prod-v1"}}
guard := &stubSnapshotGuard{err: errors.New("no fresh target knowledge base snapshot")}
publisher, err := NewSidecarPublisher(exporter, target, WithTargetSnapshots(guard, SnapshotPolicyMapped))
if err != nil {
t.Fatal(err)
}
_, publishErr := publisher.Publish(context.Background(), mappedPlanItem())
var itemErr *ItemError
if !errors.As(publishErr, &itemErr) || itemErr.Code != ErrorCodeTargetSnapshotMissing {
t.Fatalf("err=%v want code %s", publishErr, ErrorCodeTargetSnapshotMissing)
}
if len(guard.checked) != 1 || guard.checked[0] != "kb-production" {
t.Fatalf("guard must be asked about the production target: %v", guard.checked)
}
if len(calls) != 0 {
t.Fatalf("a refused release must not touch the target: %v", calls)
}
}
// 门禁未接线(target_snapshot.root 未配置)时,映射项也要拒绝——不能静默放行。
func TestSidecarPublisherFailsClosedWhenSnapshotGateUnconfigured(t *testing.T) {
calls := []string{}
manifest, err := releasebundle.Seal(testManifest())
if err != nil {
t.Fatal(err)
}
exporter := &fakeExporter{manifest: manifest, calls: &calls}
target := &fakeSidecarTarget{contract: testTargetContract(), calls: &calls, prepared: PreparedChange{PreviousRevision: "prod-v0", ProductionRevision: "prod-v1"}}
publisher, err := NewSidecarPublisher(exporter, target)
if err != nil {
t.Fatal(err)
}
_, publishErr := publisher.Publish(context.Background(), mappedPlanItem())
var itemErr *ItemError
if !errors.As(publishErr, &itemErr) || itemErr.Code != ErrorCodeTargetSnapshotMissing {
t.Fatalf("err=%v want code %s", publishErr, ErrorCodeTargetSnapshotMissing)
}
if len(calls) != 0 {
t.Fatalf("calls=%v", calls)
}
}
// 同 ID 目标(历史语义 / 本地演练)不需要快照,策略为 mapped 时门禁不介入。
func TestSidecarPublisherSkipsSnapshotGateForSameBaseTarget(t *testing.T) {
calls := []string{}
manifest, err := releasebundle.Seal(testManifest())
if err != nil {
t.Fatal(err)
}
exporter := &fakeExporter{manifest: manifest, calls: &calls}
target := &fakeSidecarTarget{contract: testTargetContract(), calls: &calls, prepared: PreparedChange{PreviousRevision: "prod-v0", ProductionRevision: "prod-v1"}}
guard := &stubSnapshotGuard{err: errors.New("must not be consulted")}
publisher, err := NewSidecarPublisher(exporter, target, WithTargetSnapshots(guard, SnapshotPolicyMapped))
if err != nil {
t.Fatal(err)
}
if _, err := publisher.Publish(context.Background(), testPlanItem()); err != nil {
t.Fatalf("same-base target must publish without a snapshot: %v", err)
}
if len(guard.checked) != 0 {
t.Fatalf("guard must not run for same-base targets: %v", guard.checked)
}
}
// policy=always 时,同 ID 目标同样要求快照。
func TestSidecarPublisherAlwaysPolicyChecksSameBaseTarget(t *testing.T) {
calls := []string{}
manifest, err := releasebundle.Seal(testManifest())
if err != nil {
t.Fatal(err)
}
exporter := &fakeExporter{manifest: manifest, calls: &calls}
target := &fakeSidecarTarget{contract: testTargetContract(), calls: &calls, prepared: PreparedChange{PreviousRevision: "prod-v0", ProductionRevision: "prod-v1"}}
guard := &stubSnapshotGuard{}
publisher, err := NewSidecarPublisher(exporter, target, WithTargetSnapshots(guard, SnapshotPolicyAlways))
if err != nil {
t.Fatal(err)
}
if _, err := publisher.Publish(context.Background(), testPlanItem()); err != nil {
t.Fatal(err)
}
if len(guard.checked) != 1 || guard.checked[0] != "kb-safe" {
t.Fatalf("always policy must check every target: %v", guard.checked)
}
}
// 撤权(delete)与回滚同样先过门禁。
func TestSidecarPublisherGatesDeleteAndRollbackToo(t *testing.T) {
for _, operation := range []string{"delete", "rollback"} {
calls := []string{}
manifest, err := releasebundle.Seal(testManifest())
if err != nil {
t.Fatal(err)
}
exporter := &fakeExporter{manifest: manifest, calls: &calls}
target := &fakeSidecarTarget{contract: testTargetContract(), calls: &calls, prepared: PreparedChange{PreviousRevision: "prod-v0", ProductionRevision: "prod-v1"}}
guard := &stubSnapshotGuard{err: errors.New("stale")}
publisher, err := NewSidecarPublisher(exporter, target, WithTargetSnapshots(guard, SnapshotPolicyMapped))
if err != nil {
t.Fatal(err)
}
item := mappedPlanItem()
item.Action = operation
var publishErr error
if operation == "delete" {
_, publishErr = publisher.Delete(context.Background(), item)
} else {
_, publishErr = publisher.Rollback(context.Background(), item)
}
var itemErr *ItemError
if !errors.As(publishErr, &itemErr) || itemErr.Code != ErrorCodeTargetSnapshotMissing {
t.Fatalf("%s: err=%v want %s", operation, publishErr, ErrorCodeTargetSnapshotMissing)
}
if len(calls) != 0 {
t.Fatalf("%s touched the target: %v", operation, calls)
}
}
}