feat(release): 发布前快照硬门禁 + 云内快照 + ES/OSS 保护
按生产级备份组合落地:
- internal/releasesnapshot:新增门禁包,要求 root 下存在覆盖目标库、未过期
(默认 24h)、含 wiki_pages.jsonl 与 sha256 的快照;本地文件再校验哈希。
7 条单测(过期/换库/篡改/缺页/未接线/半成品)。
- 发布器:写目标库之前(Publish/Delete/Rollback 三条路)先过门禁;映射目标
(源库→目标库,共享真实库)或 policy=always 时必须有快照;门禁未接线
同样拒绝(fail-closed)。新错误码 release_target_snapshot_missing(不可重试)
+ 页面中文提示。6 条发布器单测。
- worker 配置新增 target_snapshot{root,max_age_hours,policy},校验 policy/
max_age/绝对路径;compose.release-sim.yaml 只读挂载快照目录。
- scripts/target_kb_snapshot.sh:默认云内执行(与 DB 同 VPC,凭据取自 WeKnora
容器)+ 默认不导 chunks(44s 出 8 张表);修复 \copy 不能跨行/不能带 WITH、
ES 计数改用 knowledge_base_id.keyword。
- ES:注册 kb_backup 仓库(path.repo),首个快照 36s/4.7GB/0 分片失败,
SLM kb-hourly(3d)+kb-daily(7d) 已建并执行成功。
- OSS:目标桶开启版本控制(此前未开启)。
- 文档 P6 增补硬门禁与云内执行;真实快照目录上门禁实测:两个绑定库通过、
未覆盖库拒绝、未接线拒绝。
This commit is contained in:
@@ -27,7 +27,10 @@ const (
|
||||
ErrorCodeRollbackArtifactMissing = "release_rollback_artifact_missing"
|
||||
ErrorCodeRollbackFailed = "release_rollback_failed"
|
||||
ErrorCodeStoppedAfterFailure = "release_stopped_after_failure"
|
||||
ErrorCodeInternal = "release_internal_error"
|
||||
// ErrorCodeTargetSnapshotMissing:写目标库之前必须存在覆盖该库的、未过期的
|
||||
// 快照留档(scripts/target_kb_snapshot.sh 产出),否则拒绝发布。
|
||||
ErrorCodeTargetSnapshotMissing = "release_target_snapshot_missing"
|
||||
ErrorCodeInternal = "release_internal_error"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -305,7 +308,8 @@ func ValidErrorCode(code string) bool {
|
||||
switch code {
|
||||
case ErrorCodeContractMismatch, ErrorCodeBundleInvalid, ErrorCodeTargetUnavailable,
|
||||
ErrorCodeTargetWriteFailed, ErrorCodeVerificationFailed, ErrorCodeRollbackArtifactMissing,
|
||||
ErrorCodeRollbackFailed, ErrorCodeStoppedAfterFailure, ErrorCodeInternal:
|
||||
ErrorCodeRollbackFailed, ErrorCodeStoppedAfterFailure, ErrorCodeTargetSnapshotMissing,
|
||||
ErrorCodeInternal:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
|
||||
@@ -34,13 +34,73 @@ type SidecarTarget interface {
|
||||
type SidecarPublisher struct {
|
||||
exporter BundleExporter
|
||||
target SidecarTarget
|
||||
// snapshots 是"写目标库前必须存在目标库快照"的门禁;nil 表示未接线。
|
||||
snapshots TargetSnapshotGuard
|
||||
snapshotPolicy SnapshotPolicy
|
||||
}
|
||||
|
||||
func NewSidecarPublisher(exporter BundleExporter, target SidecarTarget) (*SidecarPublisher, error) {
|
||||
// TargetSnapshotGuard 由 internal/releasesnapshot.Guard 实现。
|
||||
type TargetSnapshotGuard interface {
|
||||
Check(knowledgeBaseID string) error
|
||||
}
|
||||
|
||||
// SnapshotPolicy 决定门禁对哪些项生效。
|
||||
type SnapshotPolicy int
|
||||
|
||||
const (
|
||||
// SnapshotPolicyMapped:只对"源库 → 目标库"映射的项强制(共享真实库)。
|
||||
SnapshotPolicyMapped SnapshotPolicy = iota
|
||||
// SnapshotPolicyAlways:所有写入目标库的项都要有快照。
|
||||
SnapshotPolicyAlways
|
||||
)
|
||||
|
||||
// PublisherOption 用于给发布器加装配件(保持既有两参构造的兼容)。
|
||||
type PublisherOption func(*SidecarPublisher) error
|
||||
|
||||
// WithTargetSnapshots 装上快照门禁:写入目标库之前先证明"现场已留档"。
|
||||
func WithTargetSnapshots(guard TargetSnapshotGuard, policy SnapshotPolicy) PublisherOption {
|
||||
return func(publisher *SidecarPublisher) error {
|
||||
if guard == nil {
|
||||
return errors.New("target snapshot guard is required")
|
||||
}
|
||||
publisher.snapshots = guard
|
||||
publisher.snapshotPolicy = policy
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func NewSidecarPublisher(exporter BundleExporter, target SidecarTarget, options ...PublisherOption) (*SidecarPublisher, error) {
|
||||
if exporter == nil || target == nil {
|
||||
return nil, errors.New("release sidecar dependencies are required")
|
||||
}
|
||||
return &SidecarPublisher{exporter: exporter, target: target}, nil
|
||||
publisher := &SidecarPublisher{exporter: exporter, target: target}
|
||||
for _, option := range options {
|
||||
if option == nil {
|
||||
continue
|
||||
}
|
||||
if err := option(publisher); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return publisher, nil
|
||||
}
|
||||
|
||||
// ensureTargetSnapshot 在任何写目标库的动作之前调用:目标库是共享真实库
|
||||
// (源库 → 目标库 映射)或策略要求"一律"时,必须存在覆盖该库的新快照。
|
||||
// 门禁未接线(root 未配置)时对这类项**拒绝放行**,而不是跳过校验。
|
||||
func (publisher *SidecarPublisher) ensureTargetSnapshot(item PlanItem) error {
|
||||
required := publisher.snapshotPolicy == SnapshotPolicyAlways || item.TargetBase() != item.TestKnowledgeBaseID
|
||||
if !required {
|
||||
return nil
|
||||
}
|
||||
if publisher.snapshots == nil {
|
||||
return newSidecarError(ErrorCodeTargetSnapshotMissing, false,
|
||||
errors.New("发布前快照门禁未配置(release config target_snapshot.root)"))
|
||||
}
|
||||
if err := publisher.snapshots.Check(item.TargetBase()); err != nil {
|
||||
return newSidecarError(ErrorCodeTargetSnapshotMissing, false, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// releaseItemMapping declares which production base this item must land in: the
|
||||
@@ -53,6 +113,9 @@ func releaseItemMapping(item PlanItem) releasebundle.KnowledgeBaseMapping {
|
||||
}
|
||||
|
||||
func (publisher *SidecarPublisher) Publish(ctx context.Context, item PlanItem) (ItemResult, error) {
|
||||
if err := publisher.ensureTargetSnapshot(item); err != nil {
|
||||
return ItemResult{}, err
|
||||
}
|
||||
bundle, err := publisher.exporter.Export(ctx, item)
|
||||
if err != nil {
|
||||
return ItemResult{}, newSidecarError(ErrorCodeTargetUnavailable, true, err)
|
||||
@@ -95,6 +158,9 @@ func (publisher *SidecarPublisher) Publish(ctx context.Context, item PlanItem) (
|
||||
}
|
||||
|
||||
func (publisher *SidecarPublisher) Rollback(ctx context.Context, item PlanItem) (ItemResult, error) {
|
||||
if err := publisher.ensureTargetSnapshot(item); err != nil {
|
||||
return ItemResult{}, err
|
||||
}
|
||||
if !validRevision(item.PreviousProductionRevision) {
|
||||
return ItemResult{}, newSidecarError(ErrorCodeRollbackArtifactMissing, false, errors.New("missing rollback revision"))
|
||||
}
|
||||
@@ -112,6 +178,9 @@ func (publisher *SidecarPublisher) Rollback(ctx context.Context, item PlanItem)
|
||||
}
|
||||
|
||||
func (publisher *SidecarPublisher) Delete(ctx context.Context, item PlanItem) (ItemResult, error) {
|
||||
if err := publisher.ensureTargetSnapshot(item); err != nil {
|
||||
return ItemResult{}, err
|
||||
}
|
||||
bundle, err := publisher.exporter.Export(ctx, item)
|
||||
if err != nil {
|
||||
return ItemResult{}, newSidecarError(ErrorCodeTargetUnavailable, true, err)
|
||||
|
||||
@@ -215,3 +215,148 @@ func testManifest() releasebundle.Manifest {
|
||||
func testTargetContract() releasebundle.TargetContract {
|
||||
return releasebundle.TargetContract{WeKnoraVersion: releasebundle.WeKnoraVersionV071, MigrationVersion: releasebundle.WeKnoraMigrationV071, TenantID: 10000, KnowledgeBaseID: "kb-safe", RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "bge-m3", EmbeddingDimension: releasebundle.EmbeddingDimensionV071, ElasticsearchMappingSHA256: strings.Repeat("a", 64)}
|
||||
}
|
||||
|
||||
// --- 发布前快照门禁 ---------------------------------------------------------
|
||||
|
||||
type stubSnapshotGuard struct {
|
||||
checked []string
|
||||
err error
|
||||
}
|
||||
|
||||
func (guard *stubSnapshotGuard) Check(knowledgeBaseID string) error {
|
||||
guard.checked = append(guard.checked, knowledgeBaseID)
|
||||
return guard.err
|
||||
}
|
||||
|
||||
func mappedPlanItem() PlanItem {
|
||||
item := testPlanItem()
|
||||
item.TargetKnowledgeBaseID = "kb-production"
|
||||
return item
|
||||
}
|
||||
|
||||
// 共享真实库(源库 → 目标库 映射)没有快照时必须拒绝,且不得碰目标库。
|
||||
func TestSidecarPublisherRefusesMappedItemWithoutSnapshot(t *testing.T) {
|
||||
calls := []string{}
|
||||
manifest, err := releasebundle.Seal(testManifest())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
exporter := &fakeExporter{manifest: manifest, calls: &calls}
|
||||
target := &fakeSidecarTarget{contract: testTargetContract(), calls: &calls, prepared: PreparedChange{PreviousRevision: "prod-v0", ProductionRevision: "prod-v1"}}
|
||||
guard := &stubSnapshotGuard{err: errors.New("no fresh target knowledge base snapshot")}
|
||||
publisher, err := NewSidecarPublisher(exporter, target, WithTargetSnapshots(guard, SnapshotPolicyMapped))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, publishErr := publisher.Publish(context.Background(), mappedPlanItem())
|
||||
var itemErr *ItemError
|
||||
if !errors.As(publishErr, &itemErr) || itemErr.Code != ErrorCodeTargetSnapshotMissing {
|
||||
t.Fatalf("err=%v want code %s", publishErr, ErrorCodeTargetSnapshotMissing)
|
||||
}
|
||||
if len(guard.checked) != 1 || guard.checked[0] != "kb-production" {
|
||||
t.Fatalf("guard must be asked about the production target: %v", guard.checked)
|
||||
}
|
||||
if len(calls) != 0 {
|
||||
t.Fatalf("a refused release must not touch the target: %v", calls)
|
||||
}
|
||||
}
|
||||
|
||||
// 门禁未接线(target_snapshot.root 未配置)时,映射项也要拒绝——不能静默放行。
|
||||
func TestSidecarPublisherFailsClosedWhenSnapshotGateUnconfigured(t *testing.T) {
|
||||
calls := []string{}
|
||||
manifest, err := releasebundle.Seal(testManifest())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
exporter := &fakeExporter{manifest: manifest, calls: &calls}
|
||||
target := &fakeSidecarTarget{contract: testTargetContract(), calls: &calls, prepared: PreparedChange{PreviousRevision: "prod-v0", ProductionRevision: "prod-v1"}}
|
||||
publisher, err := NewSidecarPublisher(exporter, target)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, publishErr := publisher.Publish(context.Background(), mappedPlanItem())
|
||||
var itemErr *ItemError
|
||||
if !errors.As(publishErr, &itemErr) || itemErr.Code != ErrorCodeTargetSnapshotMissing {
|
||||
t.Fatalf("err=%v want code %s", publishErr, ErrorCodeTargetSnapshotMissing)
|
||||
}
|
||||
if len(calls) != 0 {
|
||||
t.Fatalf("calls=%v", calls)
|
||||
}
|
||||
}
|
||||
|
||||
// 同 ID 目标(历史语义 / 本地演练)不需要快照,策略为 mapped 时门禁不介入。
|
||||
func TestSidecarPublisherSkipsSnapshotGateForSameBaseTarget(t *testing.T) {
|
||||
calls := []string{}
|
||||
manifest, err := releasebundle.Seal(testManifest())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
exporter := &fakeExporter{manifest: manifest, calls: &calls}
|
||||
target := &fakeSidecarTarget{contract: testTargetContract(), calls: &calls, prepared: PreparedChange{PreviousRevision: "prod-v0", ProductionRevision: "prod-v1"}}
|
||||
guard := &stubSnapshotGuard{err: errors.New("must not be consulted")}
|
||||
publisher, err := NewSidecarPublisher(exporter, target, WithTargetSnapshots(guard, SnapshotPolicyMapped))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := publisher.Publish(context.Background(), testPlanItem()); err != nil {
|
||||
t.Fatalf("same-base target must publish without a snapshot: %v", err)
|
||||
}
|
||||
if len(guard.checked) != 0 {
|
||||
t.Fatalf("guard must not run for same-base targets: %v", guard.checked)
|
||||
}
|
||||
}
|
||||
|
||||
// policy=always 时,同 ID 目标同样要求快照。
|
||||
func TestSidecarPublisherAlwaysPolicyChecksSameBaseTarget(t *testing.T) {
|
||||
calls := []string{}
|
||||
manifest, err := releasebundle.Seal(testManifest())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
exporter := &fakeExporter{manifest: manifest, calls: &calls}
|
||||
target := &fakeSidecarTarget{contract: testTargetContract(), calls: &calls, prepared: PreparedChange{PreviousRevision: "prod-v0", ProductionRevision: "prod-v1"}}
|
||||
guard := &stubSnapshotGuard{}
|
||||
publisher, err := NewSidecarPublisher(exporter, target, WithTargetSnapshots(guard, SnapshotPolicyAlways))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := publisher.Publish(context.Background(), testPlanItem()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(guard.checked) != 1 || guard.checked[0] != "kb-safe" {
|
||||
t.Fatalf("always policy must check every target: %v", guard.checked)
|
||||
}
|
||||
}
|
||||
|
||||
// 撤权(delete)与回滚同样先过门禁。
|
||||
func TestSidecarPublisherGatesDeleteAndRollbackToo(t *testing.T) {
|
||||
for _, operation := range []string{"delete", "rollback"} {
|
||||
calls := []string{}
|
||||
manifest, err := releasebundle.Seal(testManifest())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
exporter := &fakeExporter{manifest: manifest, calls: &calls}
|
||||
target := &fakeSidecarTarget{contract: testTargetContract(), calls: &calls, prepared: PreparedChange{PreviousRevision: "prod-v0", ProductionRevision: "prod-v1"}}
|
||||
guard := &stubSnapshotGuard{err: errors.New("stale")}
|
||||
publisher, err := NewSidecarPublisher(exporter, target, WithTargetSnapshots(guard, SnapshotPolicyMapped))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
item := mappedPlanItem()
|
||||
item.Action = operation
|
||||
var publishErr error
|
||||
if operation == "delete" {
|
||||
_, publishErr = publisher.Delete(context.Background(), item)
|
||||
} else {
|
||||
_, publishErr = publisher.Rollback(context.Background(), item)
|
||||
}
|
||||
var itemErr *ItemError
|
||||
if !errors.As(publishErr, &itemErr) || itemErr.Code != ErrorCodeTargetSnapshotMissing {
|
||||
t.Fatalf("%s: err=%v want %s", operation, publishErr, ErrorCodeTargetSnapshotMissing)
|
||||
}
|
||||
if len(calls) != 0 {
|
||||
t.Fatalf("%s touched the target: %v", operation, calls)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user