diff --git a/api/openapi.yaml b/api/openapi.yaml index b5d9f3e..f6cbedf 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -832,6 +832,62 @@ } } }, + "/api/v1/library/source-associations": { + "post": { + "tags": [ + "Runtime" + ], + "operationId": "associateLegacyLibrarySource", + "summary": "管理员核验原捕获证据并关联旧封存来源;不修改快照、不发布内容", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": false, + "required": [ + "scope", + "snapshot_id" + ], + "properties": { + "scope": { + "type": "string", + "enum": [ + "internal", + "external" + ] + }, + "snapshot_id": { + "type": "string" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "已持久关联并读回;associated=true,published=false,返回准确snapshot_id/source_id/source_identity" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "403": { + "description": "需要管理员登录身份" + }, + "409": { + "description": "原捕获证据缺失、来源/指纹/清单不符或已有不可变关联冲突" + }, + "500": { + "$ref": "#/components/responses/InternalError" + }, + "503": { + "description": "来源配置或关联存储不可用" + } + } + } + }, "/api/v1/library/captures": { "get": { "tags": [ diff --git a/cmd/api/main.go b/cmd/api/main.go index 65d7391..5a3b0a2 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -221,6 +221,10 @@ func run(ctx context.Context, getenv func(string) string) error { LibraryTargetJournalRoot: config.LibraryTargetJournalRoot, LibrarySourceInstance: config.SourceInstance(), LibrarySourceProject: config.SourceProject(), + LibrarySourceApp: config.LibrarySourceApp, + LibrarySourceTenantID: config.LibrarySourceTenantID, + LibrarySourceInternalKB: config.LibrarySourceInternalKB, + LibrarySourceExternalKB: config.LibrarySourceExternalKB, LibraryAgentSocket: config.LibraryAgentSocket, LibraryExecutorStatusMaxAge: config.LibraryExecutorStatusMaxAge, LibraryCapacityBudgetBytes: config.LibraryCapacityBudgetBytes, diff --git a/cmd/release-channel-e2e/main.go b/cmd/release-channel-e2e/main.go index bdacf18..26f8f9d 100644 --- a/cmd/release-channel-e2e/main.go +++ b/cmd/release-channel-e2e/main.go @@ -81,6 +81,10 @@ func secret(path string) string { } func main() { + if err := v071.CheckLegacyObjectWrites(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } configPath := flag.String("config", "", "release config json") knowledgeID := flag.String("knowledge", "", "knowledge id to release") mode := flag.String("mode", "publish", "publish | withdraw | rollback") diff --git a/cmd/release-diag/main.go b/cmd/release-diag/main.go index 9147e39..3be4c0a 100644 --- a/cmd/release-diag/main.go +++ b/cmd/release-diag/main.go @@ -69,6 +69,10 @@ func secret(path string) string { } func main() { + if err := v071.CheckLegacyObjectWrites(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } configPath := flag.String("config", "", "release config json") knowledgeID := flag.String("knowledge", "", "knowledge id to diagnose") flag.Parse() diff --git a/cmd/release-worker/runtime.go b/cmd/release-worker/runtime.go index 2b34acb..fe6a905 100644 --- a/cmd/release-worker/runtime.go +++ b/cmd/release-worker/runtime.go @@ -31,6 +31,9 @@ func (runtime *releaseRuntime) Close() { } func buildReleaseRuntime(ctx context.Context, config releaseRuntimeConfig) (*releaseRuntime, error) { + if err := v071.CheckLegacyObjectWrites(); err != nil { + return nil, err + } runtime := &releaseRuntime{} publishers := make(map[string]releaseworkflow.Publisher, len(config.Routes)) for _, route := range config.Routes { @@ -52,6 +55,9 @@ func buildReleaseRuntime(ctx context.Context, config releaseRuntimeConfig) (*rel } func buildReleaseRoute(ctx context.Context, archiveRoot string, snapshot releaseTargetSnapshotConfig, route releaseRouteConfig) (releaseworkflow.Publisher, []*pgxpool.Pool, error) { + if err := v071.CheckLegacyObjectWrites(); err != nil { + return nil, nil, err + } sourceDatabaseURL, err := readRestrictedValue(route.Source.DatabaseURLFile) if err != nil { return nil, nil, errors.New("read source database credential") diff --git a/cmd/worker/library_attempt_timing.go b/cmd/worker/library_attempt_timing.go new file mode 100644 index 0000000..2fff5ea --- /dev/null +++ b/cmd/worker/library_attempt_timing.go @@ -0,0 +1,8 @@ +//go:build !kg_isolated_drill + +package main + +import "time" + +// Production timing is not controllable through test environment variables. +const libraryAttemptLease = 10 * time.Minute diff --git a/cmd/worker/library_attempt_timing_drill.go b/cmd/worker/library_attempt_timing_drill.go new file mode 100644 index 0000000..c1338f0 --- /dev/null +++ b/cmd/worker/library_attempt_timing_drill.go @@ -0,0 +1,26 @@ +//go:build kg_isolated_drill + +package main + +import ( + "net/url" + "os" + "strings" + "time" +) + +// Only an explicitly tagged executable can shorten the real lease clock. It +// refuses non-disposable DBs; no production binary recognizes this injection. +const libraryAttemptLease = 4 * time.Second + +func init() { + if os.Getenv("KG_LIBRARY_TEST_ISOLATED") != "1" { + panic("isolated drill build requires explicit opt-in") + } + for _, name := range []string{"DATABASE_URL", "LIBRARY_PRODUCTION_DATABASE_URL"} { + u, e := url.Parse(os.Getenv(name)) + if e != nil || (u.Hostname() != "127.0.0.1" && u.Hostname() != "localhost") || !strings.HasPrefix(u.Path, "/kg_library_test_r8_") { + panic("isolated drill build refuses non-disposable database") + } + } +} diff --git a/cmd/worker/library_baseline.go b/cmd/worker/library_baseline.go index 928bfed..2f9d0db 100644 --- a/cmd/worker/library_baseline.go +++ b/cmd/worker/library_baseline.go @@ -111,10 +111,12 @@ func runLibraryBaseline(ctx context.Context, base *postgresstore.Store, config a if _, admitErr := admitLibraryJob(ctx, base, job, payload.LibraryJobID, payload.Scope, governance.LibraryJobBaseline, true); admitErr != nil { return admitErr } - writer, attemptErr := beginLibraryWriter(ctx, base, payload.LibraryJobID, libraryQueueOwner(config.WorkerID, job, payload.LibraryJobID)) - if attemptErr != nil { - return attemptErr - } + return runOwnedLibraryAttempt(ctx, base, config, job, payload.LibraryJobID, func(runCtx context.Context, writer *libraryAttemptWriter) error { + return runLibraryBaselineAttempt(runCtx, writer, config, payload) + }) +} + +func runLibraryBaselineAttempt(ctx context.Context, writer *libraryAttemptWriter, config appconfig.Config, payload libraryBaselinePayload) error { store := writer now := time.Now().UTC() recorder := libraryStageRecorder{store: store, jobID: payload.LibraryJobID, scope: payload.Scope, diff --git a/cmd/worker/library_handlers.go b/cmd/worker/library_handlers.go index 2ea8355..e75d96a 100644 --- a/cmd/worker/library_handlers.go +++ b/cmd/worker/library_handlers.go @@ -43,6 +43,40 @@ type contentTrackedTarget struct { inner libraryrelease.Target jobID string onApply func() + // revalidateGrant 由 runLibraryRelease 无条件接线:每笔副作用前按治理库里的 + // 授权行重新判定(撤销/过期即停)。测试夹具可以显式传 nil 表示"这份夹具没有 + // 持久授权行",但真实发布路径不允许为空 —— 那里的空值等于回到只信内存快照。 + revalidateGrant func(context.Context, libraryrelease.Grant) error +} + +// revalidateLibraryGrant 是"授权没有被中途撤销/过期"的唯一判定实现: +// 真实发布链路与测试都走它,避免测试自己写一份看起来相同的查询。 +func revalidateLibraryGrant(store *postgresstore.Store) func(context.Context, libraryrelease.Grant) error { + return func(ctx context.Context, current libraryrelease.Grant) error { + // 权威来源是治理库里的授权行:参数摘要一变就查不到,撤销/过期立即消失。 + found, ok, err := store.ActiveLibraryAuthorization(ctx, current.Target, current.SnapshotID, + current.ParamsDigest, time.Now().UTC()) + if err != nil { + return fmt.Errorf("读取持久授权失败:%w", err) + } + if !ok { + return errors.New("当前授权已撤销或已过期") + } + if found.AuthorizedBy != current.AuthorizedBy { + return errors.New("持久授权与本次执行身份不一致") + } + return nil + } +} + +func (t *contentTrackedTarget) RevalidateGrant(ctx context.Context, grant libraryrelease.Grant) error { + if t.revalidateGrant != nil { + return t.revalidateGrant(ctx, grant) + } + if inner, ok := t.inner.(libraryrelease.GrantRevalidator); ok { + return inner.RevalidateGrant(ctx, grant) + } + return nil } func (t *contentTrackedTarget) Current(ctx context.Context) (libraryrelease.CurrentState, error) { @@ -91,6 +125,20 @@ func (t *contentTrackedTarget) PlanWriteSet(ctx context.Context, plan libraryrel return set, err } +func (t *contentTrackedTarget) AdoptWriteSet(set libraryrelease.WriteSet) error { + adopter, ok := t.inner.(libraryrelease.WriteSetAdopter) + if !ok { + return errors.New("目标实现不支持接管已持久化写集合,拒绝重建计划") + } + if set.JobID != t.jobID { + return errors.New("写集合与 worker 作业身份不符") + } + return adopter.AdoptWriteSet(set) +} + +var _ libraryrelease.WriteSetPlanner = (*contentTrackedTarget)(nil) +var _ libraryrelease.WriteSetAdopter = (*contentTrackedTarget)(nil) + // libraryReleasePayload 是同步与恢复共用的队列载荷形状。 type libraryReleasePayload struct { LibraryJobID string `json:"library_job_id"` @@ -117,19 +165,13 @@ func runLibraryRelease(ctx context.Context, base *postgresstore.Store, config ap if _, admitErr := admitLibraryJob(ctx, base, job, payload.LibraryJobID, payload.Scope, kind, true); admitErr != nil { return admitErr } - // 执行代次:这一次投递是一个确定的执行者身份。所有治理写入都走 writer - // (同名方法带 epoch 守卫),失租的旧执行从此写不进业务状态/维护锁(F13)。 - writer, attemptErr := beginLibraryWriter(ctx, base, payload.LibraryJobID, libraryQueueOwner(config.WorkerID, job, payload.LibraryJobID)) - if attemptErr != nil { - return attemptErr - } - defer func() { - if err != nil { - writer.finish(context.WithoutCancel(ctx), governance.LibraryAttemptFailed, err.Error()) - return - } - writer.finish(context.WithoutCancel(ctx), governance.LibraryAttemptCompleted, "") - }() + return runOwnedLibraryAttempt(ctx, base, config, job, payload.LibraryJobID, func(runCtx context.Context, writer *libraryAttemptWriter) error { + return runLibraryReleaseAttempt(runCtx, writer, config, kind, payload) + }) +} + +func runLibraryReleaseAttempt(ctx context.Context, writer *libraryAttemptWriter, config appconfig.Config, + kind governance.LibraryJobKind, payload libraryReleasePayload) (err error) { store := writer now := time.Now().UTC() _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobRunning, "prebackup", "", "", "", "", now) @@ -320,7 +362,8 @@ func runLibraryRelease(ctx context.Context, base *postgresstore.Store, config ap } pending := successBoundary{} tracked := &contentTrackedTarget{inner: target, jobID: payload.LibraryJobID, - onApply: func() { contentState = contentHalfWritten }} + onApply: func() { contentState = contentHalfWritten }, + revalidateGrant: revalidateLibraryGrant(store.Store)} allowRestore := func(error) bool { return contentState == contentUntouched || contentState == contentSucceeded || contentState == contentRolledBack } @@ -549,6 +592,11 @@ func runLibraryRelease(ctx context.Context, base *postgresstore.Store, config ap // 已经持久化预备份的重新排队(同一个业务作业 id,阶段与备份都接着用), // 连预备份都没完成的才判失败——那种情况没有可回滚的基线,不能盲写。 func resumeInterruptedLibraryJobs(ctx context.Context, store libraryResumeStore, config appconfig.Config) (int, int, error) { + if recoveries, ok := store.(interface{ ResumeExpiredLibraryRecoveryQueues(context.Context) error }); ok { + if err := recoveries.ResumeExpiredLibraryRecoveryQueues(ctx); err != nil { + return 0, 0, err + } + } interrupted, err := store.InterruptedLibraryJobs(ctx) if err != nil { return 0, 0, err diff --git a/cmd/worker/library_lease.go b/cmd/worker/library_lease.go new file mode 100644 index 0000000..de149ed --- /dev/null +++ b/cmd/worker/library_lease.go @@ -0,0 +1,44 @@ +package main + +import ( + "context" + "errors" + "time" +) + +// Run remains synchronous: cancellation is a request, not proof that the last +// network write has stopped. The caller may release ownership only after run +// and its cleanup have actually returned, and the renewal goroutine is joined. +func runWithLibraryLease(parent context.Context, interval time.Duration, renew func(context.Context) error, run func(context.Context) error) error { + ctx, cancel := context.WithCancelCause(parent) + defer cancel(nil) + stop := make(chan struct{}) + done := make(chan error, 1) + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-stop: + done <- nil + return + case <-ctx.Done(): + done <- context.Cause(ctx) + return + case <-ticker.C: + bounded, finish := context.WithTimeout(ctx, interval) + err := renew(bounded) + finish() + if err != nil { + cancel(err) + done <- err + return + } + } + } + }() + err := run(ctx) + close(stop) + leaseErr := <-done + return errors.Join(err, leaseErr, context.Cause(ctx)) +} diff --git a/cmd/worker/library_owned_attempt.go b/cmd/worker/library_owned_attempt.go new file mode 100644 index 0000000..10757e0 --- /dev/null +++ b/cmd/worker/library_owned_attempt.go @@ -0,0 +1,40 @@ +package main + +import ( + "context" + "knowledge-governance/internal/appconfig" + "knowledge-governance/internal/governance" + "knowledge-governance/internal/jobs" + "knowledge-governance/internal/objectgateway" + postgresstore "knowledge-governance/internal/platform/postgres" + "time" +) + +// All target-changing handlers share one execution lock, not one lock per job or +// scope. Lease loss cancels work but cannot release ownership before cleanup joins. +func runOwnedLibraryAttempt(ctx context.Context, base *postgresstore.Store, config appconfig.Config, queue jobs.Job, jobID string, run func(context.Context, *libraryAttemptWriter) error) error { + return base.WithLibraryExecution(ctx, config.LibraryTargetInstance, func(ctx context.Context, ping func(context.Context) error) (err error) { + writer, err := beginLibraryWriter(ctx, base, jobID, libraryQueueOwner(config.WorkerID, queue, jobID)) + if err != nil { + return err + } + defer func() { + finishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + if err != nil { + writer.finish(finishCtx, governance.LibraryAttemptFailed, err.Error()) + } else { + writer.finish(finishCtx, governance.LibraryAttemptCompleted, "") + } + }() + return runWithLibraryLease(ctx, libraryAttemptLease/10, func(probe context.Context) error { + if err := ping(probe); err != nil { + return err + } + return base.RenewLibraryAttempt(probe, writer.attempt, libraryAttemptLease, time.Now().UTC()) + }, func(runCtx context.Context) error { + runCtx = objectgateway.WithClaim(runCtx, objectgateway.Claim{JobID: writer.attempt.JobID, OperationID: writer.attempt.OperationID, AttemptEpoch: int64(writer.attempt.Epoch), Owner: writer.attempt.Owner}) + return run(runCtx, writer) + }) + }) +} diff --git a/cmd/worker/library_recovery.go b/cmd/worker/library_recovery.go index 6db1a93..19c36ee 100644 --- a/cmd/worker/library_recovery.go +++ b/cmd/worker/library_recovery.go @@ -10,10 +10,12 @@ import ( "log" "time" + "bytes" "knowledge-governance/internal/appconfig" "knowledge-governance/internal/governance" "knowledge-governance/internal/jobs" "knowledge-governance/internal/libraryrelease" + "knowledge-governance/internal/objectgateway" postgresstore "knowledge-governance/internal/platform/postgres" v071 "knowledge-governance/internal/releaseadapter/v071" ) @@ -21,8 +23,13 @@ import ( // libraryRecoveryPayload 是"按预备份恢复并校验"的队列载荷。 // 它只说明动哪个窗口;恢复目标快照由 journal 里持久化的那份决定,调用方改不了。 type libraryRecoveryPayload struct { - Scope string `json:"scope"` - WindowJobID string `json:"window_job_id"` + Scope string `json:"scope"` + WindowJobID string `json:"window_job_id"` + SourceSnapshot string `json:"source_snapshot_id,omitempty"` + TargetID string `json:"target_id,omitempty"` + OriginalOperation string `json:"original_operation_id,omitempty"` + BackupSnapshot string `json:"backup_snapshot_id,omitempty"` + PlanDigest string `json:"plan_digest,omitempty"` } // runLibraryRecovery 是"内容风险未验证"窗口的唯一收尾入口: @@ -30,10 +37,15 @@ type libraryRecoveryPayload struct { // 只有校验通过才恢复服务、开入口、释放占用。失败则保持维护。 func runLibraryRecovery(ctx context.Context, store *postgresstore.Store, config appconfig.Config, job jobs.Job) error { var payload libraryRecoveryPayload - if err := json.Unmarshal(job.Payload, &payload); err != nil || + decoder := json.NewDecoder(bytes.NewReader(job.Payload)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&payload); err != nil || (payload.Scope != "internal" && payload.Scope != "external") || payload.WindowJobID == "" { return errors.New("invalid library recovery payload") } + if err := store.ValidateLibraryRecoveryQueue(ctx, job, payload.Scope, payload.WindowJobID); err != nil { + return err + } manager, err := openLibraryMaintenance(config) if err != nil { return fmt.Errorf("目标维护未接线,不能做恢复:%w", err) @@ -62,93 +74,95 @@ func runLibraryRecovery(ctx context.Context, store *postgresstore.Store, config if admitErr != nil { return admitErr } - // 恢复是一次**新执行代次**:它接管这个窗口,旧代次从此不能写业务状态/释放锁(F13)。 - writer, attemptErr := beginLibraryWriter(ctx, store, payload.WindowJobID, libraryQueueOwner(config.WorkerID, job, payload.WindowJobID)) - if attemptErr != nil { - return fmt.Errorf("无法取得恢复窗口的执行权:%w", attemptErr) - } - guarded := writer - prebackup := windowJob.SnapshotID - if prebackup == "" { - return errors.New("原作业没有持久化预备份,无法证明恢复到哪一份") - } - // 恢复是**新意图**(kind=recover):内容更旧、意图更新。把恢复事实挂在失败作业的旧 - // operation 上会被 current 的次序(按意图创建时间)判成迟到事实而拒绝——F06 就治不好。 - recoveryDigest, digestErr := governance.LibraryRequestDigest(payload.Scope, governance.LibraryJobRecover, - map[string]any{"window_job": payload.WindowJobID, "backup_snapshot": prebackup}) - if digestErr != nil { - return fmt.Errorf("无法计算恢复意图摘要:%w", digestErr) - } - recoveryKey := fmt.Sprintf("%s#%d", payload.WindowJobID, writer.attempt.Epoch) - recoveryOp, _, opErr := store.EnsureLibraryOperation(ctx, governance.LibraryOperation{ - ID: governance.LibraryOperationID(payload.Scope, governance.LibraryJobRecover, recoveryKey), - Scope: payload.Scope, Kind: governance.LibraryJobRecover, TargetSnapshotID: prebackup, - RequestDigest: recoveryDigest, OperationKey: recoveryKey, Actor: job.LeaseOwner, - CreatedAt: time.Now().UTC(), + return runOwnedLibraryAttempt(ctx, store, config, job, payload.WindowJobID, func(ctx context.Context, writer *libraryAttemptWriter) error { + guarded := writer + prebackup := windowJob.SnapshotID + if prebackup == "" { + return errors.New("原作业没有持久化预备份,无法证明恢复到哪一份") + } + // 恢复必须带着"这次作业写了什么"的持久记录:没有它就只能按目标现状推断残渣, + // 而那会误删历史孤儿/其它知识库暂未绑定的资源(快照里没有这些行,删了回不来)。 + ledger, ledgerErr := loadWriteSetForJob(ctx, store, config.LibraryArchiveRoot, payload.Scope, + payload.WindowJobID, prebackup) + if ledgerErr != nil { + note := "恢复缺少持久化的写集合记录,目标保持维护:" + ledgerErr.Error() + _ = guarded.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, time.Now().UTC()) + _ = guarded.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.WindowJobID, "needs_restore", note, time.Now().UTC()) + return errors.New(note) + } + for _, pair := range [][2]string{{payload.SourceSnapshot, ledger.TargetSnapshotID}, {payload.TargetID, config.LibraryTargetIdentity()}, {payload.OriginalOperation, windowJob.OperationID}, {payload.BackupSnapshot, prebackup}, {payload.PlanDigest, ledger.Digest}} { + if pair[0] != "" && pair[0] != pair[1] { + return errors.New("recovery supplied identity differs from original immutable binding") + } + } + recoveryDigest, digestErr := governance.LibraryRecoveryDigest(payload.Scope, job.ID, payload.WindowJobID, windowJob.OperationID, config.LibraryTargetIdentity(), ledger.TargetSnapshotID, prebackup, ledger.Digest) + if digestErr != nil { + return digestErr + } + recoveryOp, _, opErr := store.EnsureLibraryOperation(ctx, governance.LibraryOperation{ + ID: governance.LibraryOperationID(payload.Scope, governance.LibraryJobRecover, job.ID), + Scope: payload.Scope, Kind: governance.LibraryJobRecover, TargetSnapshotID: prebackup, + RequestDigest: recoveryDigest, OperationKey: job.ID, Actor: job.LeaseOwner, CreatedAt: job.CreatedAt, + }) + if opErr != nil { + return opErr + } + claim := objectgateway.ClaimFromContext(ctx) + if claim.JobID == "" { + return errors.New("recovery attempt context missing") + } + claim.RecoveryJobID, claim.RecoveryOperationID = job.ID, recoveryOp.ID + ctx = objectgateway.WithClaim(ctx, claim) + knownMapping := knownIndexMapping(ctx, store, payload.Scope) + target, _, wireErr := buildLibraryProductionStack(ctx, config, payload.Scope, prebackup, knownMapping) + if wireErr != nil { + return fmt.Errorf("恢复目标装配失败:%w", wireErr) + } + defer target.Close() + + coordinator, control, openErr := manager.Open(config.LibraryTargetJournalRoot) + if openErr != nil { + return fmt.Errorf("维护协调器不可用:%w", openErr) + } + defer control.Close() + defer coordinator.Close() + + restorer := recoveryRestorer{target: target, expected: prebackup, writeSet: ledger} + verifier := recoveryVerifier{target: target, expected: prebackup} + result, recoverErr := coordinator.RecoverWindow(ctx, payload.WindowJobID, restorer, verifier) + now := time.Now().UTC() + if recoverErr != nil { + note := "恢复/校验未通过,目标保持维护:" + recoverErr.Error() + _ = guarded.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now) + _ = guarded.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.WindowJobID, "needs_restore", note, now) + return recoverErr + } + if result.State != "recovered" { + note := "恢复流程结束但状态不是 recovered:" + result.State + _ = guarded.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now) + return errors.New(note) + } + // 恢复事实必须落成**新事件并推进 current**(F06)。旧实现只释放锁:物理内容已经 + // 是预备份那份,页面/差异/后续作业却仍以为失败的新版本生效,下一步就会拿错基线决策。 + // 事件是追加的,不改写原来那条失败作业与失败版本。 + digest := result.Digest + if digest == "" { + return errors.New("恢复完成但没有校验摘要:拒绝登记恢复事实(无法证明恢复到了哪一份内容)") + } + if err := recordRecoveryEvent(ctx, writer, recoveryOp, payload.Scope, payload.WindowJobID, result.SnapshotID, + digest, "按预备份恢复并逐类校验通过"); err != nil { + return fmt.Errorf("恢复已完成但恢复事实登记失败:%w", err) + } + // 收尾顺序:先登记事实,再释放占用。反过来会出现"锁已放、current 还指向失败版本" + // 的窗口——那正是页面把恢复成功显示成待恢复的状态。 + if err := releaseWindowLocks(ctx, guarded, identity.Instance, payload.Scope, payload.WindowJobID, + "按预备份恢复并逐类校验通过,目标已按原运行状态恢复", now); err != nil { + return fmt.Errorf("恢复已验证,但维护占用释放失败:%w", err) + } + log.Printf("library recovery succeeded window=%s scope=%s backup=%s digest=%s", + payload.WindowJobID, payload.Scope, prebackup, digest) + return nil }) - if opErr != nil { - return fmt.Errorf("无法登记恢复意图:%w", opErr) - } - // 恢复必须带着"这次作业写了什么"的持久记录:没有它就只能按目标现状推断残渣, - // 而那会误删历史孤儿/其它知识库暂未绑定的资源(快照里没有这些行,删了回不来)。 - ledger, ledgerErr := loadWriteSetForJob(ctx, store, config.LibraryArchiveRoot, payload.Scope, - payload.WindowJobID, prebackup) - if ledgerErr != nil { - note := "恢复缺少持久化的写集合记录,目标保持维护:" + ledgerErr.Error() - _ = guarded.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, time.Now().UTC()) - _ = guarded.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.WindowJobID, "needs_restore", note, time.Now().UTC()) - return errors.New(note) - } - knownMapping := knownIndexMapping(ctx, store, payload.Scope) - target, _, wireErr := buildLibraryProductionStack(ctx, config, payload.Scope, prebackup, knownMapping) - if wireErr != nil { - return fmt.Errorf("恢复目标装配失败:%w", wireErr) - } - defer target.Close() - - coordinator, control, openErr := manager.Open(config.LibraryTargetJournalRoot) - if openErr != nil { - return fmt.Errorf("维护协调器不可用:%w", openErr) - } - defer control.Close() - defer coordinator.Close() - - restorer := recoveryRestorer{target: target, expected: prebackup, writeSet: ledger} - verifier := recoveryVerifier{target: target, expected: prebackup} - result, recoverErr := coordinator.RecoverWindow(ctx, payload.WindowJobID, restorer, verifier) - now := time.Now().UTC() - if recoverErr != nil { - note := "恢复/校验未通过,目标保持维护:" + recoverErr.Error() - _ = guarded.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now) - _ = guarded.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.WindowJobID, "needs_restore", note, now) - return recoverErr - } - if result.State != "recovered" { - note := "恢复流程结束但状态不是 recovered:" + result.State - _ = guarded.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now) - return errors.New(note) - } - // 恢复事实必须落成**新事件并推进 current**(F06)。旧实现只释放锁:物理内容已经 - // 是预备份那份,页面/差异/后续作业却仍以为失败的新版本生效,下一步就会拿错基线决策。 - // 事件是追加的,不改写原来那条失败作业与失败版本。 - digest := result.Digest - if digest == "" { - return errors.New("恢复完成但没有校验摘要:拒绝登记恢复事实(无法证明恢复到了哪一份内容)") - } - if err := recordRecoveryEvent(ctx, writer, recoveryOp, payload.Scope, payload.WindowJobID, result.SnapshotID, - digest, "按预备份恢复并逐类校验通过"); err != nil { - return fmt.Errorf("恢复已完成但恢复事实登记失败:%w", err) - } - // 收尾顺序:先登记事实,再释放占用。反过来会出现"锁已放、current 还指向失败版本" - // 的窗口——那正是页面把恢复成功显示成待恢复的状态。 - if err := releaseWindowLocks(ctx, guarded, identity.Instance, payload.Scope, payload.WindowJobID, - "按预备份恢复并逐类校验通过,目标已按原运行状态恢复", now); err != nil { - return fmt.Errorf("恢复已验证,但维护占用释放失败:%w", err) - } - writer.finish(context.WithoutCancel(ctx), governance.LibraryAttemptCompleted, "recovered") - log.Printf("library recovery succeeded window=%s scope=%s backup=%s digest=%s", - payload.WindowJobID, payload.Scope, prebackup, digest) - return nil } // recoveryRestorer 只按"作业记录里那份预备份"恢复:协调器传进来的 id 与它不一致 diff --git a/cmd/worker/library_source_candidate.go b/cmd/worker/library_source_candidate.go new file mode 100644 index 0000000..e5a9395 --- /dev/null +++ b/cmd/worker/library_source_candidate.go @@ -0,0 +1,34 @@ +package main + +import ( + "errors" + "knowledge-governance/internal/appconfig" + "knowledge-governance/internal/governance" + "knowledge-governance/internal/librarysnapshot" +) + +func configuredCaptureSource(config appconfig.Config, scope string) governance.LibrarySourceIdentity { + kb := config.LibrarySourceInternalKB + if scope == "external" { + kb = config.LibrarySourceExternalKB + } + return governance.LibrarySourceIdentity{Project: config.SourceProject(), App: config.LibrarySourceApp, Scope: scope, TenantID: config.LibrarySourceTenantID, KnowledgeBaseID: kb} +} +func verifyCapturedSource(root, snapshotID string, source governance.LibrarySourceIdentity) error { + if _, err := source.ID(); err != nil { + return err + } + archive, err := librarysnapshot.Open(root, 256*1024*1024) + if err != nil { + return err + } + defer archive.Close() + manifest, err := archive.Load(snapshotID) + if err != nil { + return err + } + if manifest.Scope != source.Scope || manifest.TenantID != source.TenantID || manifest.KnowledgeBaseID != source.KnowledgeBaseID { + return errors.New("captured manifest differs from configured source") + } + return nil +} diff --git a/cmd/worker/library_writer.go b/cmd/worker/library_writer.go index 7d02b7f..393d3ba 100644 --- a/cmd/worker/library_writer.go +++ b/cmd/worker/library_writer.go @@ -37,7 +37,7 @@ func beginLibraryWriter(ctx context.Context, store *postgresstore.Store, jobID, // 迁移前的历史作业没有 operation:仍然取租约(防止双执行),但没有代次可守卫。 log.Printf("library job %s has no operation id (pre-identity row): guarded writes fall back to lease only", jobID) } - attempt, err := store.BeginLibraryAttempt(ctx, jobID, owner, 10*time.Minute, time.Now().UTC()) + attempt, err := store.BeginLibraryAttempt(ctx, jobID, owner, libraryAttemptLease, time.Now().UTC()) if err != nil { return nil, err } diff --git a/cmd/worker/main.go b/cmd/worker/main.go index efb3886..4a151cd 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -344,6 +344,11 @@ func workerHandlers(store *postgresstore.Store, config appconfig.Config) (map[st "未停任何写入者;补齐来源身份配置后重试", "", "", time.Now().UTC()) return expectErr } + source := configuredCaptureSource(config, payload.Scope) + sourceID, sourceErr := source.ID() + if sourceErr != nil { + return sourceErr + } result, err := libraryagent.CaptureReleased(ctx, agentConfig, payload.Scope, payload.CaptureJob, released, expectation) finishedAt := time.Now().UTC() if err != nil { @@ -362,18 +367,21 @@ func workerHandlers(store *postgresstore.Store, config appconfig.Config) (map[st "快照已产生但来源身份不符,未登记为测试封存;核对执行器后重新封存", "", "", finishedAt) return identityErr } + if err := verifyCapturedSource(config.LibraryArchiveRoot, result.SnapshotID, source); err != nil { + return err + } // 封存产物成版本记录:只登记"测试封存",绝不写成生产生效版本。 versionErr := store.RecordLibraryVersion(ctx, governance.LibraryVersion{ ID: governance.LibraryVersionID(governance.LibraryVersionTestSealed, payload.Scope, result.SnapshotID), Scope: payload.Scope, Kind: governance.LibraryVersionTestSealed, SnapshotID: result.SnapshotID, SourceJobID: libraryJobID, - Evidence: map[string]any{"capture_job": payload.CaptureJob, "replay": result.Replay, + Evidence: map[string]any{"source_id": sourceID, "source_identity": source, "capture_job": payload.CaptureJob, "replay": result.Replay, "requested_by": payload.RequestedBy, "expectation_fingerprint": expectationFingerprint(expectation), "agent_result": json.RawMessage(result.Payload)}, CreatedBy: payload.RequestedBy, CreatedAt: finishedAt, }) if versionErr != nil { - log.Printf("library seal job %s cannot record version: %v", libraryJobID, versionErr) + return fmt.Errorf("library seal job %s cannot record version: %w", libraryJobID, versionErr) } _ = store.UpdateLibraryStage(ctx, libraryJobID, "export", "succeeded", nil, nil, nil, nil, "已封存快照 "+result.SnapshotID, finishedAt) diff --git a/internal/appconfig/config.go b/internal/appconfig/config.go index a6d4c98..cd071e6 100644 --- a/internal/appconfig/config.go +++ b/internal/appconfig/config.go @@ -81,12 +81,15 @@ type Config struct { LibraryProductionExternalKB string // LibraryProductionS3Provider 必须与生产库 resources.provider 的取值一致: // 对象路径按 provider 路由,写错会在上传时被拒绝(而不是写错桶)。 - LibraryProductionS3Provider string - LibraryProductionS3Endpoint string - LibraryProductionS3Region string - LibraryProductionS3Bucket string - LibraryProductionS3AccessKey string - LibraryProductionS3SecretKey string + LibraryProductionS3Provider string + LibraryProductionS3Endpoint string + LibraryProductionS3Region string + LibraryProductionS3Bucket string + LibraryProductionS3AccessKey string + LibraryProductionS3SecretKey string + LibraryObjectGatewayURL string + LibraryObjectGatewayToken string + LibraryObjectGatewayGrantDirectory string // LibraryEmbeddingModel 是整库发布使用的向量模型声明(快照契约缺失时的回落值)。 LibraryEmbeddingModel string // LibraryWorkerRole 决定这个进程消费哪一段队列(见 LibraryWorkerRole 说明)。 @@ -203,54 +206,57 @@ func Load(getenv func(string) string) (Config, error) { TestAcceptancePolicyVersion: valueOrDefault(getenv("TEST_ACCEPTANCE_POLICY_VERSION"), "structural-v1"), TestAutoReviewPolicy: valueOrDefault(getenv("TEST_AUTO_REVIEW_POLICY"), string(governance.AutoReviewInternal)), // 整库逻辑备份的目标主机:三项同时给出才接线(脚本目录/备份根有默认值)。 - BackupHostSSHConfig: strings.TrimSpace(getenv("BACKUP_HOST_SSH_CONFIG")), - BackupHostTarget: strings.TrimSpace(getenv("BACKUP_HOST_TARGET")), - BackupHostScriptDir: valueOrDefault(strings.TrimSpace(getenv("BACKUP_HOST_SCRIPT_DIR")), "/opt/weknora-test"), - BackupHostRoot: valueOrDefault(strings.TrimSpace(getenv("BACKUP_HOST_ROOT")), "/srv/kb-backups"), - VersionHistoryGiteaBaseURL: strings.TrimSpace(getenv("VERSION_HISTORY_GITEA_BASE_URL")), - VersionHistoryRepo: valueOrDefault(strings.TrimSpace(getenv("VERSION_HISTORY_REPO")), "Weknora/knowledge-governance"), - VersionHistoryAllowedOrigins: splitOrigins(getenv("VERSION_HISTORY_ALLOWED_ORIGINS")), - LibraryArchiveRoot: strings.TrimSpace(getenv("LIBRARY_ARCHIVE_ROOT")), - LibraryAgentSocket: strings.TrimSpace(getenv("LIBRARY_AGENT_SOCKET")), - LibraryProductionDatabaseURL: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_DATABASE_URL")), - LibraryProductionElasticURL: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_ELASTIC_URL")), - LibraryProductionIndex: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_PRODUCTION_INDEX")), "weknora_full_chunks"), - LibraryProductionTenantID: uint64(tenantID), - LibraryProductionInternalKB: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_KB_INTERNAL")), - LibraryProductionExternalKB: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_KB_EXTERNAL")), - LibraryProductionS3Provider: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_PROVIDER")), "s3"), - LibraryProductionS3Endpoint: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_ENDPOINT")), - LibraryProductionS3Region: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_REGION")), "us-east-1"), - LibraryProductionS3Bucket: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_BUCKET")), - LibraryProductionS3AccessKey: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_ACCESS_KEY")), - LibraryProductionS3SecretKey: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_SECRET_KEY")), - LibraryTargetInstance: strings.TrimSpace(getenv("LIBRARY_TARGET_INSTANCE")), - LibrarySourceInstance: strings.TrimSpace(getenv("LIBRARY_SOURCE_INSTANCE")), - LibrarySourceProject: strings.TrimSpace(getenv("LIBRARY_SOURCE_PROJECT")), - LibrarySourceApp: strings.TrimSpace(getenv("LIBRARY_SOURCE_APP")), - LibrarySourceTenantID: uint64(sourceTenantID), - LibrarySourceInternalKB: strings.TrimSpace(getenv("LIBRARY_SOURCE_KB_INTERNAL")), - LibrarySourceExternalKB: strings.TrimSpace(getenv("LIBRARY_SOURCE_KB_EXTERNAL")), - LibraryTargetProject: strings.TrimSpace(getenv("LIBRARY_TARGET_PROJECT")), - LibraryTargetWriters: splitList(getenv("LIBRARY_TARGET_WRITERS")), - LibraryTargetNonWriters: splitDeclarations(getenv("LIBRARY_TARGET_NON_WRITERS")), - LibraryTargetDockerSocket: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_TARGET_DOCKER_SOCKET")), "/var/run/docker.sock"), - LibraryTargetIngressURL: strings.TrimSpace(getenv("LIBRARY_TARGET_INGRESS_URL")), - LibraryTargetIngressService: strings.TrimSpace(getenv("LIBRARY_TARGET_INGRESS_SERVICE")), - LibraryTargetIngressDrainFile: strings.TrimSpace(getenv("LIBRARY_TARGET_INGRESS_DRAIN_FILE")), - LibraryTargetRedisAddr: strings.TrimSpace(getenv("LIBRARY_TARGET_REDIS_ADDR")), - LibraryTargetRedisPassword: strings.TrimSpace(getenv("LIBRARY_TARGET_REDIS_PASSWORD")), - LibraryTargetRedisDB: valueOrDefaultInt(getenv("LIBRARY_TARGET_REDIS_DB"), 0), - LibraryTargetJournalRoot: strings.TrimSpace(getenv("LIBRARY_TARGET_JOURNAL_ROOT")), - LibraryEmbeddingModel: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_EMBEDDING_MODEL")), "text-embedding-v4"), - LibraryEmbeddingDimension: embeddingDimension, - LibraryCapacityBudgetBytes: capacityBudget, - LibraryRetentionVersions: retentionVersions, - LibraryRetentionDays: retentionDays, - LibraryExecutorStatusMaxAge: 2 * time.Minute, - VersionHistoryLinkOrigin: strings.TrimSpace(getenv("VERSION_HISTORY_LINK_ORIGIN")), - VersionHistoryHTTPTimeout: 15 * time.Second, - VersionHistoryMaxPages: valueOrDefaultInt(getenv("VERSION_HISTORY_MAX_PAGES"), 200), + BackupHostSSHConfig: strings.TrimSpace(getenv("BACKUP_HOST_SSH_CONFIG")), + BackupHostTarget: strings.TrimSpace(getenv("BACKUP_HOST_TARGET")), + BackupHostScriptDir: valueOrDefault(strings.TrimSpace(getenv("BACKUP_HOST_SCRIPT_DIR")), "/opt/weknora-test"), + BackupHostRoot: valueOrDefault(strings.TrimSpace(getenv("BACKUP_HOST_ROOT")), "/srv/kb-backups"), + VersionHistoryGiteaBaseURL: strings.TrimSpace(getenv("VERSION_HISTORY_GITEA_BASE_URL")), + VersionHistoryRepo: valueOrDefault(strings.TrimSpace(getenv("VERSION_HISTORY_REPO")), "Weknora/knowledge-governance"), + VersionHistoryAllowedOrigins: splitOrigins(getenv("VERSION_HISTORY_ALLOWED_ORIGINS")), + LibraryArchiveRoot: strings.TrimSpace(getenv("LIBRARY_ARCHIVE_ROOT")), + LibraryAgentSocket: strings.TrimSpace(getenv("LIBRARY_AGENT_SOCKET")), + LibraryProductionDatabaseURL: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_DATABASE_URL")), + LibraryProductionElasticURL: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_ELASTIC_URL")), + LibraryProductionIndex: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_PRODUCTION_INDEX")), "weknora_full_chunks"), + LibraryProductionTenantID: uint64(tenantID), + LibraryProductionInternalKB: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_KB_INTERNAL")), + LibraryProductionExternalKB: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_KB_EXTERNAL")), + LibraryProductionS3Provider: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_PROVIDER")), "s3"), + LibraryProductionS3Endpoint: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_ENDPOINT")), + LibraryProductionS3Region: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_REGION")), "us-east-1"), + LibraryProductionS3Bucket: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_BUCKET")), + LibraryProductionS3AccessKey: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_ACCESS_KEY")), + LibraryProductionS3SecretKey: strings.TrimSpace(getenv("LIBRARY_PRODUCTION_S3_SECRET_KEY")), + LibraryObjectGatewayURL: strings.TrimSpace(getenv("LIBRARY_OBJECT_GATEWAY_URL")), + LibraryObjectGatewayToken: strings.TrimSpace(getenv("LIBRARY_OBJECT_GATEWAY_TOKEN")), + LibraryObjectGatewayGrantDirectory: strings.TrimSpace(getenv("LIBRARY_OBJECT_GATEWAY_GRANT_DIRECTORY")), + LibraryTargetInstance: strings.TrimSpace(getenv("LIBRARY_TARGET_INSTANCE")), + LibrarySourceInstance: strings.TrimSpace(getenv("LIBRARY_SOURCE_INSTANCE")), + LibrarySourceProject: strings.TrimSpace(getenv("LIBRARY_SOURCE_PROJECT")), + LibrarySourceApp: strings.TrimSpace(getenv("LIBRARY_SOURCE_APP")), + LibrarySourceTenantID: uint64(sourceTenantID), + LibrarySourceInternalKB: strings.TrimSpace(getenv("LIBRARY_SOURCE_KB_INTERNAL")), + LibrarySourceExternalKB: strings.TrimSpace(getenv("LIBRARY_SOURCE_KB_EXTERNAL")), + LibraryTargetProject: strings.TrimSpace(getenv("LIBRARY_TARGET_PROJECT")), + LibraryTargetWriters: splitList(getenv("LIBRARY_TARGET_WRITERS")), + LibraryTargetNonWriters: splitDeclarations(getenv("LIBRARY_TARGET_NON_WRITERS")), + LibraryTargetDockerSocket: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_TARGET_DOCKER_SOCKET")), "/var/run/docker.sock"), + LibraryTargetIngressURL: strings.TrimSpace(getenv("LIBRARY_TARGET_INGRESS_URL")), + LibraryTargetIngressService: strings.TrimSpace(getenv("LIBRARY_TARGET_INGRESS_SERVICE")), + LibraryTargetIngressDrainFile: strings.TrimSpace(getenv("LIBRARY_TARGET_INGRESS_DRAIN_FILE")), + LibraryTargetRedisAddr: strings.TrimSpace(getenv("LIBRARY_TARGET_REDIS_ADDR")), + LibraryTargetRedisPassword: strings.TrimSpace(getenv("LIBRARY_TARGET_REDIS_PASSWORD")), + LibraryTargetRedisDB: valueOrDefaultInt(getenv("LIBRARY_TARGET_REDIS_DB"), 0), + LibraryTargetJournalRoot: strings.TrimSpace(getenv("LIBRARY_TARGET_JOURNAL_ROOT")), + LibraryEmbeddingModel: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_EMBEDDING_MODEL")), "text-embedding-v4"), + LibraryEmbeddingDimension: embeddingDimension, + LibraryCapacityBudgetBytes: capacityBudget, + LibraryRetentionVersions: retentionVersions, + LibraryRetentionDays: retentionDays, + LibraryExecutorStatusMaxAge: 2 * time.Minute, + VersionHistoryLinkOrigin: strings.TrimSpace(getenv("VERSION_HISTORY_LINK_ORIGIN")), + VersionHistoryHTTPTimeout: 15 * time.Second, + VersionHistoryMaxPages: valueOrDefaultInt(getenv("VERSION_HISTORY_MAX_PAGES"), 200), } if config.DatabaseURL == "" { return Config{}, errors.New("DATABASE_URL is required") diff --git a/internal/governance/library_dispatch.go b/internal/governance/library_dispatch.go new file mode 100644 index 0000000..034be0b --- /dev/null +++ b/internal/governance/library_dispatch.go @@ -0,0 +1,16 @@ +package governance + +import "encoding/json" + +func LibraryQueueRequest(handoff bool, jobID, scope, snapshotID, releaseType string) (string, string, []byte, error) { + payload := map[string]string{"library_job_id": jobID, "scope": scope, "target_snapshot_id": snapshotID} + queueID := QueueJobIDFor(jobID) + queueType := releaseType + if handoff { + payload["next_type"] = releaseType + queueID = LibraryHandoffQueueJobID(jobID) + queueType = "library.snapshot.handoff" + } + raw, err := json.Marshal(payload) + return queueID, queueType, raw, err +} diff --git a/internal/governance/library_recovery_identity.go b/internal/governance/library_recovery_identity.go new file mode 100644 index 0000000..5f13a07 --- /dev/null +++ b/internal/governance/library_recovery_identity.go @@ -0,0 +1,20 @@ +package governance + +import ( + "crypto/sha256" + "encoding/hex" +) + +// The recovery request has its own queue identity and stable operation. It +// borrows the original window's fence, never its immutable sending identity. +func LibraryRecoveryQueueID(windowJob string) string { + sum := sha256.Sum256([]byte(windowJob)) + return "lrec-" + hex.EncodeToString(sum[:14]) +} + +func LibraryRecoveryDigest(scope, queue, window, operation, target, source, backup, plan string) (string, error) { + return LibraryRequestDigest(scope, LibraryJobRecover, map[string]any{ + "recovery_queue": queue, "window_job": window, "original_operation": operation, + "target_id": target, "source_snapshot": source, "backup_snapshot": backup, "writeset_digest": plan, + }) +} diff --git a/internal/governance/library_source_identity.go b/internal/governance/library_source_identity.go new file mode 100644 index 0000000..2e29a4c --- /dev/null +++ b/internal/governance/library_source_identity.go @@ -0,0 +1,30 @@ +package governance + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" +) + +// SourceIdentity binds an immutable capture to its configured origin, not to a +// target or a page of mixed version history. It does not modify snapshot hashes. +type LibrarySourceIdentity struct { + Project string `json:"project"` + App string `json:"app"` + Scope string `json:"scope"` + TenantID uint64 `json:"tenant_id"` + KnowledgeBaseID string `json:"knowledge_base_id"` +} + +func (s LibrarySourceIdentity) ID() (string, error) { + if s.Project == "" || s.App == "" || (s.Scope != "internal" && s.Scope != "external") || s.TenantID == 0 || s.KnowledgeBaseID == "" { + return "", errors.New("source project/app/scope/tenant/knowledge base binding incomplete") + } + raw, err := json.Marshal(s) + if err != nil { + return "", err + } + sum := sha256.Sum256(raw) + return hex.EncodeToString(sum[:]), nil +} diff --git a/internal/httpapi/auth.go b/internal/httpapi/auth.go index a9f4aec..5e372f5 100644 --- a/internal/httpapi/auth.go +++ b/internal/httpapi/auth.go @@ -99,7 +99,7 @@ func isAuthEndpoint(method, path string) bool { // classify maps a request to the minimum role required. func classifyRole(method, path string) string { - if path == "/api/v1/source-connections/audit" { + if path == "/api/v1/source-connections/audit" || path == "/api/v1/library/source-associations" { return roleAdmin } // The member roster exposes account identities and roles: admin-only for diff --git a/internal/httpapi/library_baseline.go b/internal/httpapi/library_baseline.go index f93dbae..128916e 100644 --- a/internal/httpapi/library_baseline.go +++ b/internal/httpapi/library_baseline.go @@ -9,7 +9,6 @@ import ( "time" "knowledge-governance/internal/governance" - "knowledge-governance/internal/jobs" postgresstore "knowledge-governance/internal/platform/postgres" ) @@ -107,12 +106,12 @@ func libraryBaselineEnqueue(w http.ResponseWriter, r *http.Request, config Confi return } now := time.Now().UTC() - job, reused, err := store.CreateLibraryJob(r.Context(), governance.LibraryJob{ + job, reused, err := store.CreateLibraryDispatch(r.Context(), governance.LibraryJob{ ID: fmt.Sprintf("libbase-%s-%d", request.Scope, now.UnixNano()), Scope: request.Scope, Kind: governance.LibraryJobBaseline, IdempotencyKey: "library.baseline:" + request.Scope + ":" + now.Format("20060102T150405.000000000"), RequestDigest: digest, Actor: identity.Username, - }, governance.BaselineStages(), now) + }, governance.BaselineStages(), false, now) if errors.Is(err, governance.ErrLibraryJobActive) { writeJSON(w, http.StatusConflict, map[string]any{"code": "library_write_active", "error": "该目标已有在飞的写作业(同步/恢复/基线互斥),等它结束再发起"}) @@ -131,26 +130,6 @@ func libraryBaselineEnqueue(w http.ResponseWriter, r *http.Request, config Confi writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": true}) return } - payload, err := json.Marshal(map[string]string{"library_job_id": job.ID, "scope": request.Scope}) - if err != nil { - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot build baseline payload"}) - return - } - queueJob, err := config.Store.Enqueue(r.Context(), jobs.EnqueueRequest{ - ID: governance.QueueJobIDFor(job.ID), Type: libraryBaselineJobType, Payload: payload, - AvailableAt: now, MaxAttempts: 1, IdempotencyKey: "library.baseline.queue:" + job.ID, - }, now) - if err != nil || queueJob == nil { - // 作业先落库、队列投递失败就把作业取消:不留下"看起来在跑"的空作业, - // 否则它会一直占着该 scope 的写互斥。 - _ = store.CancelLibraryJob(r.Context(), job.ID, "队列投递失败,未开始建立基线", now) - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot enqueue baseline job"}) - return - } - if attachErr := store.AttachLibraryQueueJob(r.Context(), job.ID, queueJob.ID, now); attachErr != nil { - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot attach baseline job"}) - return - } writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": false, "target_counts": counts}) } diff --git a/internal/httpapi/library_flow.go b/internal/httpapi/library_flow.go index 49c72a9..b5df92f 100644 --- a/internal/httpapi/library_flow.go +++ b/internal/httpapi/library_flow.go @@ -2,12 +2,9 @@ package httpapi import ( "context" - "crypto/sha256" - "encoding/hex" "encoding/json" "errors" "fmt" - "log" "net/http" "strings" "time" @@ -360,6 +357,7 @@ type libraryVersionRef struct { } type libraryJobStore interface { + CreateLibraryDispatch(context.Context, governance.LibraryJob, []string, bool, time.Time) (governance.LibraryJob, bool, error) CreateLibraryJob(ctx context.Context, job governance.LibraryJob, stages []string, now time.Time) (governance.LibraryJob, bool, error) // EnsureLibraryOperation 幂等登记"一次明确意图":同键+同摘要 ⇒ 同一次意图(客户端重放), // 换键 ⇒ 新意图。作业是意图的执行代次,不能反过来用"历史成功作业"回答新意图(F01)。 @@ -470,8 +468,7 @@ func libraryMaintenanceRecoverHandler(w http.ResponseWriter, r *http.Request, co } // 队列作业 id 与幂等键都按窗口固定:重复点击只会得到同一个恢复作业。 // 队列 id 有长度上限(36),窗口作业 id 比它还长,所以按窗口摘要取短 id。 - sum := sha256.Sum256([]byte(held.OwnerJobID)) - queueID := "lrec-" + hex.EncodeToString(sum[:14]) + queueID := governance.LibraryRecoveryQueueID(held.OwnerJobID) queueJob, err := config.Store.Enqueue(r.Context(), jobs.EnqueueRequest{ ID: queueID, Type: libraryRecoverJobType, Payload: payload, AvailableAt: now, MaxAttempts: 1, IdempotencyKey: "library.recover:" + held.OwnerJobID, @@ -727,7 +724,11 @@ func libraryOverviewHandler(w http.ResponseWriter, r *http.Request, config Confi return } if archiveReadable { - overview.TestSealed, overview.TestSealedReason = testSealedOf(versions, byID) + var candidateErr error + overview.TestSealed, candidateErr = currentSourceCandidate(r.Context(), config, scope) + if candidateErr != nil { + overview.TestSealedReason = candidateErr.Error() + } } overview.Production = libraryKnownView{State: "unknown", Reason: "还没有生产验证通过的生效版本记录"} overview.Prebackup = libraryKnownView{State: "unknown", Reason: "这个范围还没有任何生产预备份"} @@ -873,6 +874,7 @@ func librarySyncEnqueue(w http.ResponseWriter, r *http.Request, config Config) { // OperationKey 是客户端重放键:同一次意图重发要命中同一个 operation。 // 不传就是一次新意图(同一个快照再发布一次也必须真的执行)。 OperationKey string `json:"operation_key"` + SourceID string `json:"source_id"` } if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&request); err != nil { writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"}) @@ -939,6 +941,11 @@ func librarySyncEnqueue(w http.ResponseWriter, r *http.Request, config Config) { return } + candidate, candidateErr := currentSourceCandidate(r.Context(), config, request.Scope) + if candidateErr != nil || request.SourceID == "" || candidate == nil || candidate.SourceID != request.SourceID || candidate.SnapshotID != request.Target { + writeJSON(w, http.StatusConflict, map[string]any{"code": "library_source_candidate_changed", "error": "来源候选已变化或身份未绑定,请刷新后重试", "detail": errString(candidateErr)}) + return + } // 走到这里说明闸口与证据都齐了才真正建作业;互斥与幂等由数据库索引保证。 digest, err := governance.LibraryRequestDigest(request.Scope, governance.LibraryJobSync, map[string]any{"target_snapshot_id": request.Target}) @@ -965,11 +972,11 @@ func librarySyncEnqueue(w http.ResponseWriter, r *http.Request, config Config) { writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot register library intent"}) return } - job, reused, err := store.CreateLibraryJob(r.Context(), governance.LibraryJob{ + job, reused, err := store.CreateLibraryDispatch(r.Context(), governance.LibraryJob{ ID: jobID, Scope: request.Scope, Kind: governance.LibraryJobSync, IdempotencyKey: intent.JobKey, RequestDigest: digest, OperationID: intent.Operation.ID, TargetSnapshotID: request.Target, Actor: actor, - }, governance.SyncStages(), now) + }, governance.SyncStages(), config.LibraryHandoffEnabled, now) if errors.Is(err, governance.ErrLibraryJobActive) { writeJSON(w, http.StatusConflict, map[string]any{"code": "library_sync_active", "error": "该目标已有在飞的生产写作业(同步与恢复互斥)"}) @@ -986,13 +993,13 @@ func librarySyncEnqueue(w http.ResponseWriter, r *http.Request, config Config) { // 上一次同意图请求失败/被取消后,幂等键不能再把新请求塞回那个死作业里: // 在**同一个 operation** 下开一个新的执行作业(重试代次),旧作业保留为审计记录。 if reused && (job.State == governance.LibraryJobFailed || job.State == governance.LibraryJobCancelled) { - job, reused, err = store.CreateLibraryJob(r.Context(), governance.LibraryJob{ + job, reused, err = store.CreateLibraryDispatch(r.Context(), governance.LibraryJob{ ID: fmt.Sprintf("libsync-%s-%d", request.Scope, time.Now().UTC().UnixNano()), Scope: request.Scope, Kind: governance.LibraryJobSync, IdempotencyKey: intent.JobKey + ":retry:" + now.Format("20060102T150405.000000000"), RequestDigest: digest, OperationID: intent.Operation.ID, TargetSnapshotID: request.Target, Actor: actor, - }, governance.SyncStages(), now) + }, governance.SyncStages(), config.LibraryHandoffEnabled, now) if err != nil || reused { writeJSON(w, http.StatusConflict, map[string]any{"code": "library_sync_retry_conflict", "error": "无法为重试创建新作业", "detail": errString(err)}) @@ -1003,27 +1010,6 @@ func librarySyncEnqueue(w http.ResponseWriter, r *http.Request, config Config) { writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": true}) return } - // 业务作业先落库,再投递队列:投递失败就把作业标失败,不留下"看起来在跑"的空作业。 - queueID, queueType, payload, err := libraryQueueRequest(config.LibraryHandoffEnabled, job.ID, - request.Scope, request.Target, librarySyncJobType) - if err != nil { - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot build sync payload"}) - return - } - queueJob, err := config.Store.Enqueue(r.Context(), jobs.EnqueueRequest{ - ID: queueID, Type: queueType, Payload: payload, AvailableAt: now, - MaxAttempts: 1, IdempotencyKey: queueType + ".queue:" + job.ID, - }, now) - if err != nil || queueJob == nil { - log.Printf("library sync enqueue failed job=%s: %v", job.ID, err) - _ = store.CancelLibraryJob(r.Context(), job.ID, "队列投递失败,未开始同步", now) - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot enqueue sync job"}) - return - } - if attachErr := store.AttachLibraryQueueJob(r.Context(), job.ID, queueJob.ID, now); attachErr != nil { - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot attach sync job"}) - return - } writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": false}) } @@ -1033,17 +1019,5 @@ func librarySyncEnqueue(w http.ResponseWriter, r *http.Request, config Config) { // 由它在成功后再入 releaseType;搬运未接线时保持原样。两种情况下 releaseType // 作业的 ID 都是 QueueJobIDFor(jobID),所以接管/续作的既有路径不受影响。 func libraryQueueRequest(handoff bool, jobID, scope, snapshotID, releaseType string) (string, string, []byte, error) { - payload := map[string]string{"library_job_id": jobID, "scope": scope, "target_snapshot_id": snapshotID} - queueID := governance.QueueJobIDFor(jobID) - queueType := releaseType - if handoff { - payload["next_type"] = releaseType - queueID = governance.LibraryHandoffQueueJobID(jobID) - queueType = libraryHandoffJobType - } - raw, err := json.Marshal(payload) - if err != nil { - return "", "", nil, err - } - return queueID, queueType, raw, nil + return governance.LibraryQueueRequest(handoff, jobID, scope, snapshotID, releaseType) } diff --git a/internal/httpapi/library_legacy_source.go b/internal/httpapi/library_legacy_source.go new file mode 100644 index 0000000..1beedcd --- /dev/null +++ b/internal/httpapi/library_legacy_source.go @@ -0,0 +1,48 @@ +package httpapi + +import ( + "context" + "encoding/json" + "io" + "knowledge-governance/internal/governance" + "net/http" + "time" +) + +type libraryLegacySourceBinder interface { + BindLegacyLibrarySource(context.Context, string, string, governance.LibrarySourceIdentity, string, time.Time) error +} + +func libraryLegacySourceBind(w http.ResponseWriter, r *http.Request, config Config) { + identity, ok := sessionFrom(r.Context()) + if !ok || identity.Role != roleAdmin || identity.Username == "" { + writeJSON(w, http.StatusForbidden, map[string]string{"error": "administrator session required"}) + return + } + var input struct { + Scope string `json:"scope"` + SnapshotID string `json:"snapshot_id"` + } + decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)) + decoder.DisallowUnknownFields() + if decoder.Decode(&input) != nil || decoder.Decode(new(any)) != io.EOF || (input.Scope != "internal" && input.Scope != "external") || input.SnapshotID == "" { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "exact snapshot and scope required"}) + return + } + source := configuredLibrarySource(config, input.Scope) + sourceID, err := source.ID() + if err != nil { + writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": err.Error()}) + return + } + store, ok := config.Store.(libraryLegacySourceBinder) + if !ok { + writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "source association store unavailable"}) + return + } + if err = store.BindLegacyLibrarySource(r.Context(), input.SnapshotID, config.LibraryArchiveRoot, source, identity.Username, time.Now().UTC()); err != nil { + writeJSON(w, http.StatusConflict, map[string]string{"error": err.Error()}) + return + } + writeJSON(w, http.StatusOK, map[string]any{"snapshot_id": input.SnapshotID, "source_id": sourceID, "source_identity": source, "associated": true, "published": false}) +} diff --git a/internal/httpapi/library_restore.go b/internal/httpapi/library_restore.go index f7d1578..d1eca63 100644 --- a/internal/httpapi/library_restore.go +++ b/internal/httpapi/library_restore.go @@ -104,12 +104,12 @@ func libraryRestoreEnqueue(w http.ResponseWriter, r *http.Request, config Config writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot register restore intent"}) return } - job, reused, err := store.CreateLibraryJob(r.Context(), governance.LibraryJob{ + job, reused, err := store.CreateLibraryDispatch(r.Context(), governance.LibraryJob{ ID: jobID, Scope: request.Scope, Kind: governance.LibraryJobRestore, IdempotencyKey: intent.JobKey, RequestDigest: digest, OperationID: intent.Operation.ID, TargetSnapshotID: request.SnapshotID, Actor: identity.Username, - }, governance.SyncStages(), now) + }, governance.SyncStages(), config.LibraryHandoffEnabled, now) if errors.Is(err, governance.ErrLibraryJobActive) { writeJSON(w, http.StatusConflict, map[string]any{"code": "library_restore_active", "error": "该目标已有在飞作业"}) return @@ -123,12 +123,12 @@ func libraryRestoreEnqueue(w http.ResponseWriter, r *http.Request, config Config return } if reused && (job.State == governance.LibraryJobFailed || job.State == governance.LibraryJobCancelled) { - job, reused, err = store.CreateLibraryJob(r.Context(), governance.LibraryJob{ + job, reused, err = store.CreateLibraryDispatch(r.Context(), governance.LibraryJob{ ID: jobID + "-retry", Scope: request.Scope, Kind: governance.LibraryJobRestore, IdempotencyKey: intent.JobKey + ":retry:" + strconv.FormatInt(now.UnixNano(), 10), RequestDigest: digest, OperationID: intent.Operation.ID, TargetSnapshotID: request.SnapshotID, Actor: identity.Username, - }, governance.SyncStages(), now) + }, governance.SyncStages(), config.LibraryHandoffEnabled, now) if err != nil || reused { writeJSON(w, http.StatusConflict, map[string]any{"code": "library_restore_retry_conflict", "error": "无法为重试创建新作业"}) return @@ -138,25 +138,6 @@ func libraryRestoreEnqueue(w http.ResponseWriter, r *http.Request, config Config writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": true}) return } - queueID, queueType, payload, err := libraryQueueRequest(config.LibraryHandoffEnabled, job.ID, - request.Scope, request.SnapshotID, libraryRestoreJobType) - if err != nil { - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot build restore payload"}) - return - } - queueJob, err := config.Store.Enqueue(r.Context(), jobs.EnqueueRequest{ - ID: queueID, Type: queueType, Payload: payload, AvailableAt: now, - MaxAttempts: 1, IdempotencyKey: queueType + ".queue:" + job.ID, - }, now) - if err != nil || queueJob == nil { - _ = store.CancelLibraryJob(r.Context(), job.ID, "队列投递失败,未开始恢复", now) - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot enqueue restore job"}) - return - } - if attachErr := store.AttachLibraryQueueJob(r.Context(), job.ID, queueJob.ID, now); attachErr != nil { - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot attach restore job"}) - return - } writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": false}) } diff --git a/internal/httpapi/library_snapshots.go b/internal/httpapi/library_snapshots.go index bbdab31..12f1f63 100644 --- a/internal/httpapi/library_snapshots.go +++ b/internal/httpapi/library_snapshots.go @@ -8,6 +8,7 @@ import ( "strings" "time" + "knowledge-governance/internal/governance" "knowledge-governance/internal/librarysnapshot" ) @@ -28,19 +29,21 @@ type librarySnapshotPart struct { } type librarySnapshotView struct { - SnapshotID string `json:"snapshot_id"` - Scope string `json:"scope"` - KnowledgeBaseID string `json:"knowledge_base_id"` - TenantID uint64 `json:"tenant_id"` - CreatedAt time.Time `json:"created_at"` - ValidationPolicy string `json:"validation_policy"` - Format string `json:"format"` - Compatibility librarysnapshot.Compatibility `json:"compatibility"` - Parts []librarySnapshotPart `json:"parts"` - LogicalBytes int64 `json:"logical_bytes"` - UniqueBytes int64 `json:"unique_bytes"` - SharedBytes int64 `json:"shared_bytes"` - Totals map[string]int64 `json:"totals"` + SourceID string `json:"source_id,omitempty"` + SourceIdentity *governance.LibrarySourceIdentity `json:"source_identity,omitempty"` + SnapshotID string `json:"snapshot_id"` + Scope string `json:"scope"` + KnowledgeBaseID string `json:"knowledge_base_id"` + TenantID uint64 `json:"tenant_id"` + CreatedAt time.Time `json:"created_at"` + ValidationPolicy string `json:"validation_policy"` + Format string `json:"format"` + Compatibility librarysnapshot.Compatibility `json:"compatibility"` + Parts []librarySnapshotPart `json:"parts"` + LogicalBytes int64 `json:"logical_bytes"` + UniqueBytes int64 `json:"unique_bytes"` + SharedBytes int64 `json:"shared_bytes"` + Totals map[string]int64 `json:"totals"` // Verified 只在这一份版本的每个部件都被重新哈希并核对通过时为 true。 // 列表口径下它恒为 false:枚举只证明清单可读,不是"完整性已验证"。 Verified bool `json:"verified"` diff --git a/internal/httpapi/library_source_candidate.go b/internal/httpapi/library_source_candidate.go new file mode 100644 index 0000000..23189bc --- /dev/null +++ b/internal/httpapi/library_source_candidate.go @@ -0,0 +1,58 @@ +package httpapi + +import ( + "context" + "errors" + "knowledge-governance/internal/governance" + "knowledge-governance/internal/librarysnapshot" +) + +type librarySourceCandidateStore interface { + LibrarySourceCandidate(context.Context, string, string) (governance.LibraryVersion, bool, error) +} + +func configuredLibrarySource(config Config, scope string) governance.LibrarySourceIdentity { + kb := config.LibrarySourceInternalKB + if scope == "external" { + kb = config.LibrarySourceExternalKB + } + return governance.LibrarySourceIdentity{Project: config.LibrarySourceProject, App: config.LibrarySourceApp, Scope: scope, TenantID: config.LibrarySourceTenantID, KnowledgeBaseID: kb} +} + +func currentSourceCandidate(ctx context.Context, config Config, scope string) (*librarySnapshotView, error) { + source := configuredLibrarySource(config, scope) + id, err := source.ID() + if err != nil { + return nil, err + } + store, ok := config.Store.(librarySourceCandidateStore) + if !ok { + return nil, errors.New("source candidate exact query unavailable") + } + candidate, found, err := store.LibrarySourceCandidate(ctx, scope, id) + if err != nil { + return nil, err + } + if !found { + return nil, errors.New("当前来源没有已绑定封存;旧快照须以原捕获证据核验关联,不能继承其他来源") + } + if candidate.Kind != governance.LibraryVersionTestSealed || candidate.Scope != scope || candidate.Evidence["source_id"] != id { + return nil, errors.New("candidate source binding mismatch") + } + archive, err := librarysnapshot.Open(config.LibraryArchiveRoot, 256*1024*1024) + if err != nil { + return nil, err + } + defer archive.Close() + manifest, err := archive.Load(candidate.SnapshotID) + if err != nil { + return nil, err + } + if manifest.Scope != scope || manifest.TenantID != source.TenantID || manifest.KnowledgeBaseID != source.KnowledgeBaseID { + return nil, errors.New("candidate manifest differs from configured source") + } + view := librarySnapshotViewOfNoUsage(candidate.SnapshotID, manifest) + view.SourceID = id + view.SourceIdentity = &source + return &view, nil +} diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index f852253..f6f6d1c 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -167,7 +167,11 @@ type Config struct { LibraryAgentSocket string // LibrarySourceProject 是执行器应指向的实例名(agent -project,封存源)。 // 与它不一致时按"封错库"处理:不可发起,并说明差在哪。 - LibrarySourceProject string + LibrarySourceProject string + LibrarySourceApp string + LibrarySourceTenantID uint64 + LibrarySourceInternalKB string + LibrarySourceExternalKB string // LibraryExecutorStatusMaxAge 是执行器观测的有效期;超过即"无法确认"。 LibraryExecutorStatusMaxAge time.Duration // LibraryCapacityBudgetBytes 是整库归档容量预算;0 = 未设置(容量门禁不放行)。 @@ -449,6 +453,9 @@ func NewHandler(config Config) (http.Handler, error) { librarySnapshotDetail(w, r, config) }) // 页面发起封存:只投递作业(主机侧 agent 执行),并回读作业真实状态。 + mux.HandleFunc("POST /api/v1/library/source-associations", func(w http.ResponseWriter, r *http.Request) { + libraryLegacySourceBind(w, r, config) + }) mux.HandleFunc("POST /api/v1/library/captures", func(w http.ResponseWriter, r *http.Request) { libraryCaptureEnqueue(w, r, config) }) diff --git a/internal/libraryrelease/index_plan.go b/internal/libraryrelease/index_plan.go new file mode 100644 index 0000000..aa826e8 --- /dev/null +++ b/internal/libraryrelease/index_plan.go @@ -0,0 +1,14 @@ +package libraryrelease + +import "encoding/json" + +// IndexWrite freezes the exact index/ID, original bytes and CAS generation. +// After=nil is an explicit removal, never a scope-wide delete-by-query. +type IndexWrite struct { + Index string `json:"index"` + ID string `json:"id"` + Before json.RawMessage `json:"before,omitempty"` + SeqNo *int64 `json:"seq_no,omitempty"` + PrimaryTerm *int64 `json:"primary_term,omitempty"` + After json.RawMessage `json:"after,omitempty"` +} diff --git a/internal/libraryrelease/object_plan.go b/internal/libraryrelease/object_plan.go new file mode 100644 index 0000000..4e7fc27 --- /dev/null +++ b/internal/libraryrelease/object_plan.go @@ -0,0 +1,18 @@ +package libraryrelease + +// ObjectWrite identifies a physical object, not a row ID or a guessed orphan. +// Existing objects are read-only. This protocol refuses differing overwrites. +type ObjectWrite struct { + BeforeVersionID string `json:"before_version_id,omitempty"` + ResourceID string `json:"resource_id"` + Provider string `json:"provider"` + BackendID string `json:"backend_id"` + Bucket string `json:"bucket"` + Key string `json:"key"` + Locator string `json:"locator"` + SHA256 string `json:"sha256"` + Bytes int64 `json:"bytes"` + Existing bool `json:"existing"` + BeforeETag string `json:"before_etag,omitempty"` + OwnerToken string `json:"owner_token"` +} diff --git a/internal/libraryrelease/release.go b/internal/libraryrelease/release.go index 7f30fd5..d298a11 100644 --- a/internal/libraryrelease/release.go +++ b/internal/libraryrelease/release.go @@ -115,6 +115,16 @@ type Recorder interface { Stage(ctx context.Context, stage string, state string, detail string) } +// GrantRevalidator 让编排在每笔副作用前按**持久授权**回读。 +// +// 内存里的 Grant 只是一个快照:它既不能延长有效期,也不能掩盖中途撤销。 +// 目标实现了这个接口就必须给出权威答案(读治理库的授权行);返回错误即拒绝继续。 +// 未实现时行为不变(只核静态有效期),所以这不是给"忘记接线"留的后门 —— +// 生产目标的 wrapper 由 runLibraryRelease 无条件接线。 +type GrantRevalidator interface { + RevalidateGrant(ctx context.Context, grant Grant) error +} + // BackupObserver 让调用方在预备份完成时立刻持久化快照身份: // worker 被杀后重启必须能接管这份预备份,而不是重新备份覆盖它。 type BackupObserver interface { @@ -184,6 +194,8 @@ func Run(ctx context.Context, request Request, target Target, backups BackupStor if recorder == nil { recorder = NoopRecorder{} } + // Request.Now is a diagnostic input, not a clock authority for execution. + request.Now = time.Now().UTC() if err := ValidateGrant(request); err != nil { outcome.Error = err.Error() recorder.Stage(ctx, "prebackup", "failed", err.Error()) @@ -217,6 +229,33 @@ func Run(ctx context.Context, request Request, target Target, backups BackupStor } recorder.Stage(ctx, "prebackup", "succeeded", "预备份 "+backup.ID) + // Backup may outlive the grant. Recheck before planning (which may allocate + // numbers), and again after durable plan registration before the first apply. + // With a GrantRevalidator the check is authoritative: a revoked or expired + // authorization row stops the run even though the in-memory grant still looks + // valid, and a stale in-memory expiry cannot be extended either. + checkLiveGrant := func() bool { + request.Now = time.Now().UTC() + if e := ValidateGrant(request); e != nil { + outcome.Error = e.Error() + outcome.Recovery = "预备份保留;需要有效的新授权后继续" + recorder.Stage(ctx, "prebackup", "failed", outcome.Error) + return false + } + if revalidator, ok := target.(GrantRevalidator); ok { + if e := revalidator.RevalidateGrant(ctx, request.Grant); e != nil { + outcome.Error = "持久授权已失效,未继续写入:" + e.Error() + outcome.Recovery = "预备份保留;重新签发授权后按原预备份继续" + recorder.Stage(ctx, "prebackup", "failed", outcome.Error) + return false + } + } + return true + } + if !checkLiveGrant() { + return outcome + } + // 写集合必须在第一笔写入之前落盘并登记:恢复只按"这次明确写了什么"清理, // 不按目标现状推断残渣(那会把历史孤儿/共享资源当成本次残留删掉)。 existingPlan := backup.WriteSet != nil @@ -265,6 +304,9 @@ func Run(ctx context.Context, request Request, target Target, backups BackupStor } } + if !checkLiveGrant() { + return outcome + } for _, stage := range Stages() { recorder.Stage(ctx, string(stage), "running", "") if err = target.Apply(ctx, stage, request.SnapshotID); err != nil { diff --git a/internal/libraryrelease/writeset.go b/internal/libraryrelease/writeset.go index b740ade..3d64067 100644 --- a/internal/libraryrelease/writeset.go +++ b/internal/libraryrelease/writeset.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "sort" + "strconv" "time" ) @@ -15,6 +16,9 @@ import ( // 宁可保持维护,也不要用一份"格式变了但看起来还能读"的记录去删目标数据。 const WriteSetVersion = 1 +// MigrationWriteSetVersion rejects reinterpretation by old V1 executors. +const MigrationWriteSetVersion = 2 + // WriteSetRow 是台账里的一行:本次发布明确要写的精确身份 + 写前状态。 // // 为什么需要"写前状态":恢复不能在目标里按"没人引用"推断残渣。历史孤儿资源、 @@ -75,17 +79,33 @@ type PriorSnapshot struct { // WriteSet 是"这一次发布的写集合 + 写前整实例内容证据",内容寻址、可校验、 // 与 job/target/backup 绑定。恢复只读它,不读目标现状去推断。 +// LocalIDMapping is operation-local source lineage. Decimal strings are used +// only inside this typed sidecar; archived row values remain JSON integers. +type LocalIDMapping struct { + Table string `json:"table"` + Column string `json:"column"` + Source string `json:"source"` + Target string `json:"target"` +} + type WriteSet struct { - Version int `json:"version"` - JobID string `json:"job_id"` - Target string `json:"target"` - TenantID uint64 `json:"tenant_id"` - KnowledgeBaseID string `json:"knowledge_base_id"` - BackupSnapshotID string `json:"backup_snapshot_id"` - TargetSnapshotID string `json:"target_snapshot_id"` - CreatedAt time.Time `json:"created_at"` - Rows []WriteSetRow `json:"rows"` - Universe map[string]TableFingerprint `json:"universe"` + Index []IndexWrite `json:"index,omitempty"` + Objects []ObjectWrite `json:"objects,omitempty"` + SchemaDigest string `json:"schema_digest,omitempty"` + ContractVersion int `json:"contract_version,omitempty"` + SourceTenantID uint64 `json:"source_tenant_id,omitempty"` + SourceKnowledgeBaseID string `json:"source_knowledge_base_id,omitempty"` + LocalIDs []LocalIDMapping `json:"local_ids,omitempty"` + Version int `json:"version"` + JobID string `json:"job_id"` + Target string `json:"target"` + TenantID uint64 `json:"tenant_id"` + KnowledgeBaseID string `json:"knowledge_base_id"` + BackupSnapshotID string `json:"backup_snapshot_id"` + TargetSnapshotID string `json:"target_snapshot_id"` + CreatedAt time.Time `json:"created_at"` + Rows []WriteSetRow `json:"rows"` + Universe map[string]TableFingerprint `json:"universe"` // Digest 是对除本字段外全部内容的 sha256:落盘文件名与治理库里的引用都用它, // 记录被改一个字节就对不上。 Digest string `json:"digest"` @@ -111,6 +131,12 @@ func (w *WriteSet) Seal() error { // CanonicalBytes 是不含摘要字段的规范字节:摘要覆盖的就是它,落盘文件另外 // 带上摘要本身(便于自描述),读回时用同一份规范字节重新推导。 func (w WriteSet) CanonicalBytes() ([]byte, error) { + if w.Version == WriteSetVersion && (w.ContractVersion != 0 || w.SourceTenantID != 0 || w.SourceKnowledgeBaseID != "" || len(w.LocalIDs) != 0 || w.SchemaDigest != "" || len(w.Objects) != 0 || len(w.Index) != 0) { + return nil, errors.New("historical V1 cannot carry migration contract fields") + } + if w.Version != WriteSetVersion && w.Version != MigrationWriteSetVersion { + return nil, errors.New("unsupported write set format") + } copied := w copied.Digest = "" rows := make([]WriteSetRow, len(w.Rows)) @@ -142,7 +168,28 @@ func (w *WriteSet) computeDigest() (string, error) { // Verify 校验摘要与必备身份。缺任一身份都必须失败:没有绑定就没有"这份记录属于 // 这个作业/目标/预备份"的证明,拿它去删目标数据无法追责。 func (w *WriteSet) Verify() error { - if w.Version != WriteSetVersion { + if w.ContractVersion != 0 && w.ContractVersion != 1 { + return errors.New("unsupported migration contract version") + } + if len(w.LocalIDs) > 0 && (w.ContractVersion != 1 || w.SourceTenantID == 0 || w.SourceKnowledgeBaseID == "") { + return errors.New("local mapping lacks source contract binding") + } + sources, targets := map[string]bool{}, map[string]bool{} + for _, m := range w.LocalIDs { + valid := (m.Table == "chunks" || m.Table == "knowledge_tags") && m.Column == "seq_id" || m.Table == "wiki_log_entries" && m.Column == "id" + a, e1 := strconv.ParseInt(m.Source, 10, 64) + b, e2 := strconv.ParseInt(m.Target, 10, 64) + if !valid || e1 != nil || e2 != nil || a <= 0 || b <= 0 || strconv.FormatInt(a, 10) != m.Source || strconv.FormatInt(b, 10) != m.Target { + return errors.New("invalid typed local mapping") + } + source, target := m.Table+"/"+m.Source, m.Table+"/"+m.Target + if sources[source] || targets[target] { + return errors.New("duplicate typed local mapping") + } + sources[source] = true + targets[target] = true + } + if w.Version != WriteSetVersion && w.Version != MigrationWriteSetVersion { return fmt.Errorf("写集合记录版本 %d 不受支持(当前 %d)", w.Version, WriteSetVersion) } if w.JobID == "" || w.BackupSnapshotID == "" || w.TargetSnapshotID == "" { diff --git a/internal/librarysnapshot/archive.go b/internal/librarysnapshot/archive.go index d13c21f..3fb3092 100644 --- a/internal/librarysnapshot/archive.go +++ b/internal/librarysnapshot/archive.go @@ -57,6 +57,10 @@ type Archive struct { } func Open(root string, maxBlobBytes int64) (*Archive, error) { + return openArchive(root, maxBlobBytes, syncRoot) +} + +func openArchive(root string, maxBlobBytes int64, syncDirectory func(*os.Root) error) (*Archive, error) { if !filepath.IsAbs(root) || maxBlobBytes < 1 || maxBlobBytes == math.MaxInt64 { return nil, errors.New("invalid archive configuration") } @@ -94,6 +98,22 @@ func Open(root string, maxBlobBytes int64) (*Archive, error) { a.versions = sub } } + // File and child-directory fsync cannot persist their parent entries. The + // archive root links blobs/versions, and its parent links the archive itself. + if err := syncDirectory(r); err != nil { + a.Close() + return nil, err + } + parent, err := os.OpenRoot(filepath.Dir(root)) + if err != nil { + a.Close() + return nil, err + } + err = errors.Join(syncDirectory(parent), parent.Close()) + if err != nil { + a.Close() + return nil, err + } return a, nil } func (a *Archive) Close() error { @@ -282,6 +302,22 @@ func (a *Archive) countRows(p Part, resources map[string]bool) (int64, error) { } defer f.Close() dec := json.NewDecoder(io.LimitReader(f, a.maxBlobBytes+1)) + if p.Kind == "schema" { + var catalog map[string][]map[string]json.RawMessage + if err := dec.Decode(&catalog); err != nil || len(catalog) != 6 { + return 0, errors.New("invalid source schema catalog") + } + for _, key := range []string{"columns", "constraints", "indexes", "triggers", "relations", "sequences"} { + if rows, ok := catalog[key]; !ok || rows == nil { + return 0, errors.New("incomplete source schema catalog") + } + } + var extra any + if dec.Decode(&extra) != io.EOF { + return 0, errors.New("trailing source schema catalog data") + } + return 1, nil + } token, err := dec.Token() if err != nil || token != json.Delim('[') { return 0, errors.New("snapshot component must be a JSON row array") @@ -401,7 +437,7 @@ func encodeManifest(m Manifest) ([]byte, error) { seen := map[string]bool{} for _, p := range m.Parts { key := p.Kind + "/" + p.Name - if (!required[key] && !(p.Kind == "resource" && identifier(p.Name) && p.Count == 1)) || seen[key] || !validDigest(p.SHA256) || p.Bytes < 0 || p.Count < 0 { + if (!required[key] && !(p.Kind == "resource" && identifier(p.Name) && p.Count == 1) && !(key == "schema/migration-catalog" && p.Count == 1)) || seen[key] || !validDigest(p.SHA256) || p.Bytes < 0 || p.Count < 0 { return nil, errors.New("invalid or nonknowledge snapshot component") } seen[key] = true diff --git a/internal/librarystack/stack.go b/internal/librarystack/stack.go index 0ea3788..70d9d94 100644 --- a/internal/librarystack/stack.go +++ b/internal/librarystack/stack.go @@ -16,6 +16,7 @@ import ( "knowledge-governance/internal/appconfig" "knowledge-governance/internal/librarymaintenance" "knowledge-governance/internal/librarysnapshot" + "knowledge-governance/internal/objectgateway" v071 "knowledge-governance/internal/releaseadapter/v071" "knowledge-governance/internal/releasebundle" ) @@ -93,9 +94,25 @@ func newTarget(ctx context.Context, config appconfig.Config, scope scopeConfig) if err != nil { return nil, errors.New("生产对象存储配置无效") } + if config.LibraryObjectGatewayURL == "" { + return nil, errors.New("对象写入网关未配置:发布目标不允许直写对象存储") + } + var writer v071.ObjectStore = objects + if config.LibraryObjectGatewayURL != "" || config.LibraryObjectGatewayToken != "" || config.LibraryObjectGatewayGrantDirectory != "" { + identity := objectgateway.Digest([]byte(scope.s3.Provider + "\x00" + scope.s3.Endpoint + "\x00" + scope.s3.Bucket)) + client, err := objectgateway.NewClient(config.LibraryObjectGatewayURL, config.LibraryObjectGatewayToken, identity) + if err != nil { + return nil, err + } + gateway := &v071.GatewayObjectStore{Automatic: true, Claim: objectgateway.ClaimFromContext(ctx), Client: client, Provider: scope.s3.Provider, Bucket: scope.s3.Bucket, GrantDirectory: config.LibraryObjectGatewayGrantDirectory} + if err := gateway.ConditionalCapability(ctx); err != nil { + return nil, err + } + writer = gateway + } return v071.NewProductionLibraryTarget(ctx, v071.ProductionLibraryConfig{ Postgres: config.LibraryProductionDatabaseURL, SearchURL: config.LibraryProductionElasticURL, - SearchIndex: config.LibraryProductionIndex, Objects: objects, ObjectsReader: objects, + SearchIndex: config.LibraryProductionIndex, Objects: writer, ObjectsReader: objects, ArchiveRoot: config.LibraryArchiveRoot, ObjectsBucket: config.LibraryProductionS3Bucket, MaxDocuments: 200000, diff --git a/internal/platform/postgres/library_dispatch.go b/internal/platform/postgres/library_dispatch.go new file mode 100644 index 0000000..3cc3377 --- /dev/null +++ b/internal/platform/postgres/library_dispatch.go @@ -0,0 +1,68 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" + "knowledge-governance/internal/governance" + "knowledge-governance/internal/jobs" + "time" +) + +func (s *Store) CreateLibraryDispatch(ctx context.Context, job governance.LibraryJob, stages []string, handoff bool, now time.Time) (governance.LibraryJob, bool, error) { + tx, err := s.db.Begin(ctx) + if err != nil { + return governance.LibraryJob{}, false, err + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + created, reused, err := NewStore(tx).WithLibraryTarget(s.libraryTargetID).createLibraryDispatch(ctx, job, stages, handoff, now) + if err != nil { + return governance.LibraryJob{}, false, err + } + if err = tx.Commit(ctx); err != nil { + return governance.LibraryJob{}, false, err + } + return created, reused, nil +} +func (s *Store) createLibraryDispatch(ctx context.Context, job governance.LibraryJob, stages []string, handoff bool, now time.Time) (governance.LibraryJob, bool, error) { + created, reused, err := s.CreateLibraryJob(ctx, job, stages, now) + if err != nil { + return created, reused, err + } + if reused && (created.State != governance.LibraryJobPending || created.QueueJobID != "") { + return created, true, nil + } + kind := "library." + string(created.Kind) + if created.Kind != governance.LibraryJobSync && created.Kind != governance.LibraryJobRestore && created.Kind != governance.LibraryJobBaseline { + return created, reused, errors.New("unsupported library dispatch kind") + } + queueID, kind, raw, err := governance.LibraryQueueRequest(handoff, created.ID, created.Scope, created.TargetSnapshotID, kind) + if err != nil { + return created, reused, err + } + var payload map[string]string + if err = json.Unmarshal(raw, &payload); err != nil { + return created, reused, err + } + queued, err := s.Enqueue(ctx, jobs.EnqueueRequest{ID: queueID, Type: kind, Payload: raw, AvailableAt: now, MaxAttempts: 1, IdempotencyKey: kind + ".queue:" + created.ID}, now) + if err != nil { + return created, reused, err + } + if queued == nil || queued.ID != queueID || queued.Type != kind { + return created, reused, errors.New("queue identity conflict") + } + var actual map[string]string + if json.Unmarshal(queued.Payload, &actual) != nil || len(actual) != len(payload) { + return created, reused, errors.New("queue payload mismatch") + } + for key, value := range payload { + if actual[key] != value { + return created, reused, errors.New("queue payload mismatch") + } + } + if err := s.AttachLibraryQueueJob(ctx, created.ID, queued.ID, now); err != nil { + return created, reused, err + } + created.QueueJobID = queued.ID + return created, reused, nil +} diff --git a/internal/platform/postgres/library_execution.go b/internal/platform/postgres/library_execution.go new file mode 100644 index 0000000..9486b32 --- /dev/null +++ b/internal/platform/postgres/library_execution.go @@ -0,0 +1,36 @@ +package postgres + +import ( + "context" + "errors" + "time" +) + +var ErrLibraryExecutionBusy = errors.New("target already has an active executor") + +func (s *Store) WithLibraryExecution(ctx context.Context, instance string, run func(context.Context, func(context.Context) error) error) error { + if instance == "" { + return errors.New("execution instance identity required") + } + // A lease expiry is not proof that the old writer has stopped. Keep a + // shared PG advisory lock until the synchronous callback (including cleanup) + // exits. No deferred unlock may run merely because ctx was cancelled. + tx, err := s.db.Begin(ctx) + if err != nil { + return err + } + defer func() { + cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer cancel() + _ = tx.Rollback(cleanup) + }() + var held bool + if err = tx.QueryRow(ctx, `SELECT pg_try_advisory_xact_lock(hashtextextended($1,0))`, "library-execution:"+instance).Scan(&held); err != nil { + return err + } + if !held { + return ErrLibraryExecutionBusy + } + ping := func(probe context.Context) error { var one int; return tx.QueryRow(probe, `SELECT 1`).Scan(&one) } + return run(ctx, ping) +} diff --git a/internal/platform/postgres/library_jobs.go b/internal/platform/postgres/library_jobs.go index 173eeae..cd2d7df 100644 --- a/internal/platform/postgres/library_jobs.go +++ b/internal/platform/postgres/library_jobs.go @@ -430,6 +430,7 @@ WHERE lj.kind IN ('sync','restore','baseline') OR (lj.state='pending' AND j.state='running' AND j.lease_until IS NOT NULL AND j.lease_until < now()) ) AND lj.target_id=$1 +AND NOT EXISTS (SELECT 1 FROM jobs recovery WHERE recovery.type='library.recover' AND recovery.payload->>'window_job_id'=lj.id AND recovery.payload->>'scope'=lj.scope) ORDER BY lj.created_at`, s.libraryTargetID) if err != nil { return nil, fmt.Errorf("list interrupted library jobs: %w", err) @@ -529,6 +530,7 @@ WHERE id=$1 AND state IN ('running','retry','failed','pending') func (s *Store) ClaimLibraryJobForResume(ctx context.Context, jobID string, now time.Time) (bool, error) { tag, err := s.db.Exec(ctx, `UPDATE library_jobs SET state='pending', updated_at=$2 WHERE id=$1 AND target_id=$3 AND state IN ('running','pending') + AND NOT EXISTS (SELECT 1 FROM jobs recovery WHERE recovery.type='library.recover' AND recovery.payload->>'window_job_id'=library_jobs.id AND recovery.payload->>'scope'=library_jobs.scope) AND (lease_owner='' OR lease_until IS NULL OR lease_until < $2)`, jobID, now.UTC(), s.libraryTargetID) if err != nil { return false, fmt.Errorf("claim interrupted library job: %w", err) diff --git a/internal/platform/postgres/library_legacy_source.go b/internal/platform/postgres/library_legacy_source.go new file mode 100644 index 0000000..2c3bb47 --- /dev/null +++ b/internal/platform/postgres/library_legacy_source.go @@ -0,0 +1,143 @@ +package postgres + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "strings" + "time" + + "knowledge-governance/internal/governance" + "knowledge-governance/internal/libraryagent" + "knowledge-governance/internal/librarysnapshot" +) + +// BindLegacyLibrarySource adds evidence, not a new snapshot or rewritten origin. +// Both the archived manifest and the original capture response must independently +// identify the configured source. A scope match by itself can never authorize it. +func (s *Store) BindLegacyLibrarySource(ctx context.Context, snapshotID, archiveRoot string, source governance.LibrarySourceIdentity, actor string, now time.Time) error { + sourceID, err := source.ID() + if err != nil { + return err + } + if strings.TrimSpace(actor) == "" || now.IsZero() { + return errors.New("source association requires audited actor and time") + } + archive, err := librarysnapshot.Open(archiveRoot, 256*1024*1024) + if err != nil { + return err + } + defer archive.Close() + manifest, err := archive.Load(snapshotID) + if err != nil { + return err + } + if manifest.Scope != source.Scope || manifest.TenantID != source.TenantID || manifest.KnowledgeBaseID != source.KnowledgeBaseID { + return errors.New("historical manifest differs from configured source") + } + tx, err := s.db.Begin(ctx) + if err != nil { + return err + } + defer func() { + cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer cancel() + _ = tx.Rollback(cleanup) + }() + rows, err := tx.Query(ctx, `SELECT id,source_job_id,evidence FROM library_versions WHERE scope=$1 AND snapshot_id=$2 AND kind='test_sealed' AND target_id='' FOR UPDATE`, source.Scope, snapshotID) + if err != nil { + return err + } + type sealed struct { + id, job string + evidence []byte + } + items := []sealed{} + for rows.Next() { + var v sealed + if err = rows.Scan(&v.id, &v.job, &v.evidence); err != nil { + rows.Close() + return err + } + items = append(items, v) + } + err = rows.Err() + rows.Close() + if err != nil { + return err + } + if len(items) != 1 || items[0].job == "" { + return errors.New("unique original captured seal required") + } + old := items[0] + commitAndVerify := func() error { + if err := tx.Commit(ctx); err != nil { + return err + } + var verifiedID string + if err := s.db.QueryRow(ctx, `SELECT evidence->>'source_id' FROM library_versions WHERE id=$1 AND scope=$2 AND snapshot_id=$3`, old.id, source.Scope, snapshotID).Scan(&verifiedID); err != nil { + return err + } + if verifiedID != sourceID { + return errors.New("committed source association readback mismatch") + } + return nil + } + var evidence map[string]json.RawMessage + if json.Unmarshal(old.evidence, &evidence) != nil || evidence == nil { + return errors.New("original capture evidence is not an object") + } + var captured struct { + SnapshotID string `json:"snapshot_id"` + Identity struct { + Project string `json:"project"` + App string `json:"source_app"` + Root string `json:"archive_root"` + Fingerprint string `json:"expectation_fingerprint"` + } `json:"agent_identity"` + } + var expected string + if json.Unmarshal(evidence["agent_result"], &captured) != nil || json.Unmarshal(evidence["expectation_fingerprint"], &expected) != nil { + return errors.New("original capture identity evidence missing") + } + identity := captured.Identity + fingerprint := libraryagent.NewExpectation(identity.Project, identity.App, identity.Root).Fingerprint + if captured.SnapshotID != snapshotID || identity.Project != source.Project || identity.App != source.App || identity.Root == "" || expected != fingerprint || identity.Fingerprint != expected { + return errors.New("original capture identity or fingerprint mismatch") + } + if raw, exists := evidence["source_id"]; exists { + var current string + var bound governance.LibrarySourceIdentity + if json.Unmarshal(raw, ¤t) != nil || current != sourceID || json.Unmarshal(evidence["source_identity"], &bound) != nil || bound != source { + return errors.New("sealed source association cannot be changed") + } + return commitAndVerify() + } + for _, reserved := range []string{"source_identity", "legacy_source_binding"} { + if _, exists := evidence[reserved]; exists { + return errors.New("partial source association cannot be overwritten") + } + } + sum := sha256.Sum256(old.evidence) + addition, err := json.Marshal(map[string]any{"source_id": sourceID, "source_identity": source, "legacy_source_binding": map[string]any{"snapshot_id": snapshotID, "original_event_id": old.id, "original_source_job_id": old.job, "original_evidence_sha256": hex.EncodeToString(sum[:]), "expectation_fingerprint": expected, "actor": actor, "bound_at": now.UTC()}}) + if err != nil { + return err + } + tag, err := tx.Exec(ctx, `UPDATE library_versions SET evidence=evidence || $2::jsonb WHERE id=$1 AND NOT (evidence ? 'source_id')`, old.id, addition) + if err != nil { + return err + } + if tag.RowsAffected() != 1 { + return errors.New("source association changed concurrently") + } + var actual string + if err = tx.QueryRow(ctx, `SELECT evidence->>'source_id' FROM library_versions WHERE id=$1`, old.id).Scan(&actual); err != nil { + return err + } + if actual != sourceID { + return errors.New("source association readback mismatch") + } + return commitAndVerify() +} diff --git a/internal/platform/postgres/library_operations.go b/internal/platform/postgres/library_operations.go index 42e7fbe..b89ca14 100644 --- a/internal/platform/postgres/library_operations.go +++ b/internal/platform/postgres/library_operations.go @@ -459,23 +459,35 @@ func (s *Store) LatestLibraryAttempt(ctx context.Context, operationID string) (g // RenewLibraryAttempt 续租:失租(0 行)即返回 ErrLibraryEpochStale,调用方必须停手。 func (s *Store) RenewLibraryAttempt(ctx context.Context, attempt governance.LibraryAttempt, lease time.Duration, now time.Time) error { - until := now.Add(lease).UTC() - tag, err := s.db.Exec(ctx, ` -UPDATE library_attempts SET heartbeat_at=$4, lease_until=$5 -WHERE operation_id=$1 AND epoch=$2 AND owner=$3 AND state='running'`, - attempt.OperationID, attempt.Epoch, attempt.Owner, now.UTC(), until) + if lease <= 0 { + return errors.New("positive lease required") + } + tx, err := s.db.Begin(ctx) if err != nil { - return fmt.Errorf("renew library attempt: %w", err) + return err + } + defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }() + // Lock job first, exactly like BeginLibraryAttempt. Both leases are one fact; + // never resurrect an expired owner or commit half of a renewal. + tag, err := tx.Exec(ctx, `UPDATE library_jobs SET lease_until=clock_timestamp()+$4::interval + WHERE id=$1 AND attempt_epoch=$2 AND lease_owner=$3 AND lease_until>clock_timestamp()`, attempt.JobID, attempt.Epoch, attempt.Owner, lease.String()) + if err != nil { + return err } if tag.RowsAffected() != 1 { return governance.ErrLibraryEpochStale } - if _, err := s.db.Exec(ctx, ` -UPDATE library_jobs SET lease_until=$4 WHERE id=$1 AND attempt_epoch=$2 AND lease_owner=$3`, - attempt.JobID, attempt.Epoch, attempt.Owner, until); err != nil { - return fmt.Errorf("renew library job lease: %w", err) + if attempt.OperationID != "" { + tag, err = tx.Exec(ctx, `UPDATE library_attempts SET heartbeat_at=clock_timestamp(),lease_until=clock_timestamp()+$4::interval + WHERE operation_id=$1 AND epoch=$2 AND owner=$3 AND state='running' AND lease_until>clock_timestamp()`, attempt.OperationID, attempt.Epoch, attempt.Owner, lease.String()) + if err != nil { + return err + } + if tag.RowsAffected() != 1 { + return governance.ErrLibraryEpochStale + } } - return nil + return tx.Commit(ctx) } // guardLibraryJobEpoch 在事务内核"这次写入来自当前代次"。守卫失败一律返回 diff --git a/internal/platform/postgres/library_recovery_identity.go b/internal/platform/postgres/library_recovery_identity.go new file mode 100644 index 0000000..e89522e --- /dev/null +++ b/internal/platform/postgres/library_recovery_identity.go @@ -0,0 +1,42 @@ +package postgres + +import ( + "context" + "errors" + "knowledge-governance/internal/governance" + "knowledge-governance/internal/jobs" +) + +// An arbitrary queue delivery cannot borrow another job's window. Check the +// stored request AND its current delivery lease before any maintenance action. +func (s *Store) ResumeExpiredLibraryRecoveryQueues(ctx context.Context) error { + _, err := s.db.Exec(ctx, `UPDATE jobs q SET state='pending',attempt=0,lease_owner=NULL,lease_until=NULL,available_at=clock_timestamp() + FROM library_jobs j,library_instance_maintenance m + WHERE q.type='library.recover' AND q.state='running' AND q.lease_until>'window_job_id' AND j.scope=q.payload->>'scope' AND j.target_id=$1 + AND (j.lease_until IS NULL OR j.lease_until>'window_job_id'=j.id AND q.payload->>'scope'=j.scope + AND q.idempotency_key='library.recover:'||j.id + AND q.lease_owner=$4 AND q.attempt=$5 AND q.lease_until>clock_timestamp() + AND j.scope=$6 AND j.target_id=$7 AND j.kind IN ('sync','restore','baseline'))`, q.ID, window, q.Payload, q.LeaseOwner, q.Attempt, scope, s.libraryTargetID).Scan(&valid) + if e != nil { + return e + } + if !valid { + return errors.New("recovery request/source/target/lease mismatch") + } + return nil +} diff --git a/internal/platform/postgres/library_source_candidate.go b/internal/platform/postgres/library_source_candidate.go new file mode 100644 index 0000000..088e8ff --- /dev/null +++ b/internal/platform/postgres/library_source_candidate.go @@ -0,0 +1,26 @@ +package postgres + +import ( + "context" + "errors" + "knowledge-governance/internal/governance" +) + +// Query the source identity before applying LIMIT: target/prebackup events and +// other source instances can never push the candidate out of a history window. +func (s *Store) LibrarySourceCandidate(ctx context.Context, scope, sourceID string) (governance.LibraryVersion, bool, error) { + if sourceID == "" { + return governance.LibraryVersion{}, false, errors.New("source identity required") + } + rows, err := s.db.Query(ctx, `SELECT id,scope,kind,snapshot_id,version_label,source_job_id,operation_id,attempt_epoch,evidence,created_by,created_at,superseded_at + FROM library_versions WHERE scope=$1 AND kind='test_sealed' AND evidence->>'source_id'=$2 + ORDER BY created_at DESC,id DESC LIMIT 1`, scope, sourceID) + if err != nil { + return governance.LibraryVersion{}, false, err + } + items, err := scanLibraryVersions(rows, 1) + if err != nil || len(items) == 0 { + return governance.LibraryVersion{}, false, err + } + return items[0], true, nil +} diff --git a/internal/platform/postgres/library_write_fence.go b/internal/platform/postgres/library_write_fence.go index 5e519aa..dba5b5a 100644 --- a/internal/platform/postgres/library_write_fence.go +++ b/internal/platform/postgres/library_write_fence.go @@ -33,6 +33,7 @@ func libraryFence(jobIDExpr string, epochParam, ownerParam int) string { WHERE j.id = %s AND j.attempt_epoch = $%d AND j.lease_owner = $%d + AND j.lease_until > clock_timestamp() AND ( j.operation_id = '' OR EXISTS ( @@ -41,6 +42,7 @@ func libraryFence(jobIDExpr string, epochParam, ownerParam int) string { AND a.epoch = j.attempt_epoch AND a.owner = j.lease_owner AND a.state IN ('claimed','running') + AND a.lease_until > clock_timestamp() ) ) )`, jobIDExpr, epochParam, ownerParam) diff --git a/internal/releaseadapter/v071/library_archive.go b/internal/releaseadapter/v071/library_archive.go index 750fa48..3feea94 100644 --- a/internal/releaseadapter/v071/library_archive.go +++ b/internal/releaseadapter/v071/library_archive.go @@ -22,6 +22,12 @@ func ArchiveLibraryCandidate(ctx context.Context, archive *librarysnapshot.Archi if err := validateLibraryCandidate(identity, pg, es); err != nil { return "", err } + return archiveLibraryRows(ctx, archive, scope, identity, contract, pg, es, objects, createdAt) +} + +// Private: callers must distinguish a validated source candidate from raw +// target before-images. Never run source semantic filtering on recovery bytes. +func archiveLibraryRows(ctx context.Context, archive *librarysnapshot.Archive, scope string, identity PostgresIdentity, contract librarysnapshot.Compatibility, pg PostgresSnapshot, es SearchSnapshot, objects ObjectReader, createdAt time.Time) (string, error) { // 空生产基线:索引不存在时没有映射可比,契约仍按目标快照记录; // 只要这次导出的文档是零条,就不存在"拿错映射"的风险。 emptyIndex := len(es.Documents) == 0 @@ -43,6 +49,15 @@ func ArchiveLibraryCandidate(ctx context.Context, archive *librarysnapshot.Archi manifest.Parts = append(manifest.Parts, p) return nil } + if pg.Catalog != nil { + raw, err := json.Marshal(pg.Catalog) + if err != nil { + return "", err + } + if err = put("schema", "migration-catalog", 1, raw); err != nil { + return "", err + } + } for _, table := range pg.Tables { rows := table.Rows if rows == nil { @@ -211,6 +226,9 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es return errors.New("missing library table") } } + if err := validateLibraryStructuredReferences(pg); err != nil { + return err + } for _, chunk := range tables["chunks"] { if _, ok := tables["knowledges"][chunk.KnowledgeID]; !ok { return errors.New("orphan library chunk") @@ -243,10 +261,14 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es handles[resource.Handle] = true } var bodies []json.RawMessage - for _, name := range []string{"knowledges", "chunks", "wiki_pages"} { - bodies = append(bodies, pg.Table(name).Rows...) + for _, table := range pg.Tables { + bodies = append(bodies, table.Rows...) } - for _, handle := range referencedResourceHandles(bodies) { + references, err := libraryResourceHandles(bodies) + if err != nil { + return err + } + for _, handle := range references { if !handles[handle] { return errors.New("unresolved library resource") } diff --git a/internal/releaseadapter/v071/library_backup_store.go b/internal/releaseadapter/v071/library_backup_store.go index 6970b0a..1eae127 100644 --- a/internal/releaseadapter/v071/library_backup_store.go +++ b/internal/releaseadapter/v071/library_backup_store.go @@ -17,8 +17,8 @@ import ( // (PG RepeatableRead + ES 导出一致快照 + 资源对象),密封成归档里的一个版本。 // // 为什么要有静默检查:预备份期间生产若仍在变,导出的 PG/ES/对象就不是同一个切点, -// 恢复它等于恢复到"没人见过的状态"。所以导出前后各读一次行数,不一致就失败, -// 让作业重试,而不是拿一个不一致的备份当恢复点。 +// 恢复它等于恢复到"没人见过的状态"。PG逐行指纹与对象精确版本在导出前后 +// 必须一致;执行器另行核对索引before-image。此检测不能代替独占写审核。 type ProductionBackupStore struct { pool *pgxpool.Pool search *SearchExporter @@ -119,12 +119,16 @@ func (s *ProductionBackupStore) Prepare(ctx context.Context, _ string, current l if err != nil { return libraryrelease.BackupRef{}, err } - pg, err := ExportLibrary(ctx, s.pool, s.identity) + pg, err := exportLibraryBeforeImage(ctx, s.pool, s.identity) if err != nil { return libraryrelease.BackupRef{}, fmt.Errorf("读取生产整库失败: %w", err) } - // 整库导出用 ExportLibrary(按租户+知识库,不带单文档身份), - // 与封存路径复用同一份实现,避免两套范围规则。 + // Every row the actual delete predicate can reach must be present. + for _, table := range pg.Tables { + if int64(len(table.Rows)) != before[table.Name].Count { + return libraryrelease.BackupRef{}, fmt.Errorf("before-image coverage mismatch: %s", table.Name) + } + } es, err := s.search.ExportLibrary(ctx, SearchIdentity{TenantID: s.identity.TenantID, KnowledgeBaseID: s.identity.KnowledgeBaseID}, s.maxDocuments) if err != nil { @@ -143,9 +147,13 @@ func (s *ProductionBackupStore) Prepare(ctx context.Context, _ string, current l return libraryrelease.BackupRef{}, errors.New("生产索引映射与归档契约不一致") } } + objectBefore, err := s.objectVersions(ctx, pg) + if err != nil { + return libraryrelease.BackupRef{}, err + } // ArchiveLibraryCandidate 自己按 resources 行读取对象字节并计算摘要, // 所以这里直接给它生产对象读取器,不再重复导出一次资源。 - snapshotID, err := ArchiveLibraryCandidate(ctx, archive, s.scope, s.identity, s.compatibility, pg, es, s.objects, createdAt) + snapshotID, err := archiveLibraryRows(ctx, archive, s.scope, s.identity, s.compatibility, pg, es, s.objects, createdAt) if err != nil { return libraryrelease.BackupRef{}, fmt.Errorf("密封预备份失败: %w", err) } @@ -179,14 +187,60 @@ func (s *ProductionBackupStore) Prepare(ctx context.Context, _ string, current l if err != nil { return libraryrelease.BackupRef{}, errors.New("预备份封存后无法校验") } + objectAfter, err := s.objectVersions(ctx, pg) + if err != nil { + return libraryrelease.BackupRef{}, err + } + for id, before := range objectBefore { + if objectAfter[id] != before { + return libraryrelease.BackupRef{}, errors.New("object changed during prebackup; inconsistent version boundary") + } + } var total int64 for _, part := range manifest.Parts { + if part.Kind == "resource" { + observed, ok := objectBefore[part.Name] + if !ok || observed.SHA256 != part.SHA256 || observed.Bytes != part.Bytes { + return libraryrelease.BackupRef{}, errors.New("archived object differs from prebackup version boundary") + } + } total += part.Bytes } return libraryrelease.BackupRef{ID: "backup-" + snapshotID, SnapshotID: snapshotID, Bytes: total, CreatedAt: createdAt, Universe: universeBefore}, nil } +// Exact versions, not just bytes, bind the multi-store backup read interval. +// An unchanged byte string in a new version is still a different restore base. +func (s *ProductionBackupStore) objectVersions(ctx context.Context, pg PostgresSnapshot) (map[string]objectObservation, error) { + out := map[string]objectObservation{} + reader, ok := s.objects.(interface { + ObserveObject(context.Context, string, string) (objectObservation, error) + }) + for _, raw := range pg.Table("resources").Rows { + if !ok { + return nil, errors.New("prebackup reader lacks exact object-version observations") + } + var row struct { + ID string `json:"id"` + Provider string `json:"provider"` + Path string `json:"physical_path"` + } + if e := json.Unmarshal(raw, &row); e != nil { + return nil, e + } + observed, e := reader.ObserveObject(ctx, row.Provider, row.Path) + if e != nil { + return nil, e + } + if !observed.Exists || observed.VersionID == "" || observed.VersionID == "null" || observed.SHA256 == "" { + return nil, errors.New("prebackup requires an existing versioned object") + } + out[row.ID] = observed + } + return out, nil +} + func (s *ProductionBackupStore) rowCounts(ctx context.Context) (map[string]int64, error) { counts := map[string]int64{} for _, table := range productionLibraryTables { diff --git a/internal/releaseadapter/v071/library_beforeimage.go b/internal/releaseadapter/v071/library_beforeimage.go new file mode 100644 index 0000000..6d16d8c --- /dev/null +++ b/internal/releaseadapter/v071/library_beforeimage.go @@ -0,0 +1,69 @@ +package v071 + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/jackc/pgx/v5" +) + +// Recovery bytes are not a source candidate. The selection must equal the +// destructive target predicate, including tombstones and their relationships. +// Semantic source validation would discard/reject exactly those historical +// rows that an exact rollback must preserve. +func exportLibraryBeforeImage(ctx context.Context, db librarySnapshotDB, identity PostgresIdentity) (PostgresSnapshot, error) { + tx, err := db.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadOnly}) + if err != nil { + return PostgresSnapshot{}, err + } + defer tx.Rollback(context.WithoutCancel(ctx)) + var version int + var dirty bool + if err := tx.QueryRow(ctx, `SELECT version,dirty FROM schema_migrations LIMIT 1`).Scan(&version, &dirty); err != nil || version != 74 || dirty { + return PostgresSnapshot{}, errors.New("before-image schema contract mismatch") + } + exporter := PostgresExporter{db: tx} + snapshot := PostgresSnapshot{} + for _, table := range productionLibraryTables { + rows, err := exporter.queryRows(ctx, fmt.Sprintf(`SELECT to_jsonb(%s) FROM %s WHERE %s ORDER BY (%s)::text`, table, table, scopeWhere(table), identityExpr(table)), identity.TenantID, identity.KnowledgeBaseID) + if err != nil { + return PostgresSnapshot{}, err + } + snapshot.Tables = append(snapshot.Tables, newTableSnapshot(table, rows)) + } + var bodies []json.RawMessage + for _, table := range snapshot.Tables { + bodies = append(bodies, table.Rows...) + } + handles, err := libraryResourceHandles(bodies) + if err != nil { + return PostgresSnapshot{}, err + } + resourceHandles := map[string]bool{} + resourceIDs := []string{} + for _, raw := range snapshot.Table("resources").Rows { + var row struct { + ID string `json:"id"` + Handle string `json:"handle"` + } + if err := json.Unmarshal(raw, &row); err != nil { + return PostgresSnapshot{}, err + } + resourceHandles[row.Handle] = true + resourceIDs = append(resourceIDs, row.ID) + } + for _, handle := range handles { + if !resourceHandles[handle] { + return PostgresSnapshot{}, errors.New("before-image contains unresolved resource reference") + } + } + if err := verifyLibraryResourceOwners(ctx, tx, resourceIDs); err != nil { + return PostgresSnapshot{}, err + } + if err := tx.Commit(ctx); err != nil { + return PostgresSnapshot{}, err + } + return snapshot, nil +} diff --git a/internal/releaseadapter/v071/library_evidence_directory.go b/internal/releaseadapter/v071/library_evidence_directory.go new file mode 100644 index 0000000..7478649 --- /dev/null +++ b/internal/releaseadapter/v071/library_evidence_directory.go @@ -0,0 +1,40 @@ +package v071 + +import ( + "errors" + "os" + "path/filepath" +) + +// Validate each ancestor before descending; MkdirAll can follow an ancestor +// symlink before the leaf Lstat ever sees it. Sync every directory entry's parent, +// not merely the last two levels of a newly-created effect hierarchy. +func ensureEvidenceDirectory(path string, syncDirectory func(string) error) error { + absolute, err := filepath.Abs(path) + if err != nil { + return err + } + parent := filepath.Dir(absolute) + if parent != absolute { + if err = ensureEvidenceDirectory(parent, syncDirectory); err != nil { + return err + } + } + info, err := os.Lstat(absolute) + if errors.Is(err, os.ErrNotExist) { + if err = os.Mkdir(absolute, 0700); err != nil && !errors.Is(err, os.ErrExist) { + return err + } + info, err = os.Lstat(absolute) + } + if err != nil { + return err + } + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return errors.New("evidence ancestor is not a real directory") + } + if parent != absolute { + return syncDirectory(parent) + } + return nil +} diff --git a/internal/releaseadapter/v071/library_export.go b/internal/releaseadapter/v071/library_export.go index c1ff3c1..462c7b5 100644 --- a/internal/releaseadapter/v071/library_export.go +++ b/internal/releaseadapter/v071/library_export.go @@ -58,8 +58,18 @@ func ExportLibraryWithReleases(ctx context.Context, db librarySnapshotDB, identi return PostgresSnapshot{}, errors.New("library has unfinished or failed processing; semantic acceptance is still required after settlement") } } + if err := verifyLibraryRelationalClosure(ctx, tx, identity, false); err != nil { + return PostgresSnapshot{}, err + } exporter := PostgresExporter{db: tx} var snapshot PostgresSnapshot + snapshot.Catalog, _, err = captureMigrationCatalog(ctx, tx) + if err != nil { + return PostgresSnapshot{}, err + } + if err = validateMigrationCatalog(snapshot.Catalog); err != nil { + return PostgresSnapshot{}, err + } for _, entry := range libraryContentQueries { rows, err := exporter.queryRows(ctx, entry.sql, identity.TenantID, identity.KnowledgeBaseID) if err != nil { @@ -70,10 +80,13 @@ func ExportLibraryWithReleases(ctx context.Context, db librarySnapshotDB, identi // Resources can be owned by a document/chunk or only referenced in body text. // The latter includes Wiki manual pages, which document bundles used to miss. var bodies []json.RawMessage - for _, name := range []string{"knowledges", "chunks", "wiki_pages"} { - bodies = append(bodies, snapshot.Table(name).Rows...) + for _, table := range snapshot.Tables { + bodies = append(bodies, table.Rows...) + } + handles, err := libraryResourceHandles(bodies) + if err != nil { + return PostgresSnapshot{}, err } - handles := referencedResourceHandles(bodies) resources, err := exporter.queryRows(ctx, libraryResourcesSQL, identity.TenantID, identity.KnowledgeBaseID, handles) if err != nil { return PostgresSnapshot{}, err @@ -104,6 +117,13 @@ func ExportLibraryWithReleases(ctx context.Context, db librarySnapshotDB, identi return PostgresSnapshot{}, errors.New("library binding references an unavailable resource") } } + ids := make([]string, 0, len(resourceIDs)) + for id := range resourceIDs { + ids = append(ids, id) + } + if err := verifyLibraryResourceOwners(ctx, tx, ids); err != nil { + return PostgresSnapshot{}, err + } snapshot.Tables = append(snapshot.Tables, newTableSnapshot("resources", resources)) if err = tx.Commit(ctx); err != nil { return PostgresSnapshot{}, errors.New("commit library snapshot") diff --git a/internal/releaseadapter/v071/library_identity_contract.go b/internal/releaseadapter/v071/library_identity_contract.go new file mode 100644 index 0000000..17467d0 --- /dev/null +++ b/internal/releaseadapter/v071/library_identity_contract.go @@ -0,0 +1,83 @@ +package v071 + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" +) + +// The contract names physical identities, not inferred *_id conventions. A +// string ID is preserved byte-for-byte (older archives also contain non-UUID +// varchar IDs). Database-local integers must never pass through float64. +const migrationIdentityContractVersion = 1 + +type rowIdentityContract struct { + Columns []string + Kind string + LocalColumn string + Sequence string +} + +var migrationIdentityContracts = map[string]rowIdentityContract{ + "knowledges": {Columns: []string{"id"}, Kind: "stable-string"}, + "chunks": {Columns: []string{"id"}, Kind: "stable-string", LocalColumn: "seq_id", Sequence: "chunks_seq_id_seq"}, + "knowledge_tags": {Columns: []string{"id"}, Kind: "stable-string", LocalColumn: "seq_id", Sequence: "knowledge_tags_seq_id_seq"}, + "knowledge_tag_relations": {Columns: []string{"knowledge_id", "tag_id"}, Kind: "tuple"}, + "resources": {Columns: []string{"id"}, Kind: "stable-string"}, + "resource_bindings": {Columns: []string{"id"}, Kind: "stable-string"}, + "wiki_folders": {Columns: []string{"id"}, Kind: "stable-string"}, + "wiki_pages": {Columns: []string{"id"}, Kind: "stable-string"}, + "wiki_page_issues": {Columns: []string{"id"}, Kind: "stable-string"}, + "wiki_log_entries": {Columns: []string{"id"}, Kind: "local-bigint", LocalColumn: "id", Sequence: "wiki_log_entries_id_seq"}, +} + +type typedRowIdentity struct { + Kind string + Values []string +} + +// key retains the V1 ledger representation, but tuple components are validated +// independently and used as separate SQL parameters, never as a single column. +func (id typedRowIdentity) key() string { return strings.Join(id.Values, ":") } + +func positiveBigint(raw json.RawMessage) (string, error) { + raw = bytes.TrimSpace(raw) + if len(raw) == 0 { + return "", errors.New("missing integer") + } + for _, b := range raw { + if b < '0' || b > '9' { + return "", errors.New("expected JSON integer, not string/fraction/exponent") + } + } + n, err := strconv.ParseInt(string(raw), 10, 64) + if err != nil || n <= 0 || strconv.FormatInt(n, 10) != string(raw) { + return "", errors.New("integer out of positive bigint range or noncanonical") + } + return string(raw), nil +} + +func parseRowIdentity(table string, row map[string]json.RawMessage) (typedRowIdentity, error) { + contract, ok := migrationIdentityContracts[table] + if !ok { + return typedRowIdentity{}, fmt.Errorf("unknown migration identity contract: %s", table) + } + id := typedRowIdentity{Kind: contract.Kind} + for _, column := range contract.Columns { + var value string + if contract.Kind == "local-bigint" { + n, err := positiveBigint(row[column]) + if err != nil { + return id, fmt.Errorf("%s.%s: %w", table, column, err) + } + value = n + } else if err := json.Unmarshal(row[column], &value); err != nil || !validSearchIdentifier(value) || strings.Contains(value, ":") { + return id, fmt.Errorf("invalid %s.%s identity", table, column) + } + id.Values = append(id.Values, value) + } + return id, nil +} diff --git a/internal/releaseadapter/v071/library_immutable.go b/internal/releaseadapter/v071/library_immutable.go new file mode 100644 index 0000000..c22d5f5 --- /dev/null +++ b/internal/releaseadapter/v071/library_immutable.go @@ -0,0 +1,89 @@ +package v071 + +import ( + "bytes" + "errors" + "fmt" + "os" + "path/filepath" +) + +// Publish without replacement: a second writer can only verify the already +// committed bytes. Sync both the file and directory entries before returning an +// anchor that another process may act on. This is not a power-loss test. +func persistImmutableJSON(path string, raw []byte) error { + directory := filepath.Dir(path) + if err := ensureEvidenceDirectory(directory, syncEvidenceDirectory); err != nil { + return err + } + info, err := os.Lstat(directory) + if err != nil { + return err + } + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return errors.New("immutable evidence directory is not a real directory") + } + existing := func() error { + info, err := os.Lstat(path) + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return errors.New("immutable evidence is not a regular file") + } + body, err := os.ReadFile(path) + if err != nil { + return err + } + if !bytes.Equal(body, raw) { + return errors.New("immutable evidence already exists with different bytes") + } + return syncEvidenceDirectory(directory) + } + if _, err := os.Lstat(path); err == nil { + return existing() + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + file, err := os.CreateTemp(directory, ".evidence-*") + if err != nil { + return err + } + temporary := file.Name() + defer os.Remove(temporary) + defer file.Close() + if _, err := file.Write(raw); err != nil { + return err + } + if err := file.Chmod(0444); err != nil { + return err + } + if err := file.Sync(); err != nil { + return err + } + if err := file.Close(); err != nil { + return err + } + if err := os.Link(temporary, path); err != nil { + if errors.Is(err, os.ErrExist) { + return existing() + } + return fmt.Errorf("commit immutable evidence: %w", err) + } + if err := os.Remove(temporary); err != nil { + return err + } + return syncEvidenceDirectory(directory) +} + +func syncEvidenceDirectory(path string) error { + directory, err := os.Open(path) + if err != nil { + return err + } + defer directory.Close() + if err := directory.Sync(); err != nil { + return fmt.Errorf("sync evidence directory: %w", err) + } + return nil +} diff --git a/internal/releaseadapter/v071/library_index_claim.go b/internal/releaseadapter/v071/library_index_claim.go new file mode 100644 index 0000000..c64cd4b --- /dev/null +++ b/internal/releaseadapter/v071/library_index_claim.go @@ -0,0 +1,112 @@ +package v071 + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + + "knowledge-governance/internal/libraryrelease" +) + +const indexClaimPrefix = "kg-release-transient-claim:" + +// The transient marker is only visible inside the maintenance window. Final +// content is byte-equivalent to the frozen source, without control-plane fields. +// A matching payload alone never establishes ownership of an ambiguous write. +func (t *ProductionLibraryTarget) indexClaimBody(c libraryrelease.IndexWrite) (json.RawMessage, error) { + if len(c.After) == 0 { + return nil, errors.New("deleted index document has no ownership marker") + } + var body map[string]json.RawMessage + if json.Unmarshal(c.After, &body) != nil || body == nil { + return nil, errors.New("invalid index claim source") + } + var content string + if json.Unmarshal(body["content"], &content) != nil { + return nil, errors.New("index claim requires existing string content field") + } + sum := sha256.Sum256([]byte(t.activePlan.Digest + "\x00" + c.Index + "\x00" + c.ID + "\x00apply")) + body["content"], _ = json.Marshal(indexClaimPrefix + hex.EncodeToString(sum[:]) + "\n" + content) + encoded, err := json.Marshal(body) + if err != nil { + return nil, err + } + return canonicalJSON(encoded) +} +func (t *ProductionLibraryTarget) applyIndexClaim(ctx context.Context, c libraryrelease.IndexWrite) (indexReceipt, error) { + if len(c.After) == 0 { + return t.mutatePlannedIndex(ctx, c, nil, c.SeqNo, c.PrimaryTerm, false) + } + body, err := t.indexClaimBody(c) + if err != nil { + return indexReceipt{}, err + } + receipt, err := t.mutatePlannedIndex(ctx, c, body, c.SeqNo, c.PrimaryTerm, len(c.Before) == 0) + if err != nil { + return receipt, err + } + receipt.Marked = true + if err = t.saveIndexEffect(c, "claimed", &receipt); err != nil { + return receipt, err + } + return t.finalizeIndexClaim(ctx, c) +} +func (t *ProductionLibraryTarget) reconcileIndexClaim(ctx context.Context, c libraryrelease.IndexWrite) (indexReceipt, error) { + empty := indexReceipt{} + intent, err := t.loadIndexEffect(c, "apply-intent") + if err != nil { + return empty, err + } + if intent == nil { + return empty, errors.New("index claim has no durable intent") + } + finalizing, err := t.loadIndexEffect(c, "finalize-intent") + if err != nil { + return empty, err + } + if finalizing != nil { + return empty, errors.New("unknown clean finalization cannot be attributed by appearance") + } + body, err := t.indexClaimBody(c) + if err != nil { + return empty, err + } + g, err := t.observePlannedIndex(ctx, c) + if err != nil { + return empty, err + } + if !indexBytesEqual(g, body) || g.SeqNo == nil || g.PrimaryTerm == nil { + return empty, errors.New("unknown index effect lacks exact operation marker/content/CAS") + } + claimed, err := t.loadIndexEffect(c, "claimed") + if err != nil { + return empty, err + } + if claimed != nil { + if claimed.Receipt == nil || !claimed.Receipt.Marked || !sameIndexGeneration(g, claimed.Receipt.SeqNo, claimed.Receipt.PrimaryTerm) { + return empty, errors.New("claimed index generation changed") + } + return *claimed.Receipt, nil + } + result := "updated" + if len(c.Before) == 0 { + result = "created" + } + receipt := indexReceipt{Index: c.Index, ID: c.ID, Result: result, SeqNo: g.SeqNo, PrimaryTerm: g.PrimaryTerm, Marked: true} + if err = t.saveIndexEffect(c, "claimed", &receipt); err != nil { + return empty, err + } + return receipt, nil +} +func (t *ProductionLibraryTarget) finalizeIndexClaim(ctx context.Context, c libraryrelease.IndexWrite) (indexReceipt, error) { + receipt, err := t.reconcileIndexClaim(ctx, c) + if err != nil { + return indexReceipt{}, err + } + if err = t.saveIndexEffect(c, "finalize-intent", &receipt); err != nil { + return indexReceipt{}, err + } + return t.mutatePlannedIndex(ctx, c, c.After, receipt.SeqNo, receipt.PrimaryTerm, false) +} diff --git a/internal/releaseadapter/v071/library_index_delete_reconcile.go b/internal/releaseadapter/v071/library_index_delete_reconcile.go new file mode 100644 index 0000000..bb44ab4 --- /dev/null +++ b/internal/releaseadapter/v071/library_index_delete_reconcile.go @@ -0,0 +1,57 @@ +package v071 + +import ( + "bytes" + "context" + "encoding/json" + "errors" + + "knowledge-governance/internal/libraryrelease" +) + +// Unlike an ambiguous create, deletion of a known CAS generation can settle on +// absence: a delayed delete still carries that old seq_no/primary_term and cannot +// delete a subsequently recreated generation. This never infers PUT completion. +func (t *ProductionLibraryTarget) reconcileIndexCompensationDelete(ctx context.Context, c libraryrelease.IndexWrite) (bool, error) { + if len(c.Before) != 0 || len(c.After) == 0 { + return false, nil + } + intent, e := t.loadIndexEffect(c, "restore-intent") + if e != nil || intent == nil { + return false, e + } + if intent.Receipt == nil || *intent.Receipt.SeqNo < 0 || *intent.Receipt.PrimaryTerm < 1 { + return false, errors.New("index delete compensation lacks known generation") + } + proof, e := t.loadIndexEffect(c, "applied") + if e != nil { + return false, e + } + if proof == nil { + proof, e = t.loadIndexEffect(c, "claimed") + if e != nil { + return false, e + } + } + if proof == nil || proof.Receipt == nil { + return false, errors.New("index delete compensation lacks persisted ownership") + } + want, _ := json.Marshal(proof.Receipt) + got, _ := json.Marshal(intent.Receipt) + if !bytes.Equal(want, got) { + return false, errors.New("index delete compensation CAS differs from owned receipt") + } + observation, e := t.observePlannedIndex(ctx, c) + if e != nil { + return false, e + } + if observation.Found { + return false, nil + } + // No fabricated DELETE response/generation: proof is the bound restore intent, + // persisted ownership receipt and independent authoritative absent observation. + if e = t.saveIndexEffect(c, "restored", nil); e != nil { + return false, e + } + return true, nil +} diff --git a/internal/releaseadapter/v071/library_index_effects.go b/internal/releaseadapter/v071/library_index_effects.go new file mode 100644 index 0000000..eea111a --- /dev/null +++ b/internal/releaseadapter/v071/library_index_effects.go @@ -0,0 +1,290 @@ +package v071 + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "os" + "path/filepath" + + "knowledge-governance/internal/libraryrelease" + "knowledge-governance/internal/librarysnapshot" +) + +type indexReceipt struct { + Index string `json:"_index"` + ID string `json:"_id"` + Result string `json:"result"` + SeqNo *int64 `json:"_seq_no"` + PrimaryTerm *int64 `json:"_primary_term"` + Marked bool `json:"marked,omitempty"` +} +type indexEffect struct { + Plan string `json:"plan"` + Change libraryrelease.IndexWrite `json:"change"` + Receipt *indexReceipt `json:"receipt,omitempty"` +} + +func (t *ProductionLibraryTarget) indexEffectPath(c libraryrelease.IndexWrite, phase string) string { + sum := sha256.Sum256([]byte(c.Index + "\x00" + c.ID)) + return filepath.Join(t.config.ArchiveRoot, "effects", t.activePlan.Digest, "index", hex.EncodeToString(sum[:]), phase+".json") +} +func (t *ProductionLibraryTarget) saveIndexEffect(c libraryrelease.IndexWrite, phase string, r *indexReceipt) error { + raw, err := json.Marshal(indexEffect{t.activePlan.Digest, c, r}) + if err != nil { + return err + } + return persistImmutableJSON(t.indexEffectPath(c, phase), raw) +} +func (t *ProductionLibraryTarget) loadIndexEffect(c libraryrelease.IndexWrite, phase string) (*indexEffect, error) { + path := t.indexEffectPath(c, phase) + exists, err := effectExists(path) + if err != nil || !exists { + return nil, err + } + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var e indexEffect + if json.Unmarshal(raw, &e) != nil { + return nil, errors.New("invalid index effect evidence") + } + want, _ := json.Marshal(c) + got, _ := json.Marshal(e.Change) + if e.Plan != t.activePlan.Digest || !bytes.Equal(want, got) { + return nil, errors.New("index effect binding differs from immutable plan") + } + if e.Receipt != nil && (e.Receipt.Index != c.Index || e.Receipt.ID != c.ID || e.Receipt.SeqNo == nil || e.Receipt.PrimaryTerm == nil) { + return nil, errors.New("incomplete index receipt") + } + return &e, nil +} +func (t *ProductionLibraryTarget) observePlannedIndex(ctx context.Context, c libraryrelease.IndexWrite) (indexWriteGuard, error) { + guards, err := t.search.indexWriteGuards(ctx, []librarysnapshot.IndexDocument{{ID: c.ID}}, SearchIdentity{TenantID: t.scope.TenantID, KnowledgeBaseID: t.scope.KnowledgeBaseID}) + if err != nil { + return indexWriteGuard{}, err + } + return guards[c.ID], nil +} +func indexBytesEqual(g indexWriteGuard, want json.RawMessage) bool { + if len(want) == 0 { + return !g.Found + } + if !g.Found { + return false + } + raw, err := canonicalJSON(g.Source) + return err == nil && bytes.Equal(raw, want) +} +func sameIndexGeneration(g indexWriteGuard, seq, term *int64) bool { + return g.Found && g.SeqNo != nil && g.PrimaryTerm != nil && seq != nil && term != nil && *g.SeqNo == *seq && *g.PrimaryTerm == *term +} + +// Each network mutation has an immutable intent before send and a per-ID +// server generation receipt. A lost response is UNKNOWN, never "did not write". +// No retry is issued for an unresolved intent; downstream ownership fencing is +// required before unknown operations can be conclusively reconciled. +func (t *ProductionLibraryTarget) applyPlannedIndex(ctx context.Context) error { + for _, c := range t.activePlan.Index { + if bytes.Equal(c.Before, c.After) { + continue + } + applied, err := t.loadIndexEffect(c, "applied") + if err != nil { + return err + } + if applied != nil { + if applied.Receipt == nil { + return errors.New("index receipt missing") + } + g, err := t.observePlannedIndex(ctx, c) + if err != nil { + return err + } + if !indexBytesEqual(g, c.After) || (len(c.After) > 0 && !sameIndexGeneration(g, applied.Receipt.SeqNo, applied.Receipt.PrimaryTerm)) { + return errors.New("index changed after confirmed apply") + } + continue + } + intent, err := t.loadIndexEffect(c, "apply-intent") + if err != nil { + return err + } + if intent != nil { + if _, err := t.reconcileIndexClaim(ctx, c); err != nil { + return err + } + receipt, err := t.finalizeIndexClaim(ctx, c) + if err != nil { + return err + } + if err := t.saveIndexEffect(c, "applied", &receipt); err != nil { + return err + } + continue + } + g, err := t.observePlannedIndex(ctx, c) + if err != nil { + return err + } + if !indexBytesEqual(g, c.Before) || (len(c.Before) > 0 && !sameIndexGeneration(g, c.SeqNo, c.PrimaryTerm)) { + return errors.New("index changed since frozen plan") + } + if err = t.saveIndexEffect(c, "apply-intent", nil); err != nil { + return err + } + receipt, err := t.applyIndexClaim(ctx, c) + if err != nil { + return err + } + if err = t.saveIndexEffect(c, "applied", &receipt); err != nil { + return err + } + g, err = t.observePlannedIndex(ctx, c) + if err != nil { + return err + } + if !indexBytesEqual(g, c.After) || (len(c.After) > 0 && !sameIndexGeneration(g, receipt.SeqNo, receipt.PrimaryTerm)) { + return errors.New("index readback differs from receipt") + } + } + return nil +} +func (t *ProductionLibraryTarget) mutatePlannedIndex(ctx context.Context, c libraryrelease.IndexWrite, body json.RawMessage, seq, term *int64, create bool) (indexReceipt, error) { + method := http.MethodPut + endpoint := "_doc" + if create { + endpoint = "_create" + } + if len(body) == 0 { + method = http.MethodDelete + } + query := url.Values{"refresh": {"true"}} + if !create { + if seq == nil || term == nil { + return indexReceipt{}, errors.New("index mutation lacks CAS") + } + query.Set("if_seq_no", fmt.Sprint(*seq)) + query.Set("if_primary_term", fmt.Sprint(*term)) + } + path := "/" + url.PathEscape(c.Index) + "/" + endpoint + "/" + url.PathEscape(c.ID) + "?" + query.Encode() + var payload any + if len(body) > 0 { + payload = body + } + var receipt indexReceipt + if err := t.search.requestJSON(ctx, method, path, payload, &receipt); err != nil { + return receipt, fmt.Errorf("index effect outcome unknown: %w", err) + } + if receipt.ID != c.ID || receipt.Index != c.Index || receipt.SeqNo == nil || receipt.PrimaryTerm == nil { + return receipt, errors.New("index mutation receipt incomplete") + } + if len(body) == 0 && receipt.Result != "deleted" || len(body) > 0 && receipt.Result != "created" && receipt.Result != "updated" { + return receipt, errors.New("index mutation receipt not confirmed") + } + return receipt, nil +} +func (t *ProductionLibraryTarget) restorePlannedIndex(ctx context.Context) error { + for _, c := range t.activePlan.Index { + intent, err := t.loadIndexEffect(c, "apply-intent") + if err != nil { + return err + } + if intent == nil { + continue + } + // Completed compensation is read back before attempting to reconcile an + // old marked apply; the marker was intentionally removed by restoration. + done, err := t.loadIndexEffect(c, "restored") + if err != nil { + return err + } + if done != nil { + g, err := t.observePlannedIndex(ctx, c) + if err != nil { + return err + } + if !indexBytesEqual(g, c.Before) || (len(c.Before) > 0 && (done.Receipt == nil || !sameIndexGeneration(g, done.Receipt.SeqNo, done.Receipt.PrimaryTerm))) { + return errors.New("restored index changed") + } + continue + } + if reconciled, err := t.reconcileIndexCompensationDelete(ctx, c); err != nil { + return err + } else if reconciled { + continue + } + applied, err := t.loadIndexEffect(c, "applied") + if err != nil { + return err + } + if applied == nil { + receipt, err := t.reconcileIndexClaim(ctx, c) + if err != nil { + return err + } + applied = &indexEffect{Plan: t.activePlan.Digest, Change: c, Receipt: &receipt} + } + if applied.Receipt == nil { + return errors.New("missing index compensation receipt") + } + g, err := t.observePlannedIndex(ctx, c) + if err != nil { + return err + } + restoring, err := t.loadIndexEffect(c, "restore-intent") + if err != nil { + return err + } + if restoring != nil { + if len(c.Before) > 0 { + if err := t.restoreClaimedIndexBefore(ctx, c, *applied.Receipt, false); err != nil { + return err + } + continue + } + return errors.New("unknown index compensation requires reconciliation") + } + expected := c.After + if applied.Receipt.Marked { + expected, err = t.indexClaimBody(c) + if err != nil { + return err + } + } + if !indexBytesEqual(g, expected) || (len(expected) > 0 && !sameIndexGeneration(g, applied.Receipt.SeqNo, applied.Receipt.PrimaryTerm)) { + return errors.New("another writer changed index; compensation refused") + } + if err = t.saveIndexEffect(c, "restore-intent", applied.Receipt); err != nil { + return err + } + if len(c.Before) > 0 { + if err := t.restoreClaimedIndexBefore(ctx, c, *applied.Receipt, true); err != nil { + return err + } + continue + } + receipt, err := t.mutatePlannedIndex(ctx, c, c.Before, applied.Receipt.SeqNo, applied.Receipt.PrimaryTerm, len(c.After) == 0) + if err != nil { + return err + } + if err = t.saveIndexEffect(c, "restored", &receipt); err != nil { + return err + } + g, err = t.observePlannedIndex(ctx, c) + if err != nil { + return err + } + if !indexBytesEqual(g, c.Before) { + return errors.New("index compensation readback failed") + } + } + return nil +} diff --git a/internal/releaseadapter/v071/library_index_guard.go b/internal/releaseadapter/v071/library_index_guard.go new file mode 100644 index 0000000..7f2366c --- /dev/null +++ b/internal/releaseadapter/v071/library_index_guard.go @@ -0,0 +1,106 @@ +package v071 + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + + "knowledge-governance/internal/librarysnapshot" +) + +type indexWriteGuard struct { + ID string `json:"_id"` + Index string `json:"_index"` + Found bool `json:"found"` + Source json.RawMessage `json:"_source"` + SeqNo *int64 `json:"_seq_no"` + PrimaryTerm *int64 `json:"_primary_term"` + Error json.RawMessage `json:"error"` +} + +// Elasticsearch IDs belong to the entire index, not to a KB. Every requested +// ID must be explicitly present or absent in the response, never inferred from +// a truncated/malformed response or a transport error. +func (e *SearchExporter) indexWriteGuards(ctx context.Context, documents []librarysnapshot.IndexDocument, identity SearchIdentity) (map[string]indexWriteGuard, error) { + guards := map[string]indexWriteGuard{} + for start := 0; start < len(documents); start += 500 { + end := start + 500 + if end > len(documents) { + end = len(documents) + } + ids := []string{} + wanted := map[string]bool{} + for _, doc := range documents[start:end] { + if doc.ID == "" || wanted[doc.ID] { + return nil, errors.New("invalid or duplicate index identity") + } + if _, exists := guards[doc.ID]; exists { + return nil, errors.New("duplicate index identity") + } + wanted[doc.ID] = true + ids = append(ids, doc.ID) + } + var response struct { + Docs []indexWriteGuard `json:"docs"` + } + if err := e.requestJSON(ctx, http.MethodPost, "/"+e.index+"/_mget", map[string]any{"ids": ids}, &response); err != nil { + return nil, err + } + if len(response.Docs) != len(ids) { + return nil, errors.New("incomplete index identity preflight") + } + for _, doc := range response.Docs { + if !wanted[doc.ID] || doc.Index != e.index || len(doc.Error) > 0 && string(doc.Error) != "null" { + return nil, errors.New("invalid index identity preflight") + } + if _, exists := guards[doc.ID]; exists { + return nil, errors.New("duplicate index preflight result") + } + if doc.Found { + var owner struct { + Tenant uint64 `json:"tenant_id"` + KB string `json:"knowledge_base_id"` + } + if json.Unmarshal(doc.Source, &owner) != nil || owner.Tenant != identity.TenantID || owner.KB != identity.KnowledgeBaseID { + return nil, fmt.Errorf("index-level ID conflict: %s is not owned by target tenant/KB", doc.ID) + } + if doc.SeqNo == nil || doc.PrimaryTerm == nil { + return nil, errors.New("index lacks optimistic concurrency identity") + } + } + guards[doc.ID] = doc + } + } + return guards, nil +} + +func (t *ProductionLibraryTarget) preflightArchiveIndex(ctx context.Context, archive *librarysnapshot.Archive, manifest librarysnapshot.Manifest) error { + for _, part := range manifest.Parts { + if part.Kind != "index" { + continue + } + reader, err := archive.OpenPart(part) + if err != nil { + return err + } + docs, err := librarysnapshot.ReadIndexDocuments(reader) + reader.Close() + if err != nil { + return err + } + if _, err := t.search.indexWriteGuards(ctx, docs, SearchIdentity{TenantID: t.scope.TenantID, KnowledgeBaseID: t.scope.KnowledgeBaseID}); err != nil { + return err + } + } + return nil +} + +func (e *SearchExporter) BulkIndexGuarded(ctx context.Context, documents []librarysnapshot.IndexDocument, identity SearchIdentity) error { + guards, err := e.indexWriteGuards(ctx, documents, identity) + if err != nil { + return err + } + return e.bulkIndex(ctx, documents, identity, guards) +} diff --git a/internal/releaseadapter/v071/library_index_plan.go b/internal/releaseadapter/v071/library_index_plan.go new file mode 100644 index 0000000..ec8b134 --- /dev/null +++ b/internal/releaseadapter/v071/library_index_plan.go @@ -0,0 +1,162 @@ +package v071 + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "sort" + "strings" + + "knowledge-governance/internal/libraryrelease" + "knowledge-governance/internal/librarysnapshot" +) + +func archiveIndexMap(archive *librarysnapshot.Archive, manifest librarysnapshot.Manifest) (map[string]json.RawMessage, error) { + out := map[string]json.RawMessage{} + for _, part := range manifest.Parts { + if part.Kind != "index" { + continue + } + reader, err := archive.OpenPart(part) + if err != nil { + return nil, err + } + docs, err := librarysnapshot.ReadIndexDocuments(reader) + reader.Close() + if err != nil { + return nil, err + } + for _, doc := range docs { + if doc.ID == "" { + return nil, errors.New("empty index ID") + } + if _, ok := out[doc.ID]; ok { + return nil, errors.New("duplicate index ID") + } + raw, err := canonicalJSON(doc.Source) + if err != nil { + return nil, err + } + out[doc.ID] = raw + } + } + return out, nil +} +func targetIndexSource(raw json.RawMessage, tenant uint64, kb string) (json.RawMessage, error) { + var source map[string]json.RawMessage + if json.Unmarshal(raw, &source) != nil || source == nil { + return nil, errors.New("index source must be object") + } + var content string + if json.Unmarshal(source["content"], &content) != nil || strings.HasPrefix(content, indexClaimPrefix) { + return nil, errors.New("index source lacks string content or contains an unfinished transient claim") + } + source["tenant_id"], _ = json.Marshal(tenant) + source["knowledge_base_id"], _ = json.Marshal(kb) + encoded, err := json.Marshal(source) + if err != nil { + return nil, err + } + return canonicalJSON(encoded) +} +func (t *ProductionLibraryTarget) planIndex(ctx context.Context, archive *librarysnapshot.Archive, source librarysnapshot.Manifest, set *libraryrelease.WriteSet) error { + backup, err := archive.Load(set.BackupSnapshotID) + if err != nil { + return err + } + before, err := archiveIndexMap(archive, backup) + if err != nil { + return err + } + after, err := archiveIndexMap(archive, source) + if err != nil { + return err + } + ids := map[string]bool{} + for id := range before { + ids[id] = true + } + for id := range after { + ids[id] = true + } + sorted := make([]string, 0, len(ids)) + for id := range ids { + sorted = append(sorted, id) + } + sort.Strings(sorted) + docs := make([]librarysnapshot.IndexDocument, 0, len(ids)) + for _, id := range sorted { + docs = append(docs, librarysnapshot.IndexDocument{ID: id}) + } + guards, err := t.search.indexWriteGuards(ctx, docs, SearchIdentity{TenantID: t.scope.TenantID, KnowledgeBaseID: t.scope.KnowledgeBaseID}) + if err != nil { + return err + } + for _, id := range sorted { + g := guards[id] + expected, existed := before[id] + if existed != g.Found { + return errors.New("index before-image no longer covers exact write set") + } + if existed { + raw, err := canonicalJSON(g.Source) + if err != nil || !bytes.Equal(raw, expected) { + return errors.New("index changed since before-image") + } + } + var wanted json.RawMessage + if raw, ok := after[id]; ok { + wanted, err = targetIndexSource(raw, t.scope.TenantID, t.scope.KnowledgeBaseID) + if err != nil { + return err + } + } + set.Index = append(set.Index, libraryrelease.IndexWrite{Index: t.search.index, ID: id, Before: expected, SeqNo: g.SeqNo, PrimaryTerm: g.PrimaryTerm, After: wanted}) + } + return nil +} + +func (t *ProductionLibraryTarget) validateIndexPlanBinding(archive *librarysnapshot.Archive, source, backup librarysnapshot.Manifest, set libraryrelease.WriteSet) error { + before, err := archiveIndexMap(archive, backup) + if err != nil { + return err + } + after, err := archiveIndexMap(archive, source) + if err != nil { + return err + } + ids := map[string]bool{} + for id := range before { + ids[id] = true + } + for id := range after { + ids[id] = true + } + if len(ids) != len(set.Index) { + return errors.New("index plan coverage mismatch") + } + for _, change := range set.Index { + if !ids[change.ID] || change.Index != t.search.index { + return errors.New("index plan physical identity mismatch") + } + delete(ids, change.ID) + var wanted json.RawMessage + if raw, ok := after[change.ID]; ok { + wanted, err = targetIndexSource(raw, set.TenantID, set.KnowledgeBaseID) + if err != nil { + return err + } + } + if !bytes.Equal(before[change.ID], change.Before) || !bytes.Equal(wanted, change.After) { + return errors.New("index plan bytes differ from immutable archives") + } + if len(change.Before) > 0 && (change.SeqNo == nil || change.PrimaryTerm == nil) { + return errors.New("existing index document lacks CAS identity") + } + if len(change.Before) == 0 && (change.SeqNo != nil || change.PrimaryTerm != nil) { + return errors.New("absent index document has invalid CAS identity") + } + } + return nil +} diff --git a/internal/releaseadapter/v071/library_index_restore_claim.go b/internal/releaseadapter/v071/library_index_restore_claim.go new file mode 100644 index 0000000..e938a0f --- /dev/null +++ b/internal/releaseadapter/v071/library_index_restore_claim.go @@ -0,0 +1,110 @@ +package v071 + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "knowledge-governance/internal/libraryrelease" +) + +const indexRestoreClaimPrefix = "kg-release-transient-restore-claim:" + +func (t *ProductionLibraryTarget) restoreIndexClaimBody(c libraryrelease.IndexWrite) (json.RawMessage, error) { + var body map[string]json.RawMessage + if json.Unmarshal(c.Before, &body) != nil || body == nil { + return nil, errors.New("restore claim requires frozen before-image") + } + var content string + if json.Unmarshal(body["content"], &content) != nil { + return nil, errors.New("restore claim requires existing content field") + } + sum := sha256.Sum256([]byte(t.activePlan.Digest + "\x00" + c.Index + "\x00" + c.ID + "\x00restore")) + body["content"], _ = json.Marshal(indexRestoreClaimPrefix + hex.EncodeToString(sum[:]) + "\n" + content) + encoded, err := json.Marshal(body) + if err != nil { + return nil, err + } + return canonicalJSON(encoded) +} + +// Only a plan-specific marker plus the entire frozen body can reconcile a lost +// restore-write response. Equal clean content is not proof of who wrote it. +// Old unknown writes without this pre-send intent remain held, not adopted. +func (t *ProductionLibraryTarget) restoreClaimedIndexBefore(ctx context.Context, c libraryrelease.IndexWrite, applied indexReceipt, start bool) error { + body, err := t.restoreIndexClaimBody(c) + if err != nil { + return err + } + markIntent, err := t.loadIndexEffect(c, "restore-mark-intent") + if err != nil { + return err + } + claimed, err := t.loadIndexEffect(c, "restore-claimed") + if err != nil { + return err + } + if markIntent == nil { + if !start { + return errors.New("legacy unknown index restoration lacks attributable marker intent") + } + if err = t.saveIndexEffect(c, "restore-mark-intent", &applied); err != nil { + return err + } + receipt, err := t.mutatePlannedIndex(ctx, c, body, applied.SeqNo, applied.PrimaryTerm, len(c.After) == 0) + if err != nil { + return err + } + receipt.Marked = true + if err = t.saveIndexEffect(c, "restore-claimed", &receipt); err != nil { + return err + } + claimed = &indexEffect{Plan: t.activePlan.Digest, Change: c, Receipt: &receipt} + } + finalizing, err := t.loadIndexEffect(c, "restore-finalize-intent") + if err != nil { + return err + } + if finalizing != nil { + return errors.New("unknown clean restoration cannot be attributed by appearance; held") + } + observed, err := t.observePlannedIndex(ctx, c) + if err != nil { + return err + } + if !indexBytesEqual(observed, body) || observed.SeqNo == nil || observed.PrimaryTerm == nil { + return errors.New("unknown restoration lacks exact operation marker/content/CAS; held") + } + if claimed == nil { + receipt := indexReceipt{Index: c.Index, ID: c.ID, Result: "updated", SeqNo: observed.SeqNo, PrimaryTerm: observed.PrimaryTerm, Marked: true} + if len(c.After) == 0 { + receipt.Result = "created" + } + if err = t.saveIndexEffect(c, "restore-claimed", &receipt); err != nil { + return err + } + claimed = &indexEffect{Plan: t.activePlan.Digest, Change: c, Receipt: &receipt} + } + if claimed.Receipt == nil || !claimed.Receipt.Marked || !sameIndexGeneration(observed, claimed.Receipt.SeqNo, claimed.Receipt.PrimaryTerm) { + return errors.New("restoration claim generation changed; held") + } + if err = t.saveIndexEffect(c, "restore-finalize-intent", claimed.Receipt); err != nil { + return err + } + receipt, err := t.mutatePlannedIndex(ctx, c, c.Before, claimed.Receipt.SeqNo, claimed.Receipt.PrimaryTerm, false) + if err != nil { + return err + } + if err = t.saveIndexEffect(c, "restored", &receipt); err != nil { + return err + } + observed, err = t.observePlannedIndex(ctx, c) + if err != nil { + return err + } + if !indexBytesEqual(observed, c.Before) || !sameIndexGeneration(observed, receipt.SeqNo, receipt.PrimaryTerm) { + return errors.New("restoration clean readback differs from receipt") + } + return nil +} diff --git a/internal/releaseadapter/v071/library_local_ids.go b/internal/releaseadapter/v071/library_local_ids.go new file mode 100644 index 0000000..ac9cfb7 --- /dev/null +++ b/internal/releaseadapter/v071/library_local_ids.go @@ -0,0 +1,176 @@ +package v071 + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "path/filepath" + "sort" + "strconv" + + "knowledge-governance/internal/libraryrelease" + "knowledge-governance/internal/librarysnapshot" +) + +// Compile local number requests without changing sequence state. Allocation is +// a separate, explicitly mutating phase after the backup and P0 are durable. +func (t *ProductionLibraryTarget) localIDRequests(archive *librarysnapshot.Archive, manifest librarysnapshot.Manifest) ([]libraryrelease.LocalIDMapping, error) { + var result []libraryrelease.LocalIDMapping + seen := map[string]bool{} + for _, part := range manifest.Parts { + if part.Kind != "database" && part.Kind != "wiki" { + continue + } + contract, ok := migrationIdentityContracts[part.Name] + if !ok { + if part.Count != 0 { + return nil, fmt.Errorf("unknown table %s", part.Name) + } + continue + } + if contract.LocalColumn == "" { + continue + } + reader, err := archive.OpenPart(part) + if err != nil { + return nil, err + } + var rows []map[string]json.RawMessage + err = json.NewDecoder(reader).Decode(&rows) + reader.Close() + if err != nil { + return nil, err + } + for _, row := range rows { + raw, ok := row[contract.LocalColumn] + if !ok && contract.LocalColumn != "id" { + continue + } // older snapshots lacking the added column remain explicit legacy input + source, err := positiveBigint(raw) + if err != nil { + return nil, fmt.Errorf("%s.%s: %w", part.Name, contract.LocalColumn, err) + } + key := part.Name + "/" + source + if seen[key] { + return nil, fmt.Errorf("duplicate local identity %s", key) + } + seen[key] = true + result = append(result, libraryrelease.LocalIDMapping{Table: part.Name, Column: contract.LocalColumn, Source: source}) + } + } + sort.Slice(result, func(i, j int) bool { + if result[i].Table != result[j].Table { + return result[i].Table < result[j].Table + } + a, _ := strconv.ParseInt(result[i].Source, 10, 64) + b, _ := strconv.ParseInt(result[j].Source, 10, 64) + return a < b + }) + return result, nil +} + +func (t *ProductionLibraryTarget) allocateLocalIDs(ctx context.Context, set *libraryrelease.WriteSet) error { + // Preflight every sequence before consuming even one value. No setval and no + // rewind on failure: an unanchored allocation can leave holes, never aliases. + checked := map[string]bool{} + for _, mapping := range set.LocalIDs { + if checked[mapping.Table] { + continue + } + checked[mapping.Table] = true + contract := migrationIdentityContracts[mapping.Table] + var datatype, def string + err := t.pool.QueryRow(ctx, `SELECT format_type(a.atttypid,a.atttypmod),coalesce(pg_get_expr(d.adbin,d.adrelid),'') FROM pg_attribute a JOIN pg_class c ON c.oid=a.attrelid JOIN pg_namespace n ON n.oid=c.relnamespace LEFT JOIN pg_attrdef d ON d.adrelid=c.oid AND d.adnum=a.attnum WHERE n.nspname='public' AND c.relname=$1 AND a.attname=$2 AND NOT a.attisdropped`, mapping.Table, mapping.Column).Scan(&datatype, &def) + if err != nil || datatype != "bigint" || def != "nextval('"+contract.Sequence+"'::regclass)" { + return fmt.Errorf("local sequence schema mismatch: %s.%s", mapping.Table, mapping.Column) + } + var increment, maximum, cache int64 + var cycle bool + if err := t.pool.QueryRow(ctx, `SELECT seqincrement,seqmax,seqcache,seqcycle FROM pg_sequence WHERE seqrelid=to_regclass($1)`, contract.Sequence).Scan(&increment, &maximum, &cache, &cycle); err != nil { + return err + } + if increment != 1 || cycle || cache != 1 { + return errors.New("unsupported local sequence allocation policy") + } + var last, maxUsed int64 + var called bool + if err := t.pool.QueryRow(ctx, fmt.Sprintf(`SELECT last_value,is_called FROM %s`, contract.Sequence)).Scan(&last, &called); err != nil { + return err + } + if err := t.pool.QueryRow(ctx, fmt.Sprintf(`SELECT coalesce(max(%s),0) FROM %s`, mapping.Column, mapping.Table)).Scan(&maxUsed); err != nil { + return err + } + next := last + if called { + if last >= maximum { + return errors.New("local sequence exhausted") + } + next++ + } + count := int64(0) + for _, m := range set.LocalIDs { + if m.Table == mapping.Table { + count++ + } + } + if next <= maxUsed || count-1 > maximum-next { + return fmt.Errorf("local sequence behind occupied values or exhausted: %s", contract.Sequence) + } + } + raw, err := json.Marshal(set) + if err != nil { + return err + } + sum := sha256.Sum256(raw) + if err := persistImmutableJSON(filepath.Join(t.config.ArchiveRoot, "plans-p0", hex.EncodeToString(sum[:])+".json"), raw); err != nil { + return err + } + for i := range set.LocalIDs { + mapping := &set.LocalIDs[i] + contract := migrationIdentityContracts[mapping.Table] + var allocated int64 + if err := t.pool.QueryRow(ctx, `SELECT nextval($1::regclass)`, contract.Sequence).Scan(&allocated); err != nil { + return err + } + var occupied bool + if err := t.pool.QueryRow(ctx, fmt.Sprintf(`SELECT EXISTS(SELECT 1 FROM %s WHERE %s=$1)`, mapping.Table, mapping.Column), allocated).Scan(&occupied); err != nil { + return err + } + if occupied { + return errors.New("allocated local identity already occupied; maintenance/fence lost") + } + mapping.Target = strconv.FormatInt(allocated, 10) + if contract.Kind == "local-bigint" { + set.Rows = append(set.Rows, libraryrelease.WriteSetRow{Table: mapping.Table, ID: mapping.Target}) + } + } + return nil +} + +func (t *ProductionLibraryTarget) mapLocalID(table string, row map[string]json.RawMessage) (bool, error) { + if t.activePlan == nil || t.activeSnapshot != t.activePlan.TargetSnapshotID || t.preserveChunkSequence { + return false, nil + } + contract := migrationIdentityContracts[table] + if contract.LocalColumn == "" { + return false, nil + } + raw, ok := row[contract.LocalColumn] + if !ok { + return false, nil + } + source, err := positiveBigint(raw) + if err != nil { + return false, err + } + for _, mapping := range t.activePlan.LocalIDs { + if mapping.Table == table && mapping.Column == contract.LocalColumn && mapping.Source == source { + row[contract.LocalColumn] = json.RawMessage(mapping.Target) + return true, nil + } + } + return false, fmt.Errorf("missing persisted local mapping: %s.%s", table, contract.LocalColumn) +} diff --git a/internal/releaseadapter/v071/library_natural_keys.go b/internal/releaseadapter/v071/library_natural_keys.go new file mode 100644 index 0000000..9c188bc --- /dev/null +++ b/internal/releaseadapter/v071/library_natural_keys.go @@ -0,0 +1,132 @@ +package v071 + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/jackc/pgx/v5" + "knowledge-governance/internal/librarysnapshot" +) + +type migrationUniqueKey struct { + Name string + Columns []string + Predicate string + NullsEqual bool + Expression bool +} + +// Preflight all catalog unique keys, including partial indexes. Source-local +// sequence keys are checked separately by the allocator, never compared as UUIDs. +func (t *ProductionLibraryTarget) preflightNaturalKeys(ctx context.Context, archive *librarysnapshot.Archive, manifest librarysnapshot.Manifest) error { + for _, part := range manifest.Parts { + if (part.Kind != "database" && part.Kind != "wiki") || part.Count == 0 { + continue + } + contract, known := migrationIdentityContracts[part.Name] + if !known { + return errors.New("unknown natural-key contract") + } + reader, err := archive.OpenPart(part) + if err != nil { + return err + } + var source []map[string]json.RawMessage + err = json.NewDecoder(reader).Decode(&source) + reader.Close() + if err != nil { + return err + } + for _, row := range source { + if err := t.retargetRow(part.Name, row); err != nil { + return err + } + } + raw, err := json.Marshal(source) + if err != nil { + return err + } + rows, err := t.pool.Query(ctx, `SELECT ic.relname,ARRAY(SELECT a.attname::text FROM unnest(i.indkey) WITH ORDINALITY k(attnum,ord) JOIN pg_attribute a ON a.attrelid=i.indrelid AND a.attnum=k.attnum WHERE k.ord<=i.indnkeyatts ORDER BY k.ord),coalesce(pg_get_expr(i.indpred,i.indrelid),''),i.indnullsnotdistinct,i.indexprs IS NOT NULL FROM pg_index i JOIN pg_class c ON c.oid=i.indrelid JOIN pg_namespace n ON n.oid=c.relnamespace JOIN pg_class ic ON ic.oid=i.indexrelid WHERE n.nspname='public' AND c.relname=$1 AND i.indisunique ORDER BY ic.relname`, part.Name) + if err != nil { + return err + } + var keys []migrationUniqueKey + for rows.Next() { + var key migrationUniqueKey + if err := rows.Scan(&key.Name, &key.Columns, &key.Predicate, &key.NullsEqual, &key.Expression); err != nil { + rows.Close() + return err + } + keys = append(keys, key) + } + rows.Close() + if err := rows.Err(); err != nil { + return err + } + table := pgx.Identifier{"public", part.Name}.Sanitize() + recordset := "jsonb_populate_recordset(NULL::" + table + ",$1::jsonb)" + for _, key := range keys { + if key.Expression || len(key.Columns) == 0 { + return fmt.Errorf("unreviewed unique expression: %s", key.Name) + } + skip := false + for _, column := range key.Columns { + if column == contract.LocalColumn { + skip = true + } + } + if skip { + continue + } + predicate := "true" + if key.Predicate != "" { + predicate = key.Predicate + } + cols, nonnull, join := []string{}, []string{}, []string{} + for _, column := range key.Columns { + q := pgx.Identifier{column}.Sanitize() + cols = append(cols, q) + nonnull = append(nonnull, q+" IS NOT NULL") + op := "=" + if key.NullsEqual { + op = "IS NOT DISTINCT FROM" + } + join = append(join, "s."+q+" "+op+" t."+q) + } + sourceFilter := predicate + if !key.NullsEqual { + sourceFilter += " AND " + strings.Join(nonnull, " AND ") + } + var conflict bool + query := "SELECT EXISTS(SELECT 1 FROM " + recordset + " s WHERE " + sourceFilter + " GROUP BY " + strings.Join(cols, ",") + " HAVING count(*)>1)" + if err := t.pool.QueryRow(ctx, query, raw).Scan(&conflict); err != nil { + return fmt.Errorf("source unique preflight %s: %w", key.Name, err) + } + if conflict { + return fmt.Errorf("source duplicates natural key %s", key.Name) + } + targetFilter := predicate + args := []any{raw} + if part.Name != "resources" { + scope := strings.ReplaceAll(strings.ReplaceAll(scopeWhere(part.Name), "$2", "$3"), "$1", "$2") + targetFilter += " AND NOT (" + scope + ")" + args = append(args, t.scope.TenantID, t.scope.KnowledgeBaseID) + } + query = "SELECT EXISTS(SELECT 1 FROM (SELECT * FROM " + recordset + " WHERE " + sourceFilter + ") s JOIN (SELECT * FROM " + table + " WHERE " + targetFilter + ") t ON " + strings.Join(join, " AND ") + ")" + // Equal resource ID is handled by immutable row/object reuse evidence. + if part.Name == "resources" { + query = strings.TrimSuffix(query, ")") + " WHERE s.id IS DISTINCT FROM t.id)" + } + if err := t.pool.QueryRow(ctx, query, args...).Scan(&conflict); err != nil { + return fmt.Errorf("target unique preflight %s: %w", key.Name, err) + } + if conflict { + return fmt.Errorf("target natural key conflict outside replace scope: %s", key.Name) + } + } + } + return nil +} diff --git a/internal/releaseadapter/v071/library_object_effects.go b/internal/releaseadapter/v071/library_object_effects.go new file mode 100644 index 0000000..2c3e121 --- /dev/null +++ b/internal/releaseadapter/v071/library_object_effects.go @@ -0,0 +1,254 @@ +package v071 + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "knowledge-governance/internal/libraryrelease" + "knowledge-governance/internal/librarysnapshot" +) + +func (t *ProductionLibraryTarget) planObjects(ctx context.Context, archive *librarysnapshot.Archive, manifest librarysnapshot.Manifest, set *libraryrelease.WriteSet) error { + objects := map[string]librarysnapshot.Part{} + for _, p := range manifest.Parts { + if p.Kind == "resource" { + if _, exists := objects[p.Name]; exists { + return errors.New("duplicate object resource identity") + } + objects[p.Name] = p + } + } + if len(objects) == 0 { + return nil + } + store, ok := t.config.Objects.(conditionalObjectStore) + if !ok { + return errors.New("target lacks conditional object effects") + } + if err := store.ConditionalCapability(ctx); err != nil { + return err + } + seen := map[string]bool{} + for _, part := range manifest.Parts { + if part.Kind != "database" || part.Name != "resources" { + continue + } + reader, err := archive.OpenPart(part) + if err != nil { + return err + } + var rows []map[string]json.RawMessage + err = json.NewDecoder(reader).Decode(&rows) + reader.Close() + if err != nil { + return err + } + for _, row := range rows { + var id string + if err := json.Unmarshal(row["id"], &id); err != nil { + return err + } + blob, exists := objects[id] + if !exists { + return errors.New("resource has no bound object blob") + } + if err := t.retargetRow("resources", row); err != nil { + return err + } + obj := libraryrelease.ObjectWrite{ResourceID: id, SHA256: blob.SHA256, Bytes: blob.Bytes} + if json.Unmarshal(row["provider"], &obj.Provider) != nil || json.Unmarshal(row["physical_path"], &obj.Locator) != nil { + return errors.New("resource locator missing") + } + rest := strings.TrimPrefix(obj.Locator, "storage://") + parts := strings.SplitN(rest, "/", 2) + if rest == obj.Locator || len(parts) != 2 { + return errors.New("object needs explicit backend/bucket/key") + } + obj.BackendID = parts[0] + physical := strings.TrimPrefix(parts[1], obj.Provider+"://") + location := strings.SplitN(physical, "/", 2) + if physical == parts[1] || len(location) != 2 || location[0] == "" || location[1] == "" { + return errors.New("invalid physical object identity") + } + obj.Bucket, obj.Key = location[0], location[1] + // Backend aliases cannot make a shared physical key look independent. + key := obj.Provider + "/" + obj.Bucket + "/" + obj.Key + if seen[key] { + return errors.New("duplicate physical object destination") + } + seen[key] = true + token := sha256.Sum256([]byte(set.JobID + "\x00" + set.Target + "\x00" + set.BackupSnapshotID + "\x00" + set.TargetSnapshotID + "\x00" + key)) + obj.OwnerToken = hex.EncodeToString(token[:]) + observation, err := store.ObserveObject(ctx, obj.Provider, obj.Locator) + if err != nil { + return err + } + obj.Existing, obj.BeforeETag = observation.Exists, observation.ETag + obj.BeforeVersionID = observation.VersionID + if observation.Exists && (observation.SHA256 != obj.SHA256 || observation.Bytes != obj.Bytes) { + return errors.New("different existing object cannot be recoverably overwritten") + } + set.Objects = append(set.Objects, obj) + delete(objects, id) + } + } + if len(objects) != 0 { + return errors.New("object blob lacks resource identity") + } + return nil +} + +func (t *ProductionLibraryTarget) objectEffectPath(obj libraryrelease.ObjectWrite, phase string) string { + sum := sha256.Sum256([]byte(obj.Provider + "\x00" + obj.Bucket + "\x00" + obj.Key)) + return filepath.Join(t.config.ArchiveRoot, "effects", t.activePlan.Digest, "objects", hex.EncodeToString(sum[:]), phase+".json") +} + +func effectExists(path string) (bool, error) { + info, err := os.Lstat(path) + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + if err != nil { + return false, err + } + if !info.Mode().IsRegular() { + return false, errors.New("effect evidence is not a regular file") + } + return true, nil +} + +func (t *ProductionLibraryTarget) recordObjectEffect(obj libraryrelease.ObjectWrite, phase string, observation objectObservation) error { + raw, err := json.Marshal(struct { + Plan string `json:"plan"` + Object libraryrelease.ObjectWrite `json:"object"` + Observation objectObservation `json:"observation"` + }{t.activePlan.Digest, obj, observation}) + if err != nil { + return err + } + return persistImmutableJSON(t.objectEffectPath(obj, phase), raw) +} + +func matchesObject(obj libraryrelease.ObjectWrite, observed objectObservation, owned bool) bool { + return observed.Exists && observed.Bytes == obj.Bytes && observed.SHA256 == obj.SHA256 && (!owned || (observed.Owner == obj.OwnerToken && observed.VersionID != "" && observed.VersionID != "null")) +} + +func (t *ProductionLibraryTarget) applyPlannedObject(ctx context.Context, obj libraryrelease.ObjectWrite, body []byte) error { + store, ok := t.config.Objects.(conditionalObjectStore) + if !ok { + return errors.New("object effects capability missing") + } + if obj.Existing { + observed, err := store.ObserveObject(ctx, obj.Provider, obj.Locator) + if err != nil { + return err + } + if !matchesObject(obj, observed, false) || observed.ETag != obj.BeforeETag || observed.VersionID != obj.BeforeVersionID { + return errors.New("read-only object changed since plan") + } + return nil + } + intended, err := effectExists(t.objectEffectPath(obj, "create-intent")) + if err != nil { + return err + } + if !intended { + if err := t.recordObjectEffect(obj, "create-intent", objectObservation{}); err != nil { + return err + } + err = store.CreateObject(ctx, obj.Provider, obj.Locator, bytes.NewReader(body), obj.Bytes, obj.OwnerToken) + if errors.Is(err, errObjectWriteRejected) { + if e := t.recordObjectEffect(obj, "create-rejected", objectObservation{}); e != nil { + return e + } + return err + } + // Even 2xx is not a receipt: observe the exact physical target independently. + } + observed, observeErr := store.ObserveObject(ctx, obj.Provider, obj.Locator) + if observeErr != nil { + return observeErr + } + if !matchesObject(obj, observed, true) { + return errors.New("object create unknown or conflicting; do not replay/delete without ownership proof") + } + return t.recordObjectEffect(obj, "created", observed) +} + +func (t *ProductionLibraryTarget) restoreCreatedObjects(ctx context.Context) error { + if t.activePlan == nil { + return errors.New("object restoration requires adopted plan") + } + if len(t.activePlan.Objects) == 0 { + return nil + } + store, ok := t.config.Objects.(conditionalObjectStore) + if !ok { + return errors.New("object restoration capability missing") + } + for _, obj := range t.activePlan.Objects { + if obj.Existing { + continue + } + intent, err := effectExists(t.objectEffectPath(obj, "create-intent")) + if err != nil { + return err + } + if !intent { + continue + } + rejected, err := effectExists(t.objectEffectPath(obj, "create-rejected")) + if err != nil { + return err + } + if rejected { + continue + } + observed, err := store.ObserveObject(ctx, obj.Provider, obj.Locator) + if err != nil { + return err + } + created, err := effectExists(t.objectEffectPath(obj, "created")) + if err != nil { + return err + } + if !observed.Exists { + if !created { + return errors.New("unresolved create intent: absence is not proof no delayed write remains") + } + if err := t.recordObjectEffect(obj, "removed", objectObservation{}); err != nil { + return err + } + continue + } + if !matchesObject(obj, observed, true) { + return errors.New("created object now differs or has another owner; cleanup refused") + } + if err := t.recordObjectEffect(obj, "created", observed); err != nil { + return err + } + if err := t.recordObjectEffect(obj, "delete-intent", observed); err != nil { + return err + } + deleteErr := store.RemoveObject(ctx, obj.Provider, obj.Locator, observed) + after, err := store.ObserveObject(ctx, obj.Provider, obj.Locator) + if err != nil { + return err + } + if after.Exists { + return fmt.Errorf("conditional object cleanup not confirmed: %v", deleteErr) + } + if err := t.recordObjectEffect(obj, "removed", objectObservation{}); err != nil { + return err + } + } + return nil +} diff --git a/internal/releaseadapter/v071/library_object_plan_binding.go b/internal/releaseadapter/v071/library_object_plan_binding.go new file mode 100644 index 0000000..99bae81 --- /dev/null +++ b/internal/releaseadapter/v071/library_object_plan_binding.go @@ -0,0 +1,49 @@ +package v071 + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + + "knowledge-governance/internal/libraryrelease" + "knowledge-governance/internal/librarysnapshot" +) + +func validateObjectPlanBinding(set libraryrelease.WriteSet, source librarysnapshot.Manifest) error { + blobs := map[string]librarysnapshot.Part{} + for _, part := range source.Parts { + if part.Kind == "resource" { + if _, exists := blobs[part.Name]; exists { + return errors.New("duplicate source object blob") + } + blobs[part.Name] = part + } + } + if len(blobs) != len(set.Objects) { + return errors.New("object plan coverage differs from source") + } + seen := map[string]bool{} + for _, obj := range set.Objects { + part, ok := blobs[obj.ResourceID] + if !ok || obj.Bytes != part.Bytes || obj.SHA256 != part.SHA256 { + return errors.New("object plan bytes differ from bound source blob") + } + delete(blobs, obj.ResourceID) + if obj.Provider == "" || obj.BackendID == "" || obj.Bucket == "" || obj.Key == "" || obj.Locator != "storage://"+obj.BackendID+"/"+obj.Provider+"://"+obj.Bucket+"/"+obj.Key { + return errors.New("object plan physical identity incomplete") + } + key := obj.Provider + "/" + obj.Bucket + "/" + obj.Key + if seen[key] { + return errors.New("duplicate physical object in plan") + } + seen[key] = true + token := sha256.Sum256([]byte(set.JobID + "\x00" + set.Target + "\x00" + set.BackupSnapshotID + "\x00" + set.TargetSnapshotID + "\x00" + key)) + if obj.OwnerToken != hex.EncodeToString(token[:]) { + return errors.New("object ownership token not bound to job/target/backup/snapshot") + } + if obj.Existing && obj.BeforeETag == "" { + return errors.New("existing object lacks immutable before observation") + } + } + return nil +} diff --git a/internal/releaseadapter/v071/library_object_store.go b/internal/releaseadapter/v071/library_object_store.go new file mode 100644 index 0000000..9d9d7ca --- /dev/null +++ b/internal/releaseadapter/v071/library_object_store.go @@ -0,0 +1,133 @@ +package v071 + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + smithyhttp "github.com/aws/smithy-go/transport/http" +) + +type objectObservation struct { + VersionID string `json:"version_id,omitempty"` + Exists bool `json:"exists"` + ETag string `json:"etag,omitempty"` + SHA256 string `json:"sha256,omitempty"` + Bytes int64 `json:"bytes"` + Owner string `json:"owner,omitempty"` +} + +// No fallback to blind Put/Delete is allowed when conditional effects are absent. +type conditionalObjectStore interface { + ConditionalCapability(context.Context) error + ObserveObject(context.Context, string, string) (objectObservation, error) + CreateObject(context.Context, string, string, io.Reader, int64, string) error + RemoveObject(context.Context, string, string, objectObservation) error +} + +var errObjectWriteRejected = errors.New("object write definitively rejected before send") + +func (s *S3ObjectStore) ConditionalCapability(ctx context.Context) error { + if s.provider != "minio" && s.provider != "s3" { + return errors.New("object provider has no verified conditional create/version-delete adapter") + } + status, err := s.client.GetBucketVersioning(ctx, &s3.GetBucketVersioningInput{Bucket: aws.String(s.bucket)}) + if err != nil || string(status.Status) != "Enabled" { + return errors.New("object effects require verified bucket versioning; never enable a business bucket implicitly") + } + return nil +} + +// Bind the supplied logical locator to the configured physical bucket. Legacy +// Open/Put intentionally accept bare keys; the new contract does not infer them. +func (s *S3ObjectStore) conditionalKey(provider, locator string) (string, error) { + if provider != s.provider || !validObjectKey(locator) { + return "", errors.New("conditional object provider/locator mismatch") + } + rest := strings.TrimPrefix(locator, "storage://") + parts := strings.SplitN(rest, "/", 2) + if rest == locator || len(parts) != 2 || parts[0] == "" { + return "", errors.New("conditional object requires explicit backend") + } + expected := provider + "://" + s.bucket + "/" + if !strings.HasPrefix(parts[1], expected) { + return "", errors.New("conditional object bucket/provider mismatch") + } + key := strings.TrimPrefix(parts[1], expected) + if key == "" { + return "", errors.New("empty conditional object key") + } + return key, nil +} + +func (s *S3ObjectStore) ObserveObject(ctx context.Context, provider, locator string) (objectObservation, error) { + key, err := s.conditionalKey(provider, locator) + if err != nil { + return objectObservation{}, err + } + obj, err := s.client.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key)}) + if err != nil { + var response *smithyhttp.ResponseError + if errors.As(err, &response) && response.HTTPStatusCode() == 404 { + return objectObservation{}, nil + } + return objectObservation{}, errors.New("object observation unavailable") + } + defer obj.Body.Close() + if obj.ContentLength == nil || *obj.ContentLength < 0 || *obj.ContentLength > s.maxObjectBytes || obj.ETag == nil || *obj.ETag == "" { + return objectObservation{}, errors.New("incomplete object observation") + } + hash := sha256.New() + size, err := io.Copy(hash, io.LimitReader(obj.Body, s.maxObjectBytes+1)) + if err != nil || size != *obj.ContentLength { + return objectObservation{}, errors.New("object observation body incomplete") + } + return objectObservation{VersionID: aws.ToString(obj.VersionId), Exists: true, ETag: *obj.ETag, SHA256: hex.EncodeToString(hash.Sum(nil)), Bytes: size, Owner: obj.Metadata["kg-release-owner"]}, nil +} + +func (s *S3ObjectStore) CreateObject(ctx context.Context, provider, locator string, body io.Reader, size int64, owner string) error { + if err := s.ConditionalCapability(ctx); err != nil { + return err + } + key, err := s.conditionalKey(provider, locator) + if err != nil { + return err + } + if size < 0 || size > s.maxObjectBytes || len(owner) != 64 { + return errObjectWriteRejected + } + // A durable intent represents one network attempt. Automatic retries could + // remain in flight after compensation, so ambiguity is reconciled by reads. + _, err = s.client.PutObject(ctx, &s3.PutObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key), Body: body, ContentLength: aws.Int64(size), IfNoneMatch: aws.String("*"), Metadata: map[string]string{"kg-release-owner": owner}}, func(o *s3.Options) { o.RetryMaxAttempts = 1 }) + if err != nil { + return errors.New("conditional object create outcome unknown") + } + return nil +} + +func (s *S3ObjectStore) RemoveObject(ctx context.Context, provider, locator string, expected objectObservation) error { + if err := s.ConditionalCapability(ctx); err != nil { + return err + } + key, err := s.conditionalKey(provider, locator) + if err != nil { + return err + } + if expected.ETag == "" || expected.VersionID == "" || expected.VersionID == "null" { + return errors.New("deletion requires an immutable object version") + } + before, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key), VersionId: aws.String(expected.VersionID)}) + if err != nil || aws.ToString(before.ETag) != expected.ETag || before.Metadata["kg-release-owner"] != expected.Owner { + return errors.New("exact object version no longer matches receipt") + } + _, err = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key), VersionId: aws.String(expected.VersionID)}, func(o *s3.Options) { o.RetryMaxAttempts = 1 }) + if err != nil { + return errors.New("conditional object delete outcome unknown") + } + return nil +} diff --git a/internal/releaseadapter/v071/library_plan_binding.go b/internal/releaseadapter/v071/library_plan_binding.go new file mode 100644 index 0000000..5788f0c --- /dev/null +++ b/internal/releaseadapter/v071/library_plan_binding.go @@ -0,0 +1,72 @@ +package v071 + +import ( + "errors" + "knowledge-governance/internal/libraryrelease" + "knowledge-governance/internal/librarysnapshot" +) + +// Consumption verifies the immutable source, not the half-written target. A +// correctly hashed sidecar alone does not prove complete mapping coverage. +func (t *ProductionLibraryTarget) validatePlanBindings(set libraryrelease.WriteSet) error { + if set.Version != libraryrelease.MigrationWriteSetVersion || set.ContractVersion != migrationIdentityContractVersion || len(set.SchemaDigest) != 64 { + return errors.New("legacy/incomplete plan is diagnostic-only; execution evidence missing") + } + archive, err := librarysnapshot.Open(t.config.ArchiveRoot, 256*1024*1024) + if err != nil { + return err + } + defer archive.Close() + source, err := archive.Load(set.TargetSnapshotID) + if err != nil { + return err + } + if set.ContractVersion != 0 && (source.TenantID != set.SourceTenantID || source.KnowledgeBaseID != set.SourceKnowledgeBaseID) { + return errors.New("plan source identity differs from bound manifest") + } + backup, err := archive.Load(set.BackupSnapshotID) + if err != nil { + return err + } + if backup.TenantID != set.TenantID || backup.KnowledgeBaseID != set.KnowledgeBaseID { + return errors.New("plan before-image identity differs from target") + } + if err := validateObjectPlanBinding(set, source); err != nil { + return err + } + if err := t.validateIndexPlanBinding(archive, source, backup, set); err != nil { + return err + } + requests, err := t.localIDRequests(archive, source) + if err != nil { + return err + } + if len(requests) != len(set.LocalIDs) { + return errors.New("plan local mapping coverage differs from source") + } + expected := map[string]bool{} + for _, r := range requests { + expected[r.Table+"/"+r.Column+"/"+r.Source] = true + } + wikiRows := map[string]bool{} + for _, row := range set.Rows { + if row.Table == "wiki_log_entries" { + wikiRows[row.ID] = !row.Existing + } + } + for _, m := range set.LocalIDs { + if !expected[m.Table+"/"+m.Column+"/"+m.Source] { + return errors.New("plan includes unknown local mapping") + } + if m.Table == "wiki_log_entries" { + if !wikiRows[m.Target] { + return errors.New("plan omits allocated Wiki write identity") + } + delete(wikiRows, m.Target) + } + } + if len(wikiRows) > 0 { + return errors.New("plan includes unmapped Wiki write identity") + } + return nil +} diff --git a/internal/releaseadapter/v071/library_production_target.go b/internal/releaseadapter/v071/library_production_target.go index f640c14..a576bb9 100644 --- a/internal/releaseadapter/v071/library_production_target.go +++ b/internal/releaseadapter/v071/library_production_target.go @@ -19,6 +19,7 @@ import ( "knowledge-governance/internal/libraryrelease" "knowledge-governance/internal/librarysnapshot" + "knowledge-governance/internal/testbarrier" ) // 生产整库目标:真实读、真实写、真实校验、真实恢复。 @@ -99,6 +100,8 @@ type ProductionLibraryTarget struct { reusedResources map[string]bool // 回滚必须重放目标原始序列;普通发布由目标数据库分配本地序列。 preserveChunkSequence bool + activePlan *libraryrelease.WriteSet + activeSnapshot string } // resolveStorageBackend 在目标库里找本次写入的存储后端。找不到就报错:宁可不写, @@ -227,7 +230,7 @@ func scopeWhere(table string) string { // 两种归属都要算:显式绑定(knowledge/chunk)+ 被本库 chunk 正文以 // resource:// 引用(图片/附件常见,可能没有绑定行)。 // 与归档侧收集资源部件的规则保持一致,否则校验会把导入的资源算成"多余"。 - return `(EXISTS ( + return `resources.tenant_id=$1 AND (EXISTS ( SELECT 1 FROM resource_bindings b JOIN knowledges k ON k.id=b.owner_id AND k.tenant_id=b.tenant_id WHERE b.resource_id=resources.id AND b.tenant_id=$1 AND b.owner_type='knowledge' AND k.knowledge_base_id=$2 @@ -235,11 +238,7 @@ func scopeWhere(table string) string { SELECT 1 FROM resource_bindings b JOIN chunks c ON c.id=b.owner_id AND c.tenant_id=b.tenant_id WHERE b.resource_id=resources.id AND b.tenant_id=$1 AND b.owner_type='chunk' AND c.knowledge_base_id=$2) - OR EXISTS ( - SELECT 1 FROM chunks c - WHERE c.tenant_id=$1 AND c.knowledge_base_id=$2 AND c.deleted_at IS NULL - AND coalesce(resources.handle,'') <> '' - AND position('resource://'||resources.handle IN coalesce(c.content,'')) > 0))` + OR EXISTS (` + libraryResourceBodyReferenceSQL("c.tenant_id=$1 AND c.knowledge_base_id=$2") + `))` default: return "tenant_id=$1 AND knowledge_base_id=$2" } @@ -258,7 +257,7 @@ func identityExpr(table string) string { func (t *ProductionLibraryTarget) tableState(ctx context.Context, table string) (int64, string, error) { rows, err := t.pool.Query(ctx, fmt.Sprintf(`SELECT %s FROM %s WHERE %s ORDER BY 1`, - identityExpr(table), table, scopeWhere(table)), t.scope.TenantID, t.scope.KnowledgeBaseID) + "("+identityExpr(table)+")::text", table, scopeWhere(table)), t.scope.TenantID, t.scope.KnowledgeBaseID) if err != nil { return 0, "", fmt.Errorf("读取生产 %s 失败", table) } @@ -284,6 +283,24 @@ func digestIDs(ids []string) string { // Apply 把一类数据从归档写进生产;database 与 wiki 走同一个知识域事务。 func (t *ProductionLibraryTarget) Apply(ctx context.Context, stage libraryrelease.Stage, snapshotID string) error { + if authorizer, ok := t.config.Objects.(interface{ EnsureObjectGrant(context.Context) error }); ok { + if e := authorizer.EnsureObjectGrant(ctx); e != nil { + return e + } + } + if t.activePlan != nil { + expected := t.activePlan.TargetSnapshotID + if t.preserveChunkSequence { + expected = t.activePlan.BackupSnapshotID + } + if snapshotID != expected { + return errors.New("apply snapshot differs from frozen execution plan") + } + } + if err := t.verifyPlanSchema(ctx); err != nil { + return err + } + t.activeSnapshot = snapshotID if err := t.resolveStorageBackend(ctx); err != nil { return err } @@ -401,7 +418,10 @@ func (t *ProductionLibraryTarget) insertRows(ctx context.Context, tx pgx.Tx, arc if err := t.retargetRow(table, row); err != nil { return err } - if table == "chunks" && !t.preserveChunkSequence { + if _, err := t.mapLocalID(table, row); err != nil { + return err + } + if table == "chunks" && !t.preserveChunkSequence && t.activePlan == nil { delete(row, "seq_id") } if columns == nil { @@ -416,7 +436,7 @@ func (t *ProductionLibraryTarget) insertRows(ctx context.Context, tx pgx.Tx, arc if table == "resources" { statement = fmt.Sprintf(`INSERT INTO %s SELECT * FROM jsonb_populate_record(NULL::%s, $1::jsonb) ON CONFLICT (id) DO UPDATE SET %s WHERE %s.tenant_id = $2`, table, table, updateAssignments(columns), table) - } else if table == "chunks" && !t.preserveChunkSequence { + } else if table == "chunks" && !t.preserveChunkSequence && t.activePlan == nil { // Omitting the column (not inserting NULL) invokes the target default. quoted := make([]string, len(columns)) for i, column := range columns { @@ -479,7 +499,10 @@ func (t *ProductionLibraryTarget) rowTransform(table string) func(map[string]jso } // seq_id is destination-local, not content identity. Full-universe // rollback verification still compares the original value exactly. - if table == "chunks" && !t.preserveChunkSequence { + if _, err := t.mapLocalID(table, row); err != nil { + return err + } + if table == "chunks" && !t.preserveChunkSequence && t.activePlan == nil { delete(row, "seq_id") } return nil @@ -501,7 +524,7 @@ func (t *ProductionLibraryTarget) importResources(ctx context.Context, archive * } // 复用资源的对象字节已经在计划阶段逐字节比过:不再重传(重传只会改 // 对象的写入时间,没有任何收益)。 - if t.reusedResources[part.Name] { + if t.reusedResources[part.Name] && t.activePlan == nil { continue } reader, err := archive.OpenPart(part) @@ -513,6 +536,42 @@ func (t *ProductionLibraryTarget) importResources(ctx context.Context, archive * if err != nil || int64(len(body)) != part.Bytes { return fmt.Errorf("资源 %s 字节与归档摘要不符", part.Name) } + if t.activePlan != nil { + if t.activeSnapshot == t.activePlan.BackupSnapshotID { + store, ok := t.config.Objects.(conditionalObjectStore) + if !ok { + return errors.New("rollback lacks conditional object observation") + } + observed, err := store.ObserveObject(ctx, location.Provider, location.Path) + if err != nil { + return err + } + if !observed.Exists || observed.SHA256 != part.SHA256 || observed.Bytes != part.Bytes { + return errors.New("before-image object changed; refusing blind restore overwrite") + } + continue + } + if t.activeSnapshot != t.activePlan.TargetSnapshotID { + return errors.New("object snapshot differs from plan") + } + found := false + for _, obj := range t.activePlan.Objects { + if obj.ResourceID == part.Name { + if obj.Locator != location.Path || obj.Provider != location.Provider || obj.SHA256 != part.SHA256 || obj.Bytes != part.Bytes { + return errors.New("object plan differs from resource/blob") + } + if err := t.applyPlannedObject(ctx, obj, body); err != nil { + return err + } + found = true + break + } + } + if !found { + return errors.New("resource omitted from immutable object plan") + } + continue + } if err := t.config.Objects.Put(ctx, location.Provider, location.Path, bytes.NewReader(body), int64(len(body))); err != nil { return fmt.Errorf("上传资源 %s 失败: %w", part.Name, err) } @@ -587,6 +646,12 @@ WHERE tenant_id=$1 AND id = ANY($2::text[])`, t.scope.TenantID, ids) } func (t *ProductionLibraryTarget) importIndex(ctx context.Context, archive *librarysnapshot.Archive, manifest librarysnapshot.Manifest) error { + if t.activePlan != nil { + if t.preserveChunkSequence { + return t.restorePlannedIndex(ctx) + } + return t.applyPlannedIndex(ctx) + } for _, part := range manifest.Parts { if part.Kind != "index" { continue @@ -605,7 +670,11 @@ func (t *ProductionLibraryTarget) importIndex(ctx context.Context, archive *libr } // 直接搬运原始文档与向量:不解析、不重新 Embedding。 identity := SearchIdentity{TenantID: t.scope.TenantID, KnowledgeBaseID: t.scope.KnowledgeBaseID} - if err := t.search.BulkIndex(ctx, documents, identity); err != nil { + write := t.search.BulkIndex + if t.activePlan != nil { + write = t.search.BulkIndexGuarded + } + if err := write(ctx, documents, identity); err != nil { return err } // 快照里没有的文档必须从生产索引删掉,否则撤销只在数据库生效。 @@ -622,6 +691,7 @@ func (t *ProductionLibraryTarget) importIndex(ctx context.Context, archive *libr // Verify 用行数 + 身份集合摘要 + 索引条数核对生产与预发布快照是否一致。 func (t *ProductionLibraryTarget) Verify(ctx context.Context, expect libraryrelease.Expectation) error { + t.activeSnapshot = expect.SnapshotID if err := t.resolveStorageBackend(ctx); err != nil { return err } @@ -650,6 +720,19 @@ func (t *ProductionLibraryTarget) Verify(ctx context.Context, expect libraryrele } } wantCount, wantDigest, err := archivedTableState(archive, part) + if err == nil && part.Name == "wiki_log_entries" && t.activePlan != nil && t.activeSnapshot == t.activePlan.TargetSnapshotID && !t.preserveChunkSequence { + ids := []string{} + for _, mapping := range t.activePlan.LocalIDs { + if mapping.Table == part.Name { + ids = append(ids, mapping.Target) + } + } + if int64(len(ids)) != wantCount { + return errors.New("Wiki local mapping coverage mismatch") + } + sort.Strings(ids) + wantDigest = digestIDs(ids) + } if errors.Is(err, errMissingIdentity) { // 个别表(例如日志表)没有代理主键:身份集合无从比,但行数与逐行内容摘要仍然必须一致。 wantContentCount, wantContentDigest, columns, contentErr := archivedTableContent(archive, part, t.rowTransform(part.Name)) @@ -790,19 +873,11 @@ func archivedTableState(archive *librarysnapshot.Archive, part librarysnapshot.P } func archiveRowIdentity(table string, row map[string]json.RawMessage) string { - value := func(key string) string { - var text string - _ = json.Unmarshal(row[key], &text) - return text + identity, err := parseRowIdentity(table, row) + if err != nil { + return "" } - if table == "knowledge_tag_relations" { - knowledge, tag := value("knowledge_id"), value("tag_id") - if knowledge == "" || tag == "" { - return "" - } - return knowledge + ":" + tag - } - return value("id") + return identity.key() } // AdoptWriteSet 消费已持久化的写集合(接管/恢复路径)。 @@ -831,12 +906,30 @@ func (t *ProductionLibraryTarget) AdoptWriteSet(set libraryrelease.WriteSet) err } reused[row.ID] = true } - if t.reusedResources == nil { - t.reusedResources = map[string]bool{} + if set.ContractVersion != 0 && set.ContractVersion != migrationIdentityContractVersion { + return errors.New("unsupported migration identity contract") } - for id := range reused { - t.reusedResources[id] = true + if err := t.validatePlanBindings(set); err != nil { + return err } + // Copy through JSON: the caller cannot mutate the adopted immutable plan. + raw, err := json.Marshal(set) + if err != nil { + return err + } + var adopted libraryrelease.WriteSet + if err := json.Unmarshal(raw, &adopted); err != nil { + return err + } + if binder, ok := t.config.Objects.(interface { + BindObjectPlan(libraryrelease.WriteSet) error + }); ok { + if err := binder.BindObjectPlan(adopted); err != nil { + return err + } + } + t.activePlan = &adopted + t.reusedResources = reused return nil } @@ -862,6 +955,15 @@ func (t *ProductionLibraryTarget) Restore(ctx context.Context, backup libraryrel return fmt.Errorf("写集合记录绑定的是 %d/%s,与目标 %d/%s 不一致", backup.WriteSet.TenantID, backup.WriteSet.KnowledgeBaseID, t.scope.TenantID, t.scope.KnowledgeBaseID) } + if err := t.AdoptWriteSet(*backup.WriteSet); err != nil { + return err + } + if err := ensureRestorableResourceReferences(ctx, t.pool, *backup.WriteSet); err != nil { + return err + } + if err := ensureNoResourceGrants(ctx, t.pool, backup.WriteSet.CreatedIDs("resources")); err != nil { + return err + } previous := t.preserveChunkSequence t.preserveChunkSequence = true defer func() { t.preserveChunkSequence = previous }() @@ -869,6 +971,7 @@ func (t *ProductionLibraryTarget) Restore(ctx context.Context, backup libraryrel if err := t.Apply(ctx, stage, backup.SnapshotID); err != nil { return fmt.Errorf("恢复阶段 %s 失败: %w", stage, err) } + _ = testbarrier.Wait(ctx, "restore:"+string(stage), backup.WriteSet.JobID) } // resources 是多知识库共享表,回放阶段不删它。这里只清"写前不存在 ∧ 由本次写入 ∧ // 不在本次恢复内容里"的精确身份,历史孤儿与其它知识库的资源一律不动。 @@ -886,6 +989,9 @@ func (t *ProductionLibraryTarget) Restore(ctx context.Context, backup libraryrel if err != nil { return err } + if err := t.restoreCreatedObjects(ctx); err != nil { + return err + } if err := t.clearJobCreatedRows(ctx, *backup.WriteSet, restored); err != nil { return err } diff --git a/internal/releaseadapter/v071/library_reference_closure.go b/internal/releaseadapter/v071/library_reference_closure.go new file mode 100644 index 0000000..186401d --- /dev/null +++ b/internal/releaseadapter/v071/library_reference_closure.go @@ -0,0 +1,145 @@ +package v071 + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "regexp" + "sort" + "strings" + + "github.com/jackc/pgx/v5" + "knowledge-governance/internal/librarysnapshot" +) + +type libraryClosureReader interface { + QueryRow(context.Context, string, ...any) pgx.Row +} + +// Check edges before JOIN-based export can erase a scoped orphan. Either parent +// can establish scope; an edge is not safe merely because its other parent is absent. +func verifyLibraryRelationalClosure(ctx context.Context, db libraryClosureReader, identity PostgresIdentity, includeDeleted bool) error { + var invalid bool + err := db.QueryRow(ctx, `/* library-relational-closure */ SELECT + EXISTS(SELECT 1 FROM chunks c LEFT JOIN knowledges k ON k.id=c.knowledge_id + WHERE c.tenant_id=$1 AND c.knowledge_base_id=$2 AND ($3 OR c.deleted_at IS NULL) + AND (k.id IS NULL OR k.tenant_id IS DISTINCT FROM c.tenant_id OR k.knowledge_base_id IS DISTINCT FROM c.knowledge_base_id OR (NOT $3 AND k.deleted_at IS NOT NULL))) + OR EXISTS(SELECT 1 FROM knowledge_tag_relations r LEFT JOIN knowledges k ON k.id=r.knowledge_id LEFT JOIN knowledge_tags t ON t.id=r.tag_id + WHERE ((k.tenant_id=$1 AND k.knowledge_base_id=$2 AND ($3 OR k.deleted_at IS NULL)) OR (t.tenant_id=$1 AND t.knowledge_base_id=$2 AND ($3 OR t.deleted_at IS NULL))) + AND (k.id IS NULL OR t.id IS NULL OR k.tenant_id IS DISTINCT FROM t.tenant_id OR k.knowledge_base_id IS DISTINCT FROM t.knowledge_base_id OR (NOT $3 AND (k.deleted_at IS NOT NULL OR t.deleted_at IS NOT NULL)))) + OR EXISTS(SELECT 1 FROM resource_bindings b WHERE b.tenant_id=$1 + AND (EXISTS(SELECT 1 FROM knowledges k WHERE k.id=b.owner_id AND k.tenant_id=$1 AND k.knowledge_base_id=$2 AND ($3 OR k.deleted_at IS NULL)) OR EXISTS(SELECT 1 FROM chunks c WHERE c.id=b.owner_id AND c.tenant_id=$1 AND c.knowledge_base_id=$2 AND ($3 OR c.deleted_at IS NULL))) + AND (b.owner_type NOT IN ('knowledge','chunk') OR b.owner_type IS NULL))`, identity.TenantID, identity.KnowledgeBaseID, includeDeleted).Scan(&invalid) + if err != nil { + return fmt.Errorf("read library reference closure: %w", err) + } + if invalid { + return errors.New("unresolved or cross-parent library relationship") + } + return nil +} + +func verifyLibraryResourceOwners(ctx context.Context, db libraryClosureReader, ids []string) error { + var invalid bool + err := db.QueryRow(ctx, `/* library-resource-owner-closure */ SELECT EXISTS( + SELECT 1 FROM resource_bindings b JOIN resources r ON r.id=b.resource_id WHERE r.id=ANY($1::text[]) AND ( + b.tenant_id IS DISTINCT FROM r.tenant_id OR b.owner_type IS NULL OR NOT ( + (b.owner_type='knowledge' AND EXISTS(SELECT 1 FROM knowledges k WHERE k.id=b.owner_id AND k.tenant_id=b.tenant_id)) OR + (b.owner_type='chunk' AND EXISTS(SELECT 1 FROM chunks c JOIN knowledges k ON k.id=c.knowledge_id AND k.tenant_id=c.tenant_id AND k.knowledge_base_id=c.knowledge_base_id WHERE c.id=b.owner_id AND c.tenant_id=b.tenant_id)))))`, ids).Scan(&invalid) + if err != nil { + return fmt.Errorf("read referenced resource owners: %w", err) + } + if invalid { + return errors.New("referenced resource has unknown or unresolved owner") + } + return nil +} + +var libraryResourceToken = regexp.MustCompile(`resource://([A-Za-z0-9_-]*)`) + +// Decode JSON before scanning string leaves (including nested metadata). The +// v071 reader accepts exactly 22 URL-safe characters, not URI query/path suffixes. +// Opaque escaped/encoded representations are refused rather than silently omitted. +func libraryResourceHandles(rows []json.RawMessage) ([]string, error) { + seen := map[string]bool{} + var visit func(any) error + visit = func(value any) error { + switch v := value.(type) { + case map[string]any: + for _, x := range v { + if e := visit(x); e != nil { + return e + } + } + case []any: + for _, x := range v { + if e := visit(x); e != nil { + return e + } + } + case string: + rest := v + for strings.Contains(rest, "resource:") { + start := strings.Index(rest, "resource:") + rest = rest[start:] + match := libraryResourceToken.FindStringSubmatchIndex(rest) + if match == nil || match[0] != 0 || match[3]-match[2] != 22 { + return errors.New("unsupported resource reference representation") + } + if match[1] < len(rest) && strings.ContainsRune("/?#=%\\", rune(rest[match[1]])) { + return errors.New("unsupported resource reference suffix") + } + seen[rest[match[2]:match[3]]] = true + rest = rest[match[1]:] + } + } + return nil + } + for _, raw := range rows { + var value any + decoder := json.NewDecoder(strings.NewReader(string(raw))) + decoder.UseNumber() + if e := decoder.Decode(&value); e != nil { + return nil, e + } + if e := visit(value); e != nil { + return nil, e + } + } + handles := make([]string, 0, len(seen)) + for handle := range seen { + handles = append(handles, handle) + } + sort.Strings(handles) + return handles, nil +} + +// Revalidate frozen source content at the actual planning boundary. A sealed +// manifest proves bytes, not that every row has a parent or a supported owner. +func validateArchivedLibraryClosure(archive *librarysnapshot.Archive, manifest librarysnapshot.Manifest) error { + var pg PostgresSnapshot + es := SearchSnapshot{MappingSHA256: manifest.Compatibility.IndexMappingSHA256} + for _, part := range manifest.Parts { + if part.Kind != "database" && part.Kind != "wiki" && part.Kind != "index" { + continue + } + reader, e := archive.OpenPart(part) + if e != nil { + return e + } + if part.Kind == "index" { + e = json.NewDecoder(reader).Decode(&es.Documents) + } else { + var rows []json.RawMessage + e = json.NewDecoder(reader).Decode(&rows) + pg.Tables = append(pg.Tables, newTableSnapshot(part.Name, rows)) + } + reader.Close() + if e != nil { + return e + } + } + es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents) + return validateLibraryCandidate(PostgresIdentity{TenantID: manifest.TenantID, KnowledgeBaseID: manifest.KnowledgeBaseID}, pg, es) +} diff --git a/internal/releaseadapter/v071/library_resource_fk.go b/internal/releaseadapter/v071/library_resource_fk.go new file mode 100644 index 0000000..a512b17 --- /dev/null +++ b/internal/releaseadapter/v071/library_resource_fk.go @@ -0,0 +1,34 @@ +package v071 + +import ( + "context" + "errors" + "github.com/jackc/pgx/v5" +) + +type resourceGrantQuery interface { + QueryRow(context.Context, string, ...any) pgx.Row +} + +// Grants are an external FK, not part of the migration archive. Deleting a +// resource would silently cascade a capability owned outside this operation. +func ensureNoResourceGrants(ctx context.Context, q resourceGrantQuery, ids []string) error { + if len(ids) == 0 { + return nil + } + var present bool + if err := q.QueryRow(ctx, `SELECT to_regclass('public.resource_access_grants') IS NOT NULL`).Scan(&present); err != nil { + return err + } + if !present { + return nil + } + var referenced bool + if err := q.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM public.resource_access_grants WHERE resource_id=ANY($1::text[]))`, ids).Scan(&referenced); err != nil { + return err + } + if referenced { + return errors.New("created resource has external access grants; refusing cascade and object deletion") + } + return nil +} diff --git a/internal/releaseadapter/v071/library_resource_references.go b/internal/releaseadapter/v071/library_resource_references.go new file mode 100644 index 0000000..a55fa5e --- /dev/null +++ b/internal/releaseadapter/v071/library_resource_references.go @@ -0,0 +1,76 @@ +package v071 + +import ( + "context" + "errors" + "fmt" + "strings" + + "knowledge-governance/internal/libraryrelease" +) + +// Keep capture/verification and deletion protection on the same set of bodies. +// predicate is an internal SQL fragment, never caller-supplied content. +func libraryResourceBodyReferenceSQL(predicate string) string { + queries := []string{} + for _, table := range productionLibraryTables { + if table == "resources" || table == "resource_bindings" || table == "knowledge_tag_relations" { + continue + } + queries = append(queries, fmt.Sprintf(`SELECT 1 FROM %s c WHERE (%s) AND coalesce(resources.handle,'')<>'' AND position('resource://'||resources.handle IN to_jsonb(c)::text)>0`, table, predicate)) + } + return strings.Join(queries, " UNION ALL ") +} + +// This must precede *all* restore mutations, including ES and object deletes. +// A created row is not permission to invalidate a later out-of-plan reader. +func ensureRestorableResourceReferences(ctx context.Context, q libraryClosureReader, set libraryrelease.WriteSet) error { + ids := set.CreatedIDs("resources") + if len(ids) == 0 { + return nil + } + if e := verifyLibraryResourceOwners(ctx, q, ids); e != nil { + return e + } + bindings := []string{} + for _, row := range set.Rows { + if row.Table == "resource_bindings" { + bindings = append(bindings, row.ID) + } + } + var referenced bool + err := q.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM resource_bindings b WHERE b.resource_id=ANY($1::text[]) AND + (NOT(b.id=ANY($2::text[])) OR b.tenant_id IS DISTINCT FROM $3 OR NOT ( + (b.owner_type='knowledge' AND EXISTS(SELECT 1 FROM knowledges k WHERE k.id=b.owner_id AND k.tenant_id=$3 AND k.knowledge_base_id=$4)) OR + (b.owner_type='chunk' AND EXISTS(SELECT 1 FROM chunks c WHERE c.id=b.owner_id AND c.tenant_id=$3 AND c.knowledge_base_id=$4)))))`, ids, bindings, set.TenantID, set.KnowledgeBaseID).Scan(&referenced) + if err != nil { + return err + } + if referenced { + return errors.New("created resource has out-of-plan bindings; restore held before writes") + } + bodySQL := libraryResourceBodyReferenceSQL("c.tenant_id IS DISTINCT FROM $2 OR c.knowledge_base_id IS DISTINCT FROM $3") + err = q.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM resources WHERE id=ANY($1::text[]) AND EXISTS (`+bodySQL+`))`, ids, set.TenantID, set.KnowledgeBaseID).Scan(&referenced) + if err != nil { + return err + } + if referenced { + return errors.New("created resource has foreign body/metadata references; restore held before writes") + } + return nil +} + +// Recheck under the already-held resource parent locks. This does not claim +// cross-store isolation from uncontrolled writers; deployed exclusivity is separate. +func ensureCreatedResourcesUnreferenced(ctx context.Context, q libraryClosureReader, ids []string) error { + var referenced bool + err := q.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM resources WHERE id=ANY($1::text[]) AND + (EXISTS(SELECT 1 FROM resource_bindings b WHERE b.resource_id=resources.id) OR EXISTS (`+libraryResourceBodyReferenceSQL("TRUE")+`)))`, ids).Scan(&referenced) + if err != nil { + return err + } + if referenced { + return errors.New("created resource is still referenced; refusing row deletion") + } + return nil +} diff --git a/internal/releaseadapter/v071/library_reuse.go b/internal/releaseadapter/v071/library_reuse.go index f20e808..46d0218 100644 --- a/internal/releaseadapter/v071/library_reuse.go +++ b/internal/releaseadapter/v071/library_reuse.go @@ -93,7 +93,7 @@ func (e *ResourceReuseEvidence) Seal() error { // Verify 校验版本、必备身份与摘要;缺任何一项都不许拿它去放行覆盖。 func (e *ResourceReuseEvidence) Verify() error { - if e.Version != reuseEvidenceVersion { + if e.Version != reuseEvidenceVersion && e.Version != 2 { return fmt.Errorf("复用证据版本 %d 不受支持(当前 %d)", e.Version, reuseEvidenceVersion) } if e.JobID == "" || e.Target == "" || e.TargetSnapshotID == "" || e.ResourceID == "" || @@ -107,9 +107,12 @@ func (e *ResourceReuseEvidence) Verify() error { if e.RetargetedRowDigest == "" || e.TargetRowDigest != e.RetargetedRowDigest { return errors.New("复用证据里目标行与本次要写的行不一致") } - if e.ProvenanceRowDigest != e.ArchivedRowDigest { + if e.Version == 1 && e.ProvenanceRowDigest != e.ArchivedRowDigest { return errors.New("复用证据里的来源行与本次快照行不一致") } + if e.Version == 2 && (e.ProvenanceKind != libraryrelease.PriorSnapshotPrebackup || e.ProvenanceRowDigest != e.RetargetedRowDigest) { + return errors.New("prebackup reuse proof differs from target projection") + } copied := *e copied.Digest = "" raw, err := json.Marshal(copied) @@ -133,50 +136,16 @@ func PersistReuseEvidence(archiveRoot string, evidence *ResourceReuseEvidence) ( if evidence.Digest == "" { return "", errors.New("复用证据没有摘要,拒绝落盘") } - directory := filepath.Join(archiveRoot, reuseDirName) - if err := os.MkdirAll(directory, 0700); err != nil { - return "", fmt.Errorf("创建复用证据目录失败: %w", err) - } - path := reusePath(archiveRoot, evidence.Digest) - if existing, readErr := os.ReadFile(path); readErr == nil { - var recorded ResourceReuseEvidence - if json.Unmarshal(existing, &recorded) != nil || recorded.Verify() != nil || recorded.Digest != evidence.Digest { - return "", errors.New("复用证据同名但内容不符,拒绝覆盖") - } - return path, nil - } else if !errors.Is(readErr, os.ErrNotExist) { - return "", fmt.Errorf("读取已有复用证据失败: %w", readErr) + if err := evidence.Verify(); err != nil { + return "", err } raw, err := json.Marshal(*evidence) if err != nil { - return "", fmt.Errorf("编码复用证据失败: %w", err) + return "", err } - temporary := path + ".tmp" - file, err := os.OpenFile(temporary, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) - if err != nil { - return "", fmt.Errorf("写入复用证据失败: %w", err) - } - if _, err := file.Write(raw); err != nil { - file.Close() - os.Remove(temporary) - return "", fmt.Errorf("写入复用证据失败: %w", err) - } - if err := file.Sync(); err != nil { - file.Close() - os.Remove(temporary) - return "", fmt.Errorf("复用证据未落盘: %w", err) - } - if err := file.Close(); err != nil { - os.Remove(temporary) - return "", fmt.Errorf("复用证据未落盘: %w", err) - } - if err := os.Chmod(temporary, 0444); err != nil { - os.Remove(temporary) - return "", fmt.Errorf("复用证据只读化失败: %w", err) - } - if err := os.Rename(temporary, path); err != nil { - os.Remove(temporary) - return "", fmt.Errorf("复用证据落盘失败: %w", err) + path := reusePath(archiveRoot, evidence.Digest) + if err := persistImmutableJSON(path, raw); err != nil { + return "", err } return path, nil } @@ -392,6 +361,39 @@ func (t *ProductionLibraryTarget) planReusableResource(ctx context.Context, arch if err != nil { return libraryrelease.WriteSetRow{}, err } + evidenceVersion := reuseEvidenceVersion + backupIDs, backupErr := archivedSnapshotResourceIDs(archive, plan.BackupSnapshotID) + if backupErr != nil { + return libraryrelease.WriteSetRow{}, backupErr + } + if backupIDs[id] { + baseline, err := archive.Load(plan.BackupSnapshotID) + if err != nil { + return libraryrelease.WriteSetRow{}, err + } + if baseline.TenantID != t.scope.TenantID || baseline.KnowledgeBaseID != t.scope.KnowledgeBaseID { + return libraryrelease.WriteSetRow{}, errors.New("resource prebackup scope mismatch") + } + for _, p := range baseline.Parts { + if p.Kind != "database" || p.Name != "resources" { + continue + } + row, err := archivedRowByID(archive, p, id) + if err != nil { + return libraryrelease.WriteSetRow{}, err + } + digest, err := canonicalRowDigest(projectRow(row, columns)) + if err != nil { + return libraryrelease.WriteSetRow{}, err + } + if digest != retargetedDigest { + return libraryrelease.WriteSetRow{}, errors.New("shared resource differs from immutable before-image") + } + provenance = libraryrelease.PriorSnapshot{JobID: plan.JobID, SnapshotID: plan.BackupSnapshotID, Kind: libraryrelease.PriorSnapshotPrebackup} + provenanceDigest = digest + evidenceVersion = 2 + } + } if provenance.JobID == "" { return libraryrelease.WriteSetRow{}, fmt.Errorf( "资源 ID 冲突:%s 已存在于目标租户但不在预备份范围内,且没有可审计的持久来源(成功同步/预备份)证明它属于本目标,拒绝覆盖", id) @@ -417,7 +419,7 @@ func (t *ProductionLibraryTarget) planReusableResource(ctx context.Context, arch objects = append(objects, reuseObject{Name: item.Name, Path: path, Bytes: size, SHA256: digest}) } evidence := &ResourceReuseEvidence{ - Version: reuseEvidenceVersion, JobID: plan.JobID, Target: plan.Target, + Version: evidenceVersion, JobID: plan.JobID, Target: plan.Target, TenantID: t.scope.TenantID, KnowledgeBaseID: t.scope.KnowledgeBaseID, SourceTenantID: manifest.TenantID, SourceKnowledgeBaseID: manifest.KnowledgeBaseID, TargetSnapshotID: plan.TargetSnapshotID, ResourceID: id, diff --git a/internal/releaseadapter/v071/library_schema_contract.go b/internal/releaseadapter/v071/library_schema_contract.go new file mode 100644 index 0000000..e7ad447 --- /dev/null +++ b/internal/releaseadapter/v071/library_schema_contract.go @@ -0,0 +1,208 @@ +package v071 + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + + "github.com/jackc/pgx/v5" + "knowledge-governance/internal/librarysnapshot" +) + +type migrationCatalog map[string]json.RawMessage + +// Catalog capture is read-only. Sequence *definitions*, not mutable last_value, +// belong in the fingerprint; allocating a number must not look like DDL drift. +func (t *ProductionLibraryTarget) migrationCatalog(ctx context.Context) (migrationCatalog, string, error) { + tx, err := t.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadOnly}) + if err != nil { + return nil, "", err + } + defer tx.Rollback(context.WithoutCancel(ctx)) + catalog, digest, err := captureMigrationCatalog(ctx, tx) + if err != nil { + return nil, "", err + } + if err := tx.Commit(ctx); err != nil { + return nil, "", err + } + return catalog, digest, nil +} + +// Source rows and their definitions must come from the same read-only transaction. +func captureMigrationCatalog(ctx context.Context, tx pgx.Tx) (migrationCatalog, string, error) { + queries := map[string]string{ + "columns": `SELECT c.relname AS table_name,a.attname AS column_name,format_type(a.atttypid,a.atttypmod) AS type,a.attnotnull AS not_null,a.attidentity AS identity,a.attgenerated AS generated,coalesce(pg_get_expr(d.adbin,d.adrelid),'') AS default_expr FROM pg_attribute a JOIN pg_class c ON c.oid=a.attrelid JOIN pg_namespace n ON n.oid=c.relnamespace LEFT JOIN pg_attrdef d ON d.adrelid=c.oid AND d.adnum=a.attnum WHERE n.nspname='public' AND c.relname=ANY($1) AND a.attnum>0 AND NOT a.attisdropped ORDER BY c.relname,a.attnum`, + "constraints": `SELECT ns.nspname AS namespace,c.relname AS table_name,x.conname AS name,x.contype AS type,pg_get_constraintdef(x.oid) AS definition,coalesce(r.relname,'') AS referenced_table,coalesce(rns.nspname,'') AS referenced_namespace FROM pg_constraint x JOIN pg_class c ON c.oid=x.conrelid JOIN pg_namespace ns ON ns.oid=c.relnamespace LEFT JOIN pg_class r ON r.oid=x.confrelid LEFT JOIN pg_namespace rns ON rns.oid=r.relnamespace WHERE (ns.nspname='public' AND c.relname=ANY($1)) OR (rns.nspname='public' AND r.relname=ANY($1)) ORDER BY ns.nspname,c.relname,x.conname`, + "indexes": `SELECT c.relname AS table_name,ic.relname AS name,i.indisunique AS unique_index,i.indisvalid AS valid,pg_get_indexdef(i.indexrelid) AS definition,coalesce(pg_get_expr(i.indpred,i.indrelid),'') AS predicate FROM pg_index i JOIN pg_class c ON c.oid=i.indrelid JOIN pg_namespace n ON n.oid=c.relnamespace JOIN pg_class ic ON ic.oid=i.indexrelid WHERE n.nspname='public' AND c.relname=ANY($1) ORDER BY c.relname,ic.relname`, + "triggers": `SELECT c.relname AS table_name,g.tgname AS name,pg_get_triggerdef(g.oid) AS definition FROM pg_trigger g JOIN pg_class c ON c.oid=g.tgrelid JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND c.relname=ANY($1) AND NOT g.tgisinternal ORDER BY c.relname,g.tgname`, + "relations": `SELECT c.relname AS table_name,c.relkind::text AS kind,c.relispartition AS partition,c.relrowsecurity AS row_security,EXISTS(SELECT 1 FROM pg_inherits i WHERE i.inhrelid=c.oid OR i.inhparent=c.oid) AS inherits FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND c.relname=ANY($1) ORDER BY c.relname`, + "sequences": `SELECT c.relname AS table_name,a.attname AS column_name,coalesce(s.seqrelid::regclass::text,'') AS sequence,s.seqstart,s.seqincrement,s.seqmin,s.seqmax,s.seqcache,s.seqcycle,coalesce(own.deptype::text,'') AS ownership FROM pg_attribute a JOIN pg_class c ON c.oid=a.attrelid JOIN pg_namespace n ON n.oid=c.relnamespace LEFT JOIN pg_attrdef d ON d.adrelid=c.oid AND d.adnum=a.attnum LEFT JOIN pg_depend dep ON dep.classid='pg_attrdef'::regclass AND dep.objid=d.oid AND dep.refclassid='pg_class'::regclass AND EXISTS(SELECT 1 FROM pg_sequence sq WHERE sq.seqrelid=dep.refobjid) LEFT JOIN pg_depend own ON own.refclassid='pg_class'::regclass AND own.refobjid=c.oid AND own.refobjsubid=a.attnum AND own.classid='pg_class'::regclass AND own.deptype IN ('a','i') AND EXISTS(SELECT 1 FROM pg_sequence sq WHERE sq.seqrelid=own.objid) LEFT JOIN pg_sequence s ON s.seqrelid=coalesce(dep.refobjid,own.objid) WHERE n.nspname='public' AND c.relname=ANY($1) AND a.attnum>0 AND NOT a.attisdropped AND (s.seqrelid IS NOT NULL OR a.attname='seq_id' OR (c.relname='wiki_log_entries' AND a.attname='id')) ORDER BY c.relname,a.attname,s.seqrelid`, + } + catalog := migrationCatalog{} + for key, query := range queries { + var raw []byte + if err := tx.QueryRow(ctx, "SELECT coalesce(jsonb_agg(to_jsonb(q)),'[]'::jsonb) FROM ("+query+") q", productionLibraryTables).Scan(&raw); err != nil { + return nil, "", fmt.Errorf("capture migration catalog %s: %w", key, err) + } + catalog[key] = raw + } + raw, err := json.Marshal(catalog) + if err != nil { + return nil, "", err + } + hash := sha256.Sum256(raw) + return catalog, hex.EncodeToString(hash[:]), nil +} + +func validateMigrationCatalog(catalog migrationCatalog) error { + var triggers []json.RawMessage + if err := json.Unmarshal(catalog["triggers"], &triggers); err != nil { + return err + } + if len(triggers) > 0 { + return errors.New("unreviewed migration trigger effects") + } + var relations []struct { + Table string `json:"table_name"` + Kind string `json:"kind"` + Partition bool `json:"partition"` + RLS bool `json:"row_security"` + Inherits bool `json:"inherits"` + } + if err := json.Unmarshal(catalog["relations"], &relations); err != nil { + return err + } + if len(relations) != len(productionLibraryTables) { + return errors.New("incomplete migration table catalog") + } + for _, r := range relations { + if r.Kind != "r" || r.Partition || r.RLS || r.Inherits { + return fmt.Errorf("unsupported migration relation: %s", r.Table) + } + } + var constraints []struct { + Namespace string `json:"namespace"` + Table string `json:"table_name"` + Type string `json:"type"` + Referenced string `json:"referenced_table"` + ReferencedNamespace string `json:"referenced_namespace"` + Definition string `json:"definition"` + } + if err := json.Unmarshal(catalog["constraints"], &constraints); err != nil { + return err + } + for _, c := range constraints { + if c.Type != "f" { + continue + } + // Internal-table membership does not prove the referenced identity survives + // local-number remapping. v071 has exactly these reviewed FK contracts. + if !(c.Namespace == "public" && c.ReferencedNamespace == "public" && + (c.Table == "resource_bindings" || c.Table == "resource_access_grants") && c.Referenced == "resources" && + c.Definition == "FOREIGN KEY (resource_id) REFERENCES resources(id) ON DELETE CASCADE") { + return fmt.Errorf("unreviewed incoming migration FK: %s.%s", c.Namespace, c.Table) + } + } + return nil +} + +func (t *ProductionLibraryTarget) preflightMigrationCatalog(ctx context.Context, archive *librarysnapshot.Archive, manifest librarysnapshot.Manifest) (string, error) { + catalog, digest, err := t.migrationCatalog(ctx) + if err != nil { + return "", err + } + if err := validateMigrationCatalog(catalog); err != nil { + return "", err + } + var columns []struct { + Table string `json:"table_name"` + Column string `json:"column_name"` + Type string `json:"type"` + Generated string `json:"generated"` + Identity string `json:"identity"` + } + if err := json.Unmarshal(catalog["columns"], &columns); err != nil { + return "", err + } + allowed := map[string]map[string]string{} + for _, c := range columns { + if c.Generated != "" || c.Identity != "" { + return "", fmt.Errorf("unsupported generated/identity column: %s.%s", c.Table, c.Column) + } + if allowed[c.Table] == nil { + allowed[c.Table] = map[string]string{} + } + allowed[c.Table][c.Column] = c.Type + } + for _, part := range manifest.Parts { + if part.Kind == "schema" { + if part.Name != "migration-catalog" || part.SHA256 != digest { + return "", errors.New("captured source/target schema catalog mismatch") + } + reader, err := archive.OpenPart(part) + if err != nil { + return "", err + } + reader.Close() + continue + } + if part.Kind != "database" && part.Kind != "wiki" { + continue + } + reader, err := archive.OpenPart(part) + if err != nil { + return "", err + } + var rows []map[string]json.RawMessage + err = json.NewDecoder(reader).Decode(&rows) + reader.Close() + if err != nil { + return "", err + } + contract := migrationIdentityContracts[part.Name] + for _, row := range rows { + for name := range row { + if _, ok := allowed[part.Name][name]; !ok { + return "", fmt.Errorf("source column absent from target contract: %s.%s", part.Name, name) + } + } + if contract.LocalColumn != "" && allowed[part.Name][contract.LocalColumn] != "" { + if allowed[part.Name][contract.LocalColumn] != "bigint" { + return "", fmt.Errorf("local column type mismatch: %s.%s", part.Name, contract.LocalColumn) + } + if _, err := positiveBigint(row[contract.LocalColumn]); err != nil { + return "", fmt.Errorf("missing/invalid source local number: %s.%s", part.Name, contract.LocalColumn) + } + } + } + } + raw, err := json.Marshal(catalog) + if err != nil { + return "", err + } + if err := persistImmutableJSON(t.config.ArchiveRoot+"/catalogs/"+digest+".json", raw); err != nil { + return "", err + } + return digest, nil +} + +func (t *ProductionLibraryTarget) verifyPlanSchema(ctx context.Context) error { + if t.activePlan == nil { + return nil + } + if t.activePlan.SchemaDigest == "" { + return errors.New("persisted execution plan lacks schema fingerprint") + } + _, digest, err := t.migrationCatalog(ctx) + if err != nil { + return err + } + if digest != t.activePlan.SchemaDigest { + return errors.New("target schema changed after immutable plan") + } + return nil +} diff --git a/internal/releaseadapter/v071/library_structured_references.go b/internal/releaseadapter/v071/library_structured_references.go new file mode 100644 index 0000000..6692d13 --- /dev/null +++ b/internal/releaseadapter/v071/library_structured_references.go @@ -0,0 +1,169 @@ +package v071 + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" +) + +// This is the finite v071 structured-reference contract, not a heuristic that +// treats arbitrary text/numbers as IDs. Historical log refs are display facts. +func validateLibraryStructuredReferences(pg PostgresSnapshot) error { + type row map[string]json.RawMessage + tables := map[string]map[string]row{} + for _, table := range pg.Tables { + tables[table.Name] = map[string]row{} + for _, raw := range table.Rows { + var r row + if e := json.Unmarshal(raw, &r); e != nil { + return e + } + var id string + json.Unmarshal(r["id"], &id) + if id == "" { + id = string(r["id"]) + } + tables[table.Name][id] = r + } + } + text := func(r row, key string) (string, error) { + v := bytes.TrimSpace(r[key]) + if len(v) == 0 || bytes.Equal(v, []byte("null")) { + return "", nil + } + var out string + if e := json.Unmarshal(v, &out); e != nil { + return "", fmt.Errorf("unsupported reference %s", key) + } + return out, nil + } + ref := func(r row, key, table string, sameKnowledge bool) error { + v, e := text(r, key) + if e != nil || v == "" { + return e + } + parent, ok := tables[table][v] + if !ok { + return fmt.Errorf("unresolved %s reference to %s", key, table) + } + if sameKnowledge { + a, _ := text(r, "knowledge_id") + b, _ := text(parent, "knowledge_id") + if a != b { + return fmt.Errorf("cross-parent %s reference", key) + } + } + return nil + } + refs := func(r row, key, table string, legacyTitle bool) error { + raw := bytes.TrimSpace(r[key]) + if len(raw) == 0 || bytes.Equal(raw, []byte("null")) { + return nil + } + var ids []string + if e := json.Unmarshal(raw, &ids); e != nil { + return fmt.Errorf("unsupported %s reference representation", key) + } + for _, id := range ids { + if legacyTitle { + id = strings.SplitN(id, "|", 2)[0] + } + if _, ok := tables[table][id]; !ok { + return fmt.Errorf("unresolved %s reference to %s", key, table) + } + } + return nil + } + for _, r := range tables["chunks"] { + for _, key := range []string{"pre_chunk_id", "next_chunk_id", "parent_chunk_id"} { + if e := ref(r, key, "chunks", true); e != nil { + return e + } + } + if e := ref(r, "tag_id", "knowledge_tags", false); e != nil { + return e + } + for _, key := range []string{"relation_chunks", "indirect_relation_chunks"} { + if e := refs(r, key, "chunks", false); e != nil { + return e + } + } + } + for _, r := range tables["wiki_folders"] { + if e := ref(r, "parent_id", "wiki_folders", false); e != nil { + return e + } + } + // Parent edges must be acyclic; pre/next chunk adjacency is intentionally not + // run through this check because a normal bidirectional adjacency is a cycle. + for _, kind := range []struct{ table, key string }{{"wiki_folders", "parent_id"}, {"chunks", "parent_chunk_id"}} { + for id := range tables[kind.table] { + seen := map[string]bool{} + for id != "" { + if seen[id] { + return fmt.Errorf("cyclic %s reference", kind.key) + } + seen[id] = true + next, e := text(tables[kind.table][id], kind.key) + if e != nil { + return e + } + id = next + } + } + } + for _, r := range tables["wiki_pages"] { + if e := ref(r, "folder_id", "wiki_folders", false); e != nil { + return e + } + if e := refs(r, "source_refs", "knowledges", true); e != nil { + return e + } + if e := refs(r, "chunk_refs", "chunks", false); e != nil { + return e + } + } + for _, r := range tables["wiki_log_entries"] { + raw := bytes.TrimSpace(r["pages_affected"]) + if len(raw) == 0 || bytes.Equal(raw, []byte("null")) { + continue + } + var items []json.RawMessage + if e := json.Unmarshal(raw, &items); e != nil { + return fmt.Errorf("unsupported Wiki log pages_affected") + } + shape := "" + for _, item := range items { + var slug string + if json.Unmarshal(item, &slug) == nil { + if shape == "object" { + return fmt.Errorf("mixed Wiki log reference representations") + } + shape = "string" + continue + } + var object map[string]json.RawMessage + if e := json.Unmarshal(item, &object); e != nil || object == nil { + return fmt.Errorf("unsupported Wiki log page reference") + } + if shape == "string" { + return fmt.Errorf("mixed Wiki log reference representations") + } + shape = "object" + if e := json.Unmarshal(object["slug"], &slug); e != nil { + return fmt.Errorf("Wiki log reference lacks string slug") + } + for key, value := range object { + if key != "slug" && key != "title" { + return fmt.Errorf("unreviewed Wiki log reference field: %s", key) + } + var s string + if e := json.Unmarshal(value, &s); e != nil { + return fmt.Errorf("unsupported Wiki log reference %s", key) + } + } + } + } + return nil +} diff --git a/internal/releaseadapter/v071/library_target_contract.go b/internal/releaseadapter/v071/library_target_contract.go index dd9ad3d..98ad8b9 100644 --- a/internal/releaseadapter/v071/library_target_contract.go +++ b/internal/releaseadapter/v071/library_target_contract.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "strings" "knowledge-governance/internal/librarysnapshot" ) @@ -88,7 +89,7 @@ func VerifySourceAgainstTarget(source librarysnapshot.Compatibility, live LiveTa return fmt.Errorf("目标 schema 版本 %d 与来源快照声明 %d 不一致:拒绝搬运", live.SchemaVersion, source.SchemaVersion) } - if !live.ModelBound || live.EmbeddingDimension < 1 { + if !live.ModelBound || live.EmbeddingDimension < 1 || strings.TrimSpace(live.EmbeddingModel) == "" { return errors.New("目标知识库没有可读的 embedding 模型绑定(模型行缺失或维度未知):" + "无法证明向量可迁移,先在目标上绑定模型") } diff --git a/internal/releaseadapter/v071/library_universe.go b/internal/releaseadapter/v071/library_universe.go index 08f8458..2153a09 100644 --- a/internal/releaseadapter/v071/library_universe.go +++ b/internal/releaseadapter/v071/library_universe.go @@ -57,11 +57,17 @@ func archivedRowIdentities(archive *librarysnapshot.Archive, part librarysnapsho if err := decoder.Decode(&row); err != nil { return nil, err } - identity := archiveRowIdentity(part.Name, row) - if identity == "" { - continue + identity, err := parseRowIdentity(part.Name, row) + if err != nil { + return nil, err } - ids = append(ids, identity) + key := identity.key() + for _, previous := range ids { + if previous == key { + return nil, fmt.Errorf("duplicate %s identity", part.Name) + } + } + ids = append(ids, key) } return ids, nil } @@ -109,6 +115,11 @@ func (t *ProductionLibraryTarget) assertNoSourceScopeRows(ctx context.Context, a if !tableHasTenant(part.Name) { continue } + // DB-local source numbers have no target-global ownership meaning. + // The mapped target values are checked by the typed content verifier. + if part.Name == "wiki_log_entries" && t.activePlan != nil && t.activeSnapshot == t.activePlan.TargetSnapshotID { + continue + } ids, err := archivedRowIdentities(archive, part) if err != nil { return err @@ -116,9 +127,23 @@ func (t *ProductionLibraryTarget) assertNoSourceScopeRows(ctx context.Context, a if len(ids) == 0 { continue } - statement := fmt.Sprintf(`SELECT count(*) FROM %s WHERE tenant_id=$1 AND id = ANY($2::text[])`, part.Name) + // Shared tables have no KB column. Equal tenant identity is not + // evidence of leakage; their ownership is checked by the plan. + if !tablesWithKB[part.Name] && t.snapshotTenant == t.scope.TenantID { + continue + } + cast := "text" + if part.Name == "wiki_log_entries" { + cast = "bigint" + } + statement := fmt.Sprintf(`SELECT count(*) FROM %s WHERE tenant_id=$1 AND id = ANY($2::%s[])`, part.Name, cast) + args := []any{t.snapshotTenant, ids} + if tablesWithKB[part.Name] { + statement += " AND knowledge_base_id=$3" + args = append(args, t.snapshotKB) + } var count int64 - if err := t.pool.QueryRow(ctx, statement, t.snapshotTenant, ids).Scan(&count); err != nil { + if err := t.pool.QueryRow(ctx, statement, args...).Scan(&count); err != nil { return fmt.Errorf("检查目标 %s 的来源范围残留失败: %w", part.Name, err) } if count > 0 { diff --git a/internal/releaseadapter/v071/library_verify_content.go b/internal/releaseadapter/v071/library_verify_content.go index d458246..7e41ed6 100644 --- a/internal/releaseadapter/v071/library_verify_content.go +++ b/internal/releaseadapter/v071/library_verify_content.go @@ -1,6 +1,7 @@ package v071 import ( + "bytes" "context" "crypto/sha256" "encoding/hex" @@ -22,7 +23,6 @@ var contentVolatileFields = map[string]bool{ "created_at": true, "updated_at": true, "processed_at": true, - "expires_at": true, } func canonicalRowDigest(raw map[string]json.RawMessage) (string, error) { @@ -32,7 +32,9 @@ func canonicalRowDigest(raw map[string]json.RawMessage) (string, error) { continue } var decoded any - if err := json.Unmarshal(value, &decoded); err != nil { + decoder := json.NewDecoder(bytes.NewReader(value)) + decoder.UseNumber() + if err := decoder.Decode(&decoded); err != nil { return "", fmt.Errorf("decode column %s: %w", key, err) } normalized[key] = decoded @@ -136,26 +138,29 @@ func archivedTableContent(archive *librarysnapshot.Archive, part librarysnapshot } // orphanIndexDocuments 找出"索引里有、生产分块里没有"的文档(按 chunk_id 关联)。 -func (t *ProductionLibraryTarget) orphanIndexDocuments(ctx context.Context, documents []SearchDocument) []string { +func (t *ProductionLibraryTarget) orphanIndexDocuments(ctx context.Context, documents []SearchDocument) ([]string, error) { if len(documents) == 0 { - return nil + return nil, nil } // 按文档归属判断而不是按 chunk_id:真实 WeKnora 是父子分块, // 索引里的子块 ID 本来就不在 chunks 表里,按 chunk_id 判会误报孤儿。 rows, err := t.pool.Query(ctx, `SELECT id FROM knowledges WHERE tenant_id=$1 AND knowledge_base_id=$2 AND deleted_at IS NULL`, t.scope.TenantID, t.scope.KnowledgeBaseID) if err != nil { - return nil + return nil, err } defer rows.Close() known := map[string]bool{} for rows.Next() { var id string if err := rows.Scan(&id); err != nil { - return nil + return nil, err } known[id] = true } + if err := rows.Err(); err != nil { + return nil, err + } orphans := make([]string, 0, 4) for _, document := range documents { var source struct { @@ -163,13 +168,13 @@ func (t *ProductionLibraryTarget) orphanIndexDocuments(ctx context.Context, docu KnowledgeID string `json:"knowledge_id"` } if json.Unmarshal(document.Source, &source) != nil || source.KnowledgeID == "" { - continue + return nil, errors.New("index document has invalid or missing knowledge_id") } if !known[source.KnowledgeID] { orphans = append(orphans, document.ID) } } - return orphans + return orphans, nil } // firstIndexDifference 返回第一处不一致的文档 ID 与字段,便于定位。 @@ -258,7 +263,11 @@ func (t *ProductionLibraryTarget) verifyIndexDigest(ctx context.Context, archive } // 孤儿向量:索引里有、数据库分块里没有的文档必须报出来。 // 只比"归档 vs 生产"会漏掉两边一致但都错的情况。 - if orphans := t.orphanIndexDocuments(ctx, snapshot.Documents); len(orphans) > 0 { + orphans, orphanErr := t.orphanIndexDocuments(ctx, snapshot.Documents) + if orphanErr != nil { + return fmt.Errorf("index ownership verification failed: %w", orphanErr) + } + if len(orphans) > 0 { return fmt.Errorf("生产索引存在 %d 条孤儿文档(所属文档已不在数据库),首个 %s", len(orphans), orphans[0]) } if len(snapshot.Documents) != len(archived) || snapshot.DocumentsSHA256 != wantDigest { diff --git a/internal/releaseadapter/v071/library_writeset.go b/internal/releaseadapter/v071/library_writeset.go index cee567e..f2159ef 100644 --- a/internal/releaseadapter/v071/library_writeset.go +++ b/internal/releaseadapter/v071/library_writeset.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" "sort" + "strings" "knowledge-governance/internal/libraryrelease" "knowledge-governance/internal/librarysnapshot" @@ -55,46 +56,9 @@ func PersistWriteSet(archiveRoot string, set *libraryrelease.WriteSet) (string, if err != nil { return "", fmt.Errorf("编码写集合失败: %w", err) } - directory := filepath.Join(archiveRoot, writeSetDirName) - if err := os.MkdirAll(directory, 0700); err != nil { - return "", fmt.Errorf("创建写集合目录失败: %w", err) - } path := writeSetPath(archiveRoot, digest) - if existing, readErr := os.ReadFile(path); readErr == nil { - var recorded libraryrelease.WriteSet - if json.Unmarshal(existing, &recorded) != nil || recorded.Verify() != nil || recorded.Digest != digest { - return "", errors.New("写集合记录同名但内容不符,拒绝覆盖") - } - return path, nil - } else if !errors.Is(readErr, os.ErrNotExist) { - return "", fmt.Errorf("读取已有写集合失败: %w", readErr) - } - temporary := path + ".tmp" - file, err := os.OpenFile(temporary, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) - if err != nil { - return "", fmt.Errorf("写入写集合失败: %w", err) - } - if _, err := file.Write(raw); err != nil { - file.Close() - os.Remove(temporary) - return "", fmt.Errorf("写入写集合失败: %w", err) - } - if err := file.Sync(); err != nil { - file.Close() - os.Remove(temporary) - return "", fmt.Errorf("写集合未落盘: %w", err) - } - if err := file.Close(); err != nil { - os.Remove(temporary) - return "", fmt.Errorf("写集合未落盘: %w", err) - } - if err := os.Chmod(temporary, 0444); err != nil { - os.Remove(temporary) - return "", fmt.Errorf("写集合只读化失败: %w", err) - } - if err := os.Rename(temporary, path); err != nil { - os.Remove(temporary) - return "", fmt.Errorf("写集合落盘失败: %w", err) + if err := persistImmutableJSON(path, raw); err != nil { + return "", err } return path, nil } @@ -152,11 +116,34 @@ func (t *ProductionLibraryTarget) PlanWriteSet(ctx context.Context, plan library if err := t.bindSnapshotScope(manifestScope{tenantID: manifest.TenantID, knowledgeBaseID: manifest.KnowledgeBaseID}); err != nil { return libraryrelease.WriteSet{}, err } + if err := validateArchivedLibraryClosure(archive, manifest); err != nil { + return libraryrelease.WriteSet{}, err + } + if err := verifyLibraryRelationalClosure(ctx, t.pool, PostgresIdentity{TenantID: t.scope.TenantID, KnowledgeBaseID: t.scope.KnowledgeBaseID}, true); err != nil { + return libraryrelease.WriteSet{}, err + } + schemaDigest, err := t.preflightMigrationCatalog(ctx, archive, manifest) + if err != nil { + return libraryrelease.WriteSet{}, err + } backupResources, err := archivedSnapshotResourceIDs(archive, plan.BackupSnapshotID) if err != nil { return libraryrelease.WriteSet{}, err } - set := libraryrelease.WriteSet{Version: libraryrelease.WriteSetVersion, JobID: plan.JobID, Target: plan.Target, + resourceIDs := make([]string, 0, len(backupResources)) + for id := range backupResources { + resourceIDs = append(resourceIDs, id) + } + for _, part := range manifest.Parts { + if part.Kind == "resource" { + resourceIDs = append(resourceIDs, part.Name) + } + } + if err := verifyLibraryResourceOwners(ctx, t.pool, resourceIDs); err != nil { + return libraryrelease.WriteSet{}, err + } + set := libraryrelease.WriteSet{SchemaDigest: schemaDigest, Version: libraryrelease.MigrationWriteSetVersion, ContractVersion: migrationIdentityContractVersion, + SourceTenantID: manifest.TenantID, SourceKnowledgeBaseID: manifest.KnowledgeBaseID, JobID: plan.JobID, Target: plan.Target, TenantID: t.scope.TenantID, KnowledgeBaseID: t.scope.KnowledgeBaseID, BackupSnapshotID: plan.BackupSnapshotID, TargetSnapshotID: plan.TargetSnapshotID, CreatedAt: t.now().UTC(), Universe: plan.Universe} @@ -171,6 +158,9 @@ func (t *ProductionLibraryTarget) PlanWriteSet(ctx context.Context, plan library if len(ids) == 0 { continue } + if part.Name == "wiki_log_entries" { + continue + } // source IDs are local; target IDs are assigned below owners, err := t.existingRowOwners(ctx, part.Name, ids) if err != nil { return libraryrelease.WriteSet{}, err @@ -181,7 +171,7 @@ func (t *ProductionLibraryTarget) PlanWriteSet(ctx context.Context, plan library set.Rows = append(set.Rows, libraryrelease.WriteSetRow{Table: part.Name, ID: id}) continue } - if part.Name == resourceReuseTableName && !backupResources[id] { + if part.Name == resourceReuseTableName { // 资源不在预备份范围,但可能正是这个目标上一次成功同步写进去、 // 又被恢复保留下来的那份内容。只有"内容/字节相同 + 来源可审计" // 才允许复用(不重写、不清理);否则按跨范围冲突拒绝。 @@ -201,12 +191,34 @@ func (t *ProductionLibraryTarget) PlanWriteSet(ctx context.Context, plan library set.Rows = append(set.Rows, libraryrelease.WriteSetRow{Table: part.Name, ID: id, Existing: true, Owner: owner}) } } + if err := t.preflightArchiveIndex(ctx, archive, manifest); err != nil { + return libraryrelease.WriteSet{}, err + } + if err := t.preflightNaturalKeys(ctx, archive, manifest); err != nil { + return libraryrelease.WriteSet{}, err + } + if err := t.planIndex(ctx, archive, manifest, &set); err != nil { + return libraryrelease.WriteSet{}, err + } + if err := t.planObjects(ctx, archive, manifest, &set); err != nil { + return libraryrelease.WriteSet{}, err + } + set.LocalIDs, err = t.localIDRequests(archive, manifest) + if err != nil { + return libraryrelease.WriteSet{}, err + } + if err := t.allocateLocalIDs(ctx, &set); err != nil { + return libraryrelease.WriteSet{}, err + } if err := set.Seal(); err != nil { return libraryrelease.WriteSet{}, err } if _, err := PersistWriteSet(t.config.ArchiveRoot, &set); err != nil { return libraryrelease.WriteSet{}, err } + if err := t.AdoptWriteSet(set); err != nil { + return libraryrelease.WriteSet{}, err + } return set, nil } @@ -240,11 +252,23 @@ func archivedSnapshotResourceIDs(archive *librarysnapshot.Archive, snapshotID st // 没有 id 列的关系表按父表 knowledges 判归属。 func (t *ProductionLibraryTarget) existingRowOwners(ctx context.Context, table string, ids []string) (map[string]string, error) { owners := map[string]string{} + if _, ok := migrationIdentityContracts[table]; !ok { + return nil, fmt.Errorf("unknown migration table %s", table) + } if table == "knowledge_tag_relations" { + knowledgeIDs, tagIDs := make([]string, 0, len(ids)), make([]string, 0, len(ids)) + for _, id := range ids { + parts := strings.Split(id, ":") + if len(parts) != 2 || !validSearchIdentifier(parts[0]) || !validSearchIdentifier(parts[1]) { + return nil, errors.New("invalid relation tuple") + } + knowledgeIDs = append(knowledgeIDs, parts[0]) + tagIDs = append(tagIDs, parts[1]) + } rows, err := t.pool.Query(ctx, ` SELECT r.knowledge_id, coalesce(r.tag_id,''), coalesce(k.tenant_id,0), coalesce(k.knowledge_base_id,'') FROM knowledge_tag_relations r LEFT JOIN knowledges k ON k.id=r.knowledge_id -WHERE r.knowledge_id = ANY($1::text[])`, ids) +WHERE (r.knowledge_id,r.tag_id) IN (SELECT * FROM unnest($1::text[],$2::text[]))`, knowledgeIDs, tagIDs) if err != nil { return nil, fmt.Errorf("读取目标 %s 归属失败", table) } @@ -271,6 +295,9 @@ WHERE b.id = ANY($1::text[])` } else if !tablesWithKB[table] { statement = fmt.Sprintf(`SELECT id, tenant_id, '' FROM %s WHERE id = ANY($1::text[])`, table) } + if table == "wiki_log_entries" { + statement = `SELECT id::text, tenant_id, coalesce(knowledge_base_id,'') FROM wiki_log_entries WHERE id = ANY($1::bigint[])` + } rows, err := t.pool.Query(ctx, statement, ids) if err != nil { return nil, fmt.Errorf("读取目标 %s 归属失败", table) @@ -326,6 +353,12 @@ func (t *ProductionLibraryTarget) assertWriteSetOwnership(table, id, owner strin // 等于把刚恢复的行再删一次。 func (t *ProductionLibraryTarget) clearJobCreatedRows(ctx context.Context, set libraryrelease.WriteSet, restored map[string]map[string]bool) error { + tx, err := t.pool.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(context.WithoutCancel(ctx)) + // 子表先删:resource_bindings → chunks/knowledges、wiki 子表 → wiki 页 → 资源。 for _, table := range []string{"resource_bindings", "chunks", "knowledges", "knowledge_tag_relations", "wiki_page_issues", "wiki_log_entries", "wiki_pages", "wiki_folders", "knowledge_tags", "resources"} { @@ -341,13 +374,32 @@ func (t *ProductionLibraryTarget) clearJobCreatedRows(ctx context.Context, set l } statement := fmt.Sprintf(`DELETE FROM %s WHERE (%s)::text = ANY($1::text[])`, table, identityExpr(table)) if table == "resources" { - statement += ` - AND NOT EXISTS (SELECT 1 FROM resource_bindings b WHERE b.resource_id=resources.id) - AND NOT EXISTS (SELECT 1 FROM chunks c WHERE c.deleted_at IS NULL - AND coalesce(resources.handle,'') <> '' - AND position('resource://'||resources.handle IN coalesce(c.content,'')) > 0)` + // Lock parent keys through grant check and delete; a concurrent FK insert + // must wait rather than race the protection query and be cascaded away. + locked, err := tx.Query(ctx, `SELECT id FROM resources WHERE id=ANY($1::text[]) FOR UPDATE`, ids) + if err != nil { + return err + } + for locked.Next() { + var id string + if err = locked.Scan(&id); err != nil { + locked.Close() + return err + } + } + locked.Close() + if err = locked.Err(); err != nil { + return err + } + if err = ensureNoResourceGrants(ctx, tx, ids); err != nil { + return err + } + + if err := ensureCreatedResourcesUnreferenced(ctx, tx, ids); err != nil { + return err + } } - if _, err := t.pool.Exec(ctx, statement, ids); err != nil { + if _, err := tx.Exec(ctx, statement, ids); err != nil { return fmt.Errorf("清理本次新增的 %s 失败: %w", table, err) } } @@ -367,7 +419,7 @@ func (t *ProductionLibraryTarget) clearJobCreatedRows(ctx context.Context, set l } statement := fmt.Sprintf(`SELECT (%s)::text FROM %s WHERE (%s)::text = ANY($1::text[])`, identityExpr(table), table, identityExpr(table)) - rows, err := t.pool.Query(ctx, statement, ids) + rows, err := tx.Query(ctx, statement, ids) if err != nil { return fmt.Errorf("复核本次新增的 %s 失败: %w", table, err) } @@ -389,7 +441,7 @@ func (t *ProductionLibraryTarget) clearJobCreatedRows(ctx context.Context, set l return fmt.Errorf("恢复后仍留下本次写入的 %d 行(%v):目标未回到预备份状态,不能当作已恢复", len(leftovers), leftovers) } - return nil + return tx.Commit(ctx) } // snapshotRowIdentities 读出"恢复要用这份快照"的全部行身份(按表)。 diff --git a/internal/releaseadapter/v071/object_import.go b/internal/releaseadapter/v071/object_import.go index a439a6f..0a042be 100644 --- a/internal/releaseadapter/v071/object_import.go +++ b/internal/releaseadapter/v071/object_import.go @@ -1,7 +1,6 @@ package v071 import ( - "bytes" "context" "errors" "io" @@ -47,57 +46,19 @@ func NewObjectImporter(store ObjectStore, config TargetObjectConfig) (*ObjectImp return &ObjectImporter{store: store, config: config}, nil } -func (importer *ObjectImporter) Stage(ctx context.Context, releaseRevision string, snapshot ObjectSnapshot) (ObjectStage, error) { - if !validSearchIdentifier(releaseRevision) || !validLowerSHA256(snapshot.SHA256) || snapshot.SHA256 != objectSnapshotDigest(snapshot.Objects) { - return ObjectStage{}, errors.New("invalid object import snapshot") - } - var total int64 - stage := ObjectStage{ReleaseRevision: releaseRevision, Provider: importer.config.Provider, Objects: make([]StagedObject, 0, len(snapshot.Objects))} - for _, object := range snapshot.Objects { - if !validSearchIdentifier(object.ResourceID) || len(object.Handle) != 22 || object.Size != int64(len(object.Data)) || object.SHA256 != shaBytes(object.Data) || !validObjectKey(object.PhysicalPath) { - return ObjectStage{}, errors.New("invalid object import entry") - } - providerPath, ok := providerPathFromLocator(object.PhysicalPath) - if !ok { - return ObjectStage{}, errors.New("invalid object import locator") - } - total += object.Size - stagingPath := ".governance-release-stage/" + releaseRevision + "/" + providerPath - if !validObjectLocator(stagingPath) { - return ObjectStage{}, errors.New("invalid object staging locator") - } - if err := importer.store.Put(ctx, importer.config.Provider, stagingPath, bytes.NewReader(object.Data), object.Size); err != nil { - return ObjectStage{}, errors.New("write staged release object") - } - size, digest, err := importer.store.Stat(ctx, importer.config.Provider, stagingPath) - if err != nil || size != object.Size || digest != object.SHA256 { - return ObjectStage{}, errors.New("verify staged release object") - } - stage.Objects = append(stage.Objects, StagedObject{Handle: object.Handle, StagingPath: stagingPath, FinalPath: providerPath, Size: object.Size, SHA256: object.SHA256}) - } - if total != snapshot.TotalBytes { - return ObjectStage{}, errors.New("object import total mismatch") - } - return stage, nil +// CheckLegacyObjectWrites has no configuration bypass. Document-level staging +// and promotion do not carry an immutable WriteSet/attempt grant, so they cannot +// participate in the exclusive object write domain. Read-only export is intact. +func CheckLegacyObjectWrites() error { + return errors.New("legacy object writes disabled: use the frozen-plan library worker and object gateway") } -func (importer *ObjectImporter) Promote(ctx context.Context, stage ObjectStage) error { - if stage.Provider != importer.config.Provider || !validSearchIdentifier(stage.ReleaseRevision) { - return errors.New("invalid object promotion stage") - } - for _, object := range stage.Objects { - if !validObjectLocator(object.StagingPath) || !validObjectLocator(object.FinalPath) || !validLowerSHA256(object.SHA256) || object.Size < 0 { - return errors.New("invalid object promotion entry") - } - if err := importer.store.Copy(ctx, importer.config.Provider, object.StagingPath, object.FinalPath); err != nil { - return errors.New("promote release object") - } - size, digest, err := importer.store.Stat(ctx, importer.config.Provider, object.FinalPath) - if err != nil || size != object.Size || digest != object.SHA256 { - return errors.New("verify promoted release object") - } - } - return nil +func (importer *ObjectImporter) Stage(context.Context, string, ObjectSnapshot) (ObjectStage, error) { + return ObjectStage{}, CheckLegacyObjectWrites() +} + +func (importer *ObjectImporter) Promote(context.Context, ObjectStage) error { + return CheckLegacyObjectWrites() } // rewriteLocatorBucket 把资源定位符里的桶换成目标桶,结构保持不变: diff --git a/internal/releaseadapter/v071/oss_version_list.go b/internal/releaseadapter/v071/oss_version_list.go new file mode 100644 index 0000000..a8487e2 --- /dev/null +++ b/internal/releaseadapter/v071/oss_version_list.go @@ -0,0 +1,79 @@ +package v071 + +import ( + "context" + "encoding/xml" + "errors" + "fmt" + "io" + "net/http" + "net/url" +) + +type ossListedVersion struct { + Key string `xml:"Key" json:"key"` + VersionID string `xml:"VersionId" json:"version_id"` +} +type ossExactVersionListing struct { + Versions []ossListedVersion `json:"versions"` + DeleteMarkers []ossListedVersion `json:"delete_markers"` + RequestIDs []string `json:"request_ids"` +} + +// Query the exact key as prefix; never scan a bucket to infer ownership. Listing +// is only discovery: deletion still requires full exact-version readback proof. +func (s *ossVersionStore) listExactVersions(ctx context.Context, key string) (ossExactVersionListing, error) { + result := ossExactVersionListing{Versions: []ossListedVersion{}, DeleteMarkers: []ossListedVersion{}} + if !validObjectKey(key) || key == "" { + return result, errors.New("exact OSS listing key required") + } + q := url.Values{"versions": {""}, "prefix": {key}, "max-keys": {"100"}} + seen := map[string]bool{} + seenMarkers := map[string]bool{} + for page := 0; page < 100; page++ { + resp, err := s.request(ctx, http.MethodGet, "", q, nil, nil) + if err != nil { + return result, err + } + var value struct { + XMLName xml.Name `xml:"ListVersionsResult"` + Name string `xml:"Name"` + Prefix string `xml:"Prefix"` + Truncated *bool `xml:"IsTruncated"` + NextKey string `xml:"NextKeyMarker"` + NextVersion string `xml:"NextVersionIdMarker"` + Versions []ossListedVersion `xml:"Version"` + Deletes []ossListedVersion `xml:"DeleteMarker"` + CommonPrefixes []string `xml:"CommonPrefixes>Prefix"` + } + if resp.StatusCode != 200 { + resp.Body.Close() + return result, fmt.Errorf("OSS list versions status %d", resp.StatusCode) + } + err = xml.NewDecoder(io.LimitReader(resp.Body, 1024*1024)).Decode(&value) + resp.Body.Close() + if err != nil || value.Name != s.bucket || value.Prefix != key || value.Truncated == nil || len(value.CommonPrefixes) != 0 { + return result, errors.New("OSS version listing identity/completeness not proven") + } + result.RequestIDs = append(result.RequestIDs, resp.Header.Get("x-oss-request-id")) + for _, v := range append(append([]ossListedVersion{}, value.Versions...), value.Deletes...) { + if v.Key != key || v.VersionID == "" || seen[v.VersionID] { + return result, errors.New("OSS listing contains unexpected key or duplicate version") + } + seen[v.VersionID] = true + } + result.Versions = append(result.Versions, value.Versions...) + result.DeleteMarkers = append(result.DeleteMarkers, value.Deletes...) + if !*value.Truncated { + return result, nil + } + marker := value.NextKey + "\x00" + value.NextVersion + if value.NextKey != key || value.NextVersion == "" || seenMarkers[marker] { + return result, errors.New("OSS listing pagination incomplete") + } + seenMarkers[marker] = true + q.Set("key-marker", value.NextKey) + q.Set("version-id-marker", value.NextVersion) + } + return result, errors.New("OSS version listing page budget exceeded") +} diff --git a/internal/releaseadapter/v071/oss_version_store.go b/internal/releaseadapter/v071/oss_version_store.go new file mode 100644 index 0000000..741cb1d --- /dev/null +++ b/internal/releaseadapter/v071/oss_version_store.go @@ -0,0 +1,231 @@ +package v071 + +import ( + "bytes" + "context" + "crypto/hmac" + "crypto/sha1" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/xml" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strings" + "time" +) + +// Native OSS version operations deliberately do NOT implement conditionalObjectStore: +// OSS documents forbid-overwrite as ineffective on versioned buckets. Appending a +// version is recoverable, but is not an atomic absent-key create. Keep that policy +// decision separate from this protocol rather than silently weakening the gate. +type ossVersionStore struct { + endpoint *url.URL + bucket, access, secret string + maxBytes int64 + client *http.Client +} + +func newOSSVersionStore(c S3ObjectStoreConfig) (*ossVersionStore, error) { + u, err := url.Parse(c.Endpoint) + if err != nil || u.Scheme != "https" || u.Host == "" || u.User != nil || u.Path != "" || u.RawQuery != "" || u.Fragment != "" || c.Provider != "oss" || c.ForcePathStyle || c.Bucket == "" || strings.ContainsAny(c.Bucket, "/.: ") || c.AccessKeyID == "" || c.SecretAccessKey == "" { + return nil, errors.New("invalid native OSS configuration") + } + if c.MaxObjectBytes == 0 { + c.MaxObjectBytes = 100 * 1024 * 1024 + } + if c.MaxObjectBytes < 1 { + return nil, errors.New("invalid OSS size limit") + } + client := &http.Client{Timeout: 2 * time.Minute} + if c.HTTPClient != nil { + clone := *c.HTTPClient + client = &clone + } + client.CheckRedirect = func(*http.Request, []*http.Request) error { return errors.New("OSS redirect refused") } + return &ossVersionStore{u, c.Bucket, c.AccessKeyID, c.SecretAccessKey, c.MaxObjectBytes, client}, nil +} + +func (s *ossVersionStore) request(ctx context.Context, method, key string, q url.Values, body []byte, headers http.Header) (*http.Response, error) { + if key != "" && (!validObjectKey(key) || strings.Contains(key, "://") || strings.HasPrefix(key, "/")) { + return nil, errors.New("invalid OSS key") + } + u := *s.endpoint + u.Host = s.bucket + "." + s.endpoint.Host + u.Path = "/" + key + u.RawQuery = q.Encode() + req, err := http.NewRequestWithContext(ctx, method, u.String(), bytes.NewReader(body)) + if err != nil { + return nil, errors.New("invalid OSS request") + } + req.Header = headers.Clone() + if req.Header == nil { + req.Header = make(http.Header) + } + req.Header.Set("Date", time.Now().UTC().Format(http.TimeFormat)) + resource := "/" + s.bucket + "/" + key + keys := make([]string, 0, len(q)) + for k := range q { + if k == "prefix" || k == "max-keys" || k == "key-marker" || k == "version-id-marker" { + if method != http.MethodGet || key != "" || !q.Has("versions") || q.Get("prefix") == "" { + return nil, errors.New("OSS listing must be exact-key scoped and read-only") + } + continue // These list filters are not OSS V1 signed subresources. + } + if k != "versionId" && k != "versioning" && k != "versions" { + return nil, errors.New("unsupported OSS subresource") + } + keys = append(keys, k) + } + sort.Strings(keys) + for i, k := range keys { + if i == 0 { + resource += "?" + } else { + resource += "&" + } + resource += k + if q.Get(k) != "" { + resource += "=" + q.Get(k) + } + } + oss := map[string]string{} + for k, v := range req.Header { + lower := strings.ToLower(k) + if strings.HasPrefix(lower, "x-oss-") { + oss[lower] = strings.TrimSpace(strings.Join(v, ",")) + } + } + keys = keys[:0] + for k := range oss { + keys = append(keys, k) + } + sort.Strings(keys) + canonical := "" + for _, k := range keys { + canonical += k + ":" + oss[k] + "\n" + } + toSign := method + "\n" + req.Header.Get("Content-MD5") + "\n" + req.Header.Get("Content-Type") + "\n" + req.Header.Get("Date") + "\n" + canonical + resource + mac := hmac.New(sha1.New, []byte(s.secret)) + _, _ = mac.Write([]byte(toSign)) + req.Header.Set("Authorization", "OSS "+s.access+":"+base64.StdEncoding.EncodeToString(mac.Sum(nil))) + // No retry middleware and no redirect: an ambiguous write remains one attempt. + resp, err := s.client.Do(req) + if err != nil { + return nil, errors.New("native OSS transport outcome unknown") + } + return resp, nil +} +func (s *ossVersionStore) versioning(ctx context.Context) error { + resp, err := s.request(ctx, http.MethodGet, "", url.Values{"versioning": {""}}, nil, nil) + if err != nil { + return err + } + defer resp.Body.Close() + var value struct { + Status string `xml:"Status"` + } + if resp.StatusCode != 200 || xml.NewDecoder(io.LimitReader(resp.Body, 4096)).Decode(&value) != nil || value.Status != "Enabled" { + return errors.New("OSS versioning is not confirmed Enabled") + } + return nil +} +func (s *ossVersionStore) observeVersion(ctx context.Context, key, version string) (objectObservation, error) { + q := url.Values{} + if version != "" { + if version == "null" { + return objectObservation{}, errors.New("null is not an immutable OSS version") + } + q.Set("versionId", version) + } + resp, err := s.request(ctx, http.MethodGet, key, q, nil, nil) + if err != nil { + return objectObservation{}, err + } + defer resp.Body.Close() + if resp.StatusCode == 404 { + var failure struct { + Code string `xml:"Code"` + } + if xml.NewDecoder(io.LimitReader(resp.Body, 4096)).Decode(&failure) != nil || (failure.Code != "NoSuchKey" && failure.Code != "NoSuchVersion") { + return objectObservation{}, errors.New("OSS 404 is not an authoritative missing object version") + } + return objectObservation{}, nil + } + if resp.StatusCode != 200 { + return objectObservation{}, fmt.Errorf("OSS version read status %d", resp.StatusCode) + } + got := resp.Header.Get("x-oss-version-id") + if got == "" || got == "null" || version != "" && got != version || resp.Header.Get("ETag") == "" || resp.ContentLength < 0 || resp.ContentLength > s.maxBytes { + return objectObservation{}, errors.New("OSS version identity incomplete") + } + hash := sha256.New() + size, err := io.Copy(hash, io.LimitReader(resp.Body, s.maxBytes+1)) + if err != nil || size != resp.ContentLength { + return objectObservation{}, errors.New("OSS version body incomplete") + } + return objectObservation{Exists: true, VersionID: got, ETag: resp.Header.Get("ETag"), Owner: resp.Header.Get("x-oss-meta-kg-release-owner"), Bytes: size, SHA256: hex.EncodeToString(hash.Sum(nil))}, nil +} + +// appendVersion is explicit about its semantics. Callers need an approved version- +// append plan and a persisted intent; this is not wired as CreateObject. +func (s *ossVersionStore) appendVersion(ctx context.Context, key string, body []byte, owner string) (objectObservation, error) { + if len(owner) != 64 || int64(len(body)) > s.maxBytes { + return objectObservation{}, errObjectWriteRejected + } + if _, err := hex.DecodeString(owner); err != nil { + return objectObservation{}, errObjectWriteRejected + } + if err := s.versioning(ctx); err != nil { + return objectObservation{}, err + } + resp, err := s.request(ctx, http.MethodPut, key, nil, body, http.Header{"Content-Type": {"application/octet-stream"}, "X-Oss-Meta-Kg-Release-Owner": {owner}}) + if err != nil { + return objectObservation{}, err + } + defer resp.Body.Close() + version := resp.Header.Get("x-oss-version-id") + if resp.StatusCode != 200 || version == "" || version == "null" { + return objectObservation{}, errors.New("OSS append receipt unknown") + } + observation, err := s.observeVersion(ctx, key, version) + sum := sha256.Sum256(body) + if err != nil { + return objectObservation{}, err + } + if !observation.Exists || observation.Owner != owner || observation.Bytes != int64(len(body)) || observation.SHA256 != hex.EncodeToString(sum[:]) { + return objectObservation{}, errors.New("OSS exact version readback mismatch") + } + return observation, nil +} +func (s *ossVersionStore) removeVersion(ctx context.Context, key string, want objectObservation) error { + if !want.Exists || want.VersionID == "" || want.VersionID == "null" || want.ETag == "" || want.Owner == "" || want.SHA256 == "" { + return errors.New("OSS deletion requires complete owned version receipt") + } + got, err := s.observeVersion(ctx, key, want.VersionID) + if err != nil { + return err + } + if !got.Exists { + return nil + } + if got != want { + return errors.New("OSS exact version differs from receipt") + } + resp, sendErr := s.request(ctx, http.MethodDelete, key, url.Values{"versionId": {want.VersionID}}, nil, nil) + if resp != nil { + resp.Body.Close() + } + after, readErr := s.observeVersion(ctx, key, want.VersionID) + if readErr != nil { + return readErr + } + if after.Exists { + return fmt.Errorf("OSS exact version delete unconfirmed: %v", sendErr) + } + return nil +} diff --git a/internal/releaseadapter/v071/postgres_export.go b/internal/releaseadapter/v071/postgres_export.go index 2811836..f94cdc0 100644 --- a/internal/releaseadapter/v071/postgres_export.go +++ b/internal/releaseadapter/v071/postgres_export.go @@ -28,7 +28,8 @@ type TableSnapshot struct { } type PostgresSnapshot struct { - Tables []TableSnapshot + Catalog migrationCatalog `json:"catalog,omitempty"` + Tables []TableSnapshot } func (snapshot PostgresSnapshot) Table(name string) TableSnapshot { diff --git a/internal/releaseadapter/v071/search.go b/internal/releaseadapter/v071/search.go index a9a908b..829259e 100644 --- a/internal/releaseadapter/v071/search.go +++ b/internal/releaseadapter/v071/search.go @@ -363,9 +363,14 @@ func searchDocumentsDigest(documents []SearchDocument) (string, error) { func canonicalJSON(raw json.RawMessage) (json.RawMessage, error) { var value any - if err := json.Unmarshal(raw, &value); err != nil { + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.UseNumber() + if err := decoder.Decode(&value); err != nil { return nil, err } + if !json.Valid(raw) { + return nil, errors.New("invalid JSON") + } return json.Marshal(value) } diff --git a/internal/releaseadapter/v071/search_write.go b/internal/releaseadapter/v071/search_write.go index 430e7dc..df9bdff 100644 --- a/internal/releaseadapter/v071/search_write.go +++ b/internal/releaseadapter/v071/search_write.go @@ -101,7 +101,7 @@ func (exporter *SearchExporter) PruneScope(ctx context.Context, identity SearchI end = len(stale) } payload := map[string]any{ - "query": map[string]any{"ids": map[string]any{"values": stale[start:end]}}, + "query": map[string]any{"bool": map[string]any{"filter": append(scopeFilterFor(inspection.fields, identity), map[string]any{"ids": map[string]any{"values": stale[start:end]}})}}, } var response struct { Deleted int `json:"deleted"` @@ -126,6 +126,10 @@ func scopeFilterFor(fields searchFieldNames, identity SearchIdentity) []any { // BulkIndex 把归档里的原始文档与向量直接写入生产索引(不解析、不重新 Embedding)。 // 写入前补齐 tenant_id/knowledge_base_id,避免归档文档缺字段时落到别的库。 func (exporter *SearchExporter) BulkIndex(ctx context.Context, documents []librarysnapshot.IndexDocument, identity SearchIdentity) error { + return exporter.bulkIndex(ctx, documents, identity, nil) +} + +func (exporter *SearchExporter) bulkIndex(ctx context.Context, documents []librarysnapshot.IndexDocument, identity SearchIdentity, guards map[string]indexWriteGuard) error { if len(documents) == 0 { return nil } @@ -146,9 +150,21 @@ func (exporter *SearchExporter) BulkIndex(ctx context.Context, documents []libra return err } payload = encoded - action, err := json.Marshal(map[string]any{"index": map[string]string{ - "_index": exporter.index, "_id": document.ID, - }}) + operation := "index" + metadata := map[string]any{"_index": exporter.index, "_id": document.ID} + if guards != nil { + guard, ok := guards[document.ID] + if !ok { + return errors.New("missing index write guard") + } + if !guard.Found { + operation = "create" + } else { + metadata["if_seq_no"] = *guard.SeqNo + metadata["if_primary_term"] = *guard.PrimaryTerm + } + } + action, err := json.Marshal(map[string]any{operation: metadata}) if err != nil { return err } @@ -189,6 +205,18 @@ func (exporter *SearchExporter) BulkIndex(ctx context.Context, documents []libra if err := json.Unmarshal(raw, &parsed); err != nil { return errors.New("Elasticsearch bulk response invalid") } + if len(parsed.Items) == len(documents) { + for i, item := range parsed.Items { + if len(item) != 1 { + return errors.New("invalid bulk item result") + } + for operation, result := range item { + if (operation != "index" && operation != "create") || result.ID != documents[i].ID || result.Status < 200 || result.Status >= 300 || result.Error != nil { + return fmt.Errorf("Elasticsearch item outcome not confirmed: %s status %d", documents[i].ID, result.Status) + } + } + } + } if parsed.Errors { for _, item := range parsed.Items { for _, result := range item { diff --git a/web/src/LegacySourceAssociation.test.tsx b/web/src/LegacySourceAssociation.test.tsx new file mode 100644 index 0000000..50274ad --- /dev/null +++ b/web/src/LegacySourceAssociation.test.tsx @@ -0,0 +1,45 @@ +import { afterEach, expect, it, vi } from 'vitest' +import { renderToStaticMarkup } from 'react-dom/server' +import { associateLegacySource, LegacySourceAssociation } from './LegacySourceAssociation' + +afterEach(() => vi.unstubAllGlobals()) +const reply = (status: number, data: unknown) => new Response(JSON.stringify(data), { status }) + +it('only sends exact snapshot and scope; association is not publishing', async () => { + const fetchMock = vi + .fn() + .mockResolvedValue( + reply(200, { associated: true, published: false, snapshot_id: 'snapshot', source_id: 'a'.repeat(64) }) + ) + vi.stubGlobal('fetch', fetchMock) + expect(await associateLegacySource('internal', 'snapshot')).toBe('a'.repeat(64)) + expect(fetchMock.mock.calls).toHaveLength(1) + expect(fetchMock.mock.calls[0][0]).toBe('/api/v1/library/source-associations') + expect(JSON.parse(fetchMock.mock.calls[0][1].body)).toEqual({ scope: 'internal', snapshot_id: 'snapshot' }) +}) + +it('rejects wrong-snapshot and incomplete readback, and shows evidence errors', async () => { + for (const value of [ + { associated: true, published: false, snapshot_id: 'wrong', source_id: 'a'.repeat(64) }, + { associated: true, snapshot_id: 'snapshot' }, + ]) { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(reply(200, value))) + await expect(associateLegacySource('internal', 'snapshot')).rejects.toThrow('来源关联回执') + } + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(reply(409, { error: 'original capture identity mismatch' }))) + await expect(associateLegacySource('internal', 'snapshot')).rejects.toThrow('original capture identity mismatch') +}) + +it('hides association controls from non-admin and distinguishes it from publication', () => { + expect( + renderToStaticMarkup( + {}} /> + ) + ).toBe('') + const html = renderToStaticMarkup( + {}} /> + ) + expect(html).toContain('只追加治理关联') + expect(html).toContain('disabled') + expect(html).toContain('核验原证据并关联') +}) diff --git a/web/src/LegacySourceAssociation.tsx b/web/src/LegacySourceAssociation.tsx new file mode 100644 index 0000000..5d0a872 --- /dev/null +++ b/web/src/LegacySourceAssociation.tsx @@ -0,0 +1,83 @@ +import { useState } from 'react' +import type { LibrarySnapshot } from './librarySnapshots' + +export async function associateLegacySource(scope: string, snapshotID: string): Promise { + const response = await fetch('/api/v1/library/source-associations', { + method: 'POST', + headers: { 'Content-Type': 'application/json', Accept: 'application/json' }, + body: JSON.stringify({ scope, snapshot_id: snapshotID }), + }) + const value = (await response.json().catch(() => null)) as Record | null + if (!response.ok) + throw new Error(typeof value?.error === 'string' ? value.error : `来源关联失败(HTTP ${response.status})`) + if ( + value?.associated !== true || + value.published !== false || + value.snapshot_id !== snapshotID || + typeof value.source_id !== 'string' || + !/^[a-f0-9]{64}$/.test(value.source_id) + ) { + throw new Error('来源关联回执不完整或快照身份不符') + } + return value.source_id +} + +export function LegacySourceAssociation({ + scope, + snapshots, + admin, + onBound, +}: { + scope: string + snapshots: LibrarySnapshot[] + admin: boolean + onBound: () => Promise +}) { + const [snapshotID, setSnapshotID] = useState('') + const [pending, setPending] = useState(false) + const [notice, setNotice] = useState('') + if (!admin) return null + const choices = snapshots.filter((item) => item.scope === scope) + const bind = async () => { + if (pending || !choices.some((item) => item.snapshot_id === snapshotID)) return + setPending(true) + setNotice('') + try { + const sourceID = await associateLegacySource(scope, snapshotID) + setNotice(`来源证据已核验关联:${sourceID}。没有发布内容;发布授权与写入闸门仍独立检查。`) + await onBound() + } catch (error) { + setNotice(error instanceof Error ? error.message : '来源关联失败') + } finally { + setPending(false) + } + } + return ( +
+ 关联旧封存的原始来源证据(管理员) +

+ 只追加治理关联,保留原快照与摘要。服务端核对原捕获响应、执行器指纹及清单;备份、来源不符或证据缺失均拒绝,不凭同一范围认领。 +

+ + + {notice ?

{notice}

: null} +
+ ) +} diff --git a/web/src/ReleaseTargets.tsx b/web/src/ReleaseTargets.tsx index 7308984..8c10fa1 100644 --- a/web/src/ReleaseTargets.tsx +++ b/web/src/ReleaseTargets.tsx @@ -1,3 +1,4 @@ +import { LegacySourceAssociation } from './LegacySourceAssociation' import { useCallback, useEffect, useRef, useState } from 'react' import { createLibraryRestoreAuthorization, @@ -1474,6 +1475,19 @@ export function ReleaseTargetsView(props: ViewProps) {
{target ? '完整快照' : '未接入'}
+ {target?.source_identity ? ( +
+ 来源身份与完整摘要 +

+ 来源实例:{target.source_identity.project} / {target.source_identity.app} +

+

+ 来源租户 / 知识库:{target.source_identity.tenant_id} / {target.source_identity.knowledge_base_id} +

+

来源身份:{target.source_id}

+

快照摘要:{target.snapshot_id}

+
+ ) : null}
  • {target @@ -1639,14 +1653,14 @@ export function ReleaseTargets() { } const startSync = async () => { - if (!targetSnapshot) { - setNotice('还没有可发布的测试封存') + if (!targetSnapshot || !current?.test_sealed?.source_id) { + setNotice('还没有与当前来源绑定的可发布封存,请刷新状态') return } setPendingSync(true) setNotice(null) try { - const accepted = await enqueueLibrarySync(scope, targetSnapshot) + const accepted = await enqueueLibrarySync(scope, targetSnapshot, current?.test_sealed?.source_id ?? '') setNotice(`已投递同步:${accepted.job_id}(先固定测试版本并生成预备份,再写目标)`) await load() } catch (syncError) { @@ -1801,50 +1815,59 @@ export function ReleaseTargets() { }, [pendingRecovery, maintenances, scope, load]) return ( - { - setScope(next) - setExpandedSnapshot(null) - }} - drawerOpen={drawerOpen} - onDrawer={setDrawerOpen} - expandedSnapshot={expandedSnapshot} - onExpandSnapshot={setExpandedSnapshot} - pendingCapture={pendingCapture} - canCapture={canCapture} - canWrite={canWrite} - sync={{ - canSign: canSignSync, - pendingGrant: pendingSyncGrant, - onGrant: () => void grantSync(), - canStart: current?.gate.sync_allowed === true, - pending: pendingSync, - onStart: () => void startSync(), - }} - baseline={{ - canStart: canSignSync, - pending: pendingBaseline, - onStart: () => void startBaseline(), - }} - canRestore={canRestore} - restore={restore} - restoreNotice={restoreNotice} - pendingRestore={pendingRestore} - pendingGrant={pendingGrant} - onGrantRestore={() => void grantRestore()} - onRestore={() => void startRestore()} - maintenance={maintenances[scope]} - now={now} - onRefresh={() => void load()} - onCapture={() => void startCapture()} - onRecover={(windowJobID) => void recover(windowJobID)} - /> + <> + { + setScope(next) + setExpandedSnapshot(null) + }} + drawerOpen={drawerOpen} + onDrawer={setDrawerOpen} + expandedSnapshot={expandedSnapshot} + onExpandSnapshot={setExpandedSnapshot} + pendingCapture={pendingCapture} + canCapture={canCapture} + canWrite={canWrite} + sync={{ + canSign: canSignSync, + pendingGrant: pendingSyncGrant, + onGrant: () => void grantSync(), + canStart: current?.gate.sync_allowed === true && Boolean(current?.test_sealed?.source_id), + pending: pendingSync, + onStart: () => void startSync(), + }} + baseline={{ + canStart: canSignSync, + pending: pendingBaseline, + onStart: () => void startBaseline(), + }} + canRestore={canRestore} + restore={restore} + restoreNotice={restoreNotice} + pendingRestore={pendingRestore} + pendingGrant={pendingGrant} + onGrantRestore={() => void grantRestore()} + onRestore={() => void startRestore()} + maintenance={maintenances[scope]} + now={now} + onRefresh={() => void load()} + onCapture={() => void startCapture()} + onRecover={(windowJobID) => void recover(windowJobID)} + /> + load()} + /> + ) } diff --git a/web/src/librarySnapshots.ts b/web/src/librarySnapshots.ts index d97f3b9..d99dde0 100644 --- a/web/src/librarySnapshots.ts +++ b/web/src/librarySnapshots.ts @@ -22,6 +22,8 @@ export interface LibraryCompatibility { } export interface LibrarySnapshot { + source_id?: string + source_identity?: { project: string; app: string; scope: string; tenant_id: number; knowledge_base_id: string } snapshot_id: string scope: string knowledge_base_id: string @@ -104,6 +106,16 @@ function parseSnapshot(raw: unknown): LibrarySnapshot | null { } return { snapshot_id: snapshotID, + source_id: text(raw.source_id) || undefined, + source_identity: record(raw.source_identity) + ? { + project: text(raw.source_identity.project), + app: text(raw.source_identity.app), + scope: text(raw.source_identity.scope), + tenant_id: num(raw.source_identity.tenant_id), + knowledge_base_id: text(raw.source_identity.knowledge_base_id), + } + : undefined, scope: text(raw.scope), knowledge_base_id: text(raw.knowledge_base_id), tenant_id: num(raw.tenant_id), @@ -288,11 +300,15 @@ export interface LibrarySyncAccepted { } // 启动同步:闸口未齐时接口返回 503 与逐条缺项,页面照实显示,不假装已启动。 -export async function enqueueLibrarySync(scope: string, targetSnapshotID: string): Promise { +export async function enqueueLibrarySync( + scope: string, + targetSnapshotID: string, + sourceID: string +): Promise { const response = await fetch('/api/v1/library/syncs', { method: 'POST', headers: { 'Content-Type': 'application/json', Accept: 'application/json' }, - body: JSON.stringify({ scope, target_snapshot_id: targetSnapshotID }), + body: JSON.stringify({ scope, target_snapshot_id: targetSnapshotID, source_id: sourceID }), }) const value = (await response.json().catch(() => null)) as unknown if (!response.ok) { diff --git a/web/src/librarySyncAuth.test.ts b/web/src/librarySyncAuth.test.ts index 83c7346..d5cb824 100644 --- a/web/src/librarySyncAuth.test.ts +++ b/web/src/librarySyncAuth.test.ts @@ -33,11 +33,15 @@ describe('发布页的同步授权与执行接线', () => { .fn() .mockResolvedValue(jsonResponse(202, { job: { id: 'libsync-internal-1', scope: 'internal' }, reused: false })) vi.stubGlobal('fetch', fetchMock) - const accepted = await enqueueLibrarySync('internal', 'snap-1') + const accepted = await enqueueLibrarySync('internal', 'snap-1', 'source-bound') expect(accepted.job_id).toBe('libsync-internal-1') const [url, init] = fetchMock.mock.calls[0] expect(url).toBe('/api/v1/library/syncs') - expect(JSON.parse(init.body)).toEqual({ scope: 'internal', target_snapshot_id: 'snap-1' }) + expect(JSON.parse(init.body)).toEqual({ + scope: 'internal', + target_snapshot_id: 'snap-1', + source_id: 'source-bound', + }) }) it('闸口未齐时把接口给的缺项原样抛出,而不是假装已启动', async () => { @@ -50,7 +54,7 @@ describe('发布页的同步授权与执行接线', () => { }) ) ) - await expect(enqueueLibrarySync('internal', 'snap-1')).rejects.toThrow( + await expect(enqueueLibrarySync('internal', 'snap-1', 'source-bound')).rejects.toThrow( /目标非空且没有生效版本记录;没有与目标快照匹配的有效发布授权/ ) })