fix(library): 维护收尾必须按预备份恢复并校验
build / build (push) Successful in 1m20s

删掉会自己开入口、只看服务健康的 FinalizeAfterOperatorRecovery。
新的 RecoverWindow 绑定原作业/目标指纹/持久化预备份,在维护关闭下
执行恢复与逐类校验,通过才恢复服务与入口;失败继续维护。
预备份身份与内容风险在第一笔写入前一起落盘;后台任务必须静默后才停写者;
写入者发布端口即拒绝开窗口;维护占用改为按实例互斥(000072)。
This commit is contained in:
姚章浩
2026-10-29 00:50:32 +08:00
parent ec737a9a65
commit 3643e9bf82
26 changed files with 1730 additions and 225 deletions
+34
View File
@@ -1,5 +1,39 @@
# Changelog
## 开发快照(维护收尾必须按预备份验证 / 后台任务静默 / 实例级互斥)
### Fixed
- **删除 `FinalizeAfterOperatorRecovery`**:它自称"不启动服务",实际会 `gate.OpenIngress`,
而且只看服务健康与入口 200——健康不等于恢复成功,这条路径能把半写内容重新开放。
- 新的唯一收尾入口 `Coordinator.RecoverWindow`:绑定「原作业 + 目标指纹 + journal 里持久化的预备份身份」,
在**维护关闭**状态下(入口关、写入者停、排空确认)执行 `Restore` 与**逐类校验**
(行数/身份集合/逐行内容摘要 + 索引文档 + 对象资源),校验通过才恢复服务、开入口;
校验或恢复失败则保持维护(服务停、入口关、journal 非终态)。
- 预备份身份与「内容风险」在**第一笔写入之前一起落盘**(`Window.SetBackupSnapshot` +
`MarkContentRisk`)。没有预备份身份的窗口之后既不能自动恢复,也不许被任何收尾路径开放。
- 低层恢复接口不再接受任何"已恢复/已验证"的入参:`Restorer`/`Verifier` 由协调器在窗口内调用,
`Verifier` 必须返回校验摘要并被持久化。
### Added
- 真实恢复入口 `POST /api/v1/library/maintenance/recover`:只建 `library.recover` 作业,
由 worker 按预备份恢复 + 校验后释放占用;`libraryctl finalize` 删除(不再有"手工收尾"旁路)。
- 实例级互斥:迁移 `000072_library_instance_maintenance`,维护占用按**实例**建键,
internal 持有期间 external 也进不来(旧实现按 scope 建行,两个 scope 会同时停/起同一实例)。
- 后台静默检查(`targetQuiescer`):窗口关闭入口之前,必须证明目标库 `task_pending_ops=0`、
无未完成解析、asynq 队列(pending/active/scheduled/retry)为空;否则拒绝开窗口。
HTTP 排空不等于后台任务排空——app 内的 asynq worker 会被上游 SIGTERM 的 exit(1) 切断。
- 唯一入口拓扑校验(`Manager.VerifyTopology`):写入者**不许发布任何宿主机端口**,
入口必须是独立声明的闸门且发布的端口与入口 URL 一致;不满足就拒绝开窗口。
### 限制(本轮不承诺)
- 拓扑校验只看"声明的写入者组":新增容器必须显式声明,平台不会自动猜测。
- 后台静默检查依赖目标库表 `task_pending_ops/knowledges` 与 asynq 键名(v0.7.1 现状);
上游改队列实现时需要同步调整。
- 页面只透传新增字段(实例占用/入口状态),专门文案与浏览器验收仍未做。
## 开发快照(目标维护:实例级窗口)
### Added
+52
View File
@@ -1062,6 +1062,58 @@
}
}
},
"/api/v1/library/maintenance/recover": {
"post": {
"tags": [
"Runtime"
],
"operationId": "recoverLibraryMaintenance",
"summary": "按预备份恢复并逐类校验后收尾维护窗口(仅管理员;只建恢复作业,恢复在校验通过前不开放目标)",
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"type": "object",
"required": [
"scope"
],
"properties": {
"scope": {
"type": "string",
"enum": [
"internal",
"external"
]
},
"window_job_id": {
"type": "string",
"description": "可选:必须与当前维护窗口的作业一致"
}
}
}
}
}
},
"responses": {
"202": {
"description": "已创建恢复作业并入队"
},
"400": {
"$ref": "#/components/responses/BadRequest"
},
"409": {
"description": "没有未释放的维护窗口,或窗口属于别的 scope/作业"
},
"500": {
"$ref": "#/components/responses/InternalError"
},
"503": {
"description": "目标维护未接线或维护记录存储不可用"
}
}
}
},
"/api/v1/library/restores": {
"post": {
"tags": [
+1
View File
@@ -202,6 +202,7 @@ func run(ctx context.Context, getenv func(string) string) error {
VersionHistory: versionHistoryStore,
VersionHistoryRepo: versionHistoryRepo,
LibraryArchiveRoot: config.LibraryArchiveRoot,
LibraryTargetInstance: config.LibraryTargetInstance,
LibraryAgentSocket: config.LibraryAgentSocket,
LibraryCapacityBudgetBytes: config.LibraryCapacityBudgetBytes,
LibraryProductionDatabaseURL: config.LibraryProductionDatabaseURL,
+1 -30
View File
@@ -90,8 +90,6 @@ func main() {
runErr = runVerify(opts)
case "recover":
runErr = runRecover(ctx, opts)
case "finalize":
runErr = runFinalize(ctx, opts)
default:
usage()
os.Exit(2)
@@ -103,7 +101,7 @@ func main() {
}
func usage() {
fmt.Fprint(os.Stderr, `libraryctl <status|preflight|capture|list|verify|recover|finalize> [flags]
fmt.Fprint(os.Stderr, `libraryctl <status|preflight|capture|list|verify|recover> [flags]
status probe the real source and print what is configured / blocked
preflight read the source, run settlement + candidate validation (no writes, no stop)
@@ -111,8 +109,6 @@ func usage() {
list list sealed candidates and stored bytes
verify re-verify one sealed candidate (digests, coverage, identity)
recover restore writers from an interrupted freeze journal
finalize close a held window after an operator recovered the target by hand
(verifies services + ingress are really serving; starts nothing)
Flags (defaults target the local WKNFull test stack):
-source-app WKNFull-app container whose env describes the source
@@ -821,31 +817,6 @@ func runVerify(opts options) error {
return nil
}
func runFinalize(ctx context.Context, opts options) error {
if opts.job == "" {
return errors.New("finalize requires -job")
}
coordinator, control, err := openFreeze(ctx, opts)
if err != nil {
return err
}
defer coordinator.Close()
defer control.Close()
if err := coordinator.FinalizeAfterOperatorRecovery(ctx, opts.job); err != nil {
return err
}
items, err := control.Inventory(ctx)
if err != nil {
return err
}
group := []map[string]any{}
for _, item := range items {
group = append(group, map[string]any{"id": item.ID, "running": item.Running, "healthy": item.Healthy})
}
printJSON(map[string]any{"finalized": opts.job, "code": "operator_recovered", "writers": group})
return nil
}
func runRecover(ctx context.Context, opts options) error {
coordinator, control, err := openFreeze(ctx, opts)
if err != nil {
+56 -17
View File
@@ -182,6 +182,12 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
"目标维护未接线,拒绝在挡不住入口/后台写入的情况下写目标:"+managerErr.Error(), "prebackup")
}
defer manager.Close()
// 拓扑必须先过关:写入者发布了端口 = 存在绕闸直连,窗口关的只是半扇门。
if topoErr := manager.VerifyTopology(ctx); topoErr != nil {
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
"目标拓扑不满足唯一入口约束,拒绝开维护窗口:"+topoErr.Error(), "prebackup")
}
targetIdentity := fmt.Sprintf("scope=%s db=%s index=%s bucket=%s", payload.Scope,
redactDSN(config.LibraryProductionDatabaseURL), config.LibraryProductionIndex, config.LibraryProductionS3Bucket)
originalState, stateErr := manager.OriginalState(ctx)
@@ -190,6 +196,19 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
"读不到目标写入者的原运行状态,不能开维护窗口:"+stateErr.Error(), "prebackup")
}
identity := manager.Target()
// 实例级占用:internal/external 动的是同一批容器,必须按实例互斥。
instanceLock, instanceErr := store.AcquireInstanceMaintenance(ctx, postgresstore.LibraryInstanceMaintenance{
Instance: identity.Instance, OwnerJobID: payload.LibraryJobID, OwnerScope: payload.Scope,
TargetFingerprint: identity.Fingerprint, Level: librarymaintenance.LevelInstance,
Services: identity.Writers, IngressState: "unknown", State: "acquired",
OriginalServiceState: originalState,
Note: "实例级维护:关闭目标入口并停止全部写入者,验证或回滚完成才开放",
}, now)
if instanceErr != nil {
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
"目标实例正在维护中(跨对内/对外互斥):"+instanceErr.Error(), "prebackup")
}
_ = instanceLock
maintenance, acquireErr := store.AcquireLibraryMaintenance(ctx, postgresstore.LibraryMaintenance{
Scope: payload.Scope, OwnerJobID: payload.LibraryJobID,
TargetIdentity: targetIdentity + " | " + manager.Description(),
@@ -202,6 +221,8 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
Note: "实例级维护:关闭目标入口并停止全部写入者,验证或回滚完成才开放",
}, now)
if acquireErr != nil {
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore",
"scope 占用申请失败,实例状态未确认:"+acquireErr.Error(), now)
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "目标正在维护中:"+acquireErr.Error(), "prebackup")
}
_ = maintenance
@@ -213,10 +234,11 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
if resumed {
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore",
"维护协调器不可用,接管目标的状态未确认:"+openErr.Error(), time.Now().UTC())
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore",
"维护协调器不可用,接管目标的状态未确认:"+openErr.Error(), time.Now().UTC())
} else {
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "verified",
_ = releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID,
"未进入维护窗口(协调器不可用),目标未被停用", time.Now().UTC())
_ = store.ReleaseLibraryMaintenance(ctx, payload.Scope, payload.LibraryJobID, time.Now().UTC())
}
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "维护协调器不可用:"+openErr.Error(), "prebackup")
}
@@ -348,8 +370,13 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
return manager.VerifyRunning(ctx)
}
windowWork := func(ctx context.Context, w *libraryfreeze.Window) (string, error) {
if err := w.MarkContentRisk(ctx); err != nil {
return "", err
// 预备份身份与"内容有风险"必须在第一笔写入之前一起落盘:没有预备份身份的
// 窗口之后既不能自动恢复,也不许被任何收尾路径开放。
recorder.onBackup = func(backupCtx context.Context, ref libraryrelease.BackupRef) error {
if err := w.SetBackupSnapshot(backupCtx, ref.SnapshotID); err != nil {
return err
}
return w.MarkContentRisk(backupCtx)
}
if ingressErr := store.SetLibraryMaintenanceIngress(ctx, payload.Scope, payload.LibraryJobID, "closed",
"目标入口已关闭、写入者已停止、在途请求已验证排空", time.Now().UTC()); ingressErr != nil {
@@ -369,39 +396,46 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
note += ":" + windowErr.Error()
}
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", note, now2)
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", note, now2)
if windowErr == nil {
return errors.New(note)
}
return windowErr
case contentState == contentSucceeded:
_ = store.SetLibraryMaintenanceIngress(ctx, payload.Scope, payload.LibraryJobID, "open", "目标入口已恢复并验证", now2)
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "verified",
"写入与校验完成,目标已按原运行状态恢复", now2)
if err := store.ReleaseLibraryMaintenance(ctx, payload.Scope, payload.LibraryJobID, now2); err != nil {
if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID,
"写入与校验完成,目标已按原运行状态恢复", now2); err != nil {
return fmt.Errorf("目标已恢复但维护占用释放失败: %w", err)
}
return windowErr
case contentState == contentRolledBack:
_ = store.SetLibraryMaintenanceIngress(ctx, payload.Scope, payload.LibraryJobID, "open", "已回滚并验证,目标入口恢复", now2)
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "verified",
"本次失败已回滚到预备份并验证,目标已恢复原运行状态", now2)
if err := store.ReleaseLibraryMaintenance(ctx, payload.Scope, payload.LibraryJobID, now2); err != nil {
if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID,
"本次失败已回滚到预备份并验证,目标已恢复原运行状态", now2); err != nil {
return fmt.Errorf("回滚已验证但维护占用释放失败: %w", err)
}
return windowErr
default:
// 没有写过目标(闸口/授权/接线/健康检查早退):目标没被停过,直接开放占用。
_ = store.SetLibraryMaintenanceIngress(ctx, payload.Scope, payload.LibraryJobID, "open",
"未进入维护窗口,目标未被停用", now2)
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "verified",
"未写目标,维护窗口未开启", now2)
if err := store.ReleaseLibraryMaintenance(ctx, payload.Scope, payload.LibraryJobID, now2); err != nil {
if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID,
"未写目标,维护窗口未开启", now2); err != nil {
return fmt.Errorf("维护占用释放失败: %w", err)
}
return windowErr
}
}
// releaseWindowLocks 是唯一释放路径:先记录入口已恢复,再放 scope 行与实例锁。
// 顺序反过来会出现"实例锁放了、scope 还挂着"的中间态,页面就会说正在维护。
func releaseWindowLocks(ctx context.Context, store *postgresstore.Store, instance, scope, jobID, note string, now time.Time) error {
_ = store.SetInstanceMaintenanceIngress(ctx, instance, jobID, "open", note, now)
_ = store.SetLibraryMaintenanceIngress(ctx, scope, jobID, "open", note, now)
_ = store.UpdateInstanceMaintenanceState(ctx, instance, jobID, "verified", note, now)
_ = store.UpdateLibraryMaintenanceState(ctx, scope, jobID, "verified", note, now)
if err := store.ReleaseInstanceMaintenance(ctx, instance, jobID, now); err != nil {
return err
}
return store.ReleaseLibraryMaintenance(ctx, scope, jobID, now)
}
// resumeInterruptedLibraryJobs 在 worker 启动时接管中断的写作业:
// 已经持久化预备份的重新排队(同一个业务作业 id,阶段与备份都接着用),
// 连预备份都没完成的才判失败——那种情况没有可回滚的基线,不能盲写。
@@ -485,6 +519,11 @@ func (s fixedBackupStore) Prepare(context.Context, string, libraryrelease.Curren
// 上游据此在写目标前中止,保证"没有可恢复基线就不写"。
func (r libraryStageRecorder) BackupPrepared(ctx context.Context, backup libraryrelease.BackupRef) error {
now := time.Now().UTC()
if r.onBackup != nil {
if err := r.onBackup(ctx, backup); err != nil {
return fmt.Errorf("落盘预备份身份/内容风险失败: %w", err)
}
}
if err := r.store.AdvanceLibraryJob(ctx, r.jobID, governance.LibraryJobRunning, "", "", "",
backup.SnapshotID, "", now); err != nil {
return fmt.Errorf("持久化预备份引用失败: %w", err)
+163
View File
@@ -0,0 +1,163 @@
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"log"
"time"
"knowledge-governance/internal/appconfig"
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
"knowledge-governance/internal/libraryrelease"
postgresstore "knowledge-governance/internal/platform/postgres"
v071 "knowledge-governance/internal/releaseadapter/v071"
)
// libraryRecoveryPayload 是"按预备份恢复并校验"的队列载荷。
// 它只说明动哪个窗口;恢复目标快照由 journal 里持久化的那份决定,调用方改不了。
type libraryRecoveryPayload struct {
Scope string `json:"scope"`
WindowJobID string `json:"window_job_id"`
}
// runLibraryRecovery 是"内容风险未验证"窗口的唯一收尾入口:
// 在维护关闭状态下按 journal 里的预备份恢复,再逐类校验(行身份/逐行内容/索引/资源),
// 只有校验通过才恢复服务、开入口、释放占用。失败则保持维护。
func runLibraryRecovery(ctx context.Context, store *postgresstore.Store, config appconfig.Config, job jobs.Job) error {
var payload libraryRecoveryPayload
if err := json.Unmarshal(job.Payload, &payload); err != nil ||
(payload.Scope != "internal" && payload.Scope != "external") || payload.WindowJobID == "" {
return errors.New("invalid library recovery payload")
}
manager, err := openLibraryMaintenance(config)
if err != nil {
return fmt.Errorf("目标维护未接线,不能做恢复:%w", err)
}
defer manager.Close()
identity := manager.Target()
if topoErr := manager.VerifyTopology(ctx); topoErr != nil {
return fmt.Errorf("目标拓扑不满足唯一入口约束,拒绝恢复:%w", topoErr)
}
lock, err := store.InstanceMaintenanceOf(ctx, identity.Instance)
if err != nil {
return fmt.Errorf("读不到实例维护记录:%w", err)
}
if lock == nil || lock.ReleasedAt != nil {
return errors.New("目标实例当前没有未释放的维护窗口,不需要恢复")
}
if lock.OwnerJobID != payload.WindowJobID || lock.OwnerScope != payload.Scope {
return fmt.Errorf("维护窗口属于作业 %s(scope=%s),与请求不一致", lock.OwnerJobID, lock.OwnerScope)
}
if lock.TargetFingerprint != identity.Fingerprint {
return errors.New("维护窗口的目标身份与当前配置不一致:拒绝用旧窗口恢复另一个目标")
}
// 预备份身份来自治理库里的原作业记录;协调器还会再与 journal 里的那份核对。
windowJob, err := store.LibraryJob(ctx, payload.WindowJobID)
if err != nil || windowJob.SnapshotID == "" {
return errors.New("原作业没有持久化预备份,无法证明恢复到哪一份")
}
prebackup := windowJob.SnapshotID
knownMapping := knownIndexMapping(ctx, store, payload.Scope)
target, _, wireErr := buildLibraryProductionStack(ctx, config, payload.Scope, prebackup, knownMapping)
if wireErr != nil {
return fmt.Errorf("恢复目标装配失败:%w", wireErr)
}
defer target.Close()
coordinator, control, openErr := manager.Open(config.LibraryTargetJournalRoot)
if openErr != nil {
return fmt.Errorf("维护协调器不可用:%w", openErr)
}
defer control.Close()
defer coordinator.Close()
restorer := recoveryRestorer{target: target, expected: prebackup}
verifier := recoveryVerifier{target: target, expected: prebackup}
result, recoverErr := coordinator.RecoverWindow(ctx, payload.WindowJobID, restorer, verifier)
now := time.Now().UTC()
if recoverErr != nil {
note := "恢复/校验未通过,目标保持维护:" + recoverErr.Error()
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now)
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.WindowJobID, "needs_restore", note, now)
return recoverErr
}
if result.State != "recovered" {
note := "恢复流程结束但状态不是 recovered:" + result.State
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now)
return errors.New(note)
}
if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.WindowJobID,
"按预备份恢复并逐类校验通过,目标已按原运行状态恢复", now); err != nil {
return fmt.Errorf("恢复已验证,但维护占用释放失败:%w", err)
}
log.Printf("library recovery succeeded window=%s scope=%s backup=%s", payload.WindowJobID, payload.Scope, prebackup)
return nil
}
// recoveryRestorer 只按"作业记录里那份预备份"恢复:协调器传进来的 id 与它不一致
// 就拒绝——那是两份快照,恢复出来的东西不能算数。
type recoveryRestorer struct {
target *v071.ProductionLibraryTarget
expected string
}
func (r recoveryRestorer) Restore(ctx context.Context, backupID string) error {
if backupID != r.expected {
return fmt.Errorf("预备份身份不一致(journal=%s 作业=%s)", backupID, r.expected)
}
return r.target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + backupID, SnapshotID: backupID})
}
// recoveryVerifier 做真实的内容校验,并把目标当前状态摘要作为"校验过什么"的凭证。
type recoveryVerifier struct {
target *v071.ProductionLibraryTarget
expected string
}
func (v recoveryVerifier) Verify(ctx context.Context, backupID string) (string, error) {
if backupID != v.expected {
return "", fmt.Errorf("预备份身份不一致(journal=%s 作业=%s)", backupID, v.expected)
}
// Verify 逐表比行数、身份集合、逐行内容摘要,再比索引文档与对象资源。
if err := v.target.Verify(ctx, libraryrelease.Expectation{SnapshotID: backupID}); err != nil {
return "", err
}
state, err := v.target.Current(ctx)
if err != nil {
return "", fmt.Errorf("恢复后读不到目标状态:%w", err)
}
raw, err := json.Marshal(state)
if err != nil {
return "", err
}
sum := sha256.Sum256(raw)
return hex.EncodeToString(sum[:]), nil
}
// knownIndexMapping 取"上一次登记过的索引映射",作为装配目标时的映射基线。
func knownIndexMapping(ctx context.Context, store *postgresstore.Store, scope string) string {
versions, err := store.LibraryVersions(ctx, scope, 50)
if err != nil {
return ""
}
for _, item := range versions {
if item.Kind == governance.LibraryVersionProductionEffective && item.SupersededAt == nil {
if raw, ok := item.Evidence["index_mapping_sha256"].(string); ok && raw != "" {
return raw
}
}
}
for _, item := range versions {
if raw, ok := item.Evidence["index_mapping_sha256"].(string); ok && raw != "" {
return raw
}
}
return ""
}
+9
View File
@@ -18,6 +18,7 @@ import (
"knowledge-governance/internal/governance"
"knowledge-governance/internal/jobs"
"knowledge-governance/internal/libraryagent"
"knowledge-governance/internal/libraryrelease"
"knowledge-governance/internal/librarysnapshot"
postgresstore "knowledge-governance/internal/platform/postgres"
"knowledge-governance/internal/platform/weknora"
@@ -36,6 +37,8 @@ type libraryStageRecorder struct {
totals map[string]stageSize
// mappingDigest 是本次读到的生产索引映射,随预备份一起登记成版本证据。
mappingDigest string
// onBackup 在预备份完成后、写目标之前调用:把"预备份身份 + 内容风险"落盘。
onBackup func(context.Context, libraryrelease.BackupRef) error
}
func (r libraryStageRecorder) Stage(ctx context.Context, stage, state, detail string) {
@@ -100,6 +103,9 @@ const libraryCaptureJobType = "library.capture"
// librarySyncJobType 是生产同步作业的队列类型;与封存严格分开,状态不共用。
const librarySyncJobType = "library.sync"
// libraryRecoverJobType 是"按预备份恢复并校验"的队列类型;与 API 侧常量同值。
const libraryRecoverJobType = "library.recover"
// libraryRestoreJobType 是按指定整库版本恢复的队列类型;与同步分开、授权也分开。
const libraryRestoreJobType = "library.restore"
@@ -177,6 +183,9 @@ func run(ctx context.Context, getenv func(string) string) error {
handlers[libraryRestoreJobType] = func(ctx context.Context, job jobs.Job) error {
return runLibraryRelease(ctx, store, config, job, governance.LibraryJobRestore)
}
handlers[libraryRecoverJobType] = func(ctx context.Context, job jobs.Job) error {
return runLibraryRecovery(ctx, store, config, job)
}
worker, err := jobs.NewWorker(jobs.WorkerConfig{
Queue: store,
+11 -1
View File
@@ -105,7 +105,11 @@ type Config struct {
// 入口闸门:关入口与排空在它身上做(被代理服务自己不排空)。
LibraryTargetIngressService string
LibraryTargetIngressDrainFile string
LibraryTargetJournalRoot string
// 后台静默检查:目标队列 + 目标库(HTTP 排空不等于后台任务排空)。
LibraryTargetRedisAddr string
LibraryTargetRedisPassword string
LibraryTargetRedisDB int
LibraryTargetJournalRoot string
// LibraryAgentSocket 是主机侧封存 agent 的 unix socket(只由 worker 连接)。
// 留空即"未接线":页面不显示可用的封存入口,接口返回原因而不是假装已排队。
LibraryAgentSocket string
@@ -202,6 +206,9 @@ func Load(getenv func(string) string) (Config, error) {
LibraryTargetIngressURL: strings.TrimSpace(getenv("LIBRARY_TARGET_INGRESS_URL")),
LibraryTargetIngressService: strings.TrimSpace(getenv("LIBRARY_TARGET_INGRESS_SERVICE")),
LibraryTargetIngressDrainFile: strings.TrimSpace(getenv("LIBRARY_TARGET_INGRESS_DRAIN_FILE")),
LibraryTargetRedisAddr: strings.TrimSpace(getenv("LIBRARY_TARGET_REDIS_ADDR")),
LibraryTargetRedisPassword: strings.TrimSpace(getenv("LIBRARY_TARGET_REDIS_PASSWORD")),
LibraryTargetRedisDB: valueOrDefaultInt(getenv("LIBRARY_TARGET_REDIS_DB"), 0),
LibraryTargetJournalRoot: strings.TrimSpace(getenv("LIBRARY_TARGET_JOURNAL_ROOT")),
LibraryEmbeddingModel: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_EMBEDDING_MODEL")), "text-embedding-v4"),
LibraryEmbeddingDimension: embeddingDimension,
@@ -349,6 +356,9 @@ func Load(getenv func(string) string) (Config, error) {
if config.LibraryTargetIngressDrainFile == "" {
missing = append(missing, "LIBRARY_TARGET_INGRESS_DRAIN_FILE")
}
if config.LibraryTargetRedisAddr == "" {
missing = append(missing, "LIBRARY_TARGET_REDIS_ADDR")
}
if len(missing) > 0 {
return Config{}, fmt.Errorf("目标维护配置不完整:缺少 %s", strings.Join(missing, "、"))
}
+103
View File
@@ -42,6 +42,9 @@ type libraryVersionView struct {
// librarySyncJobType 与 worker 的处理器类型一致;单独定义避免 API 依赖 worker 包。
const librarySyncJobType = "library.sync"
// libraryRecoverJobType 是"按预备份恢复并校验"的队列类型(同一份定义也在 worker 里)。
const libraryRecoverJobType = "library.recover"
type libraryKnownView struct {
State string `json:"state"` // known | unknown
Version *libraryVersionView `json:"version,omitempty"`
@@ -291,6 +294,101 @@ func libraryStore(config Config) (libraryJobStore, bool) {
return store, ok
}
// instanceMaintenanceGuard 在"实例级维护"被别的作业持有时拒绝新的写入请求。
// 只看 scope 行是不够的:internal 与 external 动的是同一批容器。
func instanceMaintenanceGuard(ctx context.Context, config Config, scope string) (map[string]any, bool) {
instance := config.LibraryTargetInstance
if instance == "" {
return nil, false
}
store, ok := config.Store.(interface {
InstanceMaintenanceOf(ctx context.Context, instance string) (*postgresstore.LibraryInstanceMaintenance, error)
})
if !ok {
return nil, false
}
held, err := store.InstanceMaintenanceOf(ctx, instance)
if err != nil || held == nil || held.ReleasedAt != nil {
return nil, false
}
return map[string]any{"code": "library_target_in_maintenance",
"error": "目标实例正在维护中,验证完成前不接受新的写入请求",
"instance": held.Instance, "owner_job": held.OwnerJobID, "owner_scope": held.OwnerScope,
"state": held.State, "ingress_state": held.IngressState}, true
}
// libraryMaintenanceRecoverHandler 是"按预备份恢复并校验"的唯一入口:
// 它只建一个恢复作业,真正的恢复与校验由 worker 在维护关闭状态下执行。
func libraryMaintenanceRecoverHandler(w http.ResponseWriter, r *http.Request, config Config) {
if r.Method != http.MethodPost {
writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"})
return
}
var request struct {
Scope string `json:"scope"`
WindowJobID string `json:"window_job_id"`
}
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&request); err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"})
return
}
if request.Scope != "internal" && request.Scope != "external" {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "scope must be internal or external"})
return
}
instance := config.LibraryTargetInstance
if instance == "" {
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"code": "library_recovery_unavailable",
"error": "目标维护未接线:无法执行按预备份恢复"})
return
}
store, ok := config.Store.(interface {
InstanceMaintenanceOf(ctx context.Context, instance string) (*postgresstore.LibraryInstanceMaintenance, error)
})
if !ok {
writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "维护记录存储不可用"})
return
}
held, err := store.InstanceMaintenanceOf(r.Context(), instance)
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "无法读取维护记录"})
return
}
if held == nil || held.ReleasedAt != nil {
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_no_held_window",
"error": "目标实例当前没有未释放的维护窗口,不需要恢复"})
return
}
if held.OwnerScope != request.Scope {
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_window_scope_mismatch",
"error": "维护窗口属于另一个 scope", "owner_scope": held.OwnerScope})
return
}
if request.WindowJobID != "" && request.WindowJobID != held.OwnerJobID {
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_window_mismatch",
"error": "维护窗口属于另一个作业", "owner_job": held.OwnerJobID})
return
}
now := time.Now().UTC()
payload, err := json.Marshal(map[string]string{"scope": held.OwnerScope, "window_job_id": held.OwnerJobID})
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot build recovery payload"})
return
}
// 队列作业 id 与幂等键都按窗口固定:重复点击只会得到同一个恢复作业。
queueJob, err := config.Store.Enqueue(r.Context(), jobs.EnqueueRequest{
ID: "librecover-" + held.OwnerJobID, Type: libraryRecoverJobType, Payload: payload,
AvailableAt: now, MaxAttempts: 1, IdempotencyKey: "library.recover:" + held.OwnerJobID,
}, now)
if err != nil || queueJob == nil {
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_recovery_enqueue_failed",
"error": "恢复作业投递失败(或已有同窗口的恢复作业)", "detail": errString(err)})
return
}
writeJSON(w, http.StatusAccepted, map[string]any{"job": queueJob, "window_job_id": held.OwnerJobID,
"scope": held.OwnerScope, "instance": held.Instance})
}
// activeGrantFor 只查"目标+快照+同一参数摘要"的未过期未撤销授权。
func activeGrantFor(ctx context.Context, config Config, scope, snapshotID string) (governance.LibraryReleaseAuthorization, bool) {
if snapshotID == "" {
@@ -597,6 +695,11 @@ func librarySyncEnqueue(w http.ResponseWriter, r *http.Request, config Config) {
}
grantFound, hasGrant := activeGrantFor(r.Context(), config, request.Scope, request.Target)
_ = grantFound
// 目标实例正在维护(跨对内/对外互斥):直接拒绝,不等到 worker 里才发现。
if body, held := instanceMaintenanceGuard(r.Context(), config, request.Scope); held {
writeJSON(w, http.StatusConflict, body)
return
}
// 目标正在维护(被别的作业持有):直接拒绝新写入,不等到 worker 里才发现。
if maintenanceStore, ok := config.Store.(interface {
LibraryMaintenanceOf(ctx context.Context, scope string) (*postgresstore.LibraryMaintenance, error)
+4
View File
@@ -54,6 +54,10 @@ func libraryRestoreEnqueue(w http.ResponseWriter, r *http.Request, config Config
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "snapshot_id required"})
return
}
if body, held := instanceMaintenanceGuard(r.Context(), config, request.Scope); held {
writeJSON(w, http.StatusConflict, body)
return
}
digest, err := libraryRestoreParamsDigest(request.Scope, request.SnapshotID)
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot digest params"})
+19 -1
View File
@@ -145,6 +145,9 @@ type Config struct {
VersionHistoryRepo string
// LibraryArchiveRoot 是整库封存产物的只读归档根;留空即未接线(页面降级)。
LibraryArchiveRoot string
// LibraryTargetInstance 是实例级维护的目标实例名;留空即"未接线",
// 页面不能发起按预备份恢复,接口返回原因而不是假装能恢复。
LibraryTargetInstance string
// LibraryAgentSocket 是主机侧封存 agent 的 unix socket;留空即页面不能发起封存。
LibraryAgentSocket string
// LibraryCapacityBudgetBytes 是整库归档容量预算;0 = 未设置(容量门禁不放行)。
@@ -461,7 +464,22 @@ func NewHandler(config Config) (http.Handler, error) {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "无法读取维护窗口"})
return
}
writeJSON(w, http.StatusOK, map[string]any{"held": held != nil && held.ReleasedAt == nil, "maintenance": held})
response := map[string]any{"held": held != nil && held.ReleasedAt == nil, "maintenance": held}
// 实例级占用才是真正的互斥来源:跨 scope 也在这里体现。
if instanceStore, ok := config.Store.(interface {
InstanceMaintenanceOf(ctx context.Context, instance string) (*postgresstore.LibraryInstanceMaintenance, error)
}); ok && config.LibraryTargetInstance != "" {
if instanceHeld, err := instanceStore.InstanceMaintenanceOf(r.Context(), config.LibraryTargetInstance); err == nil {
response["instance"] = config.LibraryTargetInstance
response["instance_maintenance"] = instanceHeld
response["held"] = instanceHeld != nil && instanceHeld.ReleasedAt == nil
}
}
writeJSON(w, http.StatusOK, response)
})
// 按预备份恢复并逐类校验:维护窗口卡住时唯一的收尾入口(worker 执行)。
mux.HandleFunc("POST /api/v1/library/maintenance/recover", func(w http.ResponseWriter, r *http.Request) {
libraryMaintenanceRecoverHandler(w, r, config)
})
mux.HandleFunc("GET /api/v1/library/jobs", func(w http.ResponseWriter, r *http.Request) {
libraryJobsHandler(w, r, config)
+71 -8
View File
@@ -47,6 +47,15 @@ type Gate interface {
OpenIngress(context.Context) error
}
// Quiescer 回答"后台写入者是不是已经空了"。
//
// 为什么 HTTP 排空不够:被代理的 WeKnora 在 SIGTERM 后会 exit(1),app 内的 asynq
// worker 正在跑的后台任务会被直接切断。要么等它们跑完,要么明确不进入窗口。
type Quiescer interface {
// Idle 返回 true 表示此刻没有排队/在跑的后台任务,也没有未结清的处理记录。
Idle(ctx context.Context) (bool, string, error)
}
// Inventory must enumerate the complete, independently scoped test writer set,
// using immutable instance IDs. Never implement it as only the currently running
// subset. Start must wait for readiness; callers still read back the result.
@@ -81,12 +90,23 @@ type journal struct {
// Gate 记录入口关闭/排空的证据;没有它就不能说"入口关了"。
// 入口恢复后证据仍保留(GateReopened=true),否则事后查不到
// "关的时候还有几个在途请求"。
Gate *GateEvidence `json:"gate,omitempty"`
GateReopened bool `json:"gate_reopened,omitempty"`
Services []serviceRecord `json:"services"`
StartedAt time.Time `json:"started_at"`
FinishedAt *time.Time `json:"finished_at,omitempty"`
Code string `json:"code,omitempty"`
Gate *GateEvidence `json:"gate,omitempty"`
GateReopened bool `json:"gate_reopened,omitempty"`
// BackupSnapshotID 是这次窗口开始写之前落盘的那份预备份:恢复只能按它来,
// 不允许事后换一份"看起来更好"的快照。
BackupSnapshotID string `json:"backup_snapshot_id,omitempty"`
// TargetFingerprint 是开窗口时的目标身份:恢复时必须还是同一个目标。
TargetFingerprint string `json:"target_fingerprint,omitempty"`
// BackgroundIdle 记录"停写入者之前后台任务队列确认为空"。
BackgroundIdle bool `json:"background_idle,omitempty"`
// 恢复证据:只有真正跑完"恢复 + 校验"才会写这三项。
RecoveredBackup string `json:"recovered_backup,omitempty"`
RecoveryDigest string `json:"recovery_digest,omitempty"`
RecoveredAt *time.Time `json:"recovered_at,omitempty"`
Services []serviceRecord `json:"services"`
StartedAt time.Time `json:"started_at"`
FinishedAt *time.Time `json:"finished_at,omitempty"`
Code string `json:"code,omitempty"`
}
type Coordinator struct {
root *os.Root
@@ -99,6 +119,9 @@ type Coordinator struct {
ingressClosed func(context.Context) error
ingressReady func(context.Context) error
gate Gate
quiescer Quiescer
fingerprint string
quiesceWait time.Duration
}
func Open(root, source string, c Controller) (*Coordinator, error) {
@@ -149,6 +172,37 @@ func (c *Coordinator) SetGate(g Gate) error {
return nil
}
// SetTargetFingerprint 绑住"这次维护动的是哪个目标"。恢复时必须与 journal
// 里记录的一致,否则说明配置换了目标,禁止拿旧窗口去收尾新目标。
func (c *Coordinator) SetTargetFingerprint(fingerprint string) error {
if len(fingerprint) != 64 {
return errors.New("invalid target fingerprint")
}
if _, err := hex.DecodeString(fingerprint); err != nil {
return errors.New("invalid target fingerprint")
}
c.fingerprint = strings.ToLower(fingerprint)
return nil
}
// SetQuiesceTimeout 调整"等后台任务静默"的上限(默认 2 分钟)。
func (c *Coordinator) SetQuiesceTimeout(d time.Duration) error {
if d <= 0 {
return errors.New("invalid quiesce timeout")
}
c.quiesceWait = d
return nil
}
// SetQuiescer 声明后台任务静默检查(HTTP 排空不等于后台任务排空)。
func (c *Coordinator) SetQuiescer(q Quiescer) error {
if q == nil {
return errors.New("invalid quiescer")
}
c.quiescer = q
return nil
}
func validIngress(raw string) bool {
parsed, err := url.Parse(raw)
return err == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") && parsed.Host != "" && parsed.Path != "" && parsed.User == nil
@@ -252,7 +306,7 @@ func validID(s string) bool {
}
return true
}
func terminal(s string) bool { return s == "succeeded" || s == "failed" }
func terminal(s string) bool { return s == "succeeded" || s == "failed" || s == "recovered" }
// A kernel lock serializes the whole test writer group, not individual KBs.
// A crash releases the lock, but leaves the write-ahead journal for recovery.
@@ -328,7 +382,7 @@ func (c *Coordinator) load(job string) (journal, error) {
return journal{}, errors.New("freeze journal identity mismatch")
}
switch j.State {
case "stopping", "held", "capturing", "restoring", "succeeded", "failed":
case "stopping", "held", "capturing", "recovering", "restoring", "succeeded", "failed", "recovered":
default:
return journal{}, errors.New("invalid freeze journal state")
}
@@ -351,6 +405,15 @@ func (c *Coordinator) load(job string) (journal, error) {
if j.GateReopened && j.Gate == nil {
return journal{}, errors.New("ingress reopened without close evidence")
}
if j.BackupSnapshotID != "" && !validDigest(j.BackupSnapshotID) {
return journal{}, errors.New("invalid backup snapshot id")
}
// 恢复证据只允许指向"这次窗口自己那份预备份",且必须带校验摘要。
if j.RecoveredAt != nil {
if j.RecoveredBackup == "" || j.RecoveredBackup != j.BackupSnapshotID || j.RecoveryDigest == "" {
return journal{}, errors.New("invalid recovery evidence")
}
}
if len(j.Services) == 0 || len(j.Services) > 256 {
return journal{}, errors.New("invalid freeze journal services")
}
@@ -0,0 +1,175 @@
package libraryfreeze
import (
"context"
"errors"
"strings"
"testing"
"time"
)
func heldJournal(job string, fingerprint, backup string) journal {
j := journal{Version: 1, Source: "target-instance", Job: job, Level: "instance",
Ingress: "http://127.0.0.1:1/health", TargetFingerprint: fingerprint,
ContentRisk: true, RestoreBlocked: true, BackupSnapshotID: backup,
Result: Result{State: "held"}, StartedAt: time.Now().UTC(),
Gate: &GateEvidence{Closed: true, Drained: true}}
j.Services = []serviceRecord{
{Service: Service{ID: "app", Running: true, Healthy: true}, StopRequested: true, StopVerified: true},
{Service: Service{ID: "store", Running: true, Healthy: true}, StopRequested: true, StopVerified: true},
}
return j
}
func stopAllServices(c *exitControl) {
for i := range c.services {
c.services[i].Running, c.services[i].Healthy = false, false
}
}
// 收尾只认"真的跑过恢复 + 校验",不认调用方自称已恢复。
func TestRecoverWindowRestoresBeforeOpening(t *testing.T) {
_, c, e := instanceFixture(t)
stopAllServices(c)
if err := e.save(heldJournal("job-rec", strings.Repeat("a", 64), strings.Repeat("b", 64))); err != nil {
t.Fatal(err)
}
if err := e.SetTargetFingerprint(strings.Repeat("a", 64)); err != nil {
t.Fatal(err)
}
gate := &fakeGate{calls: &c.calls, drain: true}
if err := e.SetGate(gate); err != nil {
t.Fatal(err)
}
var restoredWith, verifiedWith string
var order []string
restorer := restorerFunc(func(_ context.Context, id string) error {
restoredWith = id
order = append(order, "restore")
return nil
})
verifier := verifierFunc(func(_ context.Context, id string) (string, error) {
verifiedWith = id
order = append(order, "verify")
return "digest-ok", nil
})
if _, err := e.RecoverWindow(context.Background(), "job-rec", restorer, verifier); err != nil {
t.Fatal(err)
}
if restoredWith != strings.Repeat("b", 64) || verifiedWith != restoredWith {
t.Fatalf("restore=%q verify=%q", restoredWith, verifiedWith)
}
if strings.Join(order, ",") != "restore,verify" {
t.Fatalf("order=%v", order)
}
for _, s := range c.services {
if !s.Running || !s.Healthy {
t.Fatalf("service %s not restored after verified recovery", s.ID)
}
}
after, err := e.load("job-rec")
if err != nil || after.State != "recovered" || after.RecoveredBackup != restoredWith ||
after.RecoveryDigest != "digest-ok" || after.RecoveredAt == nil {
t.Fatalf("journal=%+v err=%v", after, err)
}
}
// 校验不通过 = 恢复未成功:服务保持停止、journal 非终态。
func TestRecoverWindowKeepsTargetClosedWhenVerificationFails(t *testing.T) {
_, c, e := instanceFixture(t)
stopAllServices(c)
if err := e.SetTargetFingerprint(strings.Repeat("a", 64)); err != nil {
t.Fatal(err)
}
if err := e.SetGate(&fakeGate{calls: &c.calls, drain: true}); err != nil {
t.Fatal(err)
}
if err := e.save(heldJournal("job-bad", strings.Repeat("a", 64), strings.Repeat("b", 64))); err != nil {
t.Fatal(err)
}
if _, err := e.RecoverWindow(context.Background(), "job-bad",
restorerFunc(func(context.Context, string) error { return nil }),
verifierFunc(func(context.Context, string) (string, error) {
return "", errors.New("chunks 内容与预备份不一致")
})); err == nil {
t.Fatal("verification failure reported success")
}
for _, s := range c.services {
if s.Running {
t.Fatalf("service %s was started although content verification failed", s.ID)
}
}
after, err := e.load("job-bad")
if err != nil || terminal(after.State) || !after.RestoreBlocked || after.RecoveredAt != nil {
t.Fatalf("journal=%+v err=%v", after, err)
}
}
// 没有落盘预备份身份的窗口不许"恢复":那等于凭空开放。
func TestRecoverWindowRequiresPersistedBackup(t *testing.T) {
_, c, e := instanceFixture(t)
stopAllServices(c)
j := heldJournal("job-nobak", strings.Repeat("a", 64), "")
if err := e.save(j); err != nil {
t.Fatal(err)
}
called := false
if _, err := e.RecoverWindow(context.Background(), "job-nobak",
restorerFunc(func(context.Context, string) error { called = true; return nil }),
verifierFunc(func(context.Context, string) (string, error) { return "x", nil })); err == nil || called {
t.Fatalf("recovery ran without a persisted backup: err=%v called=%v", err, called)
}
for _, s := range c.services {
if s.Running {
t.Fatal("writer reopened without a backup")
}
}
}
// 目标换了(指纹不一致)就不许拿旧窗口收尾。
func TestRecoverWindowRejectsDifferentTarget(t *testing.T) {
_, c, e := instanceFixture(t)
stopAllServices(c)
if err := e.SetTargetFingerprint(strings.Repeat("c", 64)); err != nil {
t.Fatal(err)
}
if err := e.save(heldJournal("job-other", strings.Repeat("a", 64), strings.Repeat("b", 64))); err != nil {
t.Fatal(err)
}
if _, err := e.RecoverWindow(context.Background(), "job-other",
restorerFunc(func(context.Context, string) error { return nil }),
verifierFunc(func(context.Context, string) (string, error) { return "x", nil })); err == nil {
t.Fatal("recovery accepted another target's journal")
}
}
// 后台任务还在跑时不许开窗口(HTTP 排空覆盖不到它们)。
func TestWindowRefusesWhenBackgroundBusy(t *testing.T) {
_, c, e := instanceFixture(t)
if err := e.SetQuiescer(&idleQuiescer{busy: true}); err != nil {
t.Fatal(err)
}
if err := e.SetQuiesceTimeout(100 * time.Millisecond); err != nil {
t.Fatal(err)
}
work := 0
_, err := e.RunWindow(context.Background(), "job-busy", func(context.Context) error { return nil },
func(context.Context, *Window) (string, error) { work++; return "", nil }, func(error) bool { return true })
if err == nil || work != 0 {
t.Fatalf("err=%v work=%d", err, work)
}
for _, s := range c.services {
if !s.Running {
t.Fatal("writers were stopped although the window never opened")
}
}
}
type restorerFunc func(context.Context, string) error
func (f restorerFunc) Restore(ctx context.Context, id string) error { return f(ctx, id) }
type verifierFunc func(context.Context, string) (string, error)
func (f verifierFunc) Verify(ctx context.Context, id string) (string, error) { return f(ctx, id) }
+141
View File
@@ -0,0 +1,141 @@
package libraryfreeze
import (
"context"
"errors"
"fmt"
"time"
)
// Restorer 把目标内容恢复成 backupID 描述的那一份。
// 协调器只会在"维护关闭、写入者停止"的状态下调用它,并且只传 journal 里
// 持久化的那份预备份身份——调用方没有机会换一份。
type Restorer interface {
Restore(ctx context.Context, backupID string) error
}
// Verifier 逐类比对目标与 backupID 的内容(行身份 + 逐行内容 + 索引 + 资源)。
// 返回的摘要会被持久化;返回错误即视为恢复未成功。
type Verifier interface {
Verify(ctx context.Context, backupID string) (string, error)
}
// RecoverWindow 是"内容风险未验证"窗口的唯一收尾方式:
//
// 绑定(原作业 + 目标指纹 + 预备份)→ 重新关入口/停写入者 → 恢复 → 真校验
// → 持久记录校验摘要 → 恢复服务与入口 → 才允许调用方释放占用
//
// 它不接受任何"已恢复/已验证"的入参:是否成功由这里跑出来的结果决定。
// 任何一步失败都保持关闭(服务停着、入口关着、journal 非终态),不会放行半写目标。
func (c *Coordinator) RecoverWindow(ctx context.Context, job string, restorer Restorer, verifier Verifier) (Result, error) {
if !validID(job) || restorer == nil || verifier == nil {
return Result{}, errors.New("invalid recovery request")
}
lock, err := c.lock()
if err != nil {
return Result{}, err
}
defer unlock(lock)
j, err := c.load(job)
if err != nil {
return Result{}, err
}
if terminal(j.State) {
return Result{}, errors.New("freeze journal is already terminal")
}
if !j.ContentRisk {
return Result{}, errors.New("journal has no content risk; use Recover instead")
}
// 绑定:预备份身份必须已经在写目标之前落盘,目标指纹必须还是同一个。
if !validDigest(j.BackupSnapshotID) {
return Result{}, errors.New("窗口没有落盘预备份身份,无法证明恢复目标,必须人工按归档恢复")
}
if c.fingerprint != "" && j.Level == "instance" {
if j.TargetFingerprint == "" || j.TargetFingerprint != c.fingerprint {
return Result{}, errors.New("目标身份与维护记录不一致:拒绝用旧窗口收尾另一个目标")
}
}
// 恢复必须在维护关闭状态下做:入口先关(若已关则沿用),写入者必须停住。
if c.gate != nil && (j.Gate == nil || !j.Gate.Drained) {
evidence, gateErr := c.gate.CloseIngress(ctx)
if gateErr != nil {
return j.Result, fmt.Errorf("恢复前无法关闭目标入口:%w", gateErr)
}
if !evidence.Closed || !evidence.Drained {
return j.Result, errors.New("恢复前入口未确认排空")
}
j.Gate = &evidence
if err := c.save(j); err != nil {
return j.Result, err
}
}
if err := c.stopAll(ctx, &j); err != nil {
return j.Result, fmt.Errorf("恢复前无法停下写入者:%w", err)
}
if err := c.stopped(ctx, j); err != nil {
return j.Result, fmt.Errorf("恢复前写入者没有全部停住:%w", err)
}
if err := c.verifyDrain(ctx, &j); err != nil {
return j.Result, fmt.Errorf("恢复前排空未确认:%w", err)
}
if err := c.awaitBackgroundIdle(ctx, &j); err != nil {
return j.Result, fmt.Errorf("恢复前后台任务未静默:%w", err)
}
j.State = "recovering"
if err := c.save(j); err != nil {
return j.Result, err
}
// 1) 按持久化的那份预备份恢复。
if restoreErr := restorer.Restore(ctx, j.BackupSnapshotID); restoreErr != nil {
return c.failRecovery(&j, "restore_failed", fmt.Errorf("恢复失败:%w", restoreErr))
}
// 2) 真校验(内容/资源/索引/Wiki),拿到摘要才算恢复成功。
digest, verifyErr := verifier.Verify(ctx, j.BackupSnapshotID)
if verifyErr != nil {
return c.failRecovery(&j, "verify_failed", fmt.Errorf("恢复后校验未通过:%w", verifyErr))
}
if digest == "" {
return c.failRecovery(&j, "verify_failed", errors.New("恢复后校验没有返回摘要,不能当作已验证"))
}
ended := time.Now().UTC()
j.RecoveredBackup = j.BackupSnapshotID
j.RecoveryDigest = digest
j.RecoveredAt = &ended
j.RestoreBlocked = false
if err := c.save(j); err != nil {
return j.Result, err
}
// 3) 验证成功之后才恢复服务、才开入口。
recoveryCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute)
defer cancel()
if restoreErr := c.restore(recoveryCtx, &j); restoreErr != nil {
return c.failRecovery(&j, "service_restore_failed", fmt.Errorf("内容已恢复但服务/入口未恢复:%w", restoreErr))
}
if c.ingressReady != nil {
if readyErr := c.ingressReady(recoveryCtx); readyErr != nil {
return c.failRecovery(&j, "ingress_unverified", fmt.Errorf("入口未恢复:%w", readyErr))
}
}
j.State = "recovered"
j.Code = ""
j.FinishedAt = &ended
if err := c.save(j); err != nil {
return j.Result, err
}
return j.Result, nil
}
// failRecovery 保持"目标仍不可用":服务停着、入口关着、journal 非终态。
func (c *Coordinator) failRecovery(j *journal, code string, cause error) (Result, error) {
j.State = "held"
j.Code = code
j.RestoreBlocked = true
if err := c.save(*j); err != nil {
return j.Result, errors.Join(cause, err)
}
return j.Result, cause
}
+62 -61
View File
@@ -34,6 +34,25 @@ func (w *Window) MarkContentRisk(ctx context.Context) error {
return w.c.save(*w.j)
}
// SetBackupSnapshot 把"这次窗口的预备份身份"落盘。恢复只认它:
// 没有这份身份的窗口,之后既不能自动恢复,也不许被外部收尾。
func (w *Window) SetBackupSnapshot(ctx context.Context, snapshotID string) error {
if w == nil || w.c == nil || w.j == nil {
return errors.New("invalid maintenance window")
}
if err := ctx.Err(); err != nil {
return err
}
if !validDigest(snapshotID) {
return errors.New("invalid backup snapshot id")
}
if w.j.BackupSnapshotID != "" && w.j.BackupSnapshotID != snapshotID {
return errors.New("backup snapshot changed inside the same window")
}
w.j.BackupSnapshotID = snapshotID
return w.c.save(*w.j)
}
// Job 返回窗口所属作业 id(恢复器与调用方据此对账)。
func (w *Window) Job() string {
if w == nil || w.j == nil {
@@ -113,7 +132,7 @@ func (c *Coordinator) RunWindow(ctx context.Context, job string, prepare func(co
return Result{}, err
}
j = journal{Version: 1, Source: c.source, Job: job, Level: c.level, Ingress: c.ingress,
Result: Result{State: "stopping"}, StartedAt: time.Now().UTC()}
TargetFingerprint: c.fingerprint, Result: Result{State: "stopping"}, StartedAt: time.Now().UTC()}
for _, s := range items {
if s.Running && !s.Healthy {
return Result{}, errors.New("test writer is not ready")
@@ -125,6 +144,12 @@ func (c *Coordinator) RunWindow(ctx context.Context, job string, prepare func(co
}
}
// 后台任务必须先在"还能跑"的时候跑完:入口排空只覆盖 HTTP 连接,
// app 内的 asynq worker 会被上游的 exit(1) 直接切断。
if idleErr := c.awaitBackgroundIdle(ctx, &j); idleErr != nil {
return c.abandonWindow(ctx, &j, idleErr, "background_busy")
}
// 顺序是有原因的:入口先停(新请求被拒、在途请求跑完),写者后停。
// 反过来做,正在处理的请求就会被 SIGTERM 切断——那不算排空。
if c.gate != nil && (j.Gate == nil || !j.Gate.Drained) {
@@ -267,6 +292,42 @@ func (c *Coordinator) closeWindow(ctx context.Context, j *journal, cause error,
return j.Result, cause
}
// awaitBackgroundIdle 等到后台任务队列为空再往下走。拿不到结论(查询失败)
// 也按"没空"处理:不能把"问不到"当成"没任务"。
func (c *Coordinator) awaitBackgroundIdle(ctx context.Context, j *journal) error {
if c.quiescer == nil {
if c.level == "instance" {
return errors.New("没有后台任务静默检查:不能证明停写者时后台任务已结束")
}
return nil
}
wait := c.quiesceWait
if wait <= 0 {
wait = 2 * time.Minute
}
deadline := time.Now().Add(wait)
var last string
for {
idle, detail, err := c.quiescer.Idle(ctx)
if err != nil {
return fmt.Errorf("读不到后台任务状态:%w", err)
}
if idle {
j.BackgroundIdle = true
return nil
}
last = detail
if time.Now().After(deadline) {
return fmt.Errorf("后台任务仍在运行(%s):等它结束再开窗口,否则会被 SIGTERM 切断", last)
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(500 * time.Millisecond):
}
}
}
// stopAll 只对"原本在跑"的服务发停止指令,每发一个都先落盘意图:
// 进程在停止过程中死掉时,磁盘上仍能看出"哪些服务被要求停过"。
func (c *Coordinator) stopAll(ctx context.Context, j *journal) error {
@@ -342,63 +403,3 @@ func validDigest(id string) bool {
decoded, err := hex.DecodeString(id)
return err == nil && len(decoded) == 32 && strings.ToLower(id) == id
}
// FinalizeAfterOperatorRecovery 收尾一个"内容风险未验证 + 人工已恢复"的窗口。
//
// 它不启动任何服务、也不释放治理库存的占用:只做两件事——核对"原本在跑的服务
// 现在确实在跑且健康、入口确实在服务",然后把 journal 标成 operator_recovered。
// 没有这两条核对,它就退化成"把卡住的记录清掉",那是另一种误放行。
func (c *Coordinator) FinalizeAfterOperatorRecovery(ctx context.Context, job string) error {
if !validID(job) {
return errors.New("invalid freeze job id")
}
lock, err := c.lock()
if err != nil {
return err
}
defer unlock(lock)
j, err := c.load(job)
if err != nil {
return err
}
if terminal(j.State) {
return errors.New("freeze journal is already terminal")
}
if !j.ContentRisk {
return errors.New("journal has no content risk; use Recover instead")
}
items, err := inventory(ctx, c.control)
if err != nil {
return err
}
running := map[string]Service{}
for _, item := range items {
running[item.ID] = item
}
for _, s := range j.Services {
if !s.Running {
continue
}
now, ok := running[s.ID]
if !ok || !now.Running || !now.Healthy {
return fmt.Errorf("服务 %s 还没有恢复到健康状态,不能收尾这个窗口", s.ID)
}
}
if c.gate != nil && j.Gate != nil && !j.GateReopened {
if err := c.gate.OpenIngress(ctx); err != nil {
return fmt.Errorf("入口还没有恢复:%w", err)
}
j.GateReopened = true
}
if c.ingressReady != nil {
if err := c.ingressReady(ctx); err != nil {
return fmt.Errorf("入口验证未通过:%w", err)
}
}
ended := time.Now().UTC()
j.State = "failed"
j.Code = "operator_recovered"
j.RestoreBlocked = false
j.FinishedAt = &ended
return c.save(j)
}
+13 -29
View File
@@ -29,6 +29,16 @@ func (c *exitControl) Inventory(ctx context.Context) ([]Service, error) {
return items, nil
}
// idleQuiescer:实例级窗口必须能证明后台任务已空,测试里默认给一个空的。
type idleQuiescer struct{ busy bool }
func (q *idleQuiescer) Idle(context.Context) (bool, string, error) {
if q.busy {
return false, "asynq pending=1", nil
}
return true, "asynq=0 pending_ops=0", nil
}
func instanceFixture(t *testing.T) (string, *exitControl, *Coordinator) {
t.Helper()
root := t.TempDir()
@@ -45,6 +55,9 @@ func instanceFixture(t *testing.T) (string, *exitControl, *Coordinator) {
if err := engine.SetTarget("instance", "http://127.0.0.1:1/health", nil, nil); err != nil {
t.Fatal(err)
}
if err := engine.SetQuiescer(&idleQuiescer{}); err != nil {
t.Fatal(err)
}
return root, c, engine
}
@@ -312,32 +325,3 @@ func TestWindowRefusesWhenGateNotDrained(t *testing.T) {
t.Fatalf("writers were stopped before the ingress was drained: %v", c.calls)
}
}
// 人工恢复后的收尾:服务没起来就必须拒绝(不能靠清记录放行)。
func TestFinalizeAfterOperatorRecoveryRequiresHealthyServices(t *testing.T) {
_, c, e := instanceFixture(t)
_, err := e.RunWindow(context.Background(), "job-manual", func(context.Context) error { return nil },
func(ctx context.Context, w *Window) (string, error) {
if err := w.MarkContentRisk(ctx); err != nil {
t.Fatal(err)
}
return "", errors.New("内容回滚失败")
}, func(error) bool { return false })
if err == nil {
t.Fatal("blocked window reported success")
}
// 服务还停着:收尾必须失败。
if err := e.FinalizeAfterOperatorRecovery(context.Background(), "job-manual"); err == nil {
t.Fatal("finalize accepted a still-stopped target")
}
for i := range c.services {
c.services[i].Running, c.services[i].Healthy = true, true
}
if err := e.FinalizeAfterOperatorRecovery(context.Background(), "job-manual"); err != nil {
t.Fatalf("finalize rejected a recovered target: %v", err)
}
j, err := e.load("job-manual")
if err != nil || j.State != "failed" || j.Code != "operator_recovered" || j.RestoreBlocked {
t.Fatalf("journal=%+v err=%v", j, err)
}
}
+141
View File
@@ -0,0 +1,141 @@
package librarymaintenance
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"os"
"time"
)
// dockerAPI 是目标维护用到的 Docker Engine 最小客户端(unix socket)。
// 入口闸门与拓扑校验共用同一份实现:两处各自拼一套请求只会长出两套毛病。
type dockerAPI struct {
client *http.Client
}
func newDockerAPI(socket string, timeout time.Duration) (*dockerAPI, error) {
if socket == "" || timeout <= 0 {
return nil, errors.New("invalid docker api configuration")
}
info, err := os.Stat(socket)
if err != nil || info.Mode()&os.ModeSocket == 0 {
return nil, errors.New("local Docker socket unavailable")
}
transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
return (&net.Dialer{}).DialContext(ctx, "unix", socket)
}}
return &dockerAPI{client: &http.Client{Transport: transport, Timeout: timeout,
CheckRedirect: func(*http.Request, []*http.Request) error { return errors.New("docker redirects prohibited") }}}, nil
}
func (d *dockerAPI) Close() { d.client.CloseIdleConnections() }
func (d *dockerAPI) request(ctx context.Context, method, path string, out any) error {
req, err := http.NewRequestWithContext(ctx, method, "http://docker"+path, nil)
if err != nil {
return errors.New("invalid docker request")
}
resp, err := d.client.Do(req)
if err != nil {
return errors.New("docker request failed")
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return fmt.Errorf("docker request rejected: %d", resp.StatusCode)
}
if out != nil {
data, readErr := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
if readErr != nil || json.Unmarshal(data, out) != nil {
return errors.New("invalid docker response")
}
}
return nil
}
type containerSummary struct {
ID string `json:"Id"`
Labels map[string]string `json:"Labels"`
Ports []struct {
PrivatePort int `json:"PrivatePort"`
PublicPort int `json:"PublicPort"`
Type string `json:"Type"`
} `json:"Ports"`
}
func (d *dockerAPI) listProject(ctx context.Context, project string) ([]containerSummary, error) {
filters, _ := json.Marshal(map[string][]string{"label": {"com.docker.compose.project=" + project}})
var items []containerSummary
if err := d.request(ctx, "GET", "/containers/json?all=1&filters="+url.QueryEscape(string(filters)), &items); err != nil {
return nil, err
}
return items, nil
}
type containerState struct {
ID string `json:"Id"`
Labels map[string]string `json:"Config"`
Config struct {
Labels map[string]string `json:"Labels"`
} `json:"-"`
State struct {
Running bool
Status string
ExitCode int
Health *struct{ Status string }
} `json:"State"`
HostConfig struct {
PortBindings map[string][]struct {
HostIP string `json:"HostIp"`
HostPort string `json:"HostPort"`
} `json:"PortBindings"`
} `json:"HostConfig"`
NetworkSettings struct {
Ports map[string][]struct {
HostIP string `json:"HostIp"`
HostPort string `json:"HostPort"`
} `json:"Ports"`
} `json:"NetworkSettings"`
}
func (d *dockerAPI) inspect(ctx context.Context, id string) (containerState, error) {
var data containerState
if err := d.request(ctx, "GET", "/containers/"+id+"/json", &data); err != nil {
return containerState{}, err
}
return data, nil
}
// publishedPorts 返回容器对外发布的端口(HostConfig + NetworkSettings 都算)。
// 写入者只要发布了一个端口,就存在"绕开入口闸门直连"的通道。
func (c containerState) publishedPorts() []string {
seen := map[string]bool{}
out := make([]string, 0, 2)
add := func(hostIP, hostPort, containerPort string) {
if hostPort == "" || hostPort == "0" {
return
}
key := hostIP + ":" + hostPort + "->" + containerPort
if seen[key] {
return
}
seen[key] = true
out = append(out, key)
}
for port, bindings := range c.HostConfig.PortBindings {
for _, binding := range bindings {
add(binding.HostIP, binding.HostPort, port)
}
}
for port, bindings := range c.NetworkSettings.Ports {
for _, binding := range bindings {
add(binding.HostIP, binding.HostPort, port)
}
}
return out
}
+25 -68
View File
@@ -6,11 +6,7 @@ import (
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"time"
"knowledge-governance/internal/libraryfreeze"
@@ -20,8 +16,7 @@ import (
// 闸门自己写下的 drain.json(停止接客时还有多少在途连接、最后是否归零)。
// 不看 HTTP 探测就宣布"入口关了"——那是猜。
type dockerGate struct {
client *http.Client
socket string
api *dockerAPI
project string
service string
drainFile string
@@ -29,79 +24,41 @@ type dockerGate struct {
}
func newDockerGate(socket, project, service, drainFile string, timeout time.Duration) (*dockerGate, error) {
if !filepath.IsAbs(socket) || project == "" || service == "" || !filepath.IsAbs(drainFile) {
if project == "" || service == "" || drainFile == "" || drainFile[0] != '/' {
return nil, errors.New("入口闸门配置不完整")
}
info, err := os.Stat(socket)
if err != nil || info.Mode()&os.ModeSocket == 0 {
return nil, errors.New("local Docker socket unavailable")
api, err := newDockerAPI(socket, timeout)
if err != nil {
return nil, err
}
transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
return (&net.Dialer{}).DialContext(ctx, "unix", socket)
}}
return &dockerGate{
client: &http.Client{Transport: transport, Timeout: timeout,
CheckRedirect: func(*http.Request, []*http.Request) error { return errors.New("docker redirects prohibited") }},
socket: socket, project: project, service: service, drainFile: drainFile, timeout: timeout,
}, nil
return &dockerGate{api: api, project: project, service: service, drainFile: drainFile, timeout: timeout}, nil
}
func (g *dockerGate) request(ctx context.Context, method, path string, out any) error {
req, err := http.NewRequestWithContext(ctx, method, "http://docker"+path, nil)
func (g *dockerGate) container(ctx context.Context) (containerState, error) {
items, err := g.api.listProject(ctx, g.project)
if err != nil {
return errors.New("invalid docker request")
return containerState{}, err
}
resp, err := g.client.Do(req)
if err != nil {
return errors.New("docker request failed")
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return fmt.Errorf("docker request rejected: %d", resp.StatusCode)
}
if out != nil {
data, readErr := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
if readErr != nil || json.Unmarshal(data, out) != nil {
return errors.New("invalid docker response")
id := ""
for _, item := range items {
if item.Labels["com.docker.compose.service"] != g.service {
continue
}
if id != "" {
return containerState{}, fmt.Errorf("入口闸门实例数异常:多于一个 %s", g.service)
}
id = item.ID
}
return nil
}
type gateInspection struct {
ID string `json:"Id"`
Config struct {
Labels map[string]string `json:"Labels"`
} `json:"Config"`
State struct {
Running bool
Status string
ExitCode int
Health *struct{ Status string }
} `json:"State"`
}
func (g *dockerGate) container(ctx context.Context) (gateInspection, error) {
filters, _ := json.Marshal(map[string][]string{"label": {
"com.docker.compose.project=" + g.project,
"com.docker.compose.service=" + g.service,
}})
var listed []struct {
ID string `json:"Id"`
if id == "" {
return containerState{}, fmt.Errorf("找不到入口闸门服务 %s", g.service)
}
if err := g.request(ctx, "GET", "/containers/json?all=1&filters="+url.QueryEscape(string(filters)), &listed); err != nil {
return gateInspection{}, err
}
if len(listed) != 1 {
return gateInspection{}, fmt.Errorf("入口闸门实例数异常:%d", len(listed))
}
var data gateInspection
if err := g.request(ctx, "GET", "/containers/"+listed[0].ID+"/json", &data); err != nil {
return gateInspection{}, err
data, err := g.api.inspect(ctx, id)
if err != nil {
return containerState{}, err
}
if data.Config.Labels["com.docker.compose.project"] != g.project ||
data.Config.Labels["com.docker.compose.service"] != g.service {
return gateInspection{}, errors.New("入口闸门身份不匹配")
return containerState{}, errors.New("入口闸门身份不匹配")
}
return data, nil
}
@@ -151,7 +108,7 @@ func (g *dockerGate) CloseIngress(ctx context.Context) (libraryfreeze.GateEviden
} else if !errors.Is(err, os.ErrNotExist) {
return libraryfreeze.GateEvidence{}, fmt.Errorf("无法确认排空记录已清理:%w", err)
}
if err := g.request(bounded, "POST", "/containers/"+data.ID+"/kill?signal=SIGTERM", nil); err != nil {
if err := g.api.request(bounded, "POST", "/containers/"+data.ID+"/kill?signal=SIGTERM", nil); err != nil {
return libraryfreeze.GateEvidence{}, err
}
deadline := time.Now().Add(g.timeout)
@@ -202,7 +159,7 @@ func (g *dockerGate) OpenIngress(ctx context.Context) error {
return err
}
if !data.State.Running {
if err := g.request(bounded, "POST", "/containers/"+data.ID+"/start", nil); err != nil {
if err := g.api.request(bounded, "POST", "/containers/"+data.ID+"/start", nil); err != nil {
return err
}
}
+248
View File
@@ -0,0 +1,248 @@
package librarymaintenance
import (
"bufio"
"context"
"errors"
"fmt"
"net"
"strconv"
"strings"
"time"
"github.com/jackc/pgx/v5/pgxpool"
)
// targetQuiescer 回答"后台写入者是不是已经空了":
// - 目标库:未结清的后台操作、未完成的解析(这两张表就是 app 的后台账本);
// - 目标队列:asynq 里还在排队/在跑/待重试的任务。
//
// 为什么要单独做:HTTP 入口排空只覆盖连接;app 内的 asynq worker 会被上游的
// SIGTERM exit(1) 切断,正在跑的任务就这样被腰斩。窗口必须先等它们结束。
type targetQuiescer struct {
pool *pgxpool.Pool
redisAddr string
redisPass string
redisDB int
timeout time.Duration
queueMatch string
}
func newTargetQuiescer(ctx context.Context, databaseURL, redisAddr, redisPass string, redisDB int, timeout time.Duration) (*targetQuiescer, error) {
if databaseURL == "" || redisAddr == "" {
return nil, errors.New("后台静默检查缺少目标库或队列地址")
}
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
return nil, errors.New("后台静默检查连不上目标库")
}
if timeout <= 0 {
timeout = 10 * time.Second
}
return &targetQuiescer{pool: pool, redisAddr: redisAddr, redisPass: redisPass, redisDB: redisDB,
timeout: timeout, queueMatch: "asynq:*"}, nil
}
func (q *targetQuiescer) Close() { q.pool.Close() }
func (q *targetQuiescer) Idle(ctx context.Context) (bool, string, error) {
var pendingOps, unsettled int64
// task_pending_ops / knowledges 是 app 自己的后台账本:不看它们只看 HTTP 会漏。
if err := q.pool.QueryRow(ctx, `SELECT
(SELECT count(*) FROM task_pending_ops),
(SELECT count(*) FROM knowledges WHERE deleted_at IS NULL
AND (parse_status IS DISTINCT FROM 'completed' OR pending_subtasks_count IS NULL OR pending_subtasks_count<>0))`).
Scan(&pendingOps, &unsettled); err != nil {
return false, "", fmt.Errorf("读不到后台账本:%w", err)
}
depth, err := q.queueDepth(ctx)
if err != nil {
return false, "", err
}
detail := fmt.Sprintf("pending_ops=%d unsettled_knowledges=%d asynq_active=%d", pendingOps, unsettled, depth)
return pendingOps == 0 && unsettled == 0 && depth == 0, detail, nil
}
// queueDepth 统计 asynq 里"还没结束"的任务数(排队/在跑/待重试)。
// archived/completed 是历史,不算忙。
func (q *targetQuiescer) queueDepth(ctx context.Context) (int64, error) {
conn, err := dialRedis(q.redisAddr, q.redisPass, q.redisDB, q.timeout)
if err != nil {
return 0, fmt.Errorf("连不上目标队列:%w", err)
}
defer conn.Close()
keys, err := conn.scan(ctx, q.queueMatch)
if err != nil {
return 0, fmt.Errorf("扫描目标队列失败:%w", err)
}
var total int64
for _, key := range keys {
suffix := key[strings.LastIndex(key, ":"):]
switch suffix {
case ":pending", ":active", ":inprogress":
n, err := conn.intCmd(ctx, "LLEN", key)
if err != nil {
return 0, err
}
total += n
case ":scheduled", ":retry":
n, err := conn.intCmd(ctx, "ZCARD", key)
if err != nil {
return 0, err
}
total += n
}
}
return total, nil
}
// ---- 最小 RESP 客户端:只用到 AUTH/SELECT/SCAN/LLEN/ZCARD,不引第三方库 ----
type redisConn struct {
conn net.Conn
reader *bufio.Reader
timeout time.Duration
}
func dialRedis(addr, password string, db int, timeout time.Duration) (*redisConn, error) {
dialer := &net.Dialer{Timeout: timeout}
conn, err := dialer.Dial("tcp", addr)
if err != nil {
return nil, err
}
_ = conn.SetDeadline(time.Now().Add(timeout))
c := &redisConn{conn: conn, reader: bufio.NewReader(conn), timeout: timeout}
if password != "" {
if _, err := c.do("AUTH", password); err != nil {
conn.Close()
return nil, err
}
}
if db != 0 {
if _, err := c.do("SELECT", strconv.Itoa(db)); err != nil {
conn.Close()
return nil, err
}
}
return c, nil
}
func (c *redisConn) Close() error { return c.conn.Close() }
func (c *redisConn) do(args ...string) (any, error) {
_ = c.conn.SetDeadline(time.Now().Add(c.timeout))
var builder strings.Builder
builder.WriteString("*" + strconv.Itoa(len(args)) + "\r\n")
for _, arg := range args {
builder.WriteString("$" + strconv.Itoa(len(arg)) + "\r\n" + arg + "\r\n")
}
if _, err := c.conn.Write([]byte(builder.String())); err != nil {
return nil, err
}
return c.read()
}
func (c *redisConn) read() (any, error) {
line, err := c.reader.ReadString('\n')
if err != nil {
return nil, err
}
line = strings.TrimRight(line, "\r\n")
if line == "" {
return nil, errors.New("empty redis reply")
}
switch line[0] {
case '+':
return line[1:], nil
case '-':
return nil, errors.New(line[1:])
case ':':
value, err := strconv.ParseInt(line[1:], 10, 64)
return value, err
case '$':
length, err := strconv.Atoi(line[1:])
if err != nil {
return nil, err
}
if length < 0 {
return nil, nil
}
buf := make([]byte, length+2)
if _, err := ioReadFull(c.reader, buf); err != nil {
return nil, err
}
return string(buf[:length]), nil
case '*':
count, err := strconv.Atoi(line[1:])
if err != nil {
return nil, err
}
items := make([]any, 0, count)
for i := 0; i < count; i++ {
item, err := c.read()
if err != nil {
return nil, err
}
items = append(items, item)
}
return items, nil
default:
return nil, errors.New("unsupported redis reply")
}
}
func ioReadFull(reader *bufio.Reader, buf []byte) (int, error) {
total := 0
for total < len(buf) {
n, err := reader.Read(buf[total:])
total += n
if err != nil {
return total, err
}
}
return total, nil
}
func (c *redisConn) scan(ctx context.Context, match string) ([]string, error) {
var keys []string
cursor := "0"
for {
if err := ctx.Err(); err != nil {
return nil, err
}
reply, err := c.do("SCAN", cursor, "MATCH", match, "COUNT", "1000")
if err != nil {
return nil, err
}
items, ok := reply.([]any)
if !ok || len(items) != 2 {
return nil, errors.New("unexpected SCAN reply")
}
next, _ := items[0].(string)
batch, _ := items[1].([]any)
for _, item := range batch {
if key, ok := item.(string); ok {
keys = append(keys, key)
}
}
if next == "0" || next == "" {
return keys, nil
}
cursor = next
}
}
func (c *redisConn) intCmd(ctx context.Context, cmd string, args ...string) (int64, error) {
if err := ctx.Err(); err != nil {
return 0, err
}
reply, err := c.do(append([]string{cmd}, args...)...)
if err != nil {
return 0, err
}
value, ok := reply.(int64)
if !ok {
return 0, errors.New("unexpected integer reply")
}
return value, nil
}
+126 -2
View File
@@ -56,7 +56,12 @@ type Config struct {
Database string
Index string
Bucket string
Timeout time.Duration
// 后台静默检查需要的运行时依赖(不写进维护记录)。
DatabaseURL string
RedisAddr string
RedisPassword string
RedisDB int
Timeout time.Duration
}
type Manager struct {
@@ -68,6 +73,18 @@ type Manager struct {
timeout time.Duration
client *http.Client
gate *dockerGate
api *dockerAPI
quiescer *targetQuiescer
}
// Close 释放拓扑/静默检查持有的连接。
func (m *Manager) Close() {
if m.api != nil {
m.api.Close()
}
if m.quiescer != nil {
m.quiescer.Close()
}
}
func New(cfg Config) (*Manager, error) {
@@ -102,9 +119,13 @@ func New(cfg Config) (*Manager, error) {
return nil, err
}
target.Fingerprint = fingerprint
api, err := newDockerAPI(cfg.DockerSocket, cfg.Timeout)
if err != nil {
return nil, err
}
manager := &Manager{target: target, source: project, docker: cfg.DockerSocket,
writers: writers, nonWriters: cfg.NonWriters, timeout: cfg.Timeout,
client: &http.Client{Timeout: 5 * time.Second}}
client: &http.Client{Timeout: 5 * time.Second}, api: api}
if cfg.IngressService != "" || cfg.IngressDrainFile != "" {
gate, err := newDockerGate(cfg.DockerSocket, project, cfg.IngressService, cfg.IngressDrainFile, cfg.Timeout)
if err != nil {
@@ -112,6 +133,14 @@ func New(cfg Config) (*Manager, error) {
}
manager.gate = gate
}
if cfg.DatabaseURL != "" && cfg.RedisAddr != "" {
quiescer, err := newTargetQuiescer(context.Background(), cfg.DatabaseURL, cfg.RedisAddr,
cfg.RedisPassword, cfg.RedisDB, cfg.Timeout)
if err != nil {
return nil, err
}
manager.quiescer = quiescer
}
return manager, nil
}
@@ -159,6 +188,11 @@ func (m *Manager) Open(root string) (*libraryfreeze.Coordinator, *libraryfreeze.
control.Close()
return nil, nil, err
}
if err := coordinator.SetTargetFingerprint(m.target.Fingerprint); err != nil {
coordinator.Close()
control.Close()
return nil, nil, err
}
if m.gate != nil {
if err := coordinator.SetGate(m.gate); err != nil {
coordinator.Close()
@@ -166,6 +200,17 @@ func (m *Manager) Open(root string) (*libraryfreeze.Coordinator, *libraryfreeze.
return nil, nil, err
}
}
// 后台静默检查是实例级窗口的必备条件:没有它就等于只排空了 HTTP。
if m.quiescer == nil {
coordinator.Close()
control.Close()
return nil, nil, errors.New("缺少后台静默检查(目标库/队列未配置),拒绝开实例级窗口")
}
if err := coordinator.SetQuiescer(m.quiescer); err != nil {
coordinator.Close()
control.Close()
return nil, nil, err
}
return coordinator, control, nil
}
@@ -270,3 +315,82 @@ func DatabaseIdentity(raw string) (string, error) {
}
return parsed.Host + "/" + database, nil
}
const composeServiceLabel = "com.docker.compose.service"
// VerifyTopology 强制"入口唯一":
// - 写入者不许发布任何宿主机端口(否则就存在绕闸直连,维护窗口覆盖不到);
// - 入口闸门必须存在、必须是独立服务,并且发布了入口 URL 用的那个端口。
//
// 任何一条不满足都拒绝开窗口——拓扑错了,"维护"就只是关了一半的门。
func (m *Manager) VerifyTopology(ctx context.Context) error {
items, err := m.api.listProject(ctx, m.target.Project)
if err != nil {
return err
}
byService := make(map[string]containerSummary, len(items))
for _, item := range items {
name := item.Labels[composeServiceLabel]
if name == "" {
return errors.New("项目里有容器缺少 compose service 标签,无法确认写入者边界")
}
if _, duplicated := byService[name]; duplicated {
return fmt.Errorf("项目里服务 %s 有多个实例,无法确认写入者边界", name)
}
byService[name] = item
}
for _, writer := range m.writers {
item, ok := byService[writer]
if !ok {
return fmt.Errorf("声明的写入者 %s 不存在", writer)
}
state, err := m.api.inspect(ctx, item.ID)
if err != nil {
return err
}
if ports := state.publishedPorts(); len(ports) > 0 {
return fmt.Errorf("写入者 %s 直接发布了端口 %s:存在绕开入口闸门的直连通道,拒绝开维护窗口",
writer, strings.Join(ports, ","))
}
}
if m.gate == nil {
return errors.New("实例级维护必须声明入口闸门")
}
for _, writer := range m.writers {
if writer == m.gate.service {
return errors.New("入口闸门不能同时被声明为写入者")
}
}
gateItem, ok := byService[m.gate.service]
if !ok {
return fmt.Errorf("声明的入口闸门 %s 不存在", m.gate.service)
}
gateState, err := m.api.inspect(ctx, gateItem.ID)
if err != nil {
return err
}
published := gateState.publishedPorts()
if len(published) == 0 {
return fmt.Errorf("入口闸门 %s 没有发布任何端口:外部无法访问,入口定义不成立", m.gate.service)
}
// 入口 URL 的端口必须真的是这个闸门发布的端口。
parsed, err := url.Parse(m.target.Ingress)
if err != nil || parsed.Host == "" {
return errors.New("目标入口地址无效")
}
hostPort := parsed.Port()
if hostPort == "" {
if parsed.Scheme == "https" {
hostPort = "443"
} else {
hostPort = "80"
}
}
for _, entry := range published {
if strings.Contains(entry, ":"+hostPort+"->") {
return nil
}
}
return fmt.Errorf("入口 %s 的端口 %s 不是闸门发布的端口(%s):入口指向了别的服务",
m.target.Ingress, hostPort, strings.Join(published, ","))
}
@@ -132,3 +132,24 @@ func TestDatabaseIdentityDropsCredentials(t *testing.T) {
t.Fatal("invalid DSN accepted")
}
}
// 写入者只要发布了宿主机端口,就存在绕开闸门的直连通道。
func TestPublishedPortsDetectsDirectBypass(t *testing.T) {
var state containerState
state.HostConfig.PortBindings = map[string][]struct {
HostIP string `json:"HostIp"`
HostPort string `json:"HostPort"`
}{"8080/tcp": {{HostIP: "127.0.0.1", HostPort: "18081"}}}
ports := state.publishedPorts()
if len(ports) != 1 || !strings.Contains(ports[0], "18081") {
t.Fatalf("publishedPorts=%v", ports)
}
var bare containerState
bare.HostConfig.PortBindings = map[string][]struct {
HostIP string `json:"HostIp"`
HostPort string `json:"HostPort"`
}{"8080/tcp": nil}
if got := bare.publishedPorts(); len(got) != 0 {
t.Fatalf("unpublished container reported ports: %v", got)
}
}
@@ -0,0 +1,177 @@
package postgres
import (
"context"
"encoding/json"
"errors"
"fmt"
"time"
"github.com/jackc/pgx/v5"
)
// LibraryInstanceMaintenance 是"实例级维护"的占用记录:一个目标实例同一时刻
// 只允许一个窗口,无论它来自 internal 还是 external。
//
// 为什么不是 scope 锁:stop/start 的是同一批容器,两个 scope 各持一个 scope 锁
// 时互相看不见,会同时停/起同一实例——那是假互斥,比没有锁更危险。
type LibraryInstanceMaintenance struct {
Instance string `json:"instance"`
OwnerJobID string `json:"owner_job_id"`
OwnerScope string `json:"owner_scope"`
TargetFingerprint string `json:"target_fingerprint"`
Level string `json:"level"`
Services []string `json:"services"`
IngressState string `json:"ingress_state"`
State string `json:"state"`
OriginalServiceState string `json:"original_service_state"`
Note string `json:"note"`
AcquiredAt time.Time `json:"acquired_at"`
UpdatedAt time.Time `json:"updated_at"`
ReleasedAt *time.Time `json:"released_at,omitempty"`
}
const instanceMaintenanceColumns = `instance, owner_job_id, owner_scope, target_fingerprint, level, services,
ingress_state, state, original_service_state, note, acquired_at, updated_at, released_at`
// AcquireInstanceMaintenance 原子地拿到实例维护权:
// 已释放的记录可以被新作业接管;仍被持有的只允许原作业重入。
func (s *Store) AcquireInstanceMaintenance(ctx context.Context, item LibraryInstanceMaintenance, now time.Time) (LibraryInstanceMaintenance, error) {
level := item.Level
if level == "" {
level = "instance"
}
ingress := item.IngressState
if ingress == "" {
ingress = "unknown"
}
services := item.Services
if services == nil {
services = []string{}
}
raw, err := json.Marshal(services)
if err != nil {
return LibraryInstanceMaintenance{}, fmt.Errorf("acquire instance maintenance: %w", err)
}
row := s.db.QueryRow(ctx, `
INSERT INTO library_instance_maintenance (instance, owner_job_id, owner_scope, target_fingerprint, level, services,
ingress_state, state, original_service_state, note, acquired_at, updated_at)
VALUES ($1,$2,$3,$4,$5,$6::jsonb,$7,$8,$9,$10,$11,$11)
ON CONFLICT (instance) DO UPDATE SET
owner_job_id = EXCLUDED.owner_job_id,
owner_scope = EXCLUDED.owner_scope,
target_fingerprint = EXCLUDED.target_fingerprint,
level = EXCLUDED.level,
services = EXCLUDED.services,
ingress_state = EXCLUDED.ingress_state,
state = EXCLUDED.state,
original_service_state = EXCLUDED.original_service_state,
note = EXCLUDED.note,
updated_at = EXCLUDED.updated_at,
released_at = NULL
WHERE library_instance_maintenance.released_at IS NOT NULL
OR library_instance_maintenance.owner_job_id = EXCLUDED.owner_job_id
RETURNING `+instanceMaintenanceColumns,
item.Instance, item.OwnerJobID, item.OwnerScope, item.TargetFingerprint, level, string(raw),
ingress, item.State, item.OriginalServiceState, item.Note, now.UTC())
acquired, err := scanInstanceMaintenance(row)
if errors.Is(err, pgx.ErrNoRows) {
return LibraryInstanceMaintenance{}, ErrMaintenanceHeld
}
if err != nil {
return LibraryInstanceMaintenance{}, fmt.Errorf("acquire instance maintenance: %w", err)
}
return acquired, nil
}
// UpdateInstanceMaintenanceState 只允许持有者改自己的窗口状态。
func (s *Store) UpdateInstanceMaintenanceState(ctx context.Context, instance, ownerJobID, state, note string, now time.Time) error {
tag, err := s.db.Exec(ctx, `UPDATE library_instance_maintenance SET state=$3, note=$4, updated_at=$5
WHERE instance=$1 AND owner_job_id=$2 AND released_at IS NULL`, instance, ownerJobID, state, note, now.UTC())
if err != nil {
return fmt.Errorf("update instance maintenance: %w", err)
}
if tag.RowsAffected() == 0 {
return errors.New("instance maintenance not held by this job")
}
return nil
}
// SetInstanceMaintenanceIngress 记录入口真实状态。
func (s *Store) SetInstanceMaintenanceIngress(ctx context.Context, instance, ownerJobID, state, note string, now time.Time) error {
if state != "closed" && state != "open" {
return errors.New("invalid ingress state")
}
tag, err := s.db.Exec(ctx, `UPDATE library_instance_maintenance SET ingress_state=$3, note=$4, updated_at=$5
WHERE instance=$1 AND owner_job_id=$2 AND released_at IS NULL`, instance, ownerJobID, state, note, now.UTC())
if err != nil {
return fmt.Errorf("update instance maintenance ingress: %w", err)
}
if tag.RowsAffected() == 0 {
return errors.New("instance maintenance not held by this job")
}
return nil
}
// ReleaseInstanceMaintenance 只有"内容已验证 + 服务与入口恢复"才允许调用。
func (s *Store) ReleaseInstanceMaintenance(ctx context.Context, instance, ownerJobID string, now time.Time) error {
tag, err := s.db.Exec(ctx, `UPDATE library_instance_maintenance SET state='released', released_at=$3, updated_at=$3
WHERE instance=$1 AND owner_job_id=$2 AND released_at IS NULL`, instance, ownerJobID, now.UTC())
if err != nil {
return fmt.Errorf("release instance maintenance: %w", err)
}
if tag.RowsAffected() == 0 {
return errors.New("instance maintenance not held by this job")
}
return nil
}
// InstanceMaintenanceOf 读当前实例窗口(接口互斥判断与页面展示都用它)。
func (s *Store) InstanceMaintenanceOf(ctx context.Context, instance string) (*LibraryInstanceMaintenance, error) {
row := s.db.QueryRow(ctx, `SELECT `+instanceMaintenanceColumns+` FROM library_instance_maintenance WHERE instance=$1`, instance)
item, err := scanInstanceMaintenance(row)
if errors.Is(err, pgx.ErrNoRows) {
return nil, nil
}
if err != nil {
return nil, err
}
return &item, nil
}
// HeldInstanceMaintenance 返回仍然持有(未释放)的实例窗口,供接口在不知道
// 具体实例名时也拒绝写入。
func (s *Store) HeldInstanceMaintenance(ctx context.Context) ([]LibraryInstanceMaintenance, error) {
rows, err := s.db.Query(ctx, `SELECT `+instanceMaintenanceColumns+`
FROM library_instance_maintenance WHERE released_at IS NULL ORDER BY instance`)
if err != nil {
return nil, fmt.Errorf("list instance maintenance: %w", err)
}
defer rows.Close()
items := make([]LibraryInstanceMaintenance, 0, 2)
for rows.Next() {
item, err := scanInstanceMaintenance(rows)
if err != nil {
return nil, err
}
items = append(items, item)
}
return items, rows.Err()
}
func scanInstanceMaintenance(row pgx.Row) (LibraryInstanceMaintenance, error) {
var item LibraryInstanceMaintenance
var raw []byte
err := row.Scan(&item.Instance, &item.OwnerJobID, &item.OwnerScope, &item.TargetFingerprint, &item.Level,
&raw, &item.IngressState, &item.State, &item.OriginalServiceState, &item.Note,
&item.AcquiredAt, &item.UpdatedAt, &item.ReleasedAt)
if err != nil {
return LibraryInstanceMaintenance{}, err
}
if len(raw) > 0 {
if err := json.Unmarshal(raw, &item.Services); err != nil {
return LibraryInstanceMaintenance{}, fmt.Errorf("instance maintenance services unreadable: %w", err)
}
}
return item, nil
}
@@ -71,3 +71,45 @@ func TestLibraryMaintenanceOwnershipAcrossRelease(t *testing.T) {
t.Fatalf("released window not re-acquired cleanly: %+v", second)
}
}
// 实例级占用必须跨 scope 互斥:internal 持有期间 external 也进不来。
// (旧实现按 scope 建行,两个 scope 各持一个锁,会同时停/起同一个实例。)
func TestInstanceMaintenanceExcludesAcrossScopes(t *testing.T) {
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatalf("connect: %v", err)
}
defer pool.Close()
if err := ApplyMigrations(ctx, pool); err != nil {
t.Fatalf("migrate: %v", err)
}
store := NewStore(pool)
now := time.Now().UTC()
base := LibraryInstanceMaintenance{Instance: "kgmaint-test", OwnerJobID: "job-internal",
OwnerScope: "internal", TargetFingerprint: "fp", Level: "instance", Services: []string{"app"},
State: "acquired", Note: "test"}
if _, err := store.AcquireInstanceMaintenance(ctx, base, now); err != nil {
t.Fatalf("first acquire: %v", err)
}
external := base
external.OwnerJobID = "job-external"
external.OwnerScope = "external"
if _, err := store.AcquireInstanceMaintenance(ctx, external, now.Add(time.Second)); !errors.Is(err, ErrMaintenanceHeld) {
t.Fatalf("external scope entered while internal held the instance: err=%v", err)
}
if err := store.ReleaseInstanceMaintenance(ctx, "kgmaint-test", "job-internal", now.Add(2*time.Second)); err != nil {
t.Fatalf("release: %v", err)
}
if _, err := store.AcquireInstanceMaintenance(ctx, external, now.Add(3*time.Second)); err != nil {
t.Fatalf("external scope could not take over the released instance: %v", err)
}
held, err := store.HeldInstanceMaintenance(ctx)
if err != nil || len(held) != 1 || held[0].OwnerScope != "external" {
t.Fatalf("held=%+v err=%v", held, err)
}
}
@@ -0,0 +1 @@
DROP TABLE IF EXISTS library_instance_maintenance;
@@ -0,0 +1,21 @@
-- 维护占用必须按"实例"互斥,而不是按库 scope。
-- 旧表的 scope 主键允许 internal/external 各持一个窗口,于是两个作业会同时
-- 停/起同一个实例的写入者——那是最危险的一种假互斥。
CREATE TABLE library_instance_maintenance (
instance VARCHAR(128) PRIMARY KEY,
owner_job_id VARCHAR(128) NOT NULL,
owner_scope VARCHAR(16) NOT NULL CHECK (owner_scope IN ('internal','external')),
target_fingerprint VARCHAR(64) NOT NULL,
level VARCHAR(16) NOT NULL DEFAULT 'instance' CHECK (level IN ('library','instance')),
services JSONB NOT NULL DEFAULT '[]'::jsonb,
ingress_state VARCHAR(16) NOT NULL DEFAULT 'unknown' CHECK (ingress_state IN ('unknown','closed','open')),
state VARCHAR(24) NOT NULL CHECK (state IN ('acquired','applying','verified','released','needs_restore','recovering')),
original_service_state TEXT NOT NULL DEFAULT '',
note TEXT NOT NULL DEFAULT '',
acquired_at TIMESTAMPTZ NOT NULL,
updated_at TIMESTAMPTZ NOT NULL,
released_at TIMESTAMPTZ
);
-- 把已有窗口按 scope 行回填成实例行不可行(历史 target_identity 是自由文本),
-- 这里只保证新窗口从第一刻起就按实例互斥;旧行保留为审计记录。
+13 -8
View File
@@ -11,30 +11,35 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) {
if err != nil {
t.Fatalf("loadMigrations() error = %v", err)
}
if len(migrations) != 71 {
t.Fatalf("migration count = %d, want 71", len(migrations))
if len(migrations) != 72 {
t.Fatalf("migration count = %d, want 72", len(migrations))
}
// 72 让维护占用按实例互斥(internal/external 不再各有各的锁)。
if instance := migrations[len(migrations)-1]; instance.Version != 72 ||
!strings.Contains(instance.UpSQL, "library_instance_maintenance") {
t.Fatalf("migration 72 = %#v", instance)
}
// 71 让维护记录能区分实例级维护与仅治理库互斥。
if level := migrations[len(migrations)-1]; level.Version != 71 ||
if level := migrations[len(migrations)-2]; level.Version != 71 ||
!strings.Contains(level.UpSQL, "level") || !strings.Contains(level.UpSQL, "ingress_state") {
t.Fatalf("migration 71 = %#v", level)
}
if maint := migrations[len(migrations)-2]; maint.Version != 70 || !strings.Contains(maint.UpSQL, "library_maintenance") {
if maint := migrations[len(migrations)-3]; maint.Version != 70 || !strings.Contains(maint.UpSQL, "library_maintenance") {
t.Fatalf("migration 70 = %#v", maint)
}
if tail := migrations[len(migrations)-3]; tail.Version != 69 ||
if tail := migrations[len(migrations)-4]; tail.Version != 69 ||
!strings.Contains(tail.UpSQL, "library_jobs_write_target") {
t.Fatalf("migration 69 = %#v", tail)
}
if restore := migrations[len(migrations)-4]; restore.Version != 68 || !strings.Contains(restore.UpSQL, "restore") {
if restore := migrations[len(migrations)-5]; restore.Version != 68 || !strings.Contains(restore.UpSQL, "restore") {
t.Fatalf("migration 68 = %#v", restore)
}
if last := migrations[len(migrations)-5]; last.Version != 67 ||
if last := migrations[len(migrations)-6]; last.Version != 67 ||
!strings.Contains(last.UpSQL, "library_release_authorizations") ||
!strings.Contains(last.UpSQL, "params_digest") {
t.Fatalf("migration 67 = %#v", last)
}
if flow := migrations[len(migrations)-6]; flow.Version != 66 ||
if flow := migrations[len(migrations)-7]; flow.Version != 66 ||
!strings.Contains(flow.UpSQL, "CREATE TABLE library_versions") ||
!strings.Contains(flow.UpSQL, "library_jobs_active") {
t.Fatalf("migration 66 = %#v", flow)