删掉会自己开入口、只看服务健康的 FinalizeAfterOperatorRecovery。 新的 RecoverWindow 绑定原作业/目标指纹/持久化预备份,在维护关闭下 执行恢复与逐类校验,通过才恢复服务与入口;失败继续维护。 预备份身份与内容风险在第一笔写入前一起落盘;后台任务必须静默后才停写者; 写入者发布端口即拒绝开窗口;维护占用改为按实例互斥(000072)。
This commit is contained in:
@@ -1,5 +1,39 @@
|
||||
# Changelog
|
||||
|
||||
## 开发快照(维护收尾必须按预备份验证 / 后台任务静默 / 实例级互斥)
|
||||
|
||||
### Fixed
|
||||
|
||||
- **删除 `FinalizeAfterOperatorRecovery`**:它自称"不启动服务",实际会 `gate.OpenIngress`,
|
||||
而且只看服务健康与入口 200——健康不等于恢复成功,这条路径能把半写内容重新开放。
|
||||
- 新的唯一收尾入口 `Coordinator.RecoverWindow`:绑定「原作业 + 目标指纹 + journal 里持久化的预备份身份」,
|
||||
在**维护关闭**状态下(入口关、写入者停、排空确认)执行 `Restore` 与**逐类校验**
|
||||
(行数/身份集合/逐行内容摘要 + 索引文档 + 对象资源),校验通过才恢复服务、开入口;
|
||||
校验或恢复失败则保持维护(服务停、入口关、journal 非终态)。
|
||||
- 预备份身份与「内容风险」在**第一笔写入之前一起落盘**(`Window.SetBackupSnapshot` +
|
||||
`MarkContentRisk`)。没有预备份身份的窗口之后既不能自动恢复,也不许被任何收尾路径开放。
|
||||
- 低层恢复接口不再接受任何"已恢复/已验证"的入参:`Restorer`/`Verifier` 由协调器在窗口内调用,
|
||||
`Verifier` 必须返回校验摘要并被持久化。
|
||||
|
||||
### Added
|
||||
|
||||
- 真实恢复入口 `POST /api/v1/library/maintenance/recover`:只建 `library.recover` 作业,
|
||||
由 worker 按预备份恢复 + 校验后释放占用;`libraryctl finalize` 删除(不再有"手工收尾"旁路)。
|
||||
- 实例级互斥:迁移 `000072_library_instance_maintenance`,维护占用按**实例**建键,
|
||||
internal 持有期间 external 也进不来(旧实现按 scope 建行,两个 scope 会同时停/起同一实例)。
|
||||
- 后台静默检查(`targetQuiescer`):窗口关闭入口之前,必须证明目标库 `task_pending_ops=0`、
|
||||
无未完成解析、asynq 队列(pending/active/scheduled/retry)为空;否则拒绝开窗口。
|
||||
HTTP 排空不等于后台任务排空——app 内的 asynq worker 会被上游 SIGTERM 的 exit(1) 切断。
|
||||
- 唯一入口拓扑校验(`Manager.VerifyTopology`):写入者**不许发布任何宿主机端口**,
|
||||
入口必须是独立声明的闸门且发布的端口与入口 URL 一致;不满足就拒绝开窗口。
|
||||
|
||||
### 限制(本轮不承诺)
|
||||
|
||||
- 拓扑校验只看"声明的写入者组":新增容器必须显式声明,平台不会自动猜测。
|
||||
- 后台静默检查依赖目标库表 `task_pending_ops/knowledges` 与 asynq 键名(v0.7.1 现状);
|
||||
上游改队列实现时需要同步调整。
|
||||
- 页面只透传新增字段(实例占用/入口状态),专门文案与浏览器验收仍未做。
|
||||
|
||||
## 开发快照(目标维护:实例级窗口)
|
||||
|
||||
### Added
|
||||
|
||||
@@ -1062,6 +1062,58 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/library/maintenance/recover": {
|
||||
"post": {
|
||||
"tags": [
|
||||
"Runtime"
|
||||
],
|
||||
"operationId": "recoverLibraryMaintenance",
|
||||
"summary": "按预备份恢复并逐类校验后收尾维护窗口(仅管理员;只建恢复作业,恢复在校验通过前不开放目标)",
|
||||
"requestBody": {
|
||||
"required": true,
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"scope"
|
||||
],
|
||||
"properties": {
|
||||
"scope": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"internal",
|
||||
"external"
|
||||
]
|
||||
},
|
||||
"window_job_id": {
|
||||
"type": "string",
|
||||
"description": "可选:必须与当前维护窗口的作业一致"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"responses": {
|
||||
"202": {
|
||||
"description": "已创建恢复作业并入队"
|
||||
},
|
||||
"400": {
|
||||
"$ref": "#/components/responses/BadRequest"
|
||||
},
|
||||
"409": {
|
||||
"description": "没有未释放的维护窗口,或窗口属于别的 scope/作业"
|
||||
},
|
||||
"500": {
|
||||
"$ref": "#/components/responses/InternalError"
|
||||
},
|
||||
"503": {
|
||||
"description": "目标维护未接线或维护记录存储不可用"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/library/restores": {
|
||||
"post": {
|
||||
"tags": [
|
||||
|
||||
@@ -202,6 +202,7 @@ func run(ctx context.Context, getenv func(string) string) error {
|
||||
VersionHistory: versionHistoryStore,
|
||||
VersionHistoryRepo: versionHistoryRepo,
|
||||
LibraryArchiveRoot: config.LibraryArchiveRoot,
|
||||
LibraryTargetInstance: config.LibraryTargetInstance,
|
||||
LibraryAgentSocket: config.LibraryAgentSocket,
|
||||
LibraryCapacityBudgetBytes: config.LibraryCapacityBudgetBytes,
|
||||
LibraryProductionDatabaseURL: config.LibraryProductionDatabaseURL,
|
||||
|
||||
+1
-30
@@ -90,8 +90,6 @@ func main() {
|
||||
runErr = runVerify(opts)
|
||||
case "recover":
|
||||
runErr = runRecover(ctx, opts)
|
||||
case "finalize":
|
||||
runErr = runFinalize(ctx, opts)
|
||||
default:
|
||||
usage()
|
||||
os.Exit(2)
|
||||
@@ -103,7 +101,7 @@ func main() {
|
||||
}
|
||||
|
||||
func usage() {
|
||||
fmt.Fprint(os.Stderr, `libraryctl <status|preflight|capture|list|verify|recover|finalize> [flags]
|
||||
fmt.Fprint(os.Stderr, `libraryctl <status|preflight|capture|list|verify|recover> [flags]
|
||||
|
||||
status probe the real source and print what is configured / blocked
|
||||
preflight read the source, run settlement + candidate validation (no writes, no stop)
|
||||
@@ -111,8 +109,6 @@ func usage() {
|
||||
list list sealed candidates and stored bytes
|
||||
verify re-verify one sealed candidate (digests, coverage, identity)
|
||||
recover restore writers from an interrupted freeze journal
|
||||
finalize close a held window after an operator recovered the target by hand
|
||||
(verifies services + ingress are really serving; starts nothing)
|
||||
|
||||
Flags (defaults target the local WKNFull test stack):
|
||||
-source-app WKNFull-app container whose env describes the source
|
||||
@@ -821,31 +817,6 @@ func runVerify(opts options) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func runFinalize(ctx context.Context, opts options) error {
|
||||
if opts.job == "" {
|
||||
return errors.New("finalize requires -job")
|
||||
}
|
||||
coordinator, control, err := openFreeze(ctx, opts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer coordinator.Close()
|
||||
defer control.Close()
|
||||
if err := coordinator.FinalizeAfterOperatorRecovery(ctx, opts.job); err != nil {
|
||||
return err
|
||||
}
|
||||
items, err := control.Inventory(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
group := []map[string]any{}
|
||||
for _, item := range items {
|
||||
group = append(group, map[string]any{"id": item.ID, "running": item.Running, "healthy": item.Healthy})
|
||||
}
|
||||
printJSON(map[string]any{"finalized": opts.job, "code": "operator_recovered", "writers": group})
|
||||
return nil
|
||||
}
|
||||
|
||||
func runRecover(ctx context.Context, opts options) error {
|
||||
coordinator, control, err := openFreeze(ctx, opts)
|
||||
if err != nil {
|
||||
|
||||
@@ -182,6 +182,12 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
|
||||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
|
||||
"目标维护未接线,拒绝在挡不住入口/后台写入的情况下写目标:"+managerErr.Error(), "prebackup")
|
||||
}
|
||||
defer manager.Close()
|
||||
// 拓扑必须先过关:写入者发布了端口 = 存在绕闸直连,窗口关的只是半扇门。
|
||||
if topoErr := manager.VerifyTopology(ctx); topoErr != nil {
|
||||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
|
||||
"目标拓扑不满足唯一入口约束,拒绝开维护窗口:"+topoErr.Error(), "prebackup")
|
||||
}
|
||||
targetIdentity := fmt.Sprintf("scope=%s db=%s index=%s bucket=%s", payload.Scope,
|
||||
redactDSN(config.LibraryProductionDatabaseURL), config.LibraryProductionIndex, config.LibraryProductionS3Bucket)
|
||||
originalState, stateErr := manager.OriginalState(ctx)
|
||||
@@ -190,6 +196,19 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
|
||||
"读不到目标写入者的原运行状态,不能开维护窗口:"+stateErr.Error(), "prebackup")
|
||||
}
|
||||
identity := manager.Target()
|
||||
// 实例级占用:internal/external 动的是同一批容器,必须按实例互斥。
|
||||
instanceLock, instanceErr := store.AcquireInstanceMaintenance(ctx, postgresstore.LibraryInstanceMaintenance{
|
||||
Instance: identity.Instance, OwnerJobID: payload.LibraryJobID, OwnerScope: payload.Scope,
|
||||
TargetFingerprint: identity.Fingerprint, Level: librarymaintenance.LevelInstance,
|
||||
Services: identity.Writers, IngressState: "unknown", State: "acquired",
|
||||
OriginalServiceState: originalState,
|
||||
Note: "实例级维护:关闭目标入口并停止全部写入者,验证或回滚完成才开放",
|
||||
}, now)
|
||||
if instanceErr != nil {
|
||||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID,
|
||||
"目标实例正在维护中(跨对内/对外互斥):"+instanceErr.Error(), "prebackup")
|
||||
}
|
||||
_ = instanceLock
|
||||
maintenance, acquireErr := store.AcquireLibraryMaintenance(ctx, postgresstore.LibraryMaintenance{
|
||||
Scope: payload.Scope, OwnerJobID: payload.LibraryJobID,
|
||||
TargetIdentity: targetIdentity + " | " + manager.Description(),
|
||||
@@ -202,6 +221,8 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
|
||||
Note: "实例级维护:关闭目标入口并停止全部写入者,验证或回滚完成才开放",
|
||||
}, now)
|
||||
if acquireErr != nil {
|
||||
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore",
|
||||
"scope 占用申请失败,实例状态未确认:"+acquireErr.Error(), now)
|
||||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "目标正在维护中:"+acquireErr.Error(), "prebackup")
|
||||
}
|
||||
_ = maintenance
|
||||
@@ -213,10 +234,11 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
|
||||
if resumed {
|
||||
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore",
|
||||
"维护协调器不可用,接管目标的状态未确认:"+openErr.Error(), time.Now().UTC())
|
||||
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore",
|
||||
"维护协调器不可用,接管目标的状态未确认:"+openErr.Error(), time.Now().UTC())
|
||||
} else {
|
||||
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "verified",
|
||||
_ = releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID,
|
||||
"未进入维护窗口(协调器不可用),目标未被停用", time.Now().UTC())
|
||||
_ = store.ReleaseLibraryMaintenance(ctx, payload.Scope, payload.LibraryJobID, time.Now().UTC())
|
||||
}
|
||||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "维护协调器不可用:"+openErr.Error(), "prebackup")
|
||||
}
|
||||
@@ -348,8 +370,13 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
|
||||
return manager.VerifyRunning(ctx)
|
||||
}
|
||||
windowWork := func(ctx context.Context, w *libraryfreeze.Window) (string, error) {
|
||||
if err := w.MarkContentRisk(ctx); err != nil {
|
||||
return "", err
|
||||
// 预备份身份与"内容有风险"必须在第一笔写入之前一起落盘:没有预备份身份的
|
||||
// 窗口之后既不能自动恢复,也不许被任何收尾路径开放。
|
||||
recorder.onBackup = func(backupCtx context.Context, ref libraryrelease.BackupRef) error {
|
||||
if err := w.SetBackupSnapshot(backupCtx, ref.SnapshotID); err != nil {
|
||||
return err
|
||||
}
|
||||
return w.MarkContentRisk(backupCtx)
|
||||
}
|
||||
if ingressErr := store.SetLibraryMaintenanceIngress(ctx, payload.Scope, payload.LibraryJobID, "closed",
|
||||
"目标入口已关闭、写入者已停止、在途请求已验证排空", time.Now().UTC()); ingressErr != nil {
|
||||
@@ -369,39 +396,46 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
|
||||
note += ":" + windowErr.Error()
|
||||
}
|
||||
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", note, now2)
|
||||
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.LibraryJobID, "needs_restore", note, now2)
|
||||
if windowErr == nil {
|
||||
return errors.New(note)
|
||||
}
|
||||
return windowErr
|
||||
case contentState == contentSucceeded:
|
||||
_ = store.SetLibraryMaintenanceIngress(ctx, payload.Scope, payload.LibraryJobID, "open", "目标入口已恢复并验证", now2)
|
||||
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "verified",
|
||||
"写入与校验完成,目标已按原运行状态恢复", now2)
|
||||
if err := store.ReleaseLibraryMaintenance(ctx, payload.Scope, payload.LibraryJobID, now2); err != nil {
|
||||
if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID,
|
||||
"写入与校验完成,目标已按原运行状态恢复", now2); err != nil {
|
||||
return fmt.Errorf("目标已恢复但维护占用释放失败: %w", err)
|
||||
}
|
||||
return windowErr
|
||||
case contentState == contentRolledBack:
|
||||
_ = store.SetLibraryMaintenanceIngress(ctx, payload.Scope, payload.LibraryJobID, "open", "已回滚并验证,目标入口恢复", now2)
|
||||
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "verified",
|
||||
"本次失败已回滚到预备份并验证,目标已恢复原运行状态", now2)
|
||||
if err := store.ReleaseLibraryMaintenance(ctx, payload.Scope, payload.LibraryJobID, now2); err != nil {
|
||||
if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID,
|
||||
"本次失败已回滚到预备份并验证,目标已恢复原运行状态", now2); err != nil {
|
||||
return fmt.Errorf("回滚已验证但维护占用释放失败: %w", err)
|
||||
}
|
||||
return windowErr
|
||||
default:
|
||||
// 没有写过目标(闸口/授权/接线/健康检查早退):目标没被停过,直接开放占用。
|
||||
_ = store.SetLibraryMaintenanceIngress(ctx, payload.Scope, payload.LibraryJobID, "open",
|
||||
"未进入维护窗口,目标未被停用", now2)
|
||||
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "verified",
|
||||
"未写目标,维护窗口未开启", now2)
|
||||
if err := store.ReleaseLibraryMaintenance(ctx, payload.Scope, payload.LibraryJobID, now2); err != nil {
|
||||
if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.LibraryJobID,
|
||||
"未写目标,维护窗口未开启", now2); err != nil {
|
||||
return fmt.Errorf("维护占用释放失败: %w", err)
|
||||
}
|
||||
return windowErr
|
||||
}
|
||||
}
|
||||
|
||||
// releaseWindowLocks 是唯一释放路径:先记录入口已恢复,再放 scope 行与实例锁。
|
||||
// 顺序反过来会出现"实例锁放了、scope 还挂着"的中间态,页面就会说正在维护。
|
||||
func releaseWindowLocks(ctx context.Context, store *postgresstore.Store, instance, scope, jobID, note string, now time.Time) error {
|
||||
_ = store.SetInstanceMaintenanceIngress(ctx, instance, jobID, "open", note, now)
|
||||
_ = store.SetLibraryMaintenanceIngress(ctx, scope, jobID, "open", note, now)
|
||||
_ = store.UpdateInstanceMaintenanceState(ctx, instance, jobID, "verified", note, now)
|
||||
_ = store.UpdateLibraryMaintenanceState(ctx, scope, jobID, "verified", note, now)
|
||||
if err := store.ReleaseInstanceMaintenance(ctx, instance, jobID, now); err != nil {
|
||||
return err
|
||||
}
|
||||
return store.ReleaseLibraryMaintenance(ctx, scope, jobID, now)
|
||||
}
|
||||
|
||||
// resumeInterruptedLibraryJobs 在 worker 启动时接管中断的写作业:
|
||||
// 已经持久化预备份的重新排队(同一个业务作业 id,阶段与备份都接着用),
|
||||
// 连预备份都没完成的才判失败——那种情况没有可回滚的基线,不能盲写。
|
||||
@@ -485,6 +519,11 @@ func (s fixedBackupStore) Prepare(context.Context, string, libraryrelease.Curren
|
||||
// 上游据此在写目标前中止,保证"没有可恢复基线就不写"。
|
||||
func (r libraryStageRecorder) BackupPrepared(ctx context.Context, backup libraryrelease.BackupRef) error {
|
||||
now := time.Now().UTC()
|
||||
if r.onBackup != nil {
|
||||
if err := r.onBackup(ctx, backup); err != nil {
|
||||
return fmt.Errorf("落盘预备份身份/内容风险失败: %w", err)
|
||||
}
|
||||
}
|
||||
if err := r.store.AdvanceLibraryJob(ctx, r.jobID, governance.LibraryJobRunning, "", "", "",
|
||||
backup.SnapshotID, "", now); err != nil {
|
||||
return fmt.Errorf("持久化预备份引用失败: %w", err)
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/appconfig"
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/jobs"
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
v071 "knowledge-governance/internal/releaseadapter/v071"
|
||||
)
|
||||
|
||||
// libraryRecoveryPayload 是"按预备份恢复并校验"的队列载荷。
|
||||
// 它只说明动哪个窗口;恢复目标快照由 journal 里持久化的那份决定,调用方改不了。
|
||||
type libraryRecoveryPayload struct {
|
||||
Scope string `json:"scope"`
|
||||
WindowJobID string `json:"window_job_id"`
|
||||
}
|
||||
|
||||
// runLibraryRecovery 是"内容风险未验证"窗口的唯一收尾入口:
|
||||
// 在维护关闭状态下按 journal 里的预备份恢复,再逐类校验(行身份/逐行内容/索引/资源),
|
||||
// 只有校验通过才恢复服务、开入口、释放占用。失败则保持维护。
|
||||
func runLibraryRecovery(ctx context.Context, store *postgresstore.Store, config appconfig.Config, job jobs.Job) error {
|
||||
var payload libraryRecoveryPayload
|
||||
if err := json.Unmarshal(job.Payload, &payload); err != nil ||
|
||||
(payload.Scope != "internal" && payload.Scope != "external") || payload.WindowJobID == "" {
|
||||
return errors.New("invalid library recovery payload")
|
||||
}
|
||||
manager, err := openLibraryMaintenance(config)
|
||||
if err != nil {
|
||||
return fmt.Errorf("目标维护未接线,不能做恢复:%w", err)
|
||||
}
|
||||
defer manager.Close()
|
||||
identity := manager.Target()
|
||||
if topoErr := manager.VerifyTopology(ctx); topoErr != nil {
|
||||
return fmt.Errorf("目标拓扑不满足唯一入口约束,拒绝恢复:%w", topoErr)
|
||||
}
|
||||
|
||||
lock, err := store.InstanceMaintenanceOf(ctx, identity.Instance)
|
||||
if err != nil {
|
||||
return fmt.Errorf("读不到实例维护记录:%w", err)
|
||||
}
|
||||
if lock == nil || lock.ReleasedAt != nil {
|
||||
return errors.New("目标实例当前没有未释放的维护窗口,不需要恢复")
|
||||
}
|
||||
if lock.OwnerJobID != payload.WindowJobID || lock.OwnerScope != payload.Scope {
|
||||
return fmt.Errorf("维护窗口属于作业 %s(scope=%s),与请求不一致", lock.OwnerJobID, lock.OwnerScope)
|
||||
}
|
||||
if lock.TargetFingerprint != identity.Fingerprint {
|
||||
return errors.New("维护窗口的目标身份与当前配置不一致:拒绝用旧窗口恢复另一个目标")
|
||||
}
|
||||
|
||||
// 预备份身份来自治理库里的原作业记录;协调器还会再与 journal 里的那份核对。
|
||||
windowJob, err := store.LibraryJob(ctx, payload.WindowJobID)
|
||||
if err != nil || windowJob.SnapshotID == "" {
|
||||
return errors.New("原作业没有持久化预备份,无法证明恢复到哪一份")
|
||||
}
|
||||
prebackup := windowJob.SnapshotID
|
||||
knownMapping := knownIndexMapping(ctx, store, payload.Scope)
|
||||
target, _, wireErr := buildLibraryProductionStack(ctx, config, payload.Scope, prebackup, knownMapping)
|
||||
if wireErr != nil {
|
||||
return fmt.Errorf("恢复目标装配失败:%w", wireErr)
|
||||
}
|
||||
defer target.Close()
|
||||
|
||||
coordinator, control, openErr := manager.Open(config.LibraryTargetJournalRoot)
|
||||
if openErr != nil {
|
||||
return fmt.Errorf("维护协调器不可用:%w", openErr)
|
||||
}
|
||||
defer control.Close()
|
||||
defer coordinator.Close()
|
||||
|
||||
restorer := recoveryRestorer{target: target, expected: prebackup}
|
||||
verifier := recoveryVerifier{target: target, expected: prebackup}
|
||||
result, recoverErr := coordinator.RecoverWindow(ctx, payload.WindowJobID, restorer, verifier)
|
||||
now := time.Now().UTC()
|
||||
if recoverErr != nil {
|
||||
note := "恢复/校验未通过,目标保持维护:" + recoverErr.Error()
|
||||
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now)
|
||||
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.WindowJobID, "needs_restore", note, now)
|
||||
return recoverErr
|
||||
}
|
||||
if result.State != "recovered" {
|
||||
note := "恢复流程结束但状态不是 recovered:" + result.State
|
||||
_ = store.UpdateInstanceMaintenanceState(ctx, identity.Instance, payload.WindowJobID, "needs_restore", note, now)
|
||||
return errors.New(note)
|
||||
}
|
||||
if err := releaseWindowLocks(ctx, store, identity.Instance, payload.Scope, payload.WindowJobID,
|
||||
"按预备份恢复并逐类校验通过,目标已按原运行状态恢复", now); err != nil {
|
||||
return fmt.Errorf("恢复已验证,但维护占用释放失败:%w", err)
|
||||
}
|
||||
log.Printf("library recovery succeeded window=%s scope=%s backup=%s", payload.WindowJobID, payload.Scope, prebackup)
|
||||
return nil
|
||||
}
|
||||
|
||||
// recoveryRestorer 只按"作业记录里那份预备份"恢复:协调器传进来的 id 与它不一致
|
||||
// 就拒绝——那是两份快照,恢复出来的东西不能算数。
|
||||
type recoveryRestorer struct {
|
||||
target *v071.ProductionLibraryTarget
|
||||
expected string
|
||||
}
|
||||
|
||||
func (r recoveryRestorer) Restore(ctx context.Context, backupID string) error {
|
||||
if backupID != r.expected {
|
||||
return fmt.Errorf("预备份身份不一致(journal=%s 作业=%s)", backupID, r.expected)
|
||||
}
|
||||
return r.target.Restore(ctx, libraryrelease.BackupRef{ID: "backup-" + backupID, SnapshotID: backupID})
|
||||
}
|
||||
|
||||
// recoveryVerifier 做真实的内容校验,并把目标当前状态摘要作为"校验过什么"的凭证。
|
||||
type recoveryVerifier struct {
|
||||
target *v071.ProductionLibraryTarget
|
||||
expected string
|
||||
}
|
||||
|
||||
func (v recoveryVerifier) Verify(ctx context.Context, backupID string) (string, error) {
|
||||
if backupID != v.expected {
|
||||
return "", fmt.Errorf("预备份身份不一致(journal=%s 作业=%s)", backupID, v.expected)
|
||||
}
|
||||
// Verify 逐表比行数、身份集合、逐行内容摘要,再比索引文档与对象资源。
|
||||
if err := v.target.Verify(ctx, libraryrelease.Expectation{SnapshotID: backupID}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
state, err := v.target.Current(ctx)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("恢复后读不到目标状态:%w", err)
|
||||
}
|
||||
raw, err := json.Marshal(state)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
sum := sha256.Sum256(raw)
|
||||
return hex.EncodeToString(sum[:]), nil
|
||||
}
|
||||
|
||||
// knownIndexMapping 取"上一次登记过的索引映射",作为装配目标时的映射基线。
|
||||
func knownIndexMapping(ctx context.Context, store *postgresstore.Store, scope string) string {
|
||||
versions, err := store.LibraryVersions(ctx, scope, 50)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
for _, item := range versions {
|
||||
if item.Kind == governance.LibraryVersionProductionEffective && item.SupersededAt == nil {
|
||||
if raw, ok := item.Evidence["index_mapping_sha256"].(string); ok && raw != "" {
|
||||
return raw
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, item := range versions {
|
||||
if raw, ok := item.Evidence["index_mapping_sha256"].(string); ok && raw != "" {
|
||||
return raw
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"knowledge-governance/internal/governance"
|
||||
"knowledge-governance/internal/jobs"
|
||||
"knowledge-governance/internal/libraryagent"
|
||||
"knowledge-governance/internal/libraryrelease"
|
||||
"knowledge-governance/internal/librarysnapshot"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
"knowledge-governance/internal/platform/weknora"
|
||||
@@ -36,6 +37,8 @@ type libraryStageRecorder struct {
|
||||
totals map[string]stageSize
|
||||
// mappingDigest 是本次读到的生产索引映射,随预备份一起登记成版本证据。
|
||||
mappingDigest string
|
||||
// onBackup 在预备份完成后、写目标之前调用:把"预备份身份 + 内容风险"落盘。
|
||||
onBackup func(context.Context, libraryrelease.BackupRef) error
|
||||
}
|
||||
|
||||
func (r libraryStageRecorder) Stage(ctx context.Context, stage, state, detail string) {
|
||||
@@ -100,6 +103,9 @@ const libraryCaptureJobType = "library.capture"
|
||||
// librarySyncJobType 是生产同步作业的队列类型;与封存严格分开,状态不共用。
|
||||
const librarySyncJobType = "library.sync"
|
||||
|
||||
// libraryRecoverJobType 是"按预备份恢复并校验"的队列类型;与 API 侧常量同值。
|
||||
const libraryRecoverJobType = "library.recover"
|
||||
|
||||
// libraryRestoreJobType 是按指定整库版本恢复的队列类型;与同步分开、授权也分开。
|
||||
const libraryRestoreJobType = "library.restore"
|
||||
|
||||
@@ -177,6 +183,9 @@ func run(ctx context.Context, getenv func(string) string) error {
|
||||
handlers[libraryRestoreJobType] = func(ctx context.Context, job jobs.Job) error {
|
||||
return runLibraryRelease(ctx, store, config, job, governance.LibraryJobRestore)
|
||||
}
|
||||
handlers[libraryRecoverJobType] = func(ctx context.Context, job jobs.Job) error {
|
||||
return runLibraryRecovery(ctx, store, config, job)
|
||||
}
|
||||
|
||||
worker, err := jobs.NewWorker(jobs.WorkerConfig{
|
||||
Queue: store,
|
||||
|
||||
@@ -105,7 +105,11 @@ type Config struct {
|
||||
// 入口闸门:关入口与排空在它身上做(被代理服务自己不排空)。
|
||||
LibraryTargetIngressService string
|
||||
LibraryTargetIngressDrainFile string
|
||||
LibraryTargetJournalRoot string
|
||||
// 后台静默检查:目标队列 + 目标库(HTTP 排空不等于后台任务排空)。
|
||||
LibraryTargetRedisAddr string
|
||||
LibraryTargetRedisPassword string
|
||||
LibraryTargetRedisDB int
|
||||
LibraryTargetJournalRoot string
|
||||
// LibraryAgentSocket 是主机侧封存 agent 的 unix socket(只由 worker 连接)。
|
||||
// 留空即"未接线":页面不显示可用的封存入口,接口返回原因而不是假装已排队。
|
||||
LibraryAgentSocket string
|
||||
@@ -202,6 +206,9 @@ func Load(getenv func(string) string) (Config, error) {
|
||||
LibraryTargetIngressURL: strings.TrimSpace(getenv("LIBRARY_TARGET_INGRESS_URL")),
|
||||
LibraryTargetIngressService: strings.TrimSpace(getenv("LIBRARY_TARGET_INGRESS_SERVICE")),
|
||||
LibraryTargetIngressDrainFile: strings.TrimSpace(getenv("LIBRARY_TARGET_INGRESS_DRAIN_FILE")),
|
||||
LibraryTargetRedisAddr: strings.TrimSpace(getenv("LIBRARY_TARGET_REDIS_ADDR")),
|
||||
LibraryTargetRedisPassword: strings.TrimSpace(getenv("LIBRARY_TARGET_REDIS_PASSWORD")),
|
||||
LibraryTargetRedisDB: valueOrDefaultInt(getenv("LIBRARY_TARGET_REDIS_DB"), 0),
|
||||
LibraryTargetJournalRoot: strings.TrimSpace(getenv("LIBRARY_TARGET_JOURNAL_ROOT")),
|
||||
LibraryEmbeddingModel: valueOrDefault(strings.TrimSpace(getenv("LIBRARY_EMBEDDING_MODEL")), "text-embedding-v4"),
|
||||
LibraryEmbeddingDimension: embeddingDimension,
|
||||
@@ -349,6 +356,9 @@ func Load(getenv func(string) string) (Config, error) {
|
||||
if config.LibraryTargetIngressDrainFile == "" {
|
||||
missing = append(missing, "LIBRARY_TARGET_INGRESS_DRAIN_FILE")
|
||||
}
|
||||
if config.LibraryTargetRedisAddr == "" {
|
||||
missing = append(missing, "LIBRARY_TARGET_REDIS_ADDR")
|
||||
}
|
||||
if len(missing) > 0 {
|
||||
return Config{}, fmt.Errorf("目标维护配置不完整:缺少 %s", strings.Join(missing, "、"))
|
||||
}
|
||||
|
||||
@@ -42,6 +42,9 @@ type libraryVersionView struct {
|
||||
// librarySyncJobType 与 worker 的处理器类型一致;单独定义避免 API 依赖 worker 包。
|
||||
const librarySyncJobType = "library.sync"
|
||||
|
||||
// libraryRecoverJobType 是"按预备份恢复并校验"的队列类型(同一份定义也在 worker 里)。
|
||||
const libraryRecoverJobType = "library.recover"
|
||||
|
||||
type libraryKnownView struct {
|
||||
State string `json:"state"` // known | unknown
|
||||
Version *libraryVersionView `json:"version,omitempty"`
|
||||
@@ -291,6 +294,101 @@ func libraryStore(config Config) (libraryJobStore, bool) {
|
||||
return store, ok
|
||||
}
|
||||
|
||||
// instanceMaintenanceGuard 在"实例级维护"被别的作业持有时拒绝新的写入请求。
|
||||
// 只看 scope 行是不够的:internal 与 external 动的是同一批容器。
|
||||
func instanceMaintenanceGuard(ctx context.Context, config Config, scope string) (map[string]any, bool) {
|
||||
instance := config.LibraryTargetInstance
|
||||
if instance == "" {
|
||||
return nil, false
|
||||
}
|
||||
store, ok := config.Store.(interface {
|
||||
InstanceMaintenanceOf(ctx context.Context, instance string) (*postgresstore.LibraryInstanceMaintenance, error)
|
||||
})
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
held, err := store.InstanceMaintenanceOf(ctx, instance)
|
||||
if err != nil || held == nil || held.ReleasedAt != nil {
|
||||
return nil, false
|
||||
}
|
||||
return map[string]any{"code": "library_target_in_maintenance",
|
||||
"error": "目标实例正在维护中,验证完成前不接受新的写入请求",
|
||||
"instance": held.Instance, "owner_job": held.OwnerJobID, "owner_scope": held.OwnerScope,
|
||||
"state": held.State, "ingress_state": held.IngressState}, true
|
||||
}
|
||||
|
||||
// libraryMaintenanceRecoverHandler 是"按预备份恢复并校验"的唯一入口:
|
||||
// 它只建一个恢复作业,真正的恢复与校验由 worker 在维护关闭状态下执行。
|
||||
func libraryMaintenanceRecoverHandler(w http.ResponseWriter, r *http.Request, config Config) {
|
||||
if r.Method != http.MethodPost {
|
||||
writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"})
|
||||
return
|
||||
}
|
||||
var request struct {
|
||||
Scope string `json:"scope"`
|
||||
WindowJobID string `json:"window_job_id"`
|
||||
}
|
||||
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&request); err != nil {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"})
|
||||
return
|
||||
}
|
||||
if request.Scope != "internal" && request.Scope != "external" {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "scope must be internal or external"})
|
||||
return
|
||||
}
|
||||
instance := config.LibraryTargetInstance
|
||||
if instance == "" {
|
||||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"code": "library_recovery_unavailable",
|
||||
"error": "目标维护未接线:无法执行按预备份恢复"})
|
||||
return
|
||||
}
|
||||
store, ok := config.Store.(interface {
|
||||
InstanceMaintenanceOf(ctx context.Context, instance string) (*postgresstore.LibraryInstanceMaintenance, error)
|
||||
})
|
||||
if !ok {
|
||||
writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "维护记录存储不可用"})
|
||||
return
|
||||
}
|
||||
held, err := store.InstanceMaintenanceOf(r.Context(), instance)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "无法读取维护记录"})
|
||||
return
|
||||
}
|
||||
if held == nil || held.ReleasedAt != nil {
|
||||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_no_held_window",
|
||||
"error": "目标实例当前没有未释放的维护窗口,不需要恢复"})
|
||||
return
|
||||
}
|
||||
if held.OwnerScope != request.Scope {
|
||||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_window_scope_mismatch",
|
||||
"error": "维护窗口属于另一个 scope", "owner_scope": held.OwnerScope})
|
||||
return
|
||||
}
|
||||
if request.WindowJobID != "" && request.WindowJobID != held.OwnerJobID {
|
||||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_window_mismatch",
|
||||
"error": "维护窗口属于另一个作业", "owner_job": held.OwnerJobID})
|
||||
return
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
payload, err := json.Marshal(map[string]string{"scope": held.OwnerScope, "window_job_id": held.OwnerJobID})
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot build recovery payload"})
|
||||
return
|
||||
}
|
||||
// 队列作业 id 与幂等键都按窗口固定:重复点击只会得到同一个恢复作业。
|
||||
queueJob, err := config.Store.Enqueue(r.Context(), jobs.EnqueueRequest{
|
||||
ID: "librecover-" + held.OwnerJobID, Type: libraryRecoverJobType, Payload: payload,
|
||||
AvailableAt: now, MaxAttempts: 1, IdempotencyKey: "library.recover:" + held.OwnerJobID,
|
||||
}, now)
|
||||
if err != nil || queueJob == nil {
|
||||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_recovery_enqueue_failed",
|
||||
"error": "恢复作业投递失败(或已有同窗口的恢复作业)", "detail": errString(err)})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"job": queueJob, "window_job_id": held.OwnerJobID,
|
||||
"scope": held.OwnerScope, "instance": held.Instance})
|
||||
}
|
||||
|
||||
// activeGrantFor 只查"目标+快照+同一参数摘要"的未过期未撤销授权。
|
||||
func activeGrantFor(ctx context.Context, config Config, scope, snapshotID string) (governance.LibraryReleaseAuthorization, bool) {
|
||||
if snapshotID == "" {
|
||||
@@ -597,6 +695,11 @@ func librarySyncEnqueue(w http.ResponseWriter, r *http.Request, config Config) {
|
||||
}
|
||||
grantFound, hasGrant := activeGrantFor(r.Context(), config, request.Scope, request.Target)
|
||||
_ = grantFound
|
||||
// 目标实例正在维护(跨对内/对外互斥):直接拒绝,不等到 worker 里才发现。
|
||||
if body, held := instanceMaintenanceGuard(r.Context(), config, request.Scope); held {
|
||||
writeJSON(w, http.StatusConflict, body)
|
||||
return
|
||||
}
|
||||
// 目标正在维护(被别的作业持有):直接拒绝新写入,不等到 worker 里才发现。
|
||||
if maintenanceStore, ok := config.Store.(interface {
|
||||
LibraryMaintenanceOf(ctx context.Context, scope string) (*postgresstore.LibraryMaintenance, error)
|
||||
|
||||
@@ -54,6 +54,10 @@ func libraryRestoreEnqueue(w http.ResponseWriter, r *http.Request, config Config
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "snapshot_id required"})
|
||||
return
|
||||
}
|
||||
if body, held := instanceMaintenanceGuard(r.Context(), config, request.Scope); held {
|
||||
writeJSON(w, http.StatusConflict, body)
|
||||
return
|
||||
}
|
||||
digest, err := libraryRestoreParamsDigest(request.Scope, request.SnapshotID)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot digest params"})
|
||||
|
||||
@@ -145,6 +145,9 @@ type Config struct {
|
||||
VersionHistoryRepo string
|
||||
// LibraryArchiveRoot 是整库封存产物的只读归档根;留空即未接线(页面降级)。
|
||||
LibraryArchiveRoot string
|
||||
// LibraryTargetInstance 是实例级维护的目标实例名;留空即"未接线",
|
||||
// 页面不能发起按预备份恢复,接口返回原因而不是假装能恢复。
|
||||
LibraryTargetInstance string
|
||||
// LibraryAgentSocket 是主机侧封存 agent 的 unix socket;留空即页面不能发起封存。
|
||||
LibraryAgentSocket string
|
||||
// LibraryCapacityBudgetBytes 是整库归档容量预算;0 = 未设置(容量门禁不放行)。
|
||||
@@ -461,7 +464,22 @@ func NewHandler(config Config) (http.Handler, error) {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "无法读取维护窗口"})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"held": held != nil && held.ReleasedAt == nil, "maintenance": held})
|
||||
response := map[string]any{"held": held != nil && held.ReleasedAt == nil, "maintenance": held}
|
||||
// 实例级占用才是真正的互斥来源:跨 scope 也在这里体现。
|
||||
if instanceStore, ok := config.Store.(interface {
|
||||
InstanceMaintenanceOf(ctx context.Context, instance string) (*postgresstore.LibraryInstanceMaintenance, error)
|
||||
}); ok && config.LibraryTargetInstance != "" {
|
||||
if instanceHeld, err := instanceStore.InstanceMaintenanceOf(r.Context(), config.LibraryTargetInstance); err == nil {
|
||||
response["instance"] = config.LibraryTargetInstance
|
||||
response["instance_maintenance"] = instanceHeld
|
||||
response["held"] = instanceHeld != nil && instanceHeld.ReleasedAt == nil
|
||||
}
|
||||
}
|
||||
writeJSON(w, http.StatusOK, response)
|
||||
})
|
||||
// 按预备份恢复并逐类校验:维护窗口卡住时唯一的收尾入口(worker 执行)。
|
||||
mux.HandleFunc("POST /api/v1/library/maintenance/recover", func(w http.ResponseWriter, r *http.Request) {
|
||||
libraryMaintenanceRecoverHandler(w, r, config)
|
||||
})
|
||||
mux.HandleFunc("GET /api/v1/library/jobs", func(w http.ResponseWriter, r *http.Request) {
|
||||
libraryJobsHandler(w, r, config)
|
||||
|
||||
@@ -47,6 +47,15 @@ type Gate interface {
|
||||
OpenIngress(context.Context) error
|
||||
}
|
||||
|
||||
// Quiescer 回答"后台写入者是不是已经空了"。
|
||||
//
|
||||
// 为什么 HTTP 排空不够:被代理的 WeKnora 在 SIGTERM 后会 exit(1),app 内的 asynq
|
||||
// worker 正在跑的后台任务会被直接切断。要么等它们跑完,要么明确不进入窗口。
|
||||
type Quiescer interface {
|
||||
// Idle 返回 true 表示此刻没有排队/在跑的后台任务,也没有未结清的处理记录。
|
||||
Idle(ctx context.Context) (bool, string, error)
|
||||
}
|
||||
|
||||
// Inventory must enumerate the complete, independently scoped test writer set,
|
||||
// using immutable instance IDs. Never implement it as only the currently running
|
||||
// subset. Start must wait for readiness; callers still read back the result.
|
||||
@@ -81,12 +90,23 @@ type journal struct {
|
||||
// Gate 记录入口关闭/排空的证据;没有它就不能说"入口关了"。
|
||||
// 入口恢复后证据仍保留(GateReopened=true),否则事后查不到
|
||||
// "关的时候还有几个在途请求"。
|
||||
Gate *GateEvidence `json:"gate,omitempty"`
|
||||
GateReopened bool `json:"gate_reopened,omitempty"`
|
||||
Services []serviceRecord `json:"services"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
FinishedAt *time.Time `json:"finished_at,omitempty"`
|
||||
Code string `json:"code,omitempty"`
|
||||
Gate *GateEvidence `json:"gate,omitempty"`
|
||||
GateReopened bool `json:"gate_reopened,omitempty"`
|
||||
// BackupSnapshotID 是这次窗口开始写之前落盘的那份预备份:恢复只能按它来,
|
||||
// 不允许事后换一份"看起来更好"的快照。
|
||||
BackupSnapshotID string `json:"backup_snapshot_id,omitempty"`
|
||||
// TargetFingerprint 是开窗口时的目标身份:恢复时必须还是同一个目标。
|
||||
TargetFingerprint string `json:"target_fingerprint,omitempty"`
|
||||
// BackgroundIdle 记录"停写入者之前后台任务队列确认为空"。
|
||||
BackgroundIdle bool `json:"background_idle,omitempty"`
|
||||
// 恢复证据:只有真正跑完"恢复 + 校验"才会写这三项。
|
||||
RecoveredBackup string `json:"recovered_backup,omitempty"`
|
||||
RecoveryDigest string `json:"recovery_digest,omitempty"`
|
||||
RecoveredAt *time.Time `json:"recovered_at,omitempty"`
|
||||
Services []serviceRecord `json:"services"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
FinishedAt *time.Time `json:"finished_at,omitempty"`
|
||||
Code string `json:"code,omitempty"`
|
||||
}
|
||||
type Coordinator struct {
|
||||
root *os.Root
|
||||
@@ -99,6 +119,9 @@ type Coordinator struct {
|
||||
ingressClosed func(context.Context) error
|
||||
ingressReady func(context.Context) error
|
||||
gate Gate
|
||||
quiescer Quiescer
|
||||
fingerprint string
|
||||
quiesceWait time.Duration
|
||||
}
|
||||
|
||||
func Open(root, source string, c Controller) (*Coordinator, error) {
|
||||
@@ -149,6 +172,37 @@ func (c *Coordinator) SetGate(g Gate) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetTargetFingerprint 绑住"这次维护动的是哪个目标"。恢复时必须与 journal
|
||||
// 里记录的一致,否则说明配置换了目标,禁止拿旧窗口去收尾新目标。
|
||||
func (c *Coordinator) SetTargetFingerprint(fingerprint string) error {
|
||||
if len(fingerprint) != 64 {
|
||||
return errors.New("invalid target fingerprint")
|
||||
}
|
||||
if _, err := hex.DecodeString(fingerprint); err != nil {
|
||||
return errors.New("invalid target fingerprint")
|
||||
}
|
||||
c.fingerprint = strings.ToLower(fingerprint)
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetQuiesceTimeout 调整"等后台任务静默"的上限(默认 2 分钟)。
|
||||
func (c *Coordinator) SetQuiesceTimeout(d time.Duration) error {
|
||||
if d <= 0 {
|
||||
return errors.New("invalid quiesce timeout")
|
||||
}
|
||||
c.quiesceWait = d
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetQuiescer 声明后台任务静默检查(HTTP 排空不等于后台任务排空)。
|
||||
func (c *Coordinator) SetQuiescer(q Quiescer) error {
|
||||
if q == nil {
|
||||
return errors.New("invalid quiescer")
|
||||
}
|
||||
c.quiescer = q
|
||||
return nil
|
||||
}
|
||||
|
||||
func validIngress(raw string) bool {
|
||||
parsed, err := url.Parse(raw)
|
||||
return err == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") && parsed.Host != "" && parsed.Path != "" && parsed.User == nil
|
||||
@@ -252,7 +306,7 @@ func validID(s string) bool {
|
||||
}
|
||||
return true
|
||||
}
|
||||
func terminal(s string) bool { return s == "succeeded" || s == "failed" }
|
||||
func terminal(s string) bool { return s == "succeeded" || s == "failed" || s == "recovered" }
|
||||
|
||||
// A kernel lock serializes the whole test writer group, not individual KBs.
|
||||
// A crash releases the lock, but leaves the write-ahead journal for recovery.
|
||||
@@ -328,7 +382,7 @@ func (c *Coordinator) load(job string) (journal, error) {
|
||||
return journal{}, errors.New("freeze journal identity mismatch")
|
||||
}
|
||||
switch j.State {
|
||||
case "stopping", "held", "capturing", "restoring", "succeeded", "failed":
|
||||
case "stopping", "held", "capturing", "recovering", "restoring", "succeeded", "failed", "recovered":
|
||||
default:
|
||||
return journal{}, errors.New("invalid freeze journal state")
|
||||
}
|
||||
@@ -351,6 +405,15 @@ func (c *Coordinator) load(job string) (journal, error) {
|
||||
if j.GateReopened && j.Gate == nil {
|
||||
return journal{}, errors.New("ingress reopened without close evidence")
|
||||
}
|
||||
if j.BackupSnapshotID != "" && !validDigest(j.BackupSnapshotID) {
|
||||
return journal{}, errors.New("invalid backup snapshot id")
|
||||
}
|
||||
// 恢复证据只允许指向"这次窗口自己那份预备份",且必须带校验摘要。
|
||||
if j.RecoveredAt != nil {
|
||||
if j.RecoveredBackup == "" || j.RecoveredBackup != j.BackupSnapshotID || j.RecoveryDigest == "" {
|
||||
return journal{}, errors.New("invalid recovery evidence")
|
||||
}
|
||||
}
|
||||
if len(j.Services) == 0 || len(j.Services) > 256 {
|
||||
return journal{}, errors.New("invalid freeze journal services")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
package libraryfreeze
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func heldJournal(job string, fingerprint, backup string) journal {
|
||||
j := journal{Version: 1, Source: "target-instance", Job: job, Level: "instance",
|
||||
Ingress: "http://127.0.0.1:1/health", TargetFingerprint: fingerprint,
|
||||
ContentRisk: true, RestoreBlocked: true, BackupSnapshotID: backup,
|
||||
Result: Result{State: "held"}, StartedAt: time.Now().UTC(),
|
||||
Gate: &GateEvidence{Closed: true, Drained: true}}
|
||||
j.Services = []serviceRecord{
|
||||
{Service: Service{ID: "app", Running: true, Healthy: true}, StopRequested: true, StopVerified: true},
|
||||
{Service: Service{ID: "store", Running: true, Healthy: true}, StopRequested: true, StopVerified: true},
|
||||
}
|
||||
return j
|
||||
}
|
||||
|
||||
func stopAllServices(c *exitControl) {
|
||||
for i := range c.services {
|
||||
c.services[i].Running, c.services[i].Healthy = false, false
|
||||
}
|
||||
}
|
||||
|
||||
// 收尾只认"真的跑过恢复 + 校验",不认调用方自称已恢复。
|
||||
func TestRecoverWindowRestoresBeforeOpening(t *testing.T) {
|
||||
_, c, e := instanceFixture(t)
|
||||
stopAllServices(c)
|
||||
if err := e.save(heldJournal("job-rec", strings.Repeat("a", 64), strings.Repeat("b", 64))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := e.SetTargetFingerprint(strings.Repeat("a", 64)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gate := &fakeGate{calls: &c.calls, drain: true}
|
||||
if err := e.SetGate(gate); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var restoredWith, verifiedWith string
|
||||
var order []string
|
||||
restorer := restorerFunc(func(_ context.Context, id string) error {
|
||||
restoredWith = id
|
||||
order = append(order, "restore")
|
||||
return nil
|
||||
})
|
||||
verifier := verifierFunc(func(_ context.Context, id string) (string, error) {
|
||||
verifiedWith = id
|
||||
order = append(order, "verify")
|
||||
return "digest-ok", nil
|
||||
})
|
||||
|
||||
if _, err := e.RecoverWindow(context.Background(), "job-rec", restorer, verifier); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if restoredWith != strings.Repeat("b", 64) || verifiedWith != restoredWith {
|
||||
t.Fatalf("restore=%q verify=%q", restoredWith, verifiedWith)
|
||||
}
|
||||
if strings.Join(order, ",") != "restore,verify" {
|
||||
t.Fatalf("order=%v", order)
|
||||
}
|
||||
for _, s := range c.services {
|
||||
if !s.Running || !s.Healthy {
|
||||
t.Fatalf("service %s not restored after verified recovery", s.ID)
|
||||
}
|
||||
}
|
||||
after, err := e.load("job-rec")
|
||||
if err != nil || after.State != "recovered" || after.RecoveredBackup != restoredWith ||
|
||||
after.RecoveryDigest != "digest-ok" || after.RecoveredAt == nil {
|
||||
t.Fatalf("journal=%+v err=%v", after, err)
|
||||
}
|
||||
}
|
||||
|
||||
// 校验不通过 = 恢复未成功:服务保持停止、journal 非终态。
|
||||
func TestRecoverWindowKeepsTargetClosedWhenVerificationFails(t *testing.T) {
|
||||
_, c, e := instanceFixture(t)
|
||||
stopAllServices(c)
|
||||
if err := e.SetTargetFingerprint(strings.Repeat("a", 64)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := e.SetGate(&fakeGate{calls: &c.calls, drain: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := e.save(heldJournal("job-bad", strings.Repeat("a", 64), strings.Repeat("b", 64))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := e.RecoverWindow(context.Background(), "job-bad",
|
||||
restorerFunc(func(context.Context, string) error { return nil }),
|
||||
verifierFunc(func(context.Context, string) (string, error) {
|
||||
return "", errors.New("chunks 内容与预备份不一致")
|
||||
})); err == nil {
|
||||
t.Fatal("verification failure reported success")
|
||||
}
|
||||
for _, s := range c.services {
|
||||
if s.Running {
|
||||
t.Fatalf("service %s was started although content verification failed", s.ID)
|
||||
}
|
||||
}
|
||||
after, err := e.load("job-bad")
|
||||
if err != nil || terminal(after.State) || !after.RestoreBlocked || after.RecoveredAt != nil {
|
||||
t.Fatalf("journal=%+v err=%v", after, err)
|
||||
}
|
||||
}
|
||||
|
||||
// 没有落盘预备份身份的窗口不许"恢复":那等于凭空开放。
|
||||
func TestRecoverWindowRequiresPersistedBackup(t *testing.T) {
|
||||
_, c, e := instanceFixture(t)
|
||||
stopAllServices(c)
|
||||
j := heldJournal("job-nobak", strings.Repeat("a", 64), "")
|
||||
if err := e.save(j); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
called := false
|
||||
if _, err := e.RecoverWindow(context.Background(), "job-nobak",
|
||||
restorerFunc(func(context.Context, string) error { called = true; return nil }),
|
||||
verifierFunc(func(context.Context, string) (string, error) { return "x", nil })); err == nil || called {
|
||||
t.Fatalf("recovery ran without a persisted backup: err=%v called=%v", err, called)
|
||||
}
|
||||
for _, s := range c.services {
|
||||
if s.Running {
|
||||
t.Fatal("writer reopened without a backup")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 目标换了(指纹不一致)就不许拿旧窗口收尾。
|
||||
func TestRecoverWindowRejectsDifferentTarget(t *testing.T) {
|
||||
_, c, e := instanceFixture(t)
|
||||
stopAllServices(c)
|
||||
if err := e.SetTargetFingerprint(strings.Repeat("c", 64)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := e.save(heldJournal("job-other", strings.Repeat("a", 64), strings.Repeat("b", 64))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := e.RecoverWindow(context.Background(), "job-other",
|
||||
restorerFunc(func(context.Context, string) error { return nil }),
|
||||
verifierFunc(func(context.Context, string) (string, error) { return "x", nil })); err == nil {
|
||||
t.Fatal("recovery accepted another target's journal")
|
||||
}
|
||||
}
|
||||
|
||||
// 后台任务还在跑时不许开窗口(HTTP 排空覆盖不到它们)。
|
||||
func TestWindowRefusesWhenBackgroundBusy(t *testing.T) {
|
||||
_, c, e := instanceFixture(t)
|
||||
if err := e.SetQuiescer(&idleQuiescer{busy: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := e.SetQuiesceTimeout(100 * time.Millisecond); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
work := 0
|
||||
_, err := e.RunWindow(context.Background(), "job-busy", func(context.Context) error { return nil },
|
||||
func(context.Context, *Window) (string, error) { work++; return "", nil }, func(error) bool { return true })
|
||||
if err == nil || work != 0 {
|
||||
t.Fatalf("err=%v work=%d", err, work)
|
||||
}
|
||||
for _, s := range c.services {
|
||||
if !s.Running {
|
||||
t.Fatal("writers were stopped although the window never opened")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type restorerFunc func(context.Context, string) error
|
||||
|
||||
func (f restorerFunc) Restore(ctx context.Context, id string) error { return f(ctx, id) }
|
||||
|
||||
type verifierFunc func(context.Context, string) (string, error)
|
||||
|
||||
func (f verifierFunc) Verify(ctx context.Context, id string) (string, error) { return f(ctx, id) }
|
||||
@@ -0,0 +1,141 @@
|
||||
package libraryfreeze
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Restorer 把目标内容恢复成 backupID 描述的那一份。
|
||||
// 协调器只会在"维护关闭、写入者停止"的状态下调用它,并且只传 journal 里
|
||||
// 持久化的那份预备份身份——调用方没有机会换一份。
|
||||
type Restorer interface {
|
||||
Restore(ctx context.Context, backupID string) error
|
||||
}
|
||||
|
||||
// Verifier 逐类比对目标与 backupID 的内容(行身份 + 逐行内容 + 索引 + 资源)。
|
||||
// 返回的摘要会被持久化;返回错误即视为恢复未成功。
|
||||
type Verifier interface {
|
||||
Verify(ctx context.Context, backupID string) (string, error)
|
||||
}
|
||||
|
||||
// RecoverWindow 是"内容风险未验证"窗口的唯一收尾方式:
|
||||
//
|
||||
// 绑定(原作业 + 目标指纹 + 预备份)→ 重新关入口/停写入者 → 恢复 → 真校验
|
||||
// → 持久记录校验摘要 → 恢复服务与入口 → 才允许调用方释放占用
|
||||
//
|
||||
// 它不接受任何"已恢复/已验证"的入参:是否成功由这里跑出来的结果决定。
|
||||
// 任何一步失败都保持关闭(服务停着、入口关着、journal 非终态),不会放行半写目标。
|
||||
func (c *Coordinator) RecoverWindow(ctx context.Context, job string, restorer Restorer, verifier Verifier) (Result, error) {
|
||||
if !validID(job) || restorer == nil || verifier == nil {
|
||||
return Result{}, errors.New("invalid recovery request")
|
||||
}
|
||||
lock, err := c.lock()
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
defer unlock(lock)
|
||||
|
||||
j, err := c.load(job)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
if terminal(j.State) {
|
||||
return Result{}, errors.New("freeze journal is already terminal")
|
||||
}
|
||||
if !j.ContentRisk {
|
||||
return Result{}, errors.New("journal has no content risk; use Recover instead")
|
||||
}
|
||||
// 绑定:预备份身份必须已经在写目标之前落盘,目标指纹必须还是同一个。
|
||||
if !validDigest(j.BackupSnapshotID) {
|
||||
return Result{}, errors.New("窗口没有落盘预备份身份,无法证明恢复目标,必须人工按归档恢复")
|
||||
}
|
||||
if c.fingerprint != "" && j.Level == "instance" {
|
||||
if j.TargetFingerprint == "" || j.TargetFingerprint != c.fingerprint {
|
||||
return Result{}, errors.New("目标身份与维护记录不一致:拒绝用旧窗口收尾另一个目标")
|
||||
}
|
||||
}
|
||||
|
||||
// 恢复必须在维护关闭状态下做:入口先关(若已关则沿用),写入者必须停住。
|
||||
if c.gate != nil && (j.Gate == nil || !j.Gate.Drained) {
|
||||
evidence, gateErr := c.gate.CloseIngress(ctx)
|
||||
if gateErr != nil {
|
||||
return j.Result, fmt.Errorf("恢复前无法关闭目标入口:%w", gateErr)
|
||||
}
|
||||
if !evidence.Closed || !evidence.Drained {
|
||||
return j.Result, errors.New("恢复前入口未确认排空")
|
||||
}
|
||||
j.Gate = &evidence
|
||||
if err := c.save(j); err != nil {
|
||||
return j.Result, err
|
||||
}
|
||||
}
|
||||
if err := c.stopAll(ctx, &j); err != nil {
|
||||
return j.Result, fmt.Errorf("恢复前无法停下写入者:%w", err)
|
||||
}
|
||||
if err := c.stopped(ctx, j); err != nil {
|
||||
return j.Result, fmt.Errorf("恢复前写入者没有全部停住:%w", err)
|
||||
}
|
||||
if err := c.verifyDrain(ctx, &j); err != nil {
|
||||
return j.Result, fmt.Errorf("恢复前排空未确认:%w", err)
|
||||
}
|
||||
if err := c.awaitBackgroundIdle(ctx, &j); err != nil {
|
||||
return j.Result, fmt.Errorf("恢复前后台任务未静默:%w", err)
|
||||
}
|
||||
j.State = "recovering"
|
||||
if err := c.save(j); err != nil {
|
||||
return j.Result, err
|
||||
}
|
||||
|
||||
// 1) 按持久化的那份预备份恢复。
|
||||
if restoreErr := restorer.Restore(ctx, j.BackupSnapshotID); restoreErr != nil {
|
||||
return c.failRecovery(&j, "restore_failed", fmt.Errorf("恢复失败:%w", restoreErr))
|
||||
}
|
||||
// 2) 真校验(内容/资源/索引/Wiki),拿到摘要才算恢复成功。
|
||||
digest, verifyErr := verifier.Verify(ctx, j.BackupSnapshotID)
|
||||
if verifyErr != nil {
|
||||
return c.failRecovery(&j, "verify_failed", fmt.Errorf("恢复后校验未通过:%w", verifyErr))
|
||||
}
|
||||
if digest == "" {
|
||||
return c.failRecovery(&j, "verify_failed", errors.New("恢复后校验没有返回摘要,不能当作已验证"))
|
||||
}
|
||||
ended := time.Now().UTC()
|
||||
j.RecoveredBackup = j.BackupSnapshotID
|
||||
j.RecoveryDigest = digest
|
||||
j.RecoveredAt = &ended
|
||||
j.RestoreBlocked = false
|
||||
if err := c.save(j); err != nil {
|
||||
return j.Result, err
|
||||
}
|
||||
|
||||
// 3) 验证成功之后才恢复服务、才开入口。
|
||||
recoveryCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute)
|
||||
defer cancel()
|
||||
if restoreErr := c.restore(recoveryCtx, &j); restoreErr != nil {
|
||||
return c.failRecovery(&j, "service_restore_failed", fmt.Errorf("内容已恢复但服务/入口未恢复:%w", restoreErr))
|
||||
}
|
||||
if c.ingressReady != nil {
|
||||
if readyErr := c.ingressReady(recoveryCtx); readyErr != nil {
|
||||
return c.failRecovery(&j, "ingress_unverified", fmt.Errorf("入口未恢复:%w", readyErr))
|
||||
}
|
||||
}
|
||||
j.State = "recovered"
|
||||
j.Code = ""
|
||||
j.FinishedAt = &ended
|
||||
if err := c.save(j); err != nil {
|
||||
return j.Result, err
|
||||
}
|
||||
return j.Result, nil
|
||||
}
|
||||
|
||||
// failRecovery 保持"目标仍不可用":服务停着、入口关着、journal 非终态。
|
||||
func (c *Coordinator) failRecovery(j *journal, code string, cause error) (Result, error) {
|
||||
j.State = "held"
|
||||
j.Code = code
|
||||
j.RestoreBlocked = true
|
||||
if err := c.save(*j); err != nil {
|
||||
return j.Result, errors.Join(cause, err)
|
||||
}
|
||||
return j.Result, cause
|
||||
}
|
||||
@@ -34,6 +34,25 @@ func (w *Window) MarkContentRisk(ctx context.Context) error {
|
||||
return w.c.save(*w.j)
|
||||
}
|
||||
|
||||
// SetBackupSnapshot 把"这次窗口的预备份身份"落盘。恢复只认它:
|
||||
// 没有这份身份的窗口,之后既不能自动恢复,也不许被外部收尾。
|
||||
func (w *Window) SetBackupSnapshot(ctx context.Context, snapshotID string) error {
|
||||
if w == nil || w.c == nil || w.j == nil {
|
||||
return errors.New("invalid maintenance window")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if !validDigest(snapshotID) {
|
||||
return errors.New("invalid backup snapshot id")
|
||||
}
|
||||
if w.j.BackupSnapshotID != "" && w.j.BackupSnapshotID != snapshotID {
|
||||
return errors.New("backup snapshot changed inside the same window")
|
||||
}
|
||||
w.j.BackupSnapshotID = snapshotID
|
||||
return w.c.save(*w.j)
|
||||
}
|
||||
|
||||
// Job 返回窗口所属作业 id(恢复器与调用方据此对账)。
|
||||
func (w *Window) Job() string {
|
||||
if w == nil || w.j == nil {
|
||||
@@ -113,7 +132,7 @@ func (c *Coordinator) RunWindow(ctx context.Context, job string, prepare func(co
|
||||
return Result{}, err
|
||||
}
|
||||
j = journal{Version: 1, Source: c.source, Job: job, Level: c.level, Ingress: c.ingress,
|
||||
Result: Result{State: "stopping"}, StartedAt: time.Now().UTC()}
|
||||
TargetFingerprint: c.fingerprint, Result: Result{State: "stopping"}, StartedAt: time.Now().UTC()}
|
||||
for _, s := range items {
|
||||
if s.Running && !s.Healthy {
|
||||
return Result{}, errors.New("test writer is not ready")
|
||||
@@ -125,6 +144,12 @@ func (c *Coordinator) RunWindow(ctx context.Context, job string, prepare func(co
|
||||
}
|
||||
}
|
||||
|
||||
// 后台任务必须先在"还能跑"的时候跑完:入口排空只覆盖 HTTP 连接,
|
||||
// app 内的 asynq worker 会被上游的 exit(1) 直接切断。
|
||||
if idleErr := c.awaitBackgroundIdle(ctx, &j); idleErr != nil {
|
||||
return c.abandonWindow(ctx, &j, idleErr, "background_busy")
|
||||
}
|
||||
|
||||
// 顺序是有原因的:入口先停(新请求被拒、在途请求跑完),写者后停。
|
||||
// 反过来做,正在处理的请求就会被 SIGTERM 切断——那不算排空。
|
||||
if c.gate != nil && (j.Gate == nil || !j.Gate.Drained) {
|
||||
@@ -267,6 +292,42 @@ func (c *Coordinator) closeWindow(ctx context.Context, j *journal, cause error,
|
||||
return j.Result, cause
|
||||
}
|
||||
|
||||
// awaitBackgroundIdle 等到后台任务队列为空再往下走。拿不到结论(查询失败)
|
||||
// 也按"没空"处理:不能把"问不到"当成"没任务"。
|
||||
func (c *Coordinator) awaitBackgroundIdle(ctx context.Context, j *journal) error {
|
||||
if c.quiescer == nil {
|
||||
if c.level == "instance" {
|
||||
return errors.New("没有后台任务静默检查:不能证明停写者时后台任务已结束")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
wait := c.quiesceWait
|
||||
if wait <= 0 {
|
||||
wait = 2 * time.Minute
|
||||
}
|
||||
deadline := time.Now().Add(wait)
|
||||
var last string
|
||||
for {
|
||||
idle, detail, err := c.quiescer.Idle(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("读不到后台任务状态:%w", err)
|
||||
}
|
||||
if idle {
|
||||
j.BackgroundIdle = true
|
||||
return nil
|
||||
}
|
||||
last = detail
|
||||
if time.Now().After(deadline) {
|
||||
return fmt.Errorf("后台任务仍在运行(%s):等它结束再开窗口,否则会被 SIGTERM 切断", last)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// stopAll 只对"原本在跑"的服务发停止指令,每发一个都先落盘意图:
|
||||
// 进程在停止过程中死掉时,磁盘上仍能看出"哪些服务被要求停过"。
|
||||
func (c *Coordinator) stopAll(ctx context.Context, j *journal) error {
|
||||
@@ -342,63 +403,3 @@ func validDigest(id string) bool {
|
||||
decoded, err := hex.DecodeString(id)
|
||||
return err == nil && len(decoded) == 32 && strings.ToLower(id) == id
|
||||
}
|
||||
|
||||
// FinalizeAfterOperatorRecovery 收尾一个"内容风险未验证 + 人工已恢复"的窗口。
|
||||
//
|
||||
// 它不启动任何服务、也不释放治理库存的占用:只做两件事——核对"原本在跑的服务
|
||||
// 现在确实在跑且健康、入口确实在服务",然后把 journal 标成 operator_recovered。
|
||||
// 没有这两条核对,它就退化成"把卡住的记录清掉",那是另一种误放行。
|
||||
func (c *Coordinator) FinalizeAfterOperatorRecovery(ctx context.Context, job string) error {
|
||||
if !validID(job) {
|
||||
return errors.New("invalid freeze job id")
|
||||
}
|
||||
lock, err := c.lock()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer unlock(lock)
|
||||
j, err := c.load(job)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if terminal(j.State) {
|
||||
return errors.New("freeze journal is already terminal")
|
||||
}
|
||||
if !j.ContentRisk {
|
||||
return errors.New("journal has no content risk; use Recover instead")
|
||||
}
|
||||
items, err := inventory(ctx, c.control)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
running := map[string]Service{}
|
||||
for _, item := range items {
|
||||
running[item.ID] = item
|
||||
}
|
||||
for _, s := range j.Services {
|
||||
if !s.Running {
|
||||
continue
|
||||
}
|
||||
now, ok := running[s.ID]
|
||||
if !ok || !now.Running || !now.Healthy {
|
||||
return fmt.Errorf("服务 %s 还没有恢复到健康状态,不能收尾这个窗口", s.ID)
|
||||
}
|
||||
}
|
||||
if c.gate != nil && j.Gate != nil && !j.GateReopened {
|
||||
if err := c.gate.OpenIngress(ctx); err != nil {
|
||||
return fmt.Errorf("入口还没有恢复:%w", err)
|
||||
}
|
||||
j.GateReopened = true
|
||||
}
|
||||
if c.ingressReady != nil {
|
||||
if err := c.ingressReady(ctx); err != nil {
|
||||
return fmt.Errorf("入口验证未通过:%w", err)
|
||||
}
|
||||
}
|
||||
ended := time.Now().UTC()
|
||||
j.State = "failed"
|
||||
j.Code = "operator_recovered"
|
||||
j.RestoreBlocked = false
|
||||
j.FinishedAt = &ended
|
||||
return c.save(j)
|
||||
}
|
||||
|
||||
@@ -29,6 +29,16 @@ func (c *exitControl) Inventory(ctx context.Context) ([]Service, error) {
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// idleQuiescer:实例级窗口必须能证明后台任务已空,测试里默认给一个空的。
|
||||
type idleQuiescer struct{ busy bool }
|
||||
|
||||
func (q *idleQuiescer) Idle(context.Context) (bool, string, error) {
|
||||
if q.busy {
|
||||
return false, "asynq pending=1", nil
|
||||
}
|
||||
return true, "asynq=0 pending_ops=0", nil
|
||||
}
|
||||
|
||||
func instanceFixture(t *testing.T) (string, *exitControl, *Coordinator) {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
@@ -45,6 +55,9 @@ func instanceFixture(t *testing.T) (string, *exitControl, *Coordinator) {
|
||||
if err := engine.SetTarget("instance", "http://127.0.0.1:1/health", nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := engine.SetQuiescer(&idleQuiescer{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return root, c, engine
|
||||
}
|
||||
|
||||
@@ -312,32 +325,3 @@ func TestWindowRefusesWhenGateNotDrained(t *testing.T) {
|
||||
t.Fatalf("writers were stopped before the ingress was drained: %v", c.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// 人工恢复后的收尾:服务没起来就必须拒绝(不能靠清记录放行)。
|
||||
func TestFinalizeAfterOperatorRecoveryRequiresHealthyServices(t *testing.T) {
|
||||
_, c, e := instanceFixture(t)
|
||||
_, err := e.RunWindow(context.Background(), "job-manual", func(context.Context) error { return nil },
|
||||
func(ctx context.Context, w *Window) (string, error) {
|
||||
if err := w.MarkContentRisk(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return "", errors.New("内容回滚失败")
|
||||
}, func(error) bool { return false })
|
||||
if err == nil {
|
||||
t.Fatal("blocked window reported success")
|
||||
}
|
||||
// 服务还停着:收尾必须失败。
|
||||
if err := e.FinalizeAfterOperatorRecovery(context.Background(), "job-manual"); err == nil {
|
||||
t.Fatal("finalize accepted a still-stopped target")
|
||||
}
|
||||
for i := range c.services {
|
||||
c.services[i].Running, c.services[i].Healthy = true, true
|
||||
}
|
||||
if err := e.FinalizeAfterOperatorRecovery(context.Background(), "job-manual"); err != nil {
|
||||
t.Fatalf("finalize rejected a recovered target: %v", err)
|
||||
}
|
||||
j, err := e.load("job-manual")
|
||||
if err != nil || j.State != "failed" || j.Code != "operator_recovered" || j.RestoreBlocked {
|
||||
t.Fatalf("journal=%+v err=%v", j, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
package librarymaintenance
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
// dockerAPI 是目标维护用到的 Docker Engine 最小客户端(unix socket)。
|
||||
// 入口闸门与拓扑校验共用同一份实现:两处各自拼一套请求只会长出两套毛病。
|
||||
type dockerAPI struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func newDockerAPI(socket string, timeout time.Duration) (*dockerAPI, error) {
|
||||
if socket == "" || timeout <= 0 {
|
||||
return nil, errors.New("invalid docker api configuration")
|
||||
}
|
||||
info, err := os.Stat(socket)
|
||||
if err != nil || info.Mode()&os.ModeSocket == 0 {
|
||||
return nil, errors.New("local Docker socket unavailable")
|
||||
}
|
||||
transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
|
||||
return (&net.Dialer{}).DialContext(ctx, "unix", socket)
|
||||
}}
|
||||
return &dockerAPI{client: &http.Client{Transport: transport, Timeout: timeout,
|
||||
CheckRedirect: func(*http.Request, []*http.Request) error { return errors.New("docker redirects prohibited") }}}, nil
|
||||
}
|
||||
|
||||
func (d *dockerAPI) Close() { d.client.CloseIdleConnections() }
|
||||
|
||||
func (d *dockerAPI) request(ctx context.Context, method, path string, out any) error {
|
||||
req, err := http.NewRequestWithContext(ctx, method, "http://docker"+path, nil)
|
||||
if err != nil {
|
||||
return errors.New("invalid docker request")
|
||||
}
|
||||
resp, err := d.client.Do(req)
|
||||
if err != nil {
|
||||
return errors.New("docker request failed")
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return fmt.Errorf("docker request rejected: %d", resp.StatusCode)
|
||||
}
|
||||
if out != nil {
|
||||
data, readErr := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
if readErr != nil || json.Unmarshal(data, out) != nil {
|
||||
return errors.New("invalid docker response")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type containerSummary struct {
|
||||
ID string `json:"Id"`
|
||||
Labels map[string]string `json:"Labels"`
|
||||
Ports []struct {
|
||||
PrivatePort int `json:"PrivatePort"`
|
||||
PublicPort int `json:"PublicPort"`
|
||||
Type string `json:"Type"`
|
||||
} `json:"Ports"`
|
||||
}
|
||||
|
||||
func (d *dockerAPI) listProject(ctx context.Context, project string) ([]containerSummary, error) {
|
||||
filters, _ := json.Marshal(map[string][]string{"label": {"com.docker.compose.project=" + project}})
|
||||
var items []containerSummary
|
||||
if err := d.request(ctx, "GET", "/containers/json?all=1&filters="+url.QueryEscape(string(filters)), &items); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
type containerState struct {
|
||||
ID string `json:"Id"`
|
||||
Labels map[string]string `json:"Config"`
|
||||
Config struct {
|
||||
Labels map[string]string `json:"Labels"`
|
||||
} `json:"-"`
|
||||
State struct {
|
||||
Running bool
|
||||
Status string
|
||||
ExitCode int
|
||||
Health *struct{ Status string }
|
||||
} `json:"State"`
|
||||
HostConfig struct {
|
||||
PortBindings map[string][]struct {
|
||||
HostIP string `json:"HostIp"`
|
||||
HostPort string `json:"HostPort"`
|
||||
} `json:"PortBindings"`
|
||||
} `json:"HostConfig"`
|
||||
NetworkSettings struct {
|
||||
Ports map[string][]struct {
|
||||
HostIP string `json:"HostIp"`
|
||||
HostPort string `json:"HostPort"`
|
||||
} `json:"Ports"`
|
||||
} `json:"NetworkSettings"`
|
||||
}
|
||||
|
||||
func (d *dockerAPI) inspect(ctx context.Context, id string) (containerState, error) {
|
||||
var data containerState
|
||||
if err := d.request(ctx, "GET", "/containers/"+id+"/json", &data); err != nil {
|
||||
return containerState{}, err
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// publishedPorts 返回容器对外发布的端口(HostConfig + NetworkSettings 都算)。
|
||||
// 写入者只要发布了一个端口,就存在"绕开入口闸门直连"的通道。
|
||||
func (c containerState) publishedPorts() []string {
|
||||
seen := map[string]bool{}
|
||||
out := make([]string, 0, 2)
|
||||
add := func(hostIP, hostPort, containerPort string) {
|
||||
if hostPort == "" || hostPort == "0" {
|
||||
return
|
||||
}
|
||||
key := hostIP + ":" + hostPort + "->" + containerPort
|
||||
if seen[key] {
|
||||
return
|
||||
}
|
||||
seen[key] = true
|
||||
out = append(out, key)
|
||||
}
|
||||
for port, bindings := range c.HostConfig.PortBindings {
|
||||
for _, binding := range bindings {
|
||||
add(binding.HostIP, binding.HostPort, port)
|
||||
}
|
||||
}
|
||||
for port, bindings := range c.NetworkSettings.Ports {
|
||||
for _, binding := range bindings {
|
||||
add(binding.HostIP, binding.HostPort, port)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -6,11 +6,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/libraryfreeze"
|
||||
@@ -20,8 +16,7 @@ import (
|
||||
// 闸门自己写下的 drain.json(停止接客时还有多少在途连接、最后是否归零)。
|
||||
// 不看 HTTP 探测就宣布"入口关了"——那是猜。
|
||||
type dockerGate struct {
|
||||
client *http.Client
|
||||
socket string
|
||||
api *dockerAPI
|
||||
project string
|
||||
service string
|
||||
drainFile string
|
||||
@@ -29,79 +24,41 @@ type dockerGate struct {
|
||||
}
|
||||
|
||||
func newDockerGate(socket, project, service, drainFile string, timeout time.Duration) (*dockerGate, error) {
|
||||
if !filepath.IsAbs(socket) || project == "" || service == "" || !filepath.IsAbs(drainFile) {
|
||||
if project == "" || service == "" || drainFile == "" || drainFile[0] != '/' {
|
||||
return nil, errors.New("入口闸门配置不完整")
|
||||
}
|
||||
info, err := os.Stat(socket)
|
||||
if err != nil || info.Mode()&os.ModeSocket == 0 {
|
||||
return nil, errors.New("local Docker socket unavailable")
|
||||
api, err := newDockerAPI(socket, timeout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
transport := &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
|
||||
return (&net.Dialer{}).DialContext(ctx, "unix", socket)
|
||||
}}
|
||||
return &dockerGate{
|
||||
client: &http.Client{Transport: transport, Timeout: timeout,
|
||||
CheckRedirect: func(*http.Request, []*http.Request) error { return errors.New("docker redirects prohibited") }},
|
||||
socket: socket, project: project, service: service, drainFile: drainFile, timeout: timeout,
|
||||
}, nil
|
||||
return &dockerGate{api: api, project: project, service: service, drainFile: drainFile, timeout: timeout}, nil
|
||||
}
|
||||
|
||||
func (g *dockerGate) request(ctx context.Context, method, path string, out any) error {
|
||||
req, err := http.NewRequestWithContext(ctx, method, "http://docker"+path, nil)
|
||||
func (g *dockerGate) container(ctx context.Context) (containerState, error) {
|
||||
items, err := g.api.listProject(ctx, g.project)
|
||||
if err != nil {
|
||||
return errors.New("invalid docker request")
|
||||
return containerState{}, err
|
||||
}
|
||||
resp, err := g.client.Do(req)
|
||||
if err != nil {
|
||||
return errors.New("docker request failed")
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return fmt.Errorf("docker request rejected: %d", resp.StatusCode)
|
||||
}
|
||||
if out != nil {
|
||||
data, readErr := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
if readErr != nil || json.Unmarshal(data, out) != nil {
|
||||
return errors.New("invalid docker response")
|
||||
id := ""
|
||||
for _, item := range items {
|
||||
if item.Labels["com.docker.compose.service"] != g.service {
|
||||
continue
|
||||
}
|
||||
if id != "" {
|
||||
return containerState{}, fmt.Errorf("入口闸门实例数异常:多于一个 %s", g.service)
|
||||
}
|
||||
id = item.ID
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type gateInspection struct {
|
||||
ID string `json:"Id"`
|
||||
Config struct {
|
||||
Labels map[string]string `json:"Labels"`
|
||||
} `json:"Config"`
|
||||
State struct {
|
||||
Running bool
|
||||
Status string
|
||||
ExitCode int
|
||||
Health *struct{ Status string }
|
||||
} `json:"State"`
|
||||
}
|
||||
|
||||
func (g *dockerGate) container(ctx context.Context) (gateInspection, error) {
|
||||
filters, _ := json.Marshal(map[string][]string{"label": {
|
||||
"com.docker.compose.project=" + g.project,
|
||||
"com.docker.compose.service=" + g.service,
|
||||
}})
|
||||
var listed []struct {
|
||||
ID string `json:"Id"`
|
||||
if id == "" {
|
||||
return containerState{}, fmt.Errorf("找不到入口闸门服务 %s", g.service)
|
||||
}
|
||||
if err := g.request(ctx, "GET", "/containers/json?all=1&filters="+url.QueryEscape(string(filters)), &listed); err != nil {
|
||||
return gateInspection{}, err
|
||||
}
|
||||
if len(listed) != 1 {
|
||||
return gateInspection{}, fmt.Errorf("入口闸门实例数异常:%d", len(listed))
|
||||
}
|
||||
var data gateInspection
|
||||
if err := g.request(ctx, "GET", "/containers/"+listed[0].ID+"/json", &data); err != nil {
|
||||
return gateInspection{}, err
|
||||
data, err := g.api.inspect(ctx, id)
|
||||
if err != nil {
|
||||
return containerState{}, err
|
||||
}
|
||||
if data.Config.Labels["com.docker.compose.project"] != g.project ||
|
||||
data.Config.Labels["com.docker.compose.service"] != g.service {
|
||||
return gateInspection{}, errors.New("入口闸门身份不匹配")
|
||||
return containerState{}, errors.New("入口闸门身份不匹配")
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
@@ -151,7 +108,7 @@ func (g *dockerGate) CloseIngress(ctx context.Context) (libraryfreeze.GateEviden
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
return libraryfreeze.GateEvidence{}, fmt.Errorf("无法确认排空记录已清理:%w", err)
|
||||
}
|
||||
if err := g.request(bounded, "POST", "/containers/"+data.ID+"/kill?signal=SIGTERM", nil); err != nil {
|
||||
if err := g.api.request(bounded, "POST", "/containers/"+data.ID+"/kill?signal=SIGTERM", nil); err != nil {
|
||||
return libraryfreeze.GateEvidence{}, err
|
||||
}
|
||||
deadline := time.Now().Add(g.timeout)
|
||||
@@ -202,7 +159,7 @@ func (g *dockerGate) OpenIngress(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
if !data.State.Running {
|
||||
if err := g.request(bounded, "POST", "/containers/"+data.ID+"/start", nil); err != nil {
|
||||
if err := g.api.request(bounded, "POST", "/containers/"+data.ID+"/start", nil); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
package librarymaintenance
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// targetQuiescer 回答"后台写入者是不是已经空了":
|
||||
// - 目标库:未结清的后台操作、未完成的解析(这两张表就是 app 的后台账本);
|
||||
// - 目标队列:asynq 里还在排队/在跑/待重试的任务。
|
||||
//
|
||||
// 为什么要单独做:HTTP 入口排空只覆盖连接;app 内的 asynq worker 会被上游的
|
||||
// SIGTERM exit(1) 切断,正在跑的任务就这样被腰斩。窗口必须先等它们结束。
|
||||
type targetQuiescer struct {
|
||||
pool *pgxpool.Pool
|
||||
redisAddr string
|
||||
redisPass string
|
||||
redisDB int
|
||||
timeout time.Duration
|
||||
queueMatch string
|
||||
}
|
||||
|
||||
func newTargetQuiescer(ctx context.Context, databaseURL, redisAddr, redisPass string, redisDB int, timeout time.Duration) (*targetQuiescer, error) {
|
||||
if databaseURL == "" || redisAddr == "" {
|
||||
return nil, errors.New("后台静默检查缺少目标库或队列地址")
|
||||
}
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
return nil, errors.New("后台静默检查连不上目标库")
|
||||
}
|
||||
if timeout <= 0 {
|
||||
timeout = 10 * time.Second
|
||||
}
|
||||
return &targetQuiescer{pool: pool, redisAddr: redisAddr, redisPass: redisPass, redisDB: redisDB,
|
||||
timeout: timeout, queueMatch: "asynq:*"}, nil
|
||||
}
|
||||
|
||||
func (q *targetQuiescer) Close() { q.pool.Close() }
|
||||
|
||||
func (q *targetQuiescer) Idle(ctx context.Context) (bool, string, error) {
|
||||
var pendingOps, unsettled int64
|
||||
// task_pending_ops / knowledges 是 app 自己的后台账本:不看它们只看 HTTP 会漏。
|
||||
if err := q.pool.QueryRow(ctx, `SELECT
|
||||
(SELECT count(*) FROM task_pending_ops),
|
||||
(SELECT count(*) FROM knowledges WHERE deleted_at IS NULL
|
||||
AND (parse_status IS DISTINCT FROM 'completed' OR pending_subtasks_count IS NULL OR pending_subtasks_count<>0))`).
|
||||
Scan(&pendingOps, &unsettled); err != nil {
|
||||
return false, "", fmt.Errorf("读不到后台账本:%w", err)
|
||||
}
|
||||
depth, err := q.queueDepth(ctx)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
detail := fmt.Sprintf("pending_ops=%d unsettled_knowledges=%d asynq_active=%d", pendingOps, unsettled, depth)
|
||||
return pendingOps == 0 && unsettled == 0 && depth == 0, detail, nil
|
||||
}
|
||||
|
||||
// queueDepth 统计 asynq 里"还没结束"的任务数(排队/在跑/待重试)。
|
||||
// archived/completed 是历史,不算忙。
|
||||
func (q *targetQuiescer) queueDepth(ctx context.Context) (int64, error) {
|
||||
conn, err := dialRedis(q.redisAddr, q.redisPass, q.redisDB, q.timeout)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("连不上目标队列:%w", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
keys, err := conn.scan(ctx, q.queueMatch)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("扫描目标队列失败:%w", err)
|
||||
}
|
||||
var total int64
|
||||
for _, key := range keys {
|
||||
suffix := key[strings.LastIndex(key, ":"):]
|
||||
switch suffix {
|
||||
case ":pending", ":active", ":inprogress":
|
||||
n, err := conn.intCmd(ctx, "LLEN", key)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
total += n
|
||||
case ":scheduled", ":retry":
|
||||
n, err := conn.intCmd(ctx, "ZCARD", key)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
total += n
|
||||
}
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
// ---- 最小 RESP 客户端:只用到 AUTH/SELECT/SCAN/LLEN/ZCARD,不引第三方库 ----
|
||||
|
||||
type redisConn struct {
|
||||
conn net.Conn
|
||||
reader *bufio.Reader
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
func dialRedis(addr, password string, db int, timeout time.Duration) (*redisConn, error) {
|
||||
dialer := &net.Dialer{Timeout: timeout}
|
||||
conn, err := dialer.Dial("tcp", addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = conn.SetDeadline(time.Now().Add(timeout))
|
||||
c := &redisConn{conn: conn, reader: bufio.NewReader(conn), timeout: timeout}
|
||||
if password != "" {
|
||||
if _, err := c.do("AUTH", password); err != nil {
|
||||
conn.Close()
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if db != 0 {
|
||||
if _, err := c.do("SELECT", strconv.Itoa(db)); err != nil {
|
||||
conn.Close()
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (c *redisConn) Close() error { return c.conn.Close() }
|
||||
|
||||
func (c *redisConn) do(args ...string) (any, error) {
|
||||
_ = c.conn.SetDeadline(time.Now().Add(c.timeout))
|
||||
var builder strings.Builder
|
||||
builder.WriteString("*" + strconv.Itoa(len(args)) + "\r\n")
|
||||
for _, arg := range args {
|
||||
builder.WriteString("$" + strconv.Itoa(len(arg)) + "\r\n" + arg + "\r\n")
|
||||
}
|
||||
if _, err := c.conn.Write([]byte(builder.String())); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return c.read()
|
||||
}
|
||||
|
||||
func (c *redisConn) read() (any, error) {
|
||||
line, err := c.reader.ReadString('\n')
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
line = strings.TrimRight(line, "\r\n")
|
||||
if line == "" {
|
||||
return nil, errors.New("empty redis reply")
|
||||
}
|
||||
switch line[0] {
|
||||
case '+':
|
||||
return line[1:], nil
|
||||
case '-':
|
||||
return nil, errors.New(line[1:])
|
||||
case ':':
|
||||
value, err := strconv.ParseInt(line[1:], 10, 64)
|
||||
return value, err
|
||||
case '$':
|
||||
length, err := strconv.Atoi(line[1:])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if length < 0 {
|
||||
return nil, nil
|
||||
}
|
||||
buf := make([]byte, length+2)
|
||||
if _, err := ioReadFull(c.reader, buf); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return string(buf[:length]), nil
|
||||
case '*':
|
||||
count, err := strconv.Atoi(line[1:])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items := make([]any, 0, count)
|
||||
for i := 0; i < count; i++ {
|
||||
item, err := c.read()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
return items, nil
|
||||
default:
|
||||
return nil, errors.New("unsupported redis reply")
|
||||
}
|
||||
}
|
||||
|
||||
func ioReadFull(reader *bufio.Reader, buf []byte) (int, error) {
|
||||
total := 0
|
||||
for total < len(buf) {
|
||||
n, err := reader.Read(buf[total:])
|
||||
total += n
|
||||
if err != nil {
|
||||
return total, err
|
||||
}
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func (c *redisConn) scan(ctx context.Context, match string) ([]string, error) {
|
||||
var keys []string
|
||||
cursor := "0"
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reply, err := c.do("SCAN", cursor, "MATCH", match, "COUNT", "1000")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items, ok := reply.([]any)
|
||||
if !ok || len(items) != 2 {
|
||||
return nil, errors.New("unexpected SCAN reply")
|
||||
}
|
||||
next, _ := items[0].(string)
|
||||
batch, _ := items[1].([]any)
|
||||
for _, item := range batch {
|
||||
if key, ok := item.(string); ok {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
}
|
||||
if next == "0" || next == "" {
|
||||
return keys, nil
|
||||
}
|
||||
cursor = next
|
||||
}
|
||||
}
|
||||
|
||||
func (c *redisConn) intCmd(ctx context.Context, cmd string, args ...string) (int64, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
reply, err := c.do(append([]string{cmd}, args...)...)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
value, ok := reply.(int64)
|
||||
if !ok {
|
||||
return 0, errors.New("unexpected integer reply")
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
@@ -56,7 +56,12 @@ type Config struct {
|
||||
Database string
|
||||
Index string
|
||||
Bucket string
|
||||
Timeout time.Duration
|
||||
// 后台静默检查需要的运行时依赖(不写进维护记录)。
|
||||
DatabaseURL string
|
||||
RedisAddr string
|
||||
RedisPassword string
|
||||
RedisDB int
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
type Manager struct {
|
||||
@@ -68,6 +73,18 @@ type Manager struct {
|
||||
timeout time.Duration
|
||||
client *http.Client
|
||||
gate *dockerGate
|
||||
api *dockerAPI
|
||||
quiescer *targetQuiescer
|
||||
}
|
||||
|
||||
// Close 释放拓扑/静默检查持有的连接。
|
||||
func (m *Manager) Close() {
|
||||
if m.api != nil {
|
||||
m.api.Close()
|
||||
}
|
||||
if m.quiescer != nil {
|
||||
m.quiescer.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func New(cfg Config) (*Manager, error) {
|
||||
@@ -102,9 +119,13 @@ func New(cfg Config) (*Manager, error) {
|
||||
return nil, err
|
||||
}
|
||||
target.Fingerprint = fingerprint
|
||||
api, err := newDockerAPI(cfg.DockerSocket, cfg.Timeout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
manager := &Manager{target: target, source: project, docker: cfg.DockerSocket,
|
||||
writers: writers, nonWriters: cfg.NonWriters, timeout: cfg.Timeout,
|
||||
client: &http.Client{Timeout: 5 * time.Second}}
|
||||
client: &http.Client{Timeout: 5 * time.Second}, api: api}
|
||||
if cfg.IngressService != "" || cfg.IngressDrainFile != "" {
|
||||
gate, err := newDockerGate(cfg.DockerSocket, project, cfg.IngressService, cfg.IngressDrainFile, cfg.Timeout)
|
||||
if err != nil {
|
||||
@@ -112,6 +133,14 @@ func New(cfg Config) (*Manager, error) {
|
||||
}
|
||||
manager.gate = gate
|
||||
}
|
||||
if cfg.DatabaseURL != "" && cfg.RedisAddr != "" {
|
||||
quiescer, err := newTargetQuiescer(context.Background(), cfg.DatabaseURL, cfg.RedisAddr,
|
||||
cfg.RedisPassword, cfg.RedisDB, cfg.Timeout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
manager.quiescer = quiescer
|
||||
}
|
||||
return manager, nil
|
||||
}
|
||||
|
||||
@@ -159,6 +188,11 @@ func (m *Manager) Open(root string) (*libraryfreeze.Coordinator, *libraryfreeze.
|
||||
control.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
if err := coordinator.SetTargetFingerprint(m.target.Fingerprint); err != nil {
|
||||
coordinator.Close()
|
||||
control.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
if m.gate != nil {
|
||||
if err := coordinator.SetGate(m.gate); err != nil {
|
||||
coordinator.Close()
|
||||
@@ -166,6 +200,17 @@ func (m *Manager) Open(root string) (*libraryfreeze.Coordinator, *libraryfreeze.
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
// 后台静默检查是实例级窗口的必备条件:没有它就等于只排空了 HTTP。
|
||||
if m.quiescer == nil {
|
||||
coordinator.Close()
|
||||
control.Close()
|
||||
return nil, nil, errors.New("缺少后台静默检查(目标库/队列未配置),拒绝开实例级窗口")
|
||||
}
|
||||
if err := coordinator.SetQuiescer(m.quiescer); err != nil {
|
||||
coordinator.Close()
|
||||
control.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
return coordinator, control, nil
|
||||
}
|
||||
|
||||
@@ -270,3 +315,82 @@ func DatabaseIdentity(raw string) (string, error) {
|
||||
}
|
||||
return parsed.Host + "/" + database, nil
|
||||
}
|
||||
|
||||
const composeServiceLabel = "com.docker.compose.service"
|
||||
|
||||
// VerifyTopology 强制"入口唯一":
|
||||
// - 写入者不许发布任何宿主机端口(否则就存在绕闸直连,维护窗口覆盖不到);
|
||||
// - 入口闸门必须存在、必须是独立服务,并且发布了入口 URL 用的那个端口。
|
||||
//
|
||||
// 任何一条不满足都拒绝开窗口——拓扑错了,"维护"就只是关了一半的门。
|
||||
func (m *Manager) VerifyTopology(ctx context.Context) error {
|
||||
items, err := m.api.listProject(ctx, m.target.Project)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
byService := make(map[string]containerSummary, len(items))
|
||||
for _, item := range items {
|
||||
name := item.Labels[composeServiceLabel]
|
||||
if name == "" {
|
||||
return errors.New("项目里有容器缺少 compose service 标签,无法确认写入者边界")
|
||||
}
|
||||
if _, duplicated := byService[name]; duplicated {
|
||||
return fmt.Errorf("项目里服务 %s 有多个实例,无法确认写入者边界", name)
|
||||
}
|
||||
byService[name] = item
|
||||
}
|
||||
for _, writer := range m.writers {
|
||||
item, ok := byService[writer]
|
||||
if !ok {
|
||||
return fmt.Errorf("声明的写入者 %s 不存在", writer)
|
||||
}
|
||||
state, err := m.api.inspect(ctx, item.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ports := state.publishedPorts(); len(ports) > 0 {
|
||||
return fmt.Errorf("写入者 %s 直接发布了端口 %s:存在绕开入口闸门的直连通道,拒绝开维护窗口",
|
||||
writer, strings.Join(ports, ","))
|
||||
}
|
||||
}
|
||||
if m.gate == nil {
|
||||
return errors.New("实例级维护必须声明入口闸门")
|
||||
}
|
||||
for _, writer := range m.writers {
|
||||
if writer == m.gate.service {
|
||||
return errors.New("入口闸门不能同时被声明为写入者")
|
||||
}
|
||||
}
|
||||
gateItem, ok := byService[m.gate.service]
|
||||
if !ok {
|
||||
return fmt.Errorf("声明的入口闸门 %s 不存在", m.gate.service)
|
||||
}
|
||||
gateState, err := m.api.inspect(ctx, gateItem.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
published := gateState.publishedPorts()
|
||||
if len(published) == 0 {
|
||||
return fmt.Errorf("入口闸门 %s 没有发布任何端口:外部无法访问,入口定义不成立", m.gate.service)
|
||||
}
|
||||
// 入口 URL 的端口必须真的是这个闸门发布的端口。
|
||||
parsed, err := url.Parse(m.target.Ingress)
|
||||
if err != nil || parsed.Host == "" {
|
||||
return errors.New("目标入口地址无效")
|
||||
}
|
||||
hostPort := parsed.Port()
|
||||
if hostPort == "" {
|
||||
if parsed.Scheme == "https" {
|
||||
hostPort = "443"
|
||||
} else {
|
||||
hostPort = "80"
|
||||
}
|
||||
}
|
||||
for _, entry := range published {
|
||||
if strings.Contains(entry, ":"+hostPort+"->") {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("入口 %s 的端口 %s 不是闸门发布的端口(%s):入口指向了别的服务",
|
||||
m.target.Ingress, hostPort, strings.Join(published, ","))
|
||||
}
|
||||
|
||||
@@ -132,3 +132,24 @@ func TestDatabaseIdentityDropsCredentials(t *testing.T) {
|
||||
t.Fatal("invalid DSN accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// 写入者只要发布了宿主机端口,就存在绕开闸门的直连通道。
|
||||
func TestPublishedPortsDetectsDirectBypass(t *testing.T) {
|
||||
var state containerState
|
||||
state.HostConfig.PortBindings = map[string][]struct {
|
||||
HostIP string `json:"HostIp"`
|
||||
HostPort string `json:"HostPort"`
|
||||
}{"8080/tcp": {{HostIP: "127.0.0.1", HostPort: "18081"}}}
|
||||
ports := state.publishedPorts()
|
||||
if len(ports) != 1 || !strings.Contains(ports[0], "18081") {
|
||||
t.Fatalf("publishedPorts=%v", ports)
|
||||
}
|
||||
var bare containerState
|
||||
bare.HostConfig.PortBindings = map[string][]struct {
|
||||
HostIP string `json:"HostIp"`
|
||||
HostPort string `json:"HostPort"`
|
||||
}{"8080/tcp": nil}
|
||||
if got := bare.publishedPorts(); len(got) != 0 {
|
||||
t.Fatalf("unpublished container reported ports: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// LibraryInstanceMaintenance 是"实例级维护"的占用记录:一个目标实例同一时刻
|
||||
// 只允许一个窗口,无论它来自 internal 还是 external。
|
||||
//
|
||||
// 为什么不是 scope 锁:stop/start 的是同一批容器,两个 scope 各持一个 scope 锁
|
||||
// 时互相看不见,会同时停/起同一实例——那是假互斥,比没有锁更危险。
|
||||
type LibraryInstanceMaintenance struct {
|
||||
Instance string `json:"instance"`
|
||||
OwnerJobID string `json:"owner_job_id"`
|
||||
OwnerScope string `json:"owner_scope"`
|
||||
TargetFingerprint string `json:"target_fingerprint"`
|
||||
Level string `json:"level"`
|
||||
Services []string `json:"services"`
|
||||
IngressState string `json:"ingress_state"`
|
||||
State string `json:"state"`
|
||||
OriginalServiceState string `json:"original_service_state"`
|
||||
Note string `json:"note"`
|
||||
AcquiredAt time.Time `json:"acquired_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
ReleasedAt *time.Time `json:"released_at,omitempty"`
|
||||
}
|
||||
|
||||
const instanceMaintenanceColumns = `instance, owner_job_id, owner_scope, target_fingerprint, level, services,
|
||||
ingress_state, state, original_service_state, note, acquired_at, updated_at, released_at`
|
||||
|
||||
// AcquireInstanceMaintenance 原子地拿到实例维护权:
|
||||
// 已释放的记录可以被新作业接管;仍被持有的只允许原作业重入。
|
||||
func (s *Store) AcquireInstanceMaintenance(ctx context.Context, item LibraryInstanceMaintenance, now time.Time) (LibraryInstanceMaintenance, error) {
|
||||
level := item.Level
|
||||
if level == "" {
|
||||
level = "instance"
|
||||
}
|
||||
ingress := item.IngressState
|
||||
if ingress == "" {
|
||||
ingress = "unknown"
|
||||
}
|
||||
services := item.Services
|
||||
if services == nil {
|
||||
services = []string{}
|
||||
}
|
||||
raw, err := json.Marshal(services)
|
||||
if err != nil {
|
||||
return LibraryInstanceMaintenance{}, fmt.Errorf("acquire instance maintenance: %w", err)
|
||||
}
|
||||
row := s.db.QueryRow(ctx, `
|
||||
INSERT INTO library_instance_maintenance (instance, owner_job_id, owner_scope, target_fingerprint, level, services,
|
||||
ingress_state, state, original_service_state, note, acquired_at, updated_at)
|
||||
VALUES ($1,$2,$3,$4,$5,$6::jsonb,$7,$8,$9,$10,$11,$11)
|
||||
ON CONFLICT (instance) DO UPDATE SET
|
||||
owner_job_id = EXCLUDED.owner_job_id,
|
||||
owner_scope = EXCLUDED.owner_scope,
|
||||
target_fingerprint = EXCLUDED.target_fingerprint,
|
||||
level = EXCLUDED.level,
|
||||
services = EXCLUDED.services,
|
||||
ingress_state = EXCLUDED.ingress_state,
|
||||
state = EXCLUDED.state,
|
||||
original_service_state = EXCLUDED.original_service_state,
|
||||
note = EXCLUDED.note,
|
||||
updated_at = EXCLUDED.updated_at,
|
||||
released_at = NULL
|
||||
WHERE library_instance_maintenance.released_at IS NOT NULL
|
||||
OR library_instance_maintenance.owner_job_id = EXCLUDED.owner_job_id
|
||||
RETURNING `+instanceMaintenanceColumns,
|
||||
item.Instance, item.OwnerJobID, item.OwnerScope, item.TargetFingerprint, level, string(raw),
|
||||
ingress, item.State, item.OriginalServiceState, item.Note, now.UTC())
|
||||
acquired, err := scanInstanceMaintenance(row)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return LibraryInstanceMaintenance{}, ErrMaintenanceHeld
|
||||
}
|
||||
if err != nil {
|
||||
return LibraryInstanceMaintenance{}, fmt.Errorf("acquire instance maintenance: %w", err)
|
||||
}
|
||||
return acquired, nil
|
||||
}
|
||||
|
||||
// UpdateInstanceMaintenanceState 只允许持有者改自己的窗口状态。
|
||||
func (s *Store) UpdateInstanceMaintenanceState(ctx context.Context, instance, ownerJobID, state, note string, now time.Time) error {
|
||||
tag, err := s.db.Exec(ctx, `UPDATE library_instance_maintenance SET state=$3, note=$4, updated_at=$5
|
||||
WHERE instance=$1 AND owner_job_id=$2 AND released_at IS NULL`, instance, ownerJobID, state, note, now.UTC())
|
||||
if err != nil {
|
||||
return fmt.Errorf("update instance maintenance: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return errors.New("instance maintenance not held by this job")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetInstanceMaintenanceIngress 记录入口真实状态。
|
||||
func (s *Store) SetInstanceMaintenanceIngress(ctx context.Context, instance, ownerJobID, state, note string, now time.Time) error {
|
||||
if state != "closed" && state != "open" {
|
||||
return errors.New("invalid ingress state")
|
||||
}
|
||||
tag, err := s.db.Exec(ctx, `UPDATE library_instance_maintenance SET ingress_state=$3, note=$4, updated_at=$5
|
||||
WHERE instance=$1 AND owner_job_id=$2 AND released_at IS NULL`, instance, ownerJobID, state, note, now.UTC())
|
||||
if err != nil {
|
||||
return fmt.Errorf("update instance maintenance ingress: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return errors.New("instance maintenance not held by this job")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReleaseInstanceMaintenance 只有"内容已验证 + 服务与入口恢复"才允许调用。
|
||||
func (s *Store) ReleaseInstanceMaintenance(ctx context.Context, instance, ownerJobID string, now time.Time) error {
|
||||
tag, err := s.db.Exec(ctx, `UPDATE library_instance_maintenance SET state='released', released_at=$3, updated_at=$3
|
||||
WHERE instance=$1 AND owner_job_id=$2 AND released_at IS NULL`, instance, ownerJobID, now.UTC())
|
||||
if err != nil {
|
||||
return fmt.Errorf("release instance maintenance: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return errors.New("instance maintenance not held by this job")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// InstanceMaintenanceOf 读当前实例窗口(接口互斥判断与页面展示都用它)。
|
||||
func (s *Store) InstanceMaintenanceOf(ctx context.Context, instance string) (*LibraryInstanceMaintenance, error) {
|
||||
row := s.db.QueryRow(ctx, `SELECT `+instanceMaintenanceColumns+` FROM library_instance_maintenance WHERE instance=$1`, instance)
|
||||
item, err := scanInstanceMaintenance(row)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &item, nil
|
||||
}
|
||||
|
||||
// HeldInstanceMaintenance 返回仍然持有(未释放)的实例窗口,供接口在不知道
|
||||
// 具体实例名时也拒绝写入。
|
||||
func (s *Store) HeldInstanceMaintenance(ctx context.Context) ([]LibraryInstanceMaintenance, error) {
|
||||
rows, err := s.db.Query(ctx, `SELECT `+instanceMaintenanceColumns+`
|
||||
FROM library_instance_maintenance WHERE released_at IS NULL ORDER BY instance`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list instance maintenance: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
items := make([]LibraryInstanceMaintenance, 0, 2)
|
||||
for rows.Next() {
|
||||
item, err := scanInstanceMaintenance(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
|
||||
func scanInstanceMaintenance(row pgx.Row) (LibraryInstanceMaintenance, error) {
|
||||
var item LibraryInstanceMaintenance
|
||||
var raw []byte
|
||||
err := row.Scan(&item.Instance, &item.OwnerJobID, &item.OwnerScope, &item.TargetFingerprint, &item.Level,
|
||||
&raw, &item.IngressState, &item.State, &item.OriginalServiceState, &item.Note,
|
||||
&item.AcquiredAt, &item.UpdatedAt, &item.ReleasedAt)
|
||||
if err != nil {
|
||||
return LibraryInstanceMaintenance{}, err
|
||||
}
|
||||
if len(raw) > 0 {
|
||||
if err := json.Unmarshal(raw, &item.Services); err != nil {
|
||||
return LibraryInstanceMaintenance{}, fmt.Errorf("instance maintenance services unreadable: %w", err)
|
||||
}
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
@@ -71,3 +71,45 @@ func TestLibraryMaintenanceOwnershipAcrossRelease(t *testing.T) {
|
||||
t.Fatalf("released window not re-acquired cleanly: %+v", second)
|
||||
}
|
||||
}
|
||||
|
||||
// 实例级占用必须跨 scope 互斥:internal 持有期间 external 也进不来。
|
||||
// (旧实现按 scope 建行,两个 scope 各持一个锁,会同时停/起同一个实例。)
|
||||
func TestInstanceMaintenanceExcludesAcrossScopes(t *testing.T) {
|
||||
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
|
||||
if databaseURL == "" {
|
||||
t.Skip("TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
t.Fatalf("connect: %v", err)
|
||||
}
|
||||
defer pool.Close()
|
||||
if err := ApplyMigrations(ctx, pool); err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
store := NewStore(pool)
|
||||
now := time.Now().UTC()
|
||||
base := LibraryInstanceMaintenance{Instance: "kgmaint-test", OwnerJobID: "job-internal",
|
||||
OwnerScope: "internal", TargetFingerprint: "fp", Level: "instance", Services: []string{"app"},
|
||||
State: "acquired", Note: "test"}
|
||||
if _, err := store.AcquireInstanceMaintenance(ctx, base, now); err != nil {
|
||||
t.Fatalf("first acquire: %v", err)
|
||||
}
|
||||
external := base
|
||||
external.OwnerJobID = "job-external"
|
||||
external.OwnerScope = "external"
|
||||
if _, err := store.AcquireInstanceMaintenance(ctx, external, now.Add(time.Second)); !errors.Is(err, ErrMaintenanceHeld) {
|
||||
t.Fatalf("external scope entered while internal held the instance: err=%v", err)
|
||||
}
|
||||
if err := store.ReleaseInstanceMaintenance(ctx, "kgmaint-test", "job-internal", now.Add(2*time.Second)); err != nil {
|
||||
t.Fatalf("release: %v", err)
|
||||
}
|
||||
if _, err := store.AcquireInstanceMaintenance(ctx, external, now.Add(3*time.Second)); err != nil {
|
||||
t.Fatalf("external scope could not take over the released instance: %v", err)
|
||||
}
|
||||
held, err := store.HeldInstanceMaintenance(ctx)
|
||||
if err != nil || len(held) != 1 || held[0].OwnerScope != "external" {
|
||||
t.Fatalf("held=%+v err=%v", held, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
DROP TABLE IF EXISTS library_instance_maintenance;
|
||||
@@ -0,0 +1,21 @@
|
||||
-- 维护占用必须按"实例"互斥,而不是按库 scope。
|
||||
-- 旧表的 scope 主键允许 internal/external 各持一个窗口,于是两个作业会同时
|
||||
-- 停/起同一个实例的写入者——那是最危险的一种假互斥。
|
||||
CREATE TABLE library_instance_maintenance (
|
||||
instance VARCHAR(128) PRIMARY KEY,
|
||||
owner_job_id VARCHAR(128) NOT NULL,
|
||||
owner_scope VARCHAR(16) NOT NULL CHECK (owner_scope IN ('internal','external')),
|
||||
target_fingerprint VARCHAR(64) NOT NULL,
|
||||
level VARCHAR(16) NOT NULL DEFAULT 'instance' CHECK (level IN ('library','instance')),
|
||||
services JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||
ingress_state VARCHAR(16) NOT NULL DEFAULT 'unknown' CHECK (ingress_state IN ('unknown','closed','open')),
|
||||
state VARCHAR(24) NOT NULL CHECK (state IN ('acquired','applying','verified','released','needs_restore','recovering')),
|
||||
original_service_state TEXT NOT NULL DEFAULT '',
|
||||
note TEXT NOT NULL DEFAULT '',
|
||||
acquired_at TIMESTAMPTZ NOT NULL,
|
||||
updated_at TIMESTAMPTZ NOT NULL,
|
||||
released_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
-- 把已有窗口按 scope 行回填成实例行不可行(历史 target_identity 是自由文本),
|
||||
-- 这里只保证新窗口从第一刻起就按实例互斥;旧行保留为审计记录。
|
||||
@@ -11,30 +11,35 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("loadMigrations() error = %v", err)
|
||||
}
|
||||
if len(migrations) != 71 {
|
||||
t.Fatalf("migration count = %d, want 71", len(migrations))
|
||||
if len(migrations) != 72 {
|
||||
t.Fatalf("migration count = %d, want 72", len(migrations))
|
||||
}
|
||||
// 72 让维护占用按实例互斥(internal/external 不再各有各的锁)。
|
||||
if instance := migrations[len(migrations)-1]; instance.Version != 72 ||
|
||||
!strings.Contains(instance.UpSQL, "library_instance_maintenance") {
|
||||
t.Fatalf("migration 72 = %#v", instance)
|
||||
}
|
||||
// 71 让维护记录能区分实例级维护与仅治理库互斥。
|
||||
if level := migrations[len(migrations)-1]; level.Version != 71 ||
|
||||
if level := migrations[len(migrations)-2]; level.Version != 71 ||
|
||||
!strings.Contains(level.UpSQL, "level") || !strings.Contains(level.UpSQL, "ingress_state") {
|
||||
t.Fatalf("migration 71 = %#v", level)
|
||||
}
|
||||
if maint := migrations[len(migrations)-2]; maint.Version != 70 || !strings.Contains(maint.UpSQL, "library_maintenance") {
|
||||
if maint := migrations[len(migrations)-3]; maint.Version != 70 || !strings.Contains(maint.UpSQL, "library_maintenance") {
|
||||
t.Fatalf("migration 70 = %#v", maint)
|
||||
}
|
||||
if tail := migrations[len(migrations)-3]; tail.Version != 69 ||
|
||||
if tail := migrations[len(migrations)-4]; tail.Version != 69 ||
|
||||
!strings.Contains(tail.UpSQL, "library_jobs_write_target") {
|
||||
t.Fatalf("migration 69 = %#v", tail)
|
||||
}
|
||||
if restore := migrations[len(migrations)-4]; restore.Version != 68 || !strings.Contains(restore.UpSQL, "restore") {
|
||||
if restore := migrations[len(migrations)-5]; restore.Version != 68 || !strings.Contains(restore.UpSQL, "restore") {
|
||||
t.Fatalf("migration 68 = %#v", restore)
|
||||
}
|
||||
if last := migrations[len(migrations)-5]; last.Version != 67 ||
|
||||
if last := migrations[len(migrations)-6]; last.Version != 67 ||
|
||||
!strings.Contains(last.UpSQL, "library_release_authorizations") ||
|
||||
!strings.Contains(last.UpSQL, "params_digest") {
|
||||
t.Fatalf("migration 67 = %#v", last)
|
||||
}
|
||||
if flow := migrations[len(migrations)-6]; flow.Version != 66 ||
|
||||
if flow := migrations[len(migrations)-7]; flow.Version != 66 ||
|
||||
!strings.Contains(flow.UpSQL, "CREATE TABLE library_versions") ||
|
||||
!strings.Contains(flow.UpSQL, "library_jobs_active") {
|
||||
t.Fatalf("migration 66 = %#v", flow)
|
||||
|
||||
Reference in New Issue
Block a user