feat(library): 目标维护窗口(持久所有权/独占/release-on-verify)+ known-broken 记录
build / build (push) Failing after 16s

为什么:用户确认维护窗口方案,需要真实占有目标而不是 503 页面。

- 迁移 000070 library_maintenance:scope 唯一持有者、作业/目标身份、服务原状态、阶段。
- worker 在写目标前 acquire,验证+登记成功才 release;回滚失败/未验证保持 needs_restore。
- API GET /library/maintenance + 同步入口在他人持有时 409。
- 已知缺陷如实记录:候选实例该路由 404;kill 取证脚本竞态导致三项断言为 false。
This commit is contained in:
姚章浩
2026-10-28 20:45:02 +08:00
parent b76a433574
commit 288e01a01a
10 changed files with 250 additions and 6 deletions
+19
View File
@@ -1,5 +1,24 @@
# Changelog
## 开发快照(目标维护窗口:半成品,未验证)
### Added(代码已完成,**真实取证未通过**)
- 迁移 000070 `library_maintenance`:一个 scope 一个持有者,记录作业/目标身份/服务原状态/阶段/释放时间。
- `AcquireLibraryMaintenance`(异主占用直接拒绝)、`UpdateLibraryMaintenanceState`、
`ReleaseLibraryMaintenance`、`LibraryMaintenanceOf`。
- worker:写目标前 acquire,写前置 `applying`,只有"验证通过 + 版本登记成功"才 release;
回滚失败/回滚后未验证 → `needs_restore` 且保持持有(不会被 finally 释放)。
- API:`GET /api/v1/library/maintenance`;同步入口在目标被他人持有时返回 409。
### 已知缺陷(本轮没有修完,不能当成可用)
- 候选环境实测 `GET /api/v1/library/maintenance` 返回 **404**:路由已注册在 server.go:451,
但运行实例没匹配上,根因未定位。
- 维护窗口的真实 kill/接管取证未成功:取证脚本在 kill 之前作业已跑完(竞态),
三项断言(持有中、kill 后仍持有、维护期间拒绝新写入)全部 false。
因此维护协议**没有真实证据**,不得声称已实现。
## 开发快照(安全门禁 + 崩溃接管 + 五项真实入口验收)
### Fixed
+37
View File
@@ -1025,6 +1025,43 @@
}
}
},
"/api/v1/library/maintenance": {
"get": {
"tags": [
"Runtime"
],
"operationId": "getLibraryMaintenance",
"summary": "目标维护窗口状态:谁持有、处于哪个阶段、是否已释放",
"parameters": [
{
"name": "scope",
"in": "query",
"required": true,
"schema": {
"type": "string",
"enum": [
"internal",
"external"
]
}
}
],
"responses": {
"200": {
"description": "维护窗口状态(held=true 表示目标仍被占用)"
},
"400": {
"$ref": "#/components/responses/BadRequest"
},
"500": {
"$ref": "#/components/responses/InternalError"
},
"503": {
"description": "维护窗口存储不可用"
}
}
}
},
"/api/v1/library/restores": {
"post": {
"tags": [
+22
View File
@@ -114,6 +114,19 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, wireErr.Error(), "prebackup")
}
defer target.Close()
// 维护窗口:目标在同步/恢复期间被本作业独占,任何其他写入一律拒绝。
targetIdentity := fmt.Sprintf("scope=%s db=%s index=%s bucket=%s", payload.Scope,
redactDSN(config.LibraryProductionDatabaseURL), config.LibraryProductionIndex, config.LibraryProductionS3Bucket)
maintenance, acquireErr := store.AcquireLibraryMaintenance(ctx, postgresstore.LibraryMaintenance{
Scope: payload.Scope, OwnerJobID: payload.LibraryJobID, TargetIdentity: targetIdentity,
State: "acquired", OriginalServiceState: "candidate: no external writer configured",
Note: "同步/恢复期间目标维护,验证通过才开放",
}, now)
if acquireErr != nil {
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "目标正在维护中:"+acquireErr.Error(), "prebackup")
}
_ = maintenance
if payload.FinalizeOnly {
// 写入已完成、只是版本登记失败:只做校验 + 登记,绝不重写生产。
recorder.Stage(ctx, "finalize", "running", "续作收尾:先校验生产与目标快照一致")
@@ -142,6 +155,8 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本(仅收尾,未重写生产)")
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize", "", "",
previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC())
// 只有校验+登记都完成才开放目标。
_ = store.ReleaseLibraryMaintenance(ctx, payload.Scope, payload.LibraryJobID, time.Now().UTC())
return nil
}
@@ -155,6 +170,7 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
ParamsDigest: digest, Now: now,
Grant: libraryrelease.Grant{Target: payload.Scope, SnapshotID: grant.SnapshotID,
ParamsDigest: grant.ParamsDigest, ExpiresAt: grant.ExpiresAt, AuthorizedBy: grant.AuthorizedBy}}
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "applying", "正在写目标", now)
outcome := libraryrelease.Run(ctx, request, target, backupSource, recorder)
finishedAt := time.Now().UTC()
switch outcome.State {
@@ -193,16 +209,22 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
// 状态落不下去时不要把队列作业报成功:让上层可见并等待接管。
return fmt.Errorf("作业状态写入失败: %w", advanceErr)
}
// 验证通过 + 版本登记成功 = 唯一开放目标的路径。
_ = store.ReleaseLibraryMaintenance(ctx, payload.Scope, payload.LibraryJobID, finishedAt)
log.Printf("library %s succeeded job=%s scope=%s snapshot=%s backup=%s", kind,
payload.LibraryJobID, payload.Scope, payload.TargetSnapshotID, outcome.Backup.SnapshotID)
return nil
case "rolled_back":
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback",
outcome.Error, outcome.Rollback, "", "", finishedAt)
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore",
"已回滚到上一版本,但验证未通过:目标继续维护,人工确认后再开放", finishedAt)
return errors.New(outcome.Error)
case "rollback_failed":
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback",
outcome.Error, outcome.Rollback, "", "", finishedAt)
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore",
"回滚失败:目标保持维护,必须按预备份恢复", finishedAt)
return fmt.Errorf("%s(%s)", outcome.Error, outcome.Rollback)
default:
recovery := outcome.Recovery
+16
View File
@@ -8,6 +8,7 @@ import (
"os"
"os/signal"
"path/filepath"
"strings"
"syscall"
"time"
@@ -54,6 +55,21 @@ func (r libraryStageRecorder) Stage(ctx context.Context, stage, state, detail st
func int64Ptr(value int64) *int64 { return &value }
// redactDSN 只保留库名,避免把连接串写进维护记录。
func redactDSN(dsn string) string {
if dsn == "" {
return ""
}
if index := strings.LastIndex(dsn, "/"); index >= 0 && index+1 < len(dsn) {
name := dsn[index+1:]
if cut := strings.Index(name, "?"); cut >= 0 {
name = name[:cut]
}
return name
}
return "configured"
}
// stageSize 是某一类数据在快照里的真实总量(行数/字节)。
type stageSize struct {
count int64
+12
View File
@@ -16,6 +16,7 @@ import (
"knowledge-governance/internal/jobs"
"knowledge-governance/internal/librarydiff"
"knowledge-governance/internal/librarysnapshot"
postgresstore "knowledge-governance/internal/platform/postgres"
)
// 整库版本/状态/差异读取契约 + 生产同步的 fail-closed 入口。
@@ -591,6 +592,17 @@ func librarySyncEnqueue(w http.ResponseWriter, r *http.Request, config Config) {
}
grantFound, hasGrant := activeGrantFor(r.Context(), config, request.Scope, request.Target)
_ = grantFound
// 目标正在维护(被别的作业持有):直接拒绝新写入,不等到 worker 里才发现。
if maintenanceStore, ok := config.Store.(interface {
LibraryMaintenanceOf(ctx context.Context, scope string) (*postgresstore.LibraryMaintenance, error)
}); ok {
if held, err := maintenanceStore.LibraryMaintenanceOf(r.Context(), request.Scope); err == nil && held != nil && held.ReleasedAt == nil {
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_target_in_maintenance",
"error": "目标正在维护中,验证完成前不接受新的写入请求",
"owner_job": held.OwnerJobID, "state": held.State})
return
}
}
gate := librarySyncGateWithGrant(config, overview.Production, overview.Prebackup, request.Target, hasGrant)
if !gate.SyncAllowed {
writeJSON(w, http.StatusServiceUnavailable, map[string]any{
+21
View File
@@ -447,6 +447,27 @@ func NewHandler(config Config) (http.Handler, error) {
libraryStorageHandler(w, r, config)
})
})
// 目标维护窗口状态:谁持有、处于哪个阶段、何时释放。
mux.HandleFunc("GET /api/v1/library/maintenance", func(w http.ResponseWriter, r *http.Request) {
scope := r.URL.Query().Get("scope")
if scope != "internal" && scope != "external" {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "scope must be internal or external"})
return
}
store, ok := config.Store.(interface {
LibraryMaintenanceOf(ctx context.Context, scope string) (*postgresstore.LibraryMaintenance, error)
})
if !ok {
writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "维护窗口存储不可用"})
return
}
held, err := store.LibraryMaintenanceOf(r.Context(), scope)
if err != nil {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "无法读取维护窗口"})
return
}
writeJSON(w, http.StatusOK, map[string]any{"held": held != nil && held.ReleasedAt == nil, "maintenance": held})
})
mux.HandleFunc("GET /api/v1/library/jobs", func(w http.ResponseWriter, r *http.Request) {
libraryJobsHandler(w, r, config)
})
@@ -0,0 +1,99 @@
package postgres
import (
"context"
"errors"
"fmt"
"time"
"github.com/jackc/pgx/v5"
)
// LibraryMaintenance 是目标维护窗口的持有记录:一个 scope 只能有一个持有者。
type LibraryMaintenance struct {
Scope string `json:"scope"`
OwnerJobID string `json:"owner_job_id"`
TargetIdentity string `json:"target_identity"`
State string `json:"state"`
OriginalServiceState string `json:"original_service_state"`
Note string `json:"note"`
AcquiredAt time.Time `json:"acquired_at"`
UpdatedAt time.Time `json:"updated_at"`
ReleasedAt *time.Time `json:"released_at,omitempty"`
}
// ErrMaintenanceHeld 表示目标维护窗口被别的作业占着:任何新的写入都必须拒绝。
var ErrMaintenanceHeld = errors.New("目标正在维护中,被其他作业占用")
// AcquireLibraryMaintenance 原子地拿到维护权:
// 同一作业可以重入(接管后继续),别的作业一律拒绝——这不是"尽力而为"的锁。
func (s *Store) AcquireLibraryMaintenance(ctx context.Context, item LibraryMaintenance, now time.Time) (LibraryMaintenance, error) {
row := s.db.QueryRow(ctx, `
INSERT INTO library_maintenance (scope, owner_job_id, target_identity, state, original_service_state, note, acquired_at, updated_at)
VALUES ($1,$2,$3,$4,$5,$6,$7,$7)
ON CONFLICT (scope) DO UPDATE SET
owner_job_id = EXCLUDED.owner_job_id,
target_identity = EXCLUDED.target_identity,
state = EXCLUDED.state,
note = EXCLUDED.note,
updated_at = EXCLUDED.updated_at,
released_at = NULL
WHERE library_maintenance.owner_job_id = EXCLUDED.owner_job_id
RETURNING scope, owner_job_id, target_identity, state, original_service_state, note, acquired_at, updated_at, released_at`,
item.Scope, item.OwnerJobID, item.TargetIdentity, item.State, item.OriginalServiceState, item.Note, now.UTC())
acquired, err := scanMaintenance(row)
if errors.Is(err, pgx.ErrNoRows) {
return LibraryMaintenance{}, ErrMaintenanceHeld
}
if err != nil {
return LibraryMaintenance{}, fmt.Errorf("acquire library maintenance: %w", err)
}
return acquired, nil
}
// UpdateLibraryMaintenanceState 只允许持有者改自己的窗口状态。
func (s *Store) UpdateLibraryMaintenanceState(ctx context.Context, scope, ownerJobID, state, note string, now time.Time) error {
tag, err := s.db.Exec(ctx, `UPDATE library_maintenance SET state=$3, note=$4, updated_at=$5
WHERE scope=$1 AND owner_job_id=$2 AND released_at IS NULL`, scope, ownerJobID, state, note, now.UTC())
if err != nil {
return fmt.Errorf("update library maintenance: %w", err)
}
if tag.RowsAffected() == 0 {
return errors.New("library maintenance not held by this job")
}
return nil
}
// ReleaseLibraryMaintenance 只在完整验证通过后调用;失败路径必须保持窗口。
func (s *Store) ReleaseLibraryMaintenance(ctx context.Context, scope, ownerJobID string, now time.Time) error {
tag, err := s.db.Exec(ctx, `UPDATE library_maintenance SET state='released', released_at=$3, updated_at=$3
WHERE scope=$1 AND owner_job_id=$2 AND released_at IS NULL`, scope, ownerJobID, now.UTC())
if err != nil {
return fmt.Errorf("release library maintenance: %w", err)
}
if tag.RowsAffected() == 0 {
return errors.New("library maintenance not held by this job")
}
return nil
}
// LibraryMaintenanceOf 读当前窗口状态(页面与诊断用)。
func (s *Store) LibraryMaintenanceOf(ctx context.Context, scope string) (*LibraryMaintenance, error) {
row := s.db.QueryRow(ctx, `SELECT scope, owner_job_id, target_identity, state, original_service_state, note,
acquired_at, updated_at, released_at FROM library_maintenance WHERE scope=$1`, scope)
item, err := scanMaintenance(row)
if errors.Is(err, pgx.ErrNoRows) {
return nil, nil
}
if err != nil {
return nil, err
}
return &item, nil
}
func scanMaintenance(row pgx.Row) (LibraryMaintenance, error) {
var item LibraryMaintenance
err := row.Scan(&item.Scope, &item.OwnerJobID, &item.TargetIdentity, &item.State,
&item.OriginalServiceState, &item.Note, &item.AcquiredAt, &item.UpdatedAt, &item.ReleasedAt)
return item, err
}
@@ -0,0 +1 @@
DROP TABLE IF EXISTS library_maintenance;
@@ -0,0 +1,14 @@
-- 目标维护窗口:同步/恢复期间独占目标,只有完整验证通过才开放。
-- 一个 scope 同时只能有一个维护持有者;持有者进程死掉后由下一个接管的 worker 继承,
-- 不会被"finally"无条件释放。
CREATE TABLE library_maintenance (
scope VARCHAR(16) PRIMARY KEY CHECK (scope IN ('internal','external')),
owner_job_id VARCHAR(128) NOT NULL,
target_identity TEXT NOT NULL,
state VARCHAR(24) NOT NULL CHECK (state IN ('acquired','applying','verified','released','needs_restore')),
original_service_state TEXT NOT NULL DEFAULT '',
note TEXT NOT NULL DEFAULT '',
acquired_at TIMESTAMPTZ NOT NULL,
updated_at TIMESTAMPTZ NOT NULL,
released_at TIMESTAMPTZ
);
@@ -11,22 +11,25 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) {
if err != nil {
t.Fatalf("loadMigrations() error = %v", err)
}
if len(migrations) != 69 {
t.Fatalf("migration count = %d, want 69", len(migrations))
if len(migrations) != 70 {
t.Fatalf("migration count = %d, want 70", len(migrations))
}
if tail := migrations[len(migrations)-1]; tail.Version != 69 ||
if tail := migrations[len(migrations)-2]; tail.Version != 69 ||
!strings.Contains(tail.UpSQL, "library_jobs_write_target") {
t.Fatalf("migration 69 = %#v", tail)
}
if restore := migrations[len(migrations)-2]; restore.Version != 68 || !strings.Contains(restore.UpSQL, "restore") {
if maint := migrations[len(migrations)-1]; maint.Version != 70 || !strings.Contains(maint.UpSQL, "library_maintenance") {
t.Fatalf("migration 70 = %#v", maint)
}
if restore := migrations[len(migrations)-3]; restore.Version != 68 || !strings.Contains(restore.UpSQL, "restore") {
t.Fatalf("migration 68 = %#v", restore)
}
if last := migrations[len(migrations)-3]; last.Version != 67 ||
if last := migrations[len(migrations)-4]; last.Version != 67 ||
!strings.Contains(last.UpSQL, "library_release_authorizations") ||
!strings.Contains(last.UpSQL, "params_digest") {
t.Fatalf("migration 67 = %#v", last)
}
if flow := migrations[len(migrations)-4]; flow.Version != 66 ||
if flow := migrations[len(migrations)-5]; flow.Version != 66 ||
!strings.Contains(flow.UpSQL, "CREATE TABLE library_versions") ||
!strings.Contains(flow.UpSQL, "library_jobs_active") {
t.Fatalf("migration 66 = %#v", flow)