feat(library): 目标维护窗口(持久所有权/独占/release-on-verify)+ known-broken 记录
build / build (push) Failing after 16s
build / build (push) Failing after 16s
为什么:用户确认维护窗口方案,需要真实占有目标而不是 503 页面。 - 迁移 000070 library_maintenance:scope 唯一持有者、作业/目标身份、服务原状态、阶段。 - worker 在写目标前 acquire,验证+登记成功才 release;回滚失败/未验证保持 needs_restore。 - API GET /library/maintenance + 同步入口在他人持有时 409。 - 已知缺陷如实记录:候选实例该路由 404;kill 取证脚本竞态导致三项断言为 false。
This commit is contained in:
@@ -1,5 +1,24 @@
|
||||
# Changelog
|
||||
|
||||
## 开发快照(目标维护窗口:半成品,未验证)
|
||||
|
||||
### Added(代码已完成,**真实取证未通过**)
|
||||
|
||||
- 迁移 000070 `library_maintenance`:一个 scope 一个持有者,记录作业/目标身份/服务原状态/阶段/释放时间。
|
||||
- `AcquireLibraryMaintenance`(异主占用直接拒绝)、`UpdateLibraryMaintenanceState`、
|
||||
`ReleaseLibraryMaintenance`、`LibraryMaintenanceOf`。
|
||||
- worker:写目标前 acquire,写前置 `applying`,只有"验证通过 + 版本登记成功"才 release;
|
||||
回滚失败/回滚后未验证 → `needs_restore` 且保持持有(不会被 finally 释放)。
|
||||
- API:`GET /api/v1/library/maintenance`;同步入口在目标被他人持有时返回 409。
|
||||
|
||||
### 已知缺陷(本轮没有修完,不能当成可用)
|
||||
|
||||
- 候选环境实测 `GET /api/v1/library/maintenance` 返回 **404**:路由已注册在 server.go:451,
|
||||
但运行实例没匹配上,根因未定位。
|
||||
- 维护窗口的真实 kill/接管取证未成功:取证脚本在 kill 之前作业已跑完(竞态),
|
||||
三项断言(持有中、kill 后仍持有、维护期间拒绝新写入)全部 false。
|
||||
因此维护协议**没有真实证据**,不得声称已实现。
|
||||
|
||||
## 开发快照(安全门禁 + 崩溃接管 + 五项真实入口验收)
|
||||
|
||||
### Fixed
|
||||
|
||||
@@ -1025,6 +1025,43 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/library/maintenance": {
|
||||
"get": {
|
||||
"tags": [
|
||||
"Runtime"
|
||||
],
|
||||
"operationId": "getLibraryMaintenance",
|
||||
"summary": "目标维护窗口状态:谁持有、处于哪个阶段、是否已释放",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "scope",
|
||||
"in": "query",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"internal",
|
||||
"external"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "维护窗口状态(held=true 表示目标仍被占用)"
|
||||
},
|
||||
"400": {
|
||||
"$ref": "#/components/responses/BadRequest"
|
||||
},
|
||||
"500": {
|
||||
"$ref": "#/components/responses/InternalError"
|
||||
},
|
||||
"503": {
|
||||
"description": "维护窗口存储不可用"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/library/restores": {
|
||||
"post": {
|
||||
"tags": [
|
||||
|
||||
@@ -114,6 +114,19 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
|
||||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, wireErr.Error(), "prebackup")
|
||||
}
|
||||
defer target.Close()
|
||||
// 维护窗口:目标在同步/恢复期间被本作业独占,任何其他写入一律拒绝。
|
||||
targetIdentity := fmt.Sprintf("scope=%s db=%s index=%s bucket=%s", payload.Scope,
|
||||
redactDSN(config.LibraryProductionDatabaseURL), config.LibraryProductionIndex, config.LibraryProductionS3Bucket)
|
||||
maintenance, acquireErr := store.AcquireLibraryMaintenance(ctx, postgresstore.LibraryMaintenance{
|
||||
Scope: payload.Scope, OwnerJobID: payload.LibraryJobID, TargetIdentity: targetIdentity,
|
||||
State: "acquired", OriginalServiceState: "candidate: no external writer configured",
|
||||
Note: "同步/恢复期间目标维护,验证通过才开放",
|
||||
}, now)
|
||||
if acquireErr != nil {
|
||||
return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "目标正在维护中:"+acquireErr.Error(), "prebackup")
|
||||
}
|
||||
_ = maintenance
|
||||
|
||||
if payload.FinalizeOnly {
|
||||
// 写入已完成、只是版本登记失败:只做校验 + 登记,绝不重写生产。
|
||||
recorder.Stage(ctx, "finalize", "running", "续作收尾:先校验生产与目标快照一致")
|
||||
@@ -142,6 +155,8 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
|
||||
recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本(仅收尾,未重写生产)")
|
||||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize", "", "",
|
||||
previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC())
|
||||
// 只有校验+登记都完成才开放目标。
|
||||
_ = store.ReleaseLibraryMaintenance(ctx, payload.Scope, payload.LibraryJobID, time.Now().UTC())
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -155,6 +170,7 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
|
||||
ParamsDigest: digest, Now: now,
|
||||
Grant: libraryrelease.Grant{Target: payload.Scope, SnapshotID: grant.SnapshotID,
|
||||
ParamsDigest: grant.ParamsDigest, ExpiresAt: grant.ExpiresAt, AuthorizedBy: grant.AuthorizedBy}}
|
||||
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "applying", "正在写目标", now)
|
||||
outcome := libraryrelease.Run(ctx, request, target, backupSource, recorder)
|
||||
finishedAt := time.Now().UTC()
|
||||
switch outcome.State {
|
||||
@@ -193,16 +209,22 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a
|
||||
// 状态落不下去时不要把队列作业报成功:让上层可见并等待接管。
|
||||
return fmt.Errorf("作业状态写入失败: %w", advanceErr)
|
||||
}
|
||||
// 验证通过 + 版本登记成功 = 唯一开放目标的路径。
|
||||
_ = store.ReleaseLibraryMaintenance(ctx, payload.Scope, payload.LibraryJobID, finishedAt)
|
||||
log.Printf("library %s succeeded job=%s scope=%s snapshot=%s backup=%s", kind,
|
||||
payload.LibraryJobID, payload.Scope, payload.TargetSnapshotID, outcome.Backup.SnapshotID)
|
||||
return nil
|
||||
case "rolled_back":
|
||||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback",
|
||||
outcome.Error, outcome.Rollback, "", "", finishedAt)
|
||||
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore",
|
||||
"已回滚到上一版本,但验证未通过:目标继续维护,人工确认后再开放", finishedAt)
|
||||
return errors.New(outcome.Error)
|
||||
case "rollback_failed":
|
||||
_ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback",
|
||||
outcome.Error, outcome.Rollback, "", "", finishedAt)
|
||||
_ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore",
|
||||
"回滚失败:目标保持维护,必须按预备份恢复", finishedAt)
|
||||
return fmt.Errorf("%s(%s)", outcome.Error, outcome.Rollback)
|
||||
default:
|
||||
recovery := outcome.Recovery
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@@ -54,6 +55,21 @@ func (r libraryStageRecorder) Stage(ctx context.Context, stage, state, detail st
|
||||
|
||||
func int64Ptr(value int64) *int64 { return &value }
|
||||
|
||||
// redactDSN 只保留库名,避免把连接串写进维护记录。
|
||||
func redactDSN(dsn string) string {
|
||||
if dsn == "" {
|
||||
return ""
|
||||
}
|
||||
if index := strings.LastIndex(dsn, "/"); index >= 0 && index+1 < len(dsn) {
|
||||
name := dsn[index+1:]
|
||||
if cut := strings.Index(name, "?"); cut >= 0 {
|
||||
name = name[:cut]
|
||||
}
|
||||
return name
|
||||
}
|
||||
return "configured"
|
||||
}
|
||||
|
||||
// stageSize 是某一类数据在快照里的真实总量(行数/字节)。
|
||||
type stageSize struct {
|
||||
count int64
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"knowledge-governance/internal/jobs"
|
||||
"knowledge-governance/internal/librarydiff"
|
||||
"knowledge-governance/internal/librarysnapshot"
|
||||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||||
)
|
||||
|
||||
// 整库版本/状态/差异读取契约 + 生产同步的 fail-closed 入口。
|
||||
@@ -591,6 +592,17 @@ func librarySyncEnqueue(w http.ResponseWriter, r *http.Request, config Config) {
|
||||
}
|
||||
grantFound, hasGrant := activeGrantFor(r.Context(), config, request.Scope, request.Target)
|
||||
_ = grantFound
|
||||
// 目标正在维护(被别的作业持有):直接拒绝新写入,不等到 worker 里才发现。
|
||||
if maintenanceStore, ok := config.Store.(interface {
|
||||
LibraryMaintenanceOf(ctx context.Context, scope string) (*postgresstore.LibraryMaintenance, error)
|
||||
}); ok {
|
||||
if held, err := maintenanceStore.LibraryMaintenanceOf(r.Context(), request.Scope); err == nil && held != nil && held.ReleasedAt == nil {
|
||||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_target_in_maintenance",
|
||||
"error": "目标正在维护中,验证完成前不接受新的写入请求",
|
||||
"owner_job": held.OwnerJobID, "state": held.State})
|
||||
return
|
||||
}
|
||||
}
|
||||
gate := librarySyncGateWithGrant(config, overview.Production, overview.Prebackup, request.Target, hasGrant)
|
||||
if !gate.SyncAllowed {
|
||||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{
|
||||
|
||||
@@ -447,6 +447,27 @@ func NewHandler(config Config) (http.Handler, error) {
|
||||
libraryStorageHandler(w, r, config)
|
||||
})
|
||||
})
|
||||
// 目标维护窗口状态:谁持有、处于哪个阶段、何时释放。
|
||||
mux.HandleFunc("GET /api/v1/library/maintenance", func(w http.ResponseWriter, r *http.Request) {
|
||||
scope := r.URL.Query().Get("scope")
|
||||
if scope != "internal" && scope != "external" {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "scope must be internal or external"})
|
||||
return
|
||||
}
|
||||
store, ok := config.Store.(interface {
|
||||
LibraryMaintenanceOf(ctx context.Context, scope string) (*postgresstore.LibraryMaintenance, error)
|
||||
})
|
||||
if !ok {
|
||||
writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "维护窗口存储不可用"})
|
||||
return
|
||||
}
|
||||
held, err := store.LibraryMaintenanceOf(r.Context(), scope)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "无法读取维护窗口"})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"held": held != nil && held.ReleasedAt == nil, "maintenance": held})
|
||||
})
|
||||
mux.HandleFunc("GET /api/v1/library/jobs", func(w http.ResponseWriter, r *http.Request) {
|
||||
libraryJobsHandler(w, r, config)
|
||||
})
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// LibraryMaintenance 是目标维护窗口的持有记录:一个 scope 只能有一个持有者。
|
||||
type LibraryMaintenance struct {
|
||||
Scope string `json:"scope"`
|
||||
OwnerJobID string `json:"owner_job_id"`
|
||||
TargetIdentity string `json:"target_identity"`
|
||||
State string `json:"state"`
|
||||
OriginalServiceState string `json:"original_service_state"`
|
||||
Note string `json:"note"`
|
||||
AcquiredAt time.Time `json:"acquired_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
ReleasedAt *time.Time `json:"released_at,omitempty"`
|
||||
}
|
||||
|
||||
// ErrMaintenanceHeld 表示目标维护窗口被别的作业占着:任何新的写入都必须拒绝。
|
||||
var ErrMaintenanceHeld = errors.New("目标正在维护中,被其他作业占用")
|
||||
|
||||
// AcquireLibraryMaintenance 原子地拿到维护权:
|
||||
// 同一作业可以重入(接管后继续),别的作业一律拒绝——这不是"尽力而为"的锁。
|
||||
func (s *Store) AcquireLibraryMaintenance(ctx context.Context, item LibraryMaintenance, now time.Time) (LibraryMaintenance, error) {
|
||||
row := s.db.QueryRow(ctx, `
|
||||
INSERT INTO library_maintenance (scope, owner_job_id, target_identity, state, original_service_state, note, acquired_at, updated_at)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$7)
|
||||
ON CONFLICT (scope) DO UPDATE SET
|
||||
owner_job_id = EXCLUDED.owner_job_id,
|
||||
target_identity = EXCLUDED.target_identity,
|
||||
state = EXCLUDED.state,
|
||||
note = EXCLUDED.note,
|
||||
updated_at = EXCLUDED.updated_at,
|
||||
released_at = NULL
|
||||
WHERE library_maintenance.owner_job_id = EXCLUDED.owner_job_id
|
||||
RETURNING scope, owner_job_id, target_identity, state, original_service_state, note, acquired_at, updated_at, released_at`,
|
||||
item.Scope, item.OwnerJobID, item.TargetIdentity, item.State, item.OriginalServiceState, item.Note, now.UTC())
|
||||
acquired, err := scanMaintenance(row)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return LibraryMaintenance{}, ErrMaintenanceHeld
|
||||
}
|
||||
if err != nil {
|
||||
return LibraryMaintenance{}, fmt.Errorf("acquire library maintenance: %w", err)
|
||||
}
|
||||
return acquired, nil
|
||||
}
|
||||
|
||||
// UpdateLibraryMaintenanceState 只允许持有者改自己的窗口状态。
|
||||
func (s *Store) UpdateLibraryMaintenanceState(ctx context.Context, scope, ownerJobID, state, note string, now time.Time) error {
|
||||
tag, err := s.db.Exec(ctx, `UPDATE library_maintenance SET state=$3, note=$4, updated_at=$5
|
||||
WHERE scope=$1 AND owner_job_id=$2 AND released_at IS NULL`, scope, ownerJobID, state, note, now.UTC())
|
||||
if err != nil {
|
||||
return fmt.Errorf("update library maintenance: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return errors.New("library maintenance not held by this job")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReleaseLibraryMaintenance 只在完整验证通过后调用;失败路径必须保持窗口。
|
||||
func (s *Store) ReleaseLibraryMaintenance(ctx context.Context, scope, ownerJobID string, now time.Time) error {
|
||||
tag, err := s.db.Exec(ctx, `UPDATE library_maintenance SET state='released', released_at=$3, updated_at=$3
|
||||
WHERE scope=$1 AND owner_job_id=$2 AND released_at IS NULL`, scope, ownerJobID, now.UTC())
|
||||
if err != nil {
|
||||
return fmt.Errorf("release library maintenance: %w", err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return errors.New("library maintenance not held by this job")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// LibraryMaintenanceOf 读当前窗口状态(页面与诊断用)。
|
||||
func (s *Store) LibraryMaintenanceOf(ctx context.Context, scope string) (*LibraryMaintenance, error) {
|
||||
row := s.db.QueryRow(ctx, `SELECT scope, owner_job_id, target_identity, state, original_service_state, note,
|
||||
acquired_at, updated_at, released_at FROM library_maintenance WHERE scope=$1`, scope)
|
||||
item, err := scanMaintenance(row)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &item, nil
|
||||
}
|
||||
|
||||
func scanMaintenance(row pgx.Row) (LibraryMaintenance, error) {
|
||||
var item LibraryMaintenance
|
||||
err := row.Scan(&item.Scope, &item.OwnerJobID, &item.TargetIdentity, &item.State,
|
||||
&item.OriginalServiceState, &item.Note, &item.AcquiredAt, &item.UpdatedAt, &item.ReleasedAt)
|
||||
return item, err
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
DROP TABLE IF EXISTS library_maintenance;
|
||||
@@ -0,0 +1,14 @@
|
||||
-- 目标维护窗口:同步/恢复期间独占目标,只有完整验证通过才开放。
|
||||
-- 一个 scope 同时只能有一个维护持有者;持有者进程死掉后由下一个接管的 worker 继承,
|
||||
-- 不会被"finally"无条件释放。
|
||||
CREATE TABLE library_maintenance (
|
||||
scope VARCHAR(16) PRIMARY KEY CHECK (scope IN ('internal','external')),
|
||||
owner_job_id VARCHAR(128) NOT NULL,
|
||||
target_identity TEXT NOT NULL,
|
||||
state VARCHAR(24) NOT NULL CHECK (state IN ('acquired','applying','verified','released','needs_restore')),
|
||||
original_service_state TEXT NOT NULL DEFAULT '',
|
||||
note TEXT NOT NULL DEFAULT '',
|
||||
acquired_at TIMESTAMPTZ NOT NULL,
|
||||
updated_at TIMESTAMPTZ NOT NULL,
|
||||
released_at TIMESTAMPTZ
|
||||
);
|
||||
@@ -11,22 +11,25 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("loadMigrations() error = %v", err)
|
||||
}
|
||||
if len(migrations) != 69 {
|
||||
t.Fatalf("migration count = %d, want 69", len(migrations))
|
||||
if len(migrations) != 70 {
|
||||
t.Fatalf("migration count = %d, want 70", len(migrations))
|
||||
}
|
||||
if tail := migrations[len(migrations)-1]; tail.Version != 69 ||
|
||||
if tail := migrations[len(migrations)-2]; tail.Version != 69 ||
|
||||
!strings.Contains(tail.UpSQL, "library_jobs_write_target") {
|
||||
t.Fatalf("migration 69 = %#v", tail)
|
||||
}
|
||||
if restore := migrations[len(migrations)-2]; restore.Version != 68 || !strings.Contains(restore.UpSQL, "restore") {
|
||||
if maint := migrations[len(migrations)-1]; maint.Version != 70 || !strings.Contains(maint.UpSQL, "library_maintenance") {
|
||||
t.Fatalf("migration 70 = %#v", maint)
|
||||
}
|
||||
if restore := migrations[len(migrations)-3]; restore.Version != 68 || !strings.Contains(restore.UpSQL, "restore") {
|
||||
t.Fatalf("migration 68 = %#v", restore)
|
||||
}
|
||||
if last := migrations[len(migrations)-3]; last.Version != 67 ||
|
||||
if last := migrations[len(migrations)-4]; last.Version != 67 ||
|
||||
!strings.Contains(last.UpSQL, "library_release_authorizations") ||
|
||||
!strings.Contains(last.UpSQL, "params_digest") {
|
||||
t.Fatalf("migration 67 = %#v", last)
|
||||
}
|
||||
if flow := migrations[len(migrations)-4]; flow.Version != 66 ||
|
||||
if flow := migrations[len(migrations)-5]; flow.Version != 66 ||
|
||||
!strings.Contains(flow.UpSQL, "CREATE TABLE library_versions") ||
|
||||
!strings.Contains(flow.UpSQL, "library_jobs_active") {
|
||||
t.Fatalf("migration 66 = %#v", flow)
|
||||
|
||||
Reference in New Issue
Block a user