diff --git a/CHANGELOG.md b/CHANGELOG.md index 918e937..a48cf0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Changelog +## 开发快照(目标维护窗口:半成品,未验证) + +### Added(代码已完成,**真实取证未通过**) + +- 迁移 000070 `library_maintenance`:一个 scope 一个持有者,记录作业/目标身份/服务原状态/阶段/释放时间。 +- `AcquireLibraryMaintenance`(异主占用直接拒绝)、`UpdateLibraryMaintenanceState`、 + `ReleaseLibraryMaintenance`、`LibraryMaintenanceOf`。 +- worker:写目标前 acquire,写前置 `applying`,只有"验证通过 + 版本登记成功"才 release; + 回滚失败/回滚后未验证 → `needs_restore` 且保持持有(不会被 finally 释放)。 +- API:`GET /api/v1/library/maintenance`;同步入口在目标被他人持有时返回 409。 + +### 已知缺陷(本轮没有修完,不能当成可用) + +- 候选环境实测 `GET /api/v1/library/maintenance` 返回 **404**:路由已注册在 server.go:451, + 但运行实例没匹配上,根因未定位。 +- 维护窗口的真实 kill/接管取证未成功:取证脚本在 kill 之前作业已跑完(竞态), + 三项断言(持有中、kill 后仍持有、维护期间拒绝新写入)全部 false。 + 因此维护协议**没有真实证据**,不得声称已实现。 + ## 开发快照(安全门禁 + 崩溃接管 + 五项真实入口验收) ### Fixed diff --git a/api/openapi.yaml b/api/openapi.yaml index 0ada5fe..3ba2fc8 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -1025,6 +1025,43 @@ } } }, + "/api/v1/library/maintenance": { + "get": { + "tags": [ + "Runtime" + ], + "operationId": "getLibraryMaintenance", + "summary": "目标维护窗口状态:谁持有、处于哪个阶段、是否已释放", + "parameters": [ + { + "name": "scope", + "in": "query", + "required": true, + "schema": { + "type": "string", + "enum": [ + "internal", + "external" + ] + } + } + ], + "responses": { + "200": { + "description": "维护窗口状态(held=true 表示目标仍被占用)" + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "500": { + "$ref": "#/components/responses/InternalError" + }, + "503": { + "description": "维护窗口存储不可用" + } + } + } + }, "/api/v1/library/restores": { "post": { "tags": [ diff --git a/cmd/worker/library_handlers.go b/cmd/worker/library_handlers.go index 13873dc..20e918f 100644 --- a/cmd/worker/library_handlers.go +++ b/cmd/worker/library_handlers.go @@ -114,6 +114,19 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, wireErr.Error(), "prebackup") } defer target.Close() + // 维护窗口:目标在同步/恢复期间被本作业独占,任何其他写入一律拒绝。 + targetIdentity := fmt.Sprintf("scope=%s db=%s index=%s bucket=%s", payload.Scope, + redactDSN(config.LibraryProductionDatabaseURL), config.LibraryProductionIndex, config.LibraryProductionS3Bucket) + maintenance, acquireErr := store.AcquireLibraryMaintenance(ctx, postgresstore.LibraryMaintenance{ + Scope: payload.Scope, OwnerJobID: payload.LibraryJobID, TargetIdentity: targetIdentity, + State: "acquired", OriginalServiceState: "candidate: no external writer configured", + Note: "同步/恢复期间目标维护,验证通过才开放", + }, now) + if acquireErr != nil { + return failLibraryJob(ctx, store, recorder, payload.LibraryJobID, "目标正在维护中:"+acquireErr.Error(), "prebackup") + } + _ = maintenance + if payload.FinalizeOnly { // 写入已完成、只是版本登记失败:只做校验 + 登记,绝不重写生产。 recorder.Stage(ctx, "finalize", "running", "续作收尾:先校验生产与目标快照一致") @@ -142,6 +155,8 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a recorder.Stage(ctx, "finalize", "succeeded", "已登记生效版本(仅收尾,未重写生产)") _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobSucceeded, "finalize", "", "", previous.SnapshotID, payload.TargetSnapshotID, time.Now().UTC()) + // 只有校验+登记都完成才开放目标。 + _ = store.ReleaseLibraryMaintenance(ctx, payload.Scope, payload.LibraryJobID, time.Now().UTC()) return nil } @@ -155,6 +170,7 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a ParamsDigest: digest, Now: now, Grant: libraryrelease.Grant{Target: payload.Scope, SnapshotID: grant.SnapshotID, ParamsDigest: grant.ParamsDigest, ExpiresAt: grant.ExpiresAt, AuthorizedBy: grant.AuthorizedBy}} + _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "applying", "正在写目标", now) outcome := libraryrelease.Run(ctx, request, target, backupSource, recorder) finishedAt := time.Now().UTC() switch outcome.State { @@ -193,16 +209,22 @@ func runLibraryRelease(ctx context.Context, store *postgresstore.Store, config a // 状态落不下去时不要把队列作业报成功:让上层可见并等待接管。 return fmt.Errorf("作业状态写入失败: %w", advanceErr) } + // 验证通过 + 版本登记成功 = 唯一开放目标的路径。 + _ = store.ReleaseLibraryMaintenance(ctx, payload.Scope, payload.LibraryJobID, finishedAt) log.Printf("library %s succeeded job=%s scope=%s snapshot=%s backup=%s", kind, payload.LibraryJobID, payload.Scope, payload.TargetSnapshotID, outcome.Backup.SnapshotID) return nil case "rolled_back": _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback", outcome.Error, outcome.Rollback, "", "", finishedAt) + _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", + "已回滚到上一版本,但验证未通过:目标继续维护,人工确认后再开放", finishedAt) return errors.New(outcome.Error) case "rollback_failed": _ = store.AdvanceLibraryJob(ctx, payload.LibraryJobID, governance.LibraryJobFailed, "rollback", outcome.Error, outcome.Rollback, "", "", finishedAt) + _ = store.UpdateLibraryMaintenanceState(ctx, payload.Scope, payload.LibraryJobID, "needs_restore", + "回滚失败:目标保持维护,必须按预备份恢复", finishedAt) return fmt.Errorf("%s(%s)", outcome.Error, outcome.Rollback) default: recovery := outcome.Recovery diff --git a/cmd/worker/main.go b/cmd/worker/main.go index 839840d..5dacc34 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -8,6 +8,7 @@ import ( "os" "os/signal" "path/filepath" + "strings" "syscall" "time" @@ -54,6 +55,21 @@ func (r libraryStageRecorder) Stage(ctx context.Context, stage, state, detail st func int64Ptr(value int64) *int64 { return &value } +// redactDSN 只保留库名,避免把连接串写进维护记录。 +func redactDSN(dsn string) string { + if dsn == "" { + return "" + } + if index := strings.LastIndex(dsn, "/"); index >= 0 && index+1 < len(dsn) { + name := dsn[index+1:] + if cut := strings.Index(name, "?"); cut >= 0 { + name = name[:cut] + } + return name + } + return "configured" +} + // stageSize 是某一类数据在快照里的真实总量(行数/字节)。 type stageSize struct { count int64 diff --git a/internal/httpapi/library_flow.go b/internal/httpapi/library_flow.go index 71e1a09..0db832a 100644 --- a/internal/httpapi/library_flow.go +++ b/internal/httpapi/library_flow.go @@ -16,6 +16,7 @@ import ( "knowledge-governance/internal/jobs" "knowledge-governance/internal/librarydiff" "knowledge-governance/internal/librarysnapshot" + postgresstore "knowledge-governance/internal/platform/postgres" ) // 整库版本/状态/差异读取契约 + 生产同步的 fail-closed 入口。 @@ -591,6 +592,17 @@ func librarySyncEnqueue(w http.ResponseWriter, r *http.Request, config Config) { } grantFound, hasGrant := activeGrantFor(r.Context(), config, request.Scope, request.Target) _ = grantFound + // 目标正在维护(被别的作业持有):直接拒绝新写入,不等到 worker 里才发现。 + if maintenanceStore, ok := config.Store.(interface { + LibraryMaintenanceOf(ctx context.Context, scope string) (*postgresstore.LibraryMaintenance, error) + }); ok { + if held, err := maintenanceStore.LibraryMaintenanceOf(r.Context(), request.Scope); err == nil && held != nil && held.ReleasedAt == nil { + writeJSON(w, http.StatusConflict, map[string]any{"code": "library_target_in_maintenance", + "error": "目标正在维护中,验证完成前不接受新的写入请求", + "owner_job": held.OwnerJobID, "state": held.State}) + return + } + } gate := librarySyncGateWithGrant(config, overview.Production, overview.Prebackup, request.Target, hasGrant) if !gate.SyncAllowed { writeJSON(w, http.StatusServiceUnavailable, map[string]any{ diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index d0d1d8e..791079e 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -447,6 +447,27 @@ func NewHandler(config Config) (http.Handler, error) { libraryStorageHandler(w, r, config) }) }) + // 目标维护窗口状态:谁持有、处于哪个阶段、何时释放。 + mux.HandleFunc("GET /api/v1/library/maintenance", func(w http.ResponseWriter, r *http.Request) { + scope := r.URL.Query().Get("scope") + if scope != "internal" && scope != "external" { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "scope must be internal or external"}) + return + } + store, ok := config.Store.(interface { + LibraryMaintenanceOf(ctx context.Context, scope string) (*postgresstore.LibraryMaintenance, error) + }) + if !ok { + writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "维护窗口存储不可用"}) + return + } + held, err := store.LibraryMaintenanceOf(r.Context(), scope) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "无法读取维护窗口"}) + return + } + writeJSON(w, http.StatusOK, map[string]any{"held": held != nil && held.ReleasedAt == nil, "maintenance": held}) + }) mux.HandleFunc("GET /api/v1/library/jobs", func(w http.ResponseWriter, r *http.Request) { libraryJobsHandler(w, r, config) }) diff --git a/internal/platform/postgres/library_maintenance.go b/internal/platform/postgres/library_maintenance.go new file mode 100644 index 0000000..af92b2d --- /dev/null +++ b/internal/platform/postgres/library_maintenance.go @@ -0,0 +1,99 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/jackc/pgx/v5" +) + +// LibraryMaintenance 是目标维护窗口的持有记录:一个 scope 只能有一个持有者。 +type LibraryMaintenance struct { + Scope string `json:"scope"` + OwnerJobID string `json:"owner_job_id"` + TargetIdentity string `json:"target_identity"` + State string `json:"state"` + OriginalServiceState string `json:"original_service_state"` + Note string `json:"note"` + AcquiredAt time.Time `json:"acquired_at"` + UpdatedAt time.Time `json:"updated_at"` + ReleasedAt *time.Time `json:"released_at,omitempty"` +} + +// ErrMaintenanceHeld 表示目标维护窗口被别的作业占着:任何新的写入都必须拒绝。 +var ErrMaintenanceHeld = errors.New("目标正在维护中,被其他作业占用") + +// AcquireLibraryMaintenance 原子地拿到维护权: +// 同一作业可以重入(接管后继续),别的作业一律拒绝——这不是"尽力而为"的锁。 +func (s *Store) AcquireLibraryMaintenance(ctx context.Context, item LibraryMaintenance, now time.Time) (LibraryMaintenance, error) { + row := s.db.QueryRow(ctx, ` +INSERT INTO library_maintenance (scope, owner_job_id, target_identity, state, original_service_state, note, acquired_at, updated_at) +VALUES ($1,$2,$3,$4,$5,$6,$7,$7) +ON CONFLICT (scope) DO UPDATE SET + owner_job_id = EXCLUDED.owner_job_id, + target_identity = EXCLUDED.target_identity, + state = EXCLUDED.state, + note = EXCLUDED.note, + updated_at = EXCLUDED.updated_at, + released_at = NULL +WHERE library_maintenance.owner_job_id = EXCLUDED.owner_job_id +RETURNING scope, owner_job_id, target_identity, state, original_service_state, note, acquired_at, updated_at, released_at`, + item.Scope, item.OwnerJobID, item.TargetIdentity, item.State, item.OriginalServiceState, item.Note, now.UTC()) + acquired, err := scanMaintenance(row) + if errors.Is(err, pgx.ErrNoRows) { + return LibraryMaintenance{}, ErrMaintenanceHeld + } + if err != nil { + return LibraryMaintenance{}, fmt.Errorf("acquire library maintenance: %w", err) + } + return acquired, nil +} + +// UpdateLibraryMaintenanceState 只允许持有者改自己的窗口状态。 +func (s *Store) UpdateLibraryMaintenanceState(ctx context.Context, scope, ownerJobID, state, note string, now time.Time) error { + tag, err := s.db.Exec(ctx, `UPDATE library_maintenance SET state=$3, note=$4, updated_at=$5 +WHERE scope=$1 AND owner_job_id=$2 AND released_at IS NULL`, scope, ownerJobID, state, note, now.UTC()) + if err != nil { + return fmt.Errorf("update library maintenance: %w", err) + } + if tag.RowsAffected() == 0 { + return errors.New("library maintenance not held by this job") + } + return nil +} + +// ReleaseLibraryMaintenance 只在完整验证通过后调用;失败路径必须保持窗口。 +func (s *Store) ReleaseLibraryMaintenance(ctx context.Context, scope, ownerJobID string, now time.Time) error { + tag, err := s.db.Exec(ctx, `UPDATE library_maintenance SET state='released', released_at=$3, updated_at=$3 +WHERE scope=$1 AND owner_job_id=$2 AND released_at IS NULL`, scope, ownerJobID, now.UTC()) + if err != nil { + return fmt.Errorf("release library maintenance: %w", err) + } + if tag.RowsAffected() == 0 { + return errors.New("library maintenance not held by this job") + } + return nil +} + +// LibraryMaintenanceOf 读当前窗口状态(页面与诊断用)。 +func (s *Store) LibraryMaintenanceOf(ctx context.Context, scope string) (*LibraryMaintenance, error) { + row := s.db.QueryRow(ctx, `SELECT scope, owner_job_id, target_identity, state, original_service_state, note, +acquired_at, updated_at, released_at FROM library_maintenance WHERE scope=$1`, scope) + item, err := scanMaintenance(row) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + return &item, nil +} + +func scanMaintenance(row pgx.Row) (LibraryMaintenance, error) { + var item LibraryMaintenance + err := row.Scan(&item.Scope, &item.OwnerJobID, &item.TargetIdentity, &item.State, + &item.OriginalServiceState, &item.Note, &item.AcquiredAt, &item.UpdatedAt, &item.ReleasedAt) + return item, err +} \ No newline at end of file diff --git a/internal/platform/postgres/migrations/000070_library_maintenance.down.sql b/internal/platform/postgres/migrations/000070_library_maintenance.down.sql new file mode 100644 index 0000000..c4976e0 --- /dev/null +++ b/internal/platform/postgres/migrations/000070_library_maintenance.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS library_maintenance; diff --git a/internal/platform/postgres/migrations/000070_library_maintenance.up.sql b/internal/platform/postgres/migrations/000070_library_maintenance.up.sql new file mode 100644 index 0000000..76bec48 --- /dev/null +++ b/internal/platform/postgres/migrations/000070_library_maintenance.up.sql @@ -0,0 +1,14 @@ +-- 目标维护窗口:同步/恢复期间独占目标,只有完整验证通过才开放。 +-- 一个 scope 同时只能有一个维护持有者;持有者进程死掉后由下一个接管的 worker 继承, +-- 不会被"finally"无条件释放。 +CREATE TABLE library_maintenance ( + scope VARCHAR(16) PRIMARY KEY CHECK (scope IN ('internal','external')), + owner_job_id VARCHAR(128) NOT NULL, + target_identity TEXT NOT NULL, + state VARCHAR(24) NOT NULL CHECK (state IN ('acquired','applying','verified','released','needs_restore')), + original_service_state TEXT NOT NULL DEFAULT '', + note TEXT NOT NULL DEFAULT '', + acquired_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + released_at TIMESTAMPTZ +); \ No newline at end of file diff --git a/internal/platform/postgres/migrations_test.go b/internal/platform/postgres/migrations_test.go index 0043ea0..eaa843a 100644 --- a/internal/platform/postgres/migrations_test.go +++ b/internal/platform/postgres/migrations_test.go @@ -11,22 +11,25 @@ func TestEmbeddedMigrationsAreVersionSorted(t *testing.T) { if err != nil { t.Fatalf("loadMigrations() error = %v", err) } - if len(migrations) != 69 { - t.Fatalf("migration count = %d, want 69", len(migrations)) + if len(migrations) != 70 { + t.Fatalf("migration count = %d, want 70", len(migrations)) } - if tail := migrations[len(migrations)-1]; tail.Version != 69 || + if tail := migrations[len(migrations)-2]; tail.Version != 69 || !strings.Contains(tail.UpSQL, "library_jobs_write_target") { t.Fatalf("migration 69 = %#v", tail) } - if restore := migrations[len(migrations)-2]; restore.Version != 68 || !strings.Contains(restore.UpSQL, "restore") { + if maint := migrations[len(migrations)-1]; maint.Version != 70 || !strings.Contains(maint.UpSQL, "library_maintenance") { + t.Fatalf("migration 70 = %#v", maint) + } + if restore := migrations[len(migrations)-3]; restore.Version != 68 || !strings.Contains(restore.UpSQL, "restore") { t.Fatalf("migration 68 = %#v", restore) } - if last := migrations[len(migrations)-3]; last.Version != 67 || + if last := migrations[len(migrations)-4]; last.Version != 67 || !strings.Contains(last.UpSQL, "library_release_authorizations") || !strings.Contains(last.UpSQL, "params_digest") { t.Fatalf("migration 67 = %#v", last) } - if flow := migrations[len(migrations)-4]; flow.Version != 66 || + if flow := migrations[len(migrations)-5]; flow.Version != 66 || !strings.Contains(flow.UpSQL, "CREATE TABLE library_versions") || !strings.Contains(flow.UpSQL, "library_jobs_active") { t.Fatalf("migration 66 = %#v", flow)