0088004cd6
接管分类、基线判定、已知索引映射、同步闸口的创建判据与归档保留的保护标记, 原来都用 LibraryVersions(scope, 50/100) 这个分页窗口再在内存里筛。生效版本每 scope 只有一行、可能很久以前登记,历史一长就被挤出窗口 —— 于是"已生效"被读成 "没有生效":接管会把已经上线的目标重写一遍,基线入口会给已有权威版本的目标 再建一次基线。 现在统一走两条精确查询:LibraryCurrentVersion(kind='production_effective' AND superseded_at IS NULL,每 scope 至多一行)与 LibraryVersionOfSnapshot(按快照 身份取回"已顶替"证据)。多行、自相矛盾、读不到一律 fail closed;同一含义只留 一份实现(knownIndexMapping 同步与恢复共用)。LibraryVersions 仍是 1..100 的 展示窗口,只用于列表。 验证:真实 PG 上先复现(窗口实现 ⇒ found=false),改后 7 条 store 契约用例与 worker 端到端判定用例全绿;本机 make verify 15 项 success(用例 1264 通过 / 0 失败 / 93 已声明可选跳过)。
981 lines
44 KiB
Go
981 lines
44 KiB
Go
package httpapi
|
||
|
||
import (
|
||
"context"
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"log"
|
||
"net/http"
|
||
"strings"
|
||
"time"
|
||
|
||
"github.com/jackc/pgx/v5/pgxpool"
|
||
|
||
"knowledge-governance/internal/governance"
|
||
"knowledge-governance/internal/jobs"
|
||
"knowledge-governance/internal/librarydiff"
|
||
"knowledge-governance/internal/librarysnapshot"
|
||
postgresstore "knowledge-governance/internal/platform/postgres"
|
||
v071 "knowledge-governance/internal/releaseadapter/v071"
|
||
)
|
||
|
||
// 整库版本/状态/差异读取契约 + 生产同步的 fail-closed 入口。
|
||
//
|
||
// 三类版本严格分开,不能互相顶替:
|
||
// - 测试封存(archive 里的 sealed 产物)——只证明测试库有一份完整版本;
|
||
// - 生产当前生效(library_versions.kind=production_effective)——必须有生产验证证据;
|
||
// - 生产预备份(kind=production_prebackup)——每次生产写入前的独立备份。
|
||
// 生产未建立权威版本时返回 unknown,不拿最新测试快照冒充。
|
||
|
||
type libraryVersionView struct {
|
||
ID string `json:"id"`
|
||
Kind string `json:"kind"`
|
||
SnapshotID string `json:"snapshot_id"`
|
||
VersionLabel string `json:"version_label"`
|
||
SourceJobID string `json:"source_job_id"`
|
||
Evidence map[string]any `json:"evidence"`
|
||
CreatedBy string `json:"created_by"`
|
||
CreatedAt time.Time `json:"created_at"`
|
||
SupersededAt *time.Time `json:"superseded_at,omitempty"`
|
||
}
|
||
|
||
// librarySyncJobType 与 worker 的处理器类型一致;单独定义避免 API 依赖 worker 包。
|
||
const librarySyncJobType = "library.sync"
|
||
|
||
// libraryRecoverJobType 是"按预备份恢复并校验"的队列类型(同一份定义也在 worker 里)。
|
||
const libraryRecoverJobType = "library.recover"
|
||
|
||
type libraryKnownView struct {
|
||
State string `json:"state"` // known | unknown
|
||
Version *libraryVersionView `json:"version,omitempty"`
|
||
Reason string `json:"reason,omitempty"`
|
||
// EmptyBaseline 表示"没有生效版本记录,但生产目标可读且确实是空的":
|
||
// 这是首次发布的合法起点,和"生产非空却没有权威版本"必须区分。
|
||
EmptyBaseline bool `json:"empty_baseline,omitempty"`
|
||
// BoundToTarget 只对预备份有意义:这份备份的内容是否就是当前待发布目标。
|
||
// 最近一次生产写入前的备份可能**不是**本次目标(例如刚按版本恢复过),
|
||
// 页面必须照实说是哪一次作业的备份,而不是一句"尚未创建"。
|
||
BoundToTarget bool `json:"bound_to_target,omitempty"`
|
||
}
|
||
|
||
// productionScopeState 只读探测生产目标:真正连库数一遍,不用缓存或推断。
|
||
//
|
||
// "空"必须是整库意义的空:只数 knowledges 会把"文档已删、分块/Wiki/资源还在"
|
||
// 的目标当成首次发布的空基线,从而绕过基线初始化。所以这里用适配层同一份
|
||
// 表清单与范围过滤(v071.ScopeContentEmpty),并把逐表行数交回调用方作为证据。
|
||
func productionScopeState(ctx context.Context, config Config, scope string) (bool, map[string]int64, error) {
|
||
if strings.TrimSpace(config.LibraryProductionDatabaseURL) == "" {
|
||
return false, nil, errors.New("生产整库目标未配置")
|
||
}
|
||
kb := config.LibraryProductionInternalKB
|
||
if scope == "external" {
|
||
kb = config.LibraryProductionExternalKB
|
||
}
|
||
if kb == "" {
|
||
return false, nil, errors.New("该范围的目标知识库未配置")
|
||
}
|
||
pool, err := pgxpool.New(ctx, config.LibraryProductionDatabaseURL)
|
||
if err != nil {
|
||
return false, nil, err
|
||
}
|
||
defer pool.Close()
|
||
return v071.ScopeContentEmpty(ctx, pool,
|
||
v071.ProductionScope{TenantID: config.LibraryProductionTenantID, KnowledgeBaseID: kb})
|
||
}
|
||
|
||
// productionScopeEmpty 保留"只问空不空"的调用形状;逐表行数在需要举证时用
|
||
// productionScopeState 取。
|
||
func productionScopeEmpty(ctx context.Context, config Config, scope string) (bool, error) {
|
||
empty, _, err := productionScopeState(ctx, config, scope)
|
||
return empty, err
|
||
}
|
||
|
||
type libraryDiffView struct {
|
||
State string `json:"state"` // unknown | computed
|
||
Reason string `json:"reason,omitempty"`
|
||
Added []any `json:"added"`
|
||
Updated []any `json:"updated"`
|
||
Removed []any `json:"removed"`
|
||
// BaseSnapshot/TargetSnapshot 标明这次比对用的是哪两个整库版本,
|
||
// 避免把某次历史差异当成"当前拟发布"。
|
||
BaseSnapshot string `json:"base_snapshot,omitempty"`
|
||
TargetSnapshot string `json:"target_snapshot,omitempty"`
|
||
// Wiki 是**页面级**差异:只能证明页面被增/改/撤,不能证明共享正文、实体、
|
||
// 概念或边被重新计算——后者需要 WeKnora 内部语义,本平台不做也不声称。
|
||
Wiki *libraryWikiDiffView `json:"wiki,omitempty"`
|
||
}
|
||
|
||
type libraryWikiDiffView struct {
|
||
State string `json:"state"` // computed | unknown
|
||
Reason string `json:"reason,omitempty"`
|
||
Added int `json:"added"`
|
||
Updated int `json:"updated"`
|
||
Removed int `json:"removed"`
|
||
// Sample 只列前若干页,避免把整库 Wiki 列表塞进接口。
|
||
Sample []string `json:"sample"`
|
||
Note string `json:"note"`
|
||
// Pages 是逐页的**贡献级**明细:哪一页、变了哪几类语义、掉/加了哪些来源文档。
|
||
// 页面级计数只看增删改;撤销一份共享贡献时页面通常仍是"更新",
|
||
// 只有这里的 sources_removed / changed 才能证明那份贡献确实被摘掉了。
|
||
Pages []libraryWikiPageView `json:"pages,omitempty"`
|
||
PagesTruncated bool `json:"pages_truncated,omitempty"`
|
||
SourcesAdded int `json:"sources_added"`
|
||
SourcesRemoved int `json:"sources_removed"`
|
||
BaseCount int `json:"base_count"`
|
||
TargetCount int `json:"target_count"`
|
||
}
|
||
|
||
type libraryWikiPageView struct {
|
||
Slug string `json:"slug"`
|
||
Title string `json:"title"`
|
||
PageType string `json:"page_type,omitempty"`
|
||
Status string `json:"status"` // added | updated | removed
|
||
Changed []string `json:"changed"`
|
||
SourcesAdded []string `json:"sources_added,omitempty"`
|
||
SourcesRemoved []string `json:"sources_removed,omitempty"`
|
||
CitationsBefore int `json:"citations_before"`
|
||
CitationsAfter int `json:"citations_after"`
|
||
}
|
||
|
||
// wikiPageLimit 是接口一次最多返回的页数;超出部分只给计数,不给半截明细。
|
||
const wikiPageLimit = 50
|
||
|
||
func wikiViewFrom(result librarydiff.WikiResult, limit int) *libraryWikiDiffView {
|
||
if limit <= 0 {
|
||
limit = wikiPageLimit
|
||
}
|
||
view := &libraryWikiDiffView{State: result.State, Added: result.Added, Updated: result.Updated,
|
||
Removed: result.Removed, Sample: []string{}, SourcesAdded: result.SourcesAdded,
|
||
SourcesRemoved: result.SourcesRemoved, BaseCount: result.BaseCount, TargetCount: result.TargetCount,
|
||
Note: "逐页比对正文、来源文档、引用分块、出入边与目录;语义重算发生在源侧,本平台只如实比对与保留结果"}
|
||
for index, page := range result.Pages {
|
||
if index >= limit {
|
||
view.PagesTruncated = true
|
||
break
|
||
}
|
||
view.Pages = append(view.Pages, libraryWikiPageView{Slug: page.Slug, Title: page.Title,
|
||
PageType: page.PageType, Status: page.Status, Changed: librarydiff.ChangedLabels(page.Changed),
|
||
SourcesAdded: page.SourcesAdded, SourcesRemoved: page.SourcesRemoved,
|
||
CitationsBefore: page.CitationsBefore, CitationsAfter: page.CitationsAfter})
|
||
}
|
||
for _, page := range view.Pages {
|
||
if len(view.Sample) >= 20 {
|
||
break
|
||
}
|
||
label := map[string]string{"added": "新增", "updated": "更新", "removed": "撤销"}[page.Status]
|
||
view.Sample = append(view.Sample, label+" "+page.Title)
|
||
}
|
||
return view
|
||
}
|
||
|
||
// wikiDiffOf 比较两份快照的 wiki_pages 行数组(页面级)。
|
||
// 它不能替代"删除文档后共享正文/实体/边/引用如何变化"的语义证据。
|
||
func wikiDiffOf(archive *librarysnapshot.Archive, baseID, targetID string) *libraryWikiDiffView {
|
||
unknown := func(reason string) *libraryWikiDiffView {
|
||
return &libraryWikiDiffView{State: "unknown", Reason: reason, Sample: []string{},
|
||
Note: "页面级比对缺失时不能声称 Wiki 已同步"}
|
||
}
|
||
rows := func(id string) ([]map[string]any, string) {
|
||
// 只读清单定位 wiki_pages 部件:整版 Load 会把"比两页 Wiki"变成
|
||
// 逐部件校验整库字节(79 版归档上单版就 252ms)。被读取的那一个部件
|
||
// 由 OpenPart 自己复核摘要,所以正文仍然是验过的字节。
|
||
manifest, err := archive.LoadManifest(id)
|
||
if err != nil {
|
||
return nil, "快照不可读"
|
||
}
|
||
for _, part := range manifest.Parts {
|
||
if part.Kind != "wiki" || part.Name != "wiki_pages" {
|
||
continue
|
||
}
|
||
reader, err := archive.OpenPart(part)
|
||
if err != nil {
|
||
return nil, "Wiki 正文不可读"
|
||
}
|
||
defer reader.Close()
|
||
parsed, err := librarydiff.ReadDocuments(reader)
|
||
if err != nil {
|
||
return nil, "Wiki 行格式不符"
|
||
}
|
||
return parsed, ""
|
||
}
|
||
return nil, "快照里没有 wiki_pages 部件"
|
||
}
|
||
base, reason := rows(baseID)
|
||
if reason != "" {
|
||
return unknown("生产版本:" + reason)
|
||
}
|
||
target, reason := rows(targetID)
|
||
if reason != "" {
|
||
return unknown("测试版本:" + reason)
|
||
}
|
||
result, err := librarydiff.CompareWiki(base, target)
|
||
if err != nil {
|
||
return unknown("Wiki 逐页比对失败:" + err.Error())
|
||
}
|
||
return wikiViewFrom(result, wikiPageLimit)
|
||
}
|
||
|
||
// computeLibraryDiff 用归档里的 knowledges 行数组比较两个快照:
|
||
// 身份是文档 ID,内容是排除易变列后的整行摘要;任一侧读不到就返回 false,
|
||
// 由调用方保持 unknown 并写明原因,而不是给出空差异冒充"无变化"。
|
||
func computeLibraryDiff(root, baseID, targetID string) (libraryDiffView, bool) {
|
||
unknown := libraryDiffView{State: "unknown", Added: []any{}, Updated: []any{}, Removed: []any{},
|
||
BaseSnapshot: baseID, TargetSnapshot: targetID}
|
||
if root == "" {
|
||
unknown.Reason = "整库归档未配置,无法比较生产与测试版本"
|
||
return unknown, false
|
||
}
|
||
if baseID == "" || targetID == "" {
|
||
unknown.Reason = "生产当前版本或测试封存缺失,没有可比的两份整库版本"
|
||
return unknown, false
|
||
}
|
||
archive, err := librarysnapshot.Open(root, 256*1024*1024)
|
||
if err != nil {
|
||
unknown.Reason = "归档根不可用,无法比较"
|
||
return unknown, false
|
||
}
|
||
defer archive.Close()
|
||
read := func(id string) ([]map[string]any, string) {
|
||
// 清单级定位 + OpenPart 复核被读取部件:差异只需要 knowledges 一列部件,
|
||
// 没有理由为它重算整版所有 blob。
|
||
manifest, loadErr := archive.LoadManifest(id)
|
||
if loadErr != nil {
|
||
return nil, "快照不在本地归档或校验失败"
|
||
}
|
||
for _, part := range manifest.Parts {
|
||
if part.Kind != "database" || part.Name != "knowledges" {
|
||
continue
|
||
}
|
||
reader, openErr := archive.OpenPart(part)
|
||
if openErr != nil {
|
||
return nil, "归档正文不可读"
|
||
}
|
||
defer reader.Close()
|
||
rows, readErr := librarydiff.ReadDocuments(reader)
|
||
if readErr != nil {
|
||
return nil, "归档正文格式不符"
|
||
}
|
||
return rows, ""
|
||
}
|
||
return nil, "快照里没有 knowledges 部件"
|
||
}
|
||
base, reason := read(baseID)
|
||
if reason != "" {
|
||
unknown.Reason = "生产版本:" + reason
|
||
return unknown, false
|
||
}
|
||
target, reason := read(targetID)
|
||
if reason != "" {
|
||
unknown.Reason = "测试版本:" + reason
|
||
return unknown, false
|
||
}
|
||
result, err := librarydiff.Compare(base, target)
|
||
if err != nil {
|
||
unknown.Reason = "差异计算失败:" + err.Error()
|
||
return unknown, false
|
||
}
|
||
view := libraryDiffView{State: "computed", Added: []any{}, Updated: []any{}, Removed: []any{},
|
||
BaseSnapshot: baseID, TargetSnapshot: targetID}
|
||
view.Wiki = wikiDiffOf(archive, baseID, targetID)
|
||
for _, item := range result.Added {
|
||
view.Added = append(view.Added, item)
|
||
}
|
||
for _, item := range result.Updated {
|
||
view.Updated = append(view.Updated, item)
|
||
}
|
||
for _, item := range result.Removed {
|
||
view.Removed = append(view.Removed, item)
|
||
}
|
||
return view, true
|
||
}
|
||
|
||
// libraryAuthorizationView 让页面能直说"有没有授权、有效期到什么时候"。
|
||
type libraryAuthorizationView struct {
|
||
State string `json:"state"` // active | none
|
||
AuthorizedBy string `json:"authorized_by,omitempty"`
|
||
ExpiresAt *time.Time `json:"expires_at,omitempty"`
|
||
Reason string `json:"reason,omitempty"`
|
||
}
|
||
|
||
type libraryGateView struct {
|
||
SyncAllowed bool `json:"sync_allowed"`
|
||
Reasons []string `json:"reasons"`
|
||
}
|
||
|
||
type libraryOverview struct {
|
||
Scope string `json:"scope"`
|
||
TestSealed *librarySnapshotView `json:"test_sealed,omitempty"`
|
||
TestSealedReason string `json:"test_sealed_reason,omitempty"`
|
||
Production libraryKnownView `json:"production"`
|
||
Prebackup libraryKnownView `json:"prebackup"`
|
||
// Versions 是已登记版本的轻量清单(不含 evidence):页面要按快照判断
|
||
// "这份归档是哪一类版本",不能把所有归档都写成测试封存。
|
||
Versions []libraryVersionRef `json:"versions"`
|
||
Diff libraryDiffView `json:"diff"`
|
||
Job *governance.LibraryJob `json:"job,omitempty"`
|
||
Jobs []governance.LibraryJob `json:"jobs"`
|
||
Gate libraryGateView `json:"gate"`
|
||
Authorization libraryAuthorizationView `json:"authorization"`
|
||
CaptureAvailable bool `json:"capture_available"`
|
||
CaptureReason string `json:"capture_reason,omitempty"`
|
||
Executor libraryExecutorView `json:"executor"`
|
||
Now time.Time `json:"now"`
|
||
}
|
||
|
||
// libraryVersionRef 只回答"这个快照被登记成哪一类版本、由哪个作业登记"。
|
||
type libraryVersionRef struct {
|
||
ID string `json:"id"`
|
||
Kind string `json:"kind"`
|
||
SnapshotID string `json:"snapshot_id"`
|
||
VersionLabel string `json:"version_label"`
|
||
SourceJobID string `json:"source_job_id"`
|
||
CreatedAt time.Time `json:"created_at"`
|
||
SupersededAt *time.Time `json:"superseded_at,omitempty"`
|
||
}
|
||
|
||
type libraryJobStore interface {
|
||
CreateLibraryJob(ctx context.Context, job governance.LibraryJob, stages []string, now time.Time) (governance.LibraryJob, bool, error)
|
||
LibraryJob(ctx context.Context, id string) (governance.LibraryJob, error)
|
||
ListLibraryJobs(ctx context.Context, scope string, limit int) ([]governance.LibraryJob, error)
|
||
RecordLibraryVersion(ctx context.Context, version governance.LibraryVersion) error
|
||
LibraryVersions(ctx context.Context, scope string, limit int) ([]governance.LibraryVersion, error)
|
||
// 当前生效版本与"某快照自己的登记"必须是精确查询:LibraryVersions 是分页展示窗口,
|
||
// 窗口一满就会把"已生效"读成"没有",同步闸口/基线入口就会放行一次不必要的重写。
|
||
LibraryCurrentVersion(ctx context.Context, scope string) (governance.LibraryVersion, bool, error)
|
||
LibraryVersionOfSnapshot(ctx context.Context, scope string, kind governance.LibraryVersionKind,
|
||
snapshotID string) (governance.LibraryVersion, bool, error)
|
||
UpdateLibraryStage(ctx context.Context, jobID, stage, state string, total, completed, bytesTotal, bytesDone *int64, detail string, now time.Time) error
|
||
AdvanceLibraryJob(ctx context.Context, jobID string, state governance.LibraryJobState, stage, message, recovery, snapshotID, targetSnapshotID string, now time.Time) error
|
||
AttachLibraryQueueJob(ctx context.Context, jobID, queueJobID string, now time.Time) error
|
||
CancelLibraryJob(ctx context.Context, jobID, reason string, now time.Time) error
|
||
}
|
||
|
||
func libraryStore(config Config) (libraryJobStore, bool) {
|
||
store, ok := config.Store.(libraryJobStore)
|
||
return store, ok
|
||
}
|
||
|
||
// instanceMaintenanceGuard 在"实例级维护"被别的作业持有时拒绝新的写入请求。
|
||
// 只看 scope 行是不够的:internal 与 external 动的是同一批容器。
|
||
func instanceMaintenanceGuard(ctx context.Context, config Config, scope string) (map[string]any, bool) {
|
||
instance := config.LibraryTargetInstance
|
||
if instance == "" {
|
||
return nil, false
|
||
}
|
||
store, ok := config.Store.(interface {
|
||
InstanceMaintenanceOf(ctx context.Context, instance string) (*postgresstore.LibraryInstanceMaintenance, error)
|
||
})
|
||
if !ok {
|
||
return nil, false
|
||
}
|
||
held, err := store.InstanceMaintenanceOf(ctx, instance)
|
||
if err != nil || held == nil || held.ReleasedAt != nil {
|
||
return nil, false
|
||
}
|
||
return map[string]any{"code": "library_target_in_maintenance",
|
||
"error": "目标实例正在维护中,验证完成前不接受新的写入请求",
|
||
"instance": held.Instance, "owner_job": held.OwnerJobID, "owner_scope": held.OwnerScope,
|
||
"state": held.State, "ingress_state": held.IngressState}, true
|
||
}
|
||
|
||
// libraryMaintenanceRecoverHandler 是"按预备份恢复并校验"的唯一入口:
|
||
// 它只建一个恢复作业,真正的恢复与校验由 worker 在维护关闭状态下执行。
|
||
func libraryMaintenanceRecoverHandler(w http.ResponseWriter, r *http.Request, config Config) {
|
||
if r.Method != http.MethodPost {
|
||
writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"})
|
||
return
|
||
}
|
||
var request struct {
|
||
Scope string `json:"scope"`
|
||
WindowJobID string `json:"window_job_id"`
|
||
}
|
||
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&request); err != nil {
|
||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"})
|
||
return
|
||
}
|
||
if request.Scope != "internal" && request.Scope != "external" {
|
||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "scope must be internal or external"})
|
||
return
|
||
}
|
||
instance := config.LibraryTargetInstance
|
||
if instance == "" {
|
||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"code": "library_recovery_unavailable",
|
||
"error": "目标维护未接线:无法执行按预备份恢复"})
|
||
return
|
||
}
|
||
store, ok := config.Store.(interface {
|
||
InstanceMaintenanceOf(ctx context.Context, instance string) (*postgresstore.LibraryInstanceMaintenance, error)
|
||
})
|
||
if !ok {
|
||
writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "维护记录存储不可用"})
|
||
return
|
||
}
|
||
held, err := store.InstanceMaintenanceOf(r.Context(), instance)
|
||
if err != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "无法读取维护记录"})
|
||
return
|
||
}
|
||
if held == nil || held.ReleasedAt != nil {
|
||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_no_held_window",
|
||
"error": "目标实例当前没有未释放的维护窗口,不需要恢复"})
|
||
return
|
||
}
|
||
if held.OwnerScope != request.Scope {
|
||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_window_scope_mismatch",
|
||
"error": "维护窗口属于另一个 scope", "owner_scope": held.OwnerScope})
|
||
return
|
||
}
|
||
if request.WindowJobID != "" && request.WindowJobID != held.OwnerJobID {
|
||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_window_mismatch",
|
||
"error": "维护窗口属于另一个作业", "owner_job": held.OwnerJobID})
|
||
return
|
||
}
|
||
now := time.Now().UTC()
|
||
payload, err := json.Marshal(map[string]string{"scope": held.OwnerScope, "window_job_id": held.OwnerJobID})
|
||
if err != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot build recovery payload"})
|
||
return
|
||
}
|
||
// 队列作业 id 与幂等键都按窗口固定:重复点击只会得到同一个恢复作业。
|
||
// 队列 id 有长度上限(36),窗口作业 id 比它还长,所以按窗口摘要取短 id。
|
||
sum := sha256.Sum256([]byte(held.OwnerJobID))
|
||
queueID := "lrec-" + hex.EncodeToString(sum[:14])
|
||
queueJob, err := config.Store.Enqueue(r.Context(), jobs.EnqueueRequest{
|
||
ID: queueID, Type: libraryRecoverJobType, Payload: payload,
|
||
AvailableAt: now, MaxAttempts: 1, IdempotencyKey: "library.recover:" + held.OwnerJobID,
|
||
}, now)
|
||
if err != nil || queueJob == nil {
|
||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_recovery_enqueue_failed",
|
||
"error": "恢复作业投递失败(或已有同窗口的恢复作业)", "detail": errString(err)})
|
||
return
|
||
}
|
||
// 上一次恢复失败(比如故障还没修完)不该让这个窗口永久卡死:
|
||
// 运维修好之后可以再点一次,作业会被重新排队;但同一窗口不许并发两份。
|
||
if queueJob.State == jobs.StateFailed || queueJob.State == jobs.StateCancelled {
|
||
retrier, ok := config.Store.(interface {
|
||
RetryLibraryRecovery(ctx context.Context, id, windowJobID string, now time.Time) (bool, error)
|
||
})
|
||
if !ok {
|
||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_recovery_stuck",
|
||
"error": "上一次恢复失败,且当前存储不支持重新排队"})
|
||
return
|
||
}
|
||
retried, retryErr := retrier.RetryLibraryRecovery(r.Context(), queueJob.ID, held.OwnerJobID, now)
|
||
if retryErr != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]any{"code": "library_recovery_retry_failed",
|
||
"error": "重新排队恢复作业失败", "detail": errString(retryErr)})
|
||
return
|
||
}
|
||
if !retried {
|
||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_recovery_busy",
|
||
"error": "同一窗口已有恢复作业在跑", "job": queueJob})
|
||
return
|
||
}
|
||
queueJob.State = jobs.StatePending
|
||
queueJob.Attempt = 0
|
||
queueJob.LastError = ""
|
||
}
|
||
writeJSON(w, http.StatusAccepted, map[string]any{"job": queueJob, "window_job_id": held.OwnerJobID,
|
||
"scope": held.OwnerScope, "instance": held.Instance})
|
||
}
|
||
|
||
// activeGrantFor 只查"目标+快照+同一参数摘要"的未过期未撤销授权。
|
||
func activeGrantFor(ctx context.Context, config Config, scope, snapshotID string) (governance.LibraryReleaseAuthorization, bool) {
|
||
if snapshotID == "" {
|
||
return governance.LibraryReleaseAuthorization{}, false
|
||
}
|
||
store, ok := config.Store.(interface {
|
||
ActiveLibraryAuthorization(ctx context.Context, scope, snapshotID, paramsDigest string, now time.Time) (governance.LibraryReleaseAuthorization, bool, error)
|
||
})
|
||
if !ok {
|
||
return governance.LibraryReleaseAuthorization{}, false
|
||
}
|
||
digest, err := librarySyncParamsDigest(scope, snapshotID)
|
||
if err != nil {
|
||
return governance.LibraryReleaseAuthorization{}, false
|
||
}
|
||
grant, found, err := store.ActiveLibraryAuthorization(ctx, scope, snapshotID, digest, time.Now().UTC())
|
||
if err != nil || !found {
|
||
return governance.LibraryReleaseAuthorization{}, false
|
||
}
|
||
return grant, true
|
||
}
|
||
|
||
func authorizationViewOf(grant governance.LibraryReleaseAuthorization, found bool) libraryAuthorizationView {
|
||
if !found {
|
||
return libraryAuthorizationView{State: "none", Reason: "没有与目标快照匹配的有效发布授权"}
|
||
}
|
||
expires := grant.ExpiresAt
|
||
return libraryAuthorizationView{State: "active", AuthorizedBy: grant.AuthorizedBy, ExpiresAt: &expires}
|
||
}
|
||
|
||
func scopeFromQuery(r *http.Request) (string, error) {
|
||
scope := strings.TrimSpace(r.URL.Query().Get("scope"))
|
||
if scope == "" {
|
||
scope = "internal"
|
||
}
|
||
if scope != "internal" && scope != "external" {
|
||
return "", errors.New("scope must be internal or external")
|
||
}
|
||
return scope, nil
|
||
}
|
||
|
||
func libraryVersionViewOf(item governance.LibraryVersion) libraryVersionView {
|
||
return libraryVersionView{ID: item.ID, Kind: string(item.Kind), SnapshotID: item.SnapshotID,
|
||
VersionLabel: item.VersionLabel, SourceJobID: item.SourceJobID, Evidence: item.Evidence,
|
||
CreatedBy: item.CreatedBy, CreatedAt: item.CreatedAt, SupersededAt: item.SupersededAt}
|
||
}
|
||
|
||
// librarySyncGate 只依据真实存在的证据判断能不能写生产:闸口、生产端点、
|
||
// 生产权威版本、目标快照的预备份。任何一项缺失都给出具体原因,不默认放行。
|
||
func errString(err error) string {
|
||
if err == nil {
|
||
return ""
|
||
}
|
||
return err.Error()
|
||
}
|
||
|
||
// shortSnapshot 只用于给人看的提示:完整摘要留给快照详情,别在原因句里塞 64 位。
|
||
func shortSnapshot(id string) string {
|
||
if len(id) <= 12 {
|
||
return id
|
||
}
|
||
return id[:12]
|
||
}
|
||
|
||
// testSealedOf 选"拟发布的测试封存":只有登记过 test_sealed 的那一份才算候选。
|
||
//
|
||
// 归档是共享的:生产预备份与目标基线快照同样带着 scope 写进同一个归档根,
|
||
// 按"这个 scope 最新一份"挑会把目标自己的内容当成测试候选(页面会显示一份
|
||
// 永远不该被发布的东西)。所以候选来自版本登记,再用归档确认它真的还在本地。
|
||
func testSealedOf(versions []governance.LibraryVersion, byID map[string]librarySnapshotView) (*librarySnapshotView, string) {
|
||
for _, item := range versions {
|
||
if item.Kind != governance.LibraryVersionTestSealed {
|
||
continue
|
||
}
|
||
snapshot, ok := byID[item.SnapshotID]
|
||
if !ok {
|
||
return nil, "最近一次测试封存 " + shortSnapshot(item.SnapshotID) + " 不在本机归档里(归档被清理或换了归档根)"
|
||
}
|
||
return &snapshot, ""
|
||
}
|
||
return nil, "还没有该目标的测试封存"
|
||
}
|
||
|
||
// libraryMaintenanceCapability 用配置事实回答"这台机器上维护是否接线"。
|
||
// 它刻意只说配置:能停写入者/关入口/证明排空的真实能力在 worker 侧(docker socket、
|
||
// 闸门、排空文件),API 若声称"能力可用"就是拿配置字符串冒充执行证据。
|
||
func libraryMaintenanceCapability(config Config) map[string]any {
|
||
missing := make([]string, 0, 6)
|
||
if config.LibraryTargetInstance == "" {
|
||
missing = append(missing, "LIBRARY_TARGET_INSTANCE")
|
||
}
|
||
if config.LibraryTargetProject == "" {
|
||
missing = append(missing, "LIBRARY_TARGET_PROJECT")
|
||
}
|
||
if len(config.LibraryTargetWriters) == 0 {
|
||
missing = append(missing, "LIBRARY_TARGET_WRITERS")
|
||
}
|
||
if config.LibraryTargetIngressURL == "" {
|
||
missing = append(missing, "LIBRARY_TARGET_INGRESS_URL")
|
||
}
|
||
if config.LibraryTargetIngressService == "" {
|
||
missing = append(missing, "LIBRARY_TARGET_INGRESS_SERVICE")
|
||
}
|
||
if config.LibraryTargetJournalRoot == "" {
|
||
missing = append(missing, "LIBRARY_TARGET_JOURNAL_ROOT")
|
||
}
|
||
if config.LibraryTargetDockerSocket == "" {
|
||
missing = append(missing, "LIBRARY_TARGET_DOCKER_SOCKET")
|
||
}
|
||
return map[string]any{
|
||
"configured": len(missing) == 0,
|
||
"missing": missing,
|
||
"instance": config.LibraryTargetInstance,
|
||
"project": config.LibraryTargetProject,
|
||
"writers": config.LibraryTargetWriters,
|
||
"ingress_service": config.LibraryTargetIngressService,
|
||
"note": "配置齐全只说明治理侧接线完整;停写入者、关入口与排空由 worker 在作业内校验并执行",
|
||
}
|
||
}
|
||
|
||
func librarySyncGate(config Config, production libraryKnownView, prebackup libraryKnownView, target string) libraryGateView {
|
||
return librarySyncGateWithGrant(config, production, prebackup, target, false)
|
||
}
|
||
|
||
// librarySyncGateWithGrant 在原有证据之外再要求一份有效授权:
|
||
// 授权绑定了目标、快照、参数与有效期,缺它就不能写生产。
|
||
func librarySyncGateWithGrant(config Config, production libraryKnownView, prebackup libraryKnownView, target string, hasGrant bool) libraryGateView {
|
||
gate := libraryGateView{SyncAllowed: false, Reasons: []string{}}
|
||
if !hasGrant {
|
||
gate.Reasons = append(gate.Reasons, "没有与目标快照匹配的有效发布授权")
|
||
}
|
||
if !config.ProductionReleaseRequestsEnabled {
|
||
gate.Reasons = append(gate.Reasons, "生产写入闸口关闭(PRODUCTION_RELEASE_REQUESTS_ENABLED=false)")
|
||
}
|
||
if !config.ProductionBackupRestoreVerified {
|
||
gate.Reasons = append(gate.Reasons, "没有已核验的生产恢复演练证据(PRODUCTION_BACKUP_RESTORE_VERIFIED=false)")
|
||
}
|
||
// 生产端点以"整库目标是否配置"为准:整库同步直接写 PG/ES/对象,
|
||
// 不依赖 WeKnora 的 HTTP 端,所以不能拿它是否配置当门禁。
|
||
if strings.TrimSpace(config.LibraryProductionDatabaseURL) == "" {
|
||
gate.Reasons = append(gate.Reasons, "生产整库目标未配置(LIBRARY_PRODUCTION_DATABASE_URL),无法核验生产当前状态")
|
||
}
|
||
// 首次发布:生产还没有"生效版本"记录,但只要生产目标可读且是空的,
|
||
// 这就是一个合法的"空基线",不应该被永久挡住。非空但无权威版本仍然阻断,
|
||
// 指向真实可行的入口(在维护隔离下捕获目标整库并校验后登记基线),
|
||
// 而不是让人去手写版本或清空目标。
|
||
if production.State != "known" && !production.EmptyBaseline {
|
||
gate.Reasons = append(gate.Reasons, "目标非空且没有生效版本记录:先建立目标权威基线(维护隔离下捕获整库并逐类校验),或确认目标确实是空库")
|
||
}
|
||
if target == "" {
|
||
gate.Reasons = append(gate.Reasons, "还没有可同步的测试封存版本")
|
||
}
|
||
// 预备份是作业内第一步生成的:启动前要求它已存在会造成死循环。
|
||
// 这里只在作业已经被判定失败或回滚失败时提示,不作为启动前的硬门禁。
|
||
if prebackup.State == "failed" {
|
||
gate.Reasons = append(gate.Reasons, "上一次预备份失败,先处理后才能重试")
|
||
}
|
||
if config.LibraryCapacityBudgetBytes <= 0 {
|
||
gate.Reasons = append(gate.Reasons, "容量预算未设置(LIBRARY_CAPACITY_BUDGET_BYTES),无法证明装得下")
|
||
}
|
||
gate.SyncAllowed = len(gate.Reasons) == 0
|
||
return gate
|
||
}
|
||
|
||
func libraryOverviewHandler(w http.ResponseWriter, r *http.Request, config Config) {
|
||
scope, err := scopeFromQuery(r)
|
||
if err != nil {
|
||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()})
|
||
return
|
||
}
|
||
store, ok := libraryStore(config)
|
||
if !ok {
|
||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"available": false,
|
||
"reason": "版本登记存储不可用"})
|
||
return
|
||
}
|
||
now := time.Now().UTC()
|
||
overview := libraryOverview{Scope: scope, Jobs: []governance.LibraryJob{}, Now: now,
|
||
Gate: libraryGateView{Reasons: []string{}},
|
||
Diff: libraryDiffView{State: "unknown", Reason: "生产与测试的文档级差异读取未接线", Added: []any{}, Updated: []any{}, Removed: []any{}}}
|
||
// 可发起封存 = worker 真实探测到执行器可用且指向本实例;原因逐条给全。
|
||
overview.CaptureAvailable, overview.CaptureReason, overview.Executor = libraryExecutorAvailability(r.Context(), config)
|
||
|
||
// 测试封存来自归档 + 版本登记:归档里**不止**测试封存——生产预备份与目标基线
|
||
// 快照同样带 scope,如果只按"这个 scope 最新一份"挑,页面就会把目标自己的内容
|
||
// 当成拟发布的测试候选。所以候选只认登记过 kind=test_sealed 的那一份。
|
||
snapshots, snapErr := readLibrarySnapshots(r.Context(), config.LibraryArchiveRoot)
|
||
byID := map[string]librarySnapshotView{}
|
||
archiveReadable := false
|
||
if snapErr != nil {
|
||
_, overview.TestSealedReason = libraryArchiveErrorStatus(snapErr, "归档不可读")
|
||
} else if !snapshots.Available {
|
||
overview.TestSealedReason = snapshots.Reason
|
||
} else {
|
||
archiveReadable = true
|
||
for _, snapshot := range snapshots.Snapshots {
|
||
byID[snapshot.SnapshotID] = snapshot
|
||
}
|
||
}
|
||
|
||
// 生产生效 / 预备份 / 作业来自持久登记。
|
||
// 历史列表仍是分页展示(窗口),但"当前生效版本"必须精确查:窗口一满,
|
||
// 生效版本会掉出列表,页面就会显示成"没有生效版本",与真实状态不符。
|
||
versions, err := store.LibraryVersions(r.Context(), scope, 50)
|
||
if err != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot read library versions"})
|
||
return
|
||
}
|
||
current, hasCurrent, currentErr := store.LibraryCurrentVersion(r.Context(), scope)
|
||
if currentErr != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot read current library version"})
|
||
return
|
||
}
|
||
if archiveReadable {
|
||
overview.TestSealed, overview.TestSealedReason = testSealedOf(versions, byID)
|
||
}
|
||
overview.Production = libraryKnownView{State: "unknown", Reason: "还没有生产验证通过的生效版本记录"}
|
||
overview.Prebackup = libraryKnownView{State: "unknown", Reason: "这个范围还没有任何生产预备份"}
|
||
if hasCurrent {
|
||
view := libraryVersionViewOf(current)
|
||
overview.Production = libraryKnownView{State: "known", Version: &view}
|
||
}
|
||
target := ""
|
||
if overview.TestSealed != nil {
|
||
target = overview.TestSealed.SnapshotID
|
||
}
|
||
for index := range versions {
|
||
item := versions[index]
|
||
view := libraryVersionViewOf(item)
|
||
overview.Versions = append(overview.Versions, libraryVersionRef{
|
||
ID: item.ID, Kind: string(item.Kind), SnapshotID: item.SnapshotID, VersionLabel: item.VersionLabel,
|
||
SourceJobID: item.SourceJobID, CreatedAt: item.CreatedAt, SupersededAt: item.SupersededAt,
|
||
})
|
||
switch item.Kind {
|
||
case governance.LibraryVersionProductionEffective:
|
||
// 生效状态只认上面那条精确查询;这里只负责把历史行放进列表。
|
||
case governance.LibraryVersionProductionPrebackup:
|
||
// 页面要显示"最近一次生产写入前的备份",不是"与拟发布快照同内容的那一份":
|
||
// 恢复作业登记的是**恢复前**的备份,它和测试封存快照本来就不同;
|
||
// 只按目标匹配会让真登记过的预备份显示成"尚未创建"。
|
||
// versions 已按时间倒序,第一份就是最近一次;是否与本次目标同快照另给标志。
|
||
if overview.Prebackup.State != "known" {
|
||
overview.Prebackup = libraryKnownView{State: "known", Version: &view,
|
||
BoundToTarget: target != "" && item.SnapshotID == target}
|
||
}
|
||
}
|
||
}
|
||
|
||
recent, err := store.ListLibraryJobs(r.Context(), scope, 20)
|
||
if err != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot read library jobs"})
|
||
return
|
||
}
|
||
overview.Jobs = recent
|
||
for index := range recent {
|
||
if recent[index].State == governance.LibraryJobPending || recent[index].State == governance.LibraryJobRunning {
|
||
overview.Job = &recent[index]
|
||
break
|
||
}
|
||
}
|
||
if overview.Job == nil && len(recent) > 0 {
|
||
overview.Job = &recent[0]
|
||
}
|
||
|
||
// 差异是真实计算:生产权威版本 + 测试封存都在本地归档时给出逐篇增改撤销,
|
||
// 否则保持 unknown 并说明缺哪一侧,不用空数组冒充"无变化"。
|
||
baseSnapshot := ""
|
||
if overview.Production.State == "known" && overview.Production.Version != nil {
|
||
baseSnapshot = overview.Production.Version.SnapshotID
|
||
}
|
||
// 归档可能很大:差异计算放进看门狗,超时就保持 unknown,别让页面请求挂死。
|
||
diffResult := make(chan struct {
|
||
view libraryDiffView
|
||
ok bool
|
||
}, 1)
|
||
go func() {
|
||
view, ok := computeLibraryDiff(config.LibraryArchiveRoot, baseSnapshot, target)
|
||
diffResult <- struct {
|
||
view libraryDiffView
|
||
ok bool
|
||
}{view, ok}
|
||
}()
|
||
var view libraryDiffView
|
||
var diffOK bool
|
||
select {
|
||
case outcome := <-diffResult:
|
||
view, diffOK = outcome.view, outcome.ok
|
||
case <-time.After(8 * time.Second):
|
||
view = libraryDiffView{State: "unknown", Reason: "差异计算超时(归档较大),本次保持不变更结论",
|
||
Added: []any{}, Updated: []any{}, Removed: []any{}, BaseSnapshot: baseSnapshot, TargetSnapshot: target}
|
||
diffOK = false
|
||
}
|
||
if diffOK || view.Reason != "" {
|
||
overview.Diff = view
|
||
}
|
||
|
||
grant, hasGrant := activeGrantFor(r.Context(), config, scope, target)
|
||
overview.Authorization = authorizationViewOf(grant, hasGrant)
|
||
overview.Gate = librarySyncGateWithGrant(config, overview.Production, overview.Prebackup, target, hasGrant)
|
||
writeJSON(w, http.StatusOK, overview)
|
||
}
|
||
|
||
func libraryJobsHandler(w http.ResponseWriter, r *http.Request, config Config) {
|
||
scope, err := scopeFromQuery(r)
|
||
if err != nil {
|
||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()})
|
||
return
|
||
}
|
||
store, ok := libraryStore(config)
|
||
if !ok {
|
||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"available": false, "reason": "版本登记存储不可用", "jobs": []any{}})
|
||
return
|
||
}
|
||
items, err := store.ListLibraryJobs(r.Context(), scope, 20)
|
||
if err != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot read library jobs"})
|
||
return
|
||
}
|
||
writeJSON(w, http.StatusOK, map[string]any{"available": true, "jobs": items})
|
||
}
|
||
|
||
func libraryJobDetailHandler(w http.ResponseWriter, r *http.Request, config Config) {
|
||
store, ok := libraryStore(config)
|
||
if !ok {
|
||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"available": false, "reason": "版本登记存储不可用"})
|
||
return
|
||
}
|
||
id := strings.TrimSpace(r.PathValue("id"))
|
||
if id == "" {
|
||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "job id required"})
|
||
return
|
||
}
|
||
job, err := store.LibraryJob(r.Context(), id)
|
||
if err != nil {
|
||
writeJSON(w, http.StatusNotFound, map[string]any{"available": false, "reason": "作业不存在"})
|
||
return
|
||
}
|
||
writeJSON(w, http.StatusOK, map[string]any{"available": true, "job": job})
|
||
}
|
||
|
||
// librarySyncEnqueue 是生产同步的唯一入口:闸口未齐时**不建作业**,直接说明缺项。
|
||
// 这样按钮永远不会假接通,也不会留下一个永远失败的空作业。
|
||
func librarySyncEnqueue(w http.ResponseWriter, r *http.Request, config Config) {
|
||
if r.Method != http.MethodPost {
|
||
writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"})
|
||
return
|
||
}
|
||
store, ok := libraryStore(config)
|
||
if !ok {
|
||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"code": "library_sync_unavailable", "reasons": []string{"版本登记存储不可用"}})
|
||
return
|
||
}
|
||
var request struct {
|
||
Scope string `json:"scope"`
|
||
Target string `json:"target_snapshot_id"`
|
||
}
|
||
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&request); err != nil {
|
||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"})
|
||
return
|
||
}
|
||
if request.Scope != "internal" && request.Scope != "external" {
|
||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "scope must be internal or external"})
|
||
return
|
||
}
|
||
overview := libraryOverview{Scope: request.Scope}
|
||
// 闸口依赖"生产是否已有生效版本":这是**创建**判据,必须精确查。
|
||
// 用窗口列表会把它读成"没有",于是把一次不必要的覆盖当成合法首发;
|
||
// 查询失败同样拒绝(fail closed),不猜。
|
||
current, hasCurrent, currentErr := store.LibraryCurrentVersion(r.Context(), request.Scope)
|
||
if currentErr != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot read current library version"})
|
||
return
|
||
}
|
||
versions, err := store.LibraryVersions(r.Context(), request.Scope, 50)
|
||
if err != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot read library versions"})
|
||
return
|
||
}
|
||
overview.Production = libraryKnownView{State: "unknown"}
|
||
if hasCurrent {
|
||
view := libraryVersionViewOf(current)
|
||
overview.Production = libraryKnownView{State: "known", Version: &view}
|
||
}
|
||
overview.Prebackup = libraryKnownView{State: "unknown"}
|
||
if empty, probeErr := productionScopeEmpty(r.Context(), config, request.Scope); probeErr == nil && empty {
|
||
overview.Production = libraryKnownView{State: "unknown", EmptyBaseline: true,
|
||
Reason: "生产目标可读且为空:本次是首次发布,写入前仍会生成完整预备份"}
|
||
}
|
||
for index := range versions {
|
||
item := versions[index]
|
||
if item.Kind == governance.LibraryVersionProductionPrebackup && request.Target != "" && item.SnapshotID == request.Target {
|
||
view := libraryVersionViewOf(item)
|
||
overview.Prebackup = libraryKnownView{State: "known", Version: &view}
|
||
}
|
||
}
|
||
grantFound, hasGrant := activeGrantFor(r.Context(), config, request.Scope, request.Target)
|
||
_ = grantFound
|
||
// 目标实例正在维护(跨对内/对外互斥):直接拒绝,不等到 worker 里才发现。
|
||
if body, held := instanceMaintenanceGuard(r.Context(), config, request.Scope); held {
|
||
writeJSON(w, http.StatusConflict, body)
|
||
return
|
||
}
|
||
// 目标正在维护(被别的作业持有):直接拒绝新写入,不等到 worker 里才发现。
|
||
if maintenanceStore, ok := config.Store.(interface {
|
||
LibraryMaintenanceOf(ctx context.Context, scope string) (*postgresstore.LibraryMaintenance, error)
|
||
}); ok {
|
||
if held, err := maintenanceStore.LibraryMaintenanceOf(r.Context(), request.Scope); err == nil && held != nil && held.ReleasedAt == nil {
|
||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_target_in_maintenance",
|
||
"error": "目标正在维护中,验证完成前不接受新的写入请求",
|
||
"owner_job": held.OwnerJobID, "state": held.State})
|
||
return
|
||
}
|
||
}
|
||
gate := librarySyncGateWithGrant(config, overview.Production, overview.Prebackup, request.Target, hasGrant)
|
||
if !gate.SyncAllowed {
|
||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{
|
||
"code": "library_sync_blocked", "reasons": gate.Reasons,
|
||
"production": overview.Production, "prebackup": overview.Prebackup,
|
||
})
|
||
return
|
||
}
|
||
|
||
// 走到这里说明闸口与证据都齐了才真正建作业;互斥与幂等由数据库索引保证。
|
||
digest, err := governance.LibraryRequestDigest(request.Scope, governance.LibraryJobSync,
|
||
map[string]any{"target_snapshot_id": request.Target})
|
||
if err != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot digest request"})
|
||
return
|
||
}
|
||
actor := ""
|
||
if identity, ok := sessionFrom(r.Context()); ok {
|
||
actor = identity.Username
|
||
}
|
||
now := time.Now().UTC()
|
||
jobID := fmt.Sprintf("libsync-%s-%d", request.Scope, now.UnixNano())
|
||
job, reused, err := store.CreateLibraryJob(r.Context(), governance.LibraryJob{
|
||
ID: jobID, Scope: request.Scope, Kind: governance.LibraryJobSync,
|
||
IdempotencyKey: "library.sync:" + request.Scope + ":" + request.Target, RequestDigest: digest,
|
||
TargetSnapshotID: request.Target, Actor: actor,
|
||
}, governance.SyncStages(), now)
|
||
if errors.Is(err, governance.ErrLibraryJobActive) {
|
||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_sync_active",
|
||
"error": "该目标已有在飞的生产写作业(同步与恢复互斥)"})
|
||
return
|
||
}
|
||
if errors.Is(err, governance.ErrLibraryJobIdempotencyConflict) {
|
||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_sync_conflict", "error": "同一幂等键对应了不同的请求"})
|
||
return
|
||
}
|
||
if err != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot create library sync job"})
|
||
return
|
||
}
|
||
// 上一次同参数请求失败/被取消后,幂等键不能再把新请求塞回那个死作业里:
|
||
// 换一个"重试代次"键重新建作业,旧作业保留为审计记录。
|
||
if reused && (job.State == governance.LibraryJobFailed || job.State == governance.LibraryJobCancelled) {
|
||
job, reused, err = store.CreateLibraryJob(r.Context(), governance.LibraryJob{
|
||
ID: fmt.Sprintf("libsync-%s-%d", request.Scope, now.UnixNano()), Scope: request.Scope,
|
||
Kind: governance.LibraryJobSync,
|
||
IdempotencyKey: "library.sync:" + request.Scope + ":" + request.Target + ":retry:" + now.Format("20060102T150405.000000000"),
|
||
RequestDigest: digest, TargetSnapshotID: request.Target, Actor: actor,
|
||
}, governance.SyncStages(), now)
|
||
if err != nil || reused {
|
||
writeJSON(w, http.StatusConflict, map[string]any{"code": "library_sync_retry_conflict",
|
||
"error": "无法为重试创建新作业", "detail": errString(err)})
|
||
return
|
||
}
|
||
}
|
||
if reused {
|
||
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": true})
|
||
return
|
||
}
|
||
// 业务作业先落库,再投递队列:投递失败就把作业标失败,不留下"看起来在跑"的空作业。
|
||
payload, err := json.Marshal(map[string]string{
|
||
"library_job_id": job.ID, "scope": request.Scope, "target_snapshot_id": request.Target,
|
||
})
|
||
if err != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot build sync payload"})
|
||
return
|
||
}
|
||
queueJob, err := config.Store.Enqueue(r.Context(), jobs.EnqueueRequest{
|
||
ID: governance.QueueJobIDFor(job.ID), Type: librarySyncJobType, Payload: payload, AvailableAt: now,
|
||
MaxAttempts: 1, IdempotencyKey: "library.sync.queue:" + job.ID,
|
||
}, now)
|
||
if err != nil || queueJob == nil {
|
||
log.Printf("library sync enqueue failed job=%s: %v", job.ID, err)
|
||
_ = store.CancelLibraryJob(r.Context(), job.ID, "队列投递失败,未开始同步", now)
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot enqueue sync job"})
|
||
return
|
||
}
|
||
if attachErr := store.AttachLibraryQueueJob(r.Context(), job.ID, queueJob.ID, now); attachErr != nil {
|
||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "cannot attach sync job"})
|
||
return
|
||
}
|
||
writeJSON(w, http.StatusAccepted, map[string]any{"job": job, "reused": false})
|
||
}
|