姚章浩
47eb23cf68
页面上的"可发起封存"原来只判 LIBRARY_AGENT_SOCKET 非空：正式栈的 API 根本没挂
那个 socket，而 agent 曾顶着正式单元名冻结候选实例——配置非空、页面可点、真发起
却封错库。
现在：agent 新增 health 动作（不占封存操作锁、不跑子进程，只回答身份/降级/忙），
worker 每 30 秒探测并写入迁移 074 的 library_executor_status；overview、快照列表与
POST /library/captures 共用同一份判定（过期、不可达、实例不符、降级、正忙都拒绝并
给原因）。封存结果由 agent 附上自报身份，worker 在停写入者之前再探测一次、执行后
用结果复核，不一致就不登记测试封存——心跳只能证明"刚才"。
期望身份用 LIBRARY_SOURCE_*（封存源），不是 LIBRARY_TARGET_*（同步目标），否则
正确的执行器会被判成不匹配。另加 GET /library/restore-availability：恢复能否执行由
双闸、该快照的恢复授权与实例维护状态决定，页面不再写死 disabled。
真跑（候选隔离栈，两处 socket 隔离不变）：overview 返回 executor
{kgmaint-weknora/kgmaint-app}；停 agent 后转不可达并 503 带原因；重启后恢复。
275 lines
12 KiB
Go
275 lines
12 KiB
Go
package httpapi
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"net/http"
|
||
"sort"
|
||
"strings"
|
||
"time"
|
||
|
||
"knowledge-governance/internal/librarysnapshot"
|
||
)
|
||
|
||
// 「整库版本」只读视图：把已经封存的候选产物（artifact-acceptance-v1）如实列出来。
|
||
//
|
||
// 边界（刻意不做的事）：
|
||
// - 不触发封存、不停/起写入者：封存需要维护窗口与来源凭据，只能由运维命令
|
||
// scripts/library-capture.sh 在目标主机执行；
|
||
// - 不把完整性当发布：候选证明"这一份内容被完整、可核验地封存了"（A04 口径），
|
||
// 而授权与生产同步是另外的、需要独立授权的步骤（A06 效果测试已由用户取消）；
|
||
// - 归档根没接线时返回可用性=false 与原因，不返回空表冒充"没有快照"。
|
||
type librarySnapshotPart struct {
|
||
Kind string `json:"kind"`
|
||
Name string `json:"name"`
|
||
SHA256 string `json:"sha256"`
|
||
Count int64 `json:"count"`
|
||
Bytes int64 `json:"bytes"`
|
||
}
|
||
|
||
type librarySnapshotView struct {
|
||
SnapshotID string `json:"snapshot_id"`
|
||
Scope string `json:"scope"`
|
||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||
TenantID uint64 `json:"tenant_id"`
|
||
CreatedAt time.Time `json:"created_at"`
|
||
ValidationPolicy string `json:"validation_policy"`
|
||
Format string `json:"format"`
|
||
Compatibility librarysnapshot.Compatibility `json:"compatibility"`
|
||
Parts []librarySnapshotPart `json:"parts"`
|
||
LogicalBytes int64 `json:"logical_bytes"`
|
||
UniqueBytes int64 `json:"unique_bytes"`
|
||
SharedBytes int64 `json:"shared_bytes"`
|
||
Totals map[string]int64 `json:"totals"`
|
||
// Verified 只在这一份版本的每个部件都被重新哈希并核对通过时为 true。
|
||
// 列表口径下它恒为 false：枚举只证明清单可读，不是"完整性已验证"。
|
||
Verified bool `json:"verified"`
|
||
}
|
||
|
||
// 两次读取各自能证明什么，页面必须能看见，不能混成一句"完整性已验证"。
|
||
const (
|
||
libraryVerificationManifest = "manifest"
|
||
libraryVerificationContent = "content"
|
||
// libraryVerificationNote 说明枚举口径：清单自身（文件名摘要 + 结构）校验通过，
|
||
// 被引用字节是否完好要等到执行（同步/恢复）时再逐部件复核。
|
||
libraryVerificationNote = "列表只校验每份清单自身（文件名摘要与结构）：被引用的字节在枚举时不做逐部件复核，" +
|
||
"所以这里的版本不能当作\"完整性已验证\"或\"可发布\"。执行（同步/恢复）前仍会逐部件校验字节，坏 blob 会被拒绝。"
|
||
)
|
||
|
||
type librarySnapshotList struct {
|
||
Available bool `json:"available"`
|
||
Reason string `json:"reason,omitempty"`
|
||
ArchiveRoot string `json:"archive_root,omitempty"`
|
||
Snapshots []librarySnapshotView `json:"snapshots"`
|
||
// Verification 是这份列表的口径（固定 manifest），配合 VerificationNote 说明它证明了什么。
|
||
Verification string `json:"verification"`
|
||
VerificationNote string `json:"verification_note"`
|
||
// 归档级口径：ContentBytes 是**跨版本去重后、被版本引用的内容字节**。
|
||
// 它不是磁盘占用（清单、未引用 blob、块对齐与元数据都不在里面），
|
||
// 更不是可回收空间——清理需要保留策略、依赖保护与在线作业引用。
|
||
Versions int `json:"versions"`
|
||
ContentBytes int64 `json:"content_bytes"`
|
||
BlobFileBytes int64 `json:"blob_file_bytes"`
|
||
BlobAllocatedBytes int64 `json:"blob_allocated_bytes"`
|
||
UnreferencedBlobs int `json:"unreferenced_blobs"`
|
||
UnreferencedBytes int64 `json:"unreferenced_bytes"`
|
||
ManifestFileBytes int64 `json:"manifest_file_bytes"`
|
||
ManifestAllocatedBytes int64 `json:"manifest_allocated_bytes"`
|
||
// 页面必须能说清"封存由谁执行"；能不能从这里发起，取决于主机侧 agent 是否接线。
|
||
CaptureNote string `json:"capture_note"`
|
||
CaptureAvailable bool `json:"capture_available"`
|
||
CaptureReason string `json:"capture_reason,omitempty"`
|
||
}
|
||
|
||
const libraryCaptureNote = "封存由运维命令 scripts/library-capture.sh capture -scope internal|external 在目标主机执行（停写入者→导出→恢复）。页面只读展示已封存的完整版本：它通过了数据完整性检查（身份、双向正文覆盖、生成问题、资源与摘要），这已满足封版所需的完整性口径；授权与生产同步仍是独立步骤。"
|
||
|
||
func librarySnapshotsList(w http.ResponseWriter, r *http.Request, config Config) {
|
||
view, err := readLibrarySnapshots(r.Context(), config.LibraryArchiveRoot)
|
||
// 能不能发起封存只认 worker 的真实观测，不认"配置里写了 socket 路径"。
|
||
view.CaptureAvailable, view.CaptureReason, _ = libraryExecutorAvailability(r.Context(), config)
|
||
if err != nil {
|
||
status, reason := libraryArchiveErrorStatus(err, "归档不可读")
|
||
writeJSON(w, status, librarySnapshotList{Available: false, Reason: reason,
|
||
Snapshots: []librarySnapshotView{}, Verification: libraryVerificationManifest,
|
||
VerificationNote: libraryVerificationNote, CaptureNote: libraryCaptureNote,
|
||
CaptureAvailable: view.CaptureAvailable, CaptureReason: view.CaptureReason})
|
||
return
|
||
}
|
||
writeJSON(w, http.StatusOK, view)
|
||
}
|
||
|
||
// libraryArchiveErrorStatus 把"归档读不动"的两种原因分开：请求被期限中止要说超时，
|
||
// 其余（清单坏了、目录结构不对）仍然按不可读报，都不算成功。
|
||
func libraryArchiveErrorStatus(err error, prefix string) (int, string) {
|
||
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
|
||
return http.StatusServiceUnavailable, "归档读取超时（超过请求期限），本次没有返回数据"
|
||
}
|
||
return http.StatusServiceUnavailable, prefix + "：" + err.Error()
|
||
}
|
||
|
||
// libraryManifests 按版本号逐份读清单，并在版本之间检查请求期限：
|
||
// 归档版本多时请求可以被真的中止，而不是把连接挂到读完为止。
|
||
func libraryManifests(ctx context.Context, archive *librarysnapshot.Archive, ids []string) ([]librarysnapshot.Manifest, error) {
|
||
manifests := make([]librarysnapshot.Manifest, 0, len(ids))
|
||
for _, id := range ids {
|
||
if err := ctx.Err(); err != nil {
|
||
return nil, err
|
||
}
|
||
manifest, err := archive.LoadManifest(id)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
manifests = append(manifests, manifest)
|
||
}
|
||
return manifests, nil
|
||
}
|
||
|
||
func librarySnapshotDetail(w http.ResponseWriter, r *http.Request, config Config) {
|
||
id := strings.TrimSpace(r.PathValue("id"))
|
||
if id == "" {
|
||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "snapshot id required"})
|
||
return
|
||
}
|
||
if config.LibraryArchiveRoot == "" {
|
||
writeJSON(w, http.StatusOK, map[string]any{"available": false,
|
||
"reason": "未接线：LIBRARY_ARCHIVE_ROOT 未配置，本平台没有整库归档可读"})
|
||
return
|
||
}
|
||
archive, err := librarysnapshot.Open(config.LibraryArchiveRoot, 256*1024*1024)
|
||
if err != nil {
|
||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"available": false, "reason": "归档不可读"})
|
||
return
|
||
}
|
||
defer archive.Close()
|
||
// 详情是"要动这个版本"的入口：仍然整版复核字节（单版约 0.25s），失败即 404，
|
||
// 不把坏 blob 的版本展示成可用于同步的候选。
|
||
manifest, err := archive.Load(id)
|
||
if err != nil {
|
||
writeJSON(w, http.StatusNotFound, map[string]any{"available": false, "reason": "快照不存在或校验失败"})
|
||
return
|
||
}
|
||
usage, err := archive.UsageFromManifests()
|
||
if err != nil {
|
||
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"available": false, "reason": "归档用量不可读"})
|
||
return
|
||
}
|
||
view, ok := librarySnapshotViewOf(id, manifest, usage)
|
||
if !ok {
|
||
view = librarySnapshotViewOfNoUsage(id, manifest)
|
||
}
|
||
view.Verified = true
|
||
writeJSON(w, http.StatusOK, map[string]any{"available": true, "snapshot": view,
|
||
"verification": libraryVerificationContent, "capture_note": libraryCaptureNote})
|
||
}
|
||
|
||
func readLibrarySnapshots(ctx context.Context, root string) (librarySnapshotList, error) {
|
||
view := librarySnapshotList{Snapshots: []librarySnapshotView{}, CaptureNote: libraryCaptureNote,
|
||
Verification: libraryVerificationManifest, VerificationNote: libraryVerificationNote}
|
||
if root == "" {
|
||
view.Reason = "未接线：LIBRARY_ARCHIVE_ROOT 未配置，本平台没有整库归档可读"
|
||
return view, nil
|
||
}
|
||
view.ArchiveRoot = root
|
||
archive, err := librarysnapshot.Open(root, 256*1024*1024)
|
||
if err != nil {
|
||
return view, errors.New("归档根不可用")
|
||
}
|
||
defer archive.Close()
|
||
// 列表只做清单级读取：逐版 Load 会把每次枚举放大成 O(全部版本字节)
|
||
// （实测 79 版 13.8s → 接口 10s 超时）。执行前仍走 Load/OpenPart。
|
||
usage, err := archive.UsageFromManifests()
|
||
if err != nil {
|
||
return view, errors.New("归档用量不可读")
|
||
}
|
||
// 这段跨版本去重口径与 Usage 共用同一个实现：清单级读取不能改变数字含义。
|
||
total, err := archive.ContentUsage()
|
||
if err != nil {
|
||
return view, errors.New("归档用量不可读")
|
||
}
|
||
view.Versions = total.Versions
|
||
view.ContentBytes = total.ContentBytes
|
||
view.BlobFileBytes = total.BlobFileBytes
|
||
view.BlobAllocatedBytes = total.BlobAllocatedBytes
|
||
view.UnreferencedBlobs = total.UnreferencedBlobs
|
||
view.UnreferencedBytes = total.UnreferencedBytes
|
||
view.ManifestFileBytes = total.ManifestFileBytes
|
||
view.ManifestAllocatedBytes = total.ManifestAllocatedBytes
|
||
byID := map[string]librarysnapshot.Usage{}
|
||
ids := make([]string, 0, len(usage))
|
||
for _, item := range usage {
|
||
if _, seen := byID[item.VersionID]; !seen {
|
||
ids = append(ids, item.VersionID)
|
||
}
|
||
byID[item.VersionID] = item
|
||
}
|
||
sort.Strings(ids)
|
||
manifests, err := libraryManifests(ctx, archive, ids)
|
||
if err != nil {
|
||
return view, err
|
||
}
|
||
for index, id := range ids {
|
||
snapshot, ok := librarySnapshotViewOf(id, manifests[index], []librarysnapshot.Usage{byID[id]})
|
||
if !ok {
|
||
snapshot = librarySnapshotViewOfNoUsage(id, manifests[index])
|
||
}
|
||
view.Snapshots = append(view.Snapshots, snapshot)
|
||
}
|
||
sort.Slice(view.Snapshots, func(i, j int) bool {
|
||
if view.Snapshots[i].CreatedAt.Equal(view.Snapshots[j].CreatedAt) {
|
||
return view.Snapshots[i].SnapshotID > view.Snapshots[j].SnapshotID
|
||
}
|
||
return view.Snapshots[i].CreatedAt.After(view.Snapshots[j].CreatedAt)
|
||
})
|
||
view.Available = true
|
||
return view, nil
|
||
}
|
||
|
||
func librarySnapshotViewOf(id string, manifest librarysnapshot.Manifest, usage []librarysnapshot.Usage) (librarySnapshotView, bool) {
|
||
snapshot := librarySnapshotViewOfNoUsage(id, manifest)
|
||
for _, item := range usage {
|
||
if item.VersionID == id {
|
||
snapshot.LogicalBytes = item.LogicalBytes
|
||
snapshot.UniqueBytes = item.UniqueBytes
|
||
snapshot.SharedBytes = item.SharedBytes
|
||
return snapshot, true
|
||
}
|
||
}
|
||
return snapshot, false
|
||
}
|
||
|
||
func librarySnapshotViewOfNoUsage(id string, manifest librarysnapshot.Manifest) librarySnapshotView {
|
||
snapshot := librarySnapshotView{SnapshotID: id, Scope: manifest.Scope, KnowledgeBaseID: manifest.KnowledgeBaseID,
|
||
TenantID: manifest.TenantID, CreatedAt: manifest.CreatedAt, ValidationPolicy: manifest.ValidationPolicy,
|
||
Format: manifest.Format, Compatibility: manifest.Compatibility, Parts: make([]librarySnapshotPart, 0, len(manifest.Parts)),
|
||
Totals: map[string]int64{}}
|
||
for _, part := range manifest.Parts {
|
||
snapshot.Parts = append(snapshot.Parts, librarySnapshotPart{Kind: part.Kind, Name: part.Name,
|
||
SHA256: part.SHA256, Count: part.Count, Bytes: part.Bytes})
|
||
snapshot.LogicalBytes += part.Bytes
|
||
// 汇总按"数量字段"给出人看得懂的四个数字：文档/分块/索引/资源/Wiki 页。
|
||
switch part.Name {
|
||
case "knowledges":
|
||
snapshot.Totals["documents"] = part.Count
|
||
case "chunks":
|
||
if part.Kind == "database" {
|
||
snapshot.Totals["chunks"] = part.Count
|
||
} else {
|
||
snapshot.Totals["index_documents"] = part.Count
|
||
}
|
||
case "resources":
|
||
snapshot.Totals["resources"] = part.Count
|
||
case "wiki_pages":
|
||
snapshot.Totals["wiki_pages"] = part.Count
|
||
case "wiki_folders":
|
||
snapshot.Totals["wiki_folders"] = part.Count
|
||
}
|
||
}
|
||
sort.Slice(snapshot.Parts, func(i, j int) bool {
|
||
if snapshot.Parts[i].Kind == snapshot.Parts[j].Kind {
|
||
return snapshot.Parts[i].Name < snapshot.Parts[j].Name
|
||
}
|
||
return snapshot.Parts[i].Kind < snapshot.Parts[j].Kind
|
||
})
|
||
return snapshot
|
||
}
|