姚章浩
910c0fcd76
build / build (push) Failing after 16s
- libraryagent：0700 目录内 0600 unix socket + SO_PEERCRED 同 uid， 固定 binary/归档/日志/项目/写入者声明，请求只能选范围与作业号 - API：POST/GET /api/v1/library/captures（入队 library.capture + 读队列状态） - worker：library.capture handler 走 socket；启动先 recover；不挂 Docker socket、 不持有来源凭据 - web：整库版本页显示真实作业状态并可发起封存（未接线/无权限给出原因） - scripts/library-agent-install.sh：构建、安装 systemd user 服务并读回权限
222 lines
9.3 KiB
Go
222 lines
9.3 KiB
Go
package httpapi
|
|
|
|
import (
|
|
"errors"
|
|
"net/http"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"knowledge-governance/internal/librarysnapshot"
|
|
)
|
|
|
|
// 「整库版本」只读视图：把已经封存的候选产物（artifact-acceptance-v1）如实列出来。
|
|
//
|
|
// 边界（刻意不做的事）：
|
|
// - 不触发封存、不停/起写入者：封存需要维护窗口与来源凭据，只能由运维命令
|
|
// scripts/library-capture.sh 在目标主机执行；
|
|
// - 不把完整性当发布：候选证明"这一份内容被完整、可核验地封存了"（A04 口径），
|
|
// 而授权与生产同步是另外的、需要独立授权的步骤（A06 效果测试已由用户取消）；
|
|
// - 归档根没接线时返回可用性=false 与原因，不返回空表冒充"没有快照"。
|
|
type librarySnapshotPart struct {
|
|
Kind string `json:"kind"`
|
|
Name string `json:"name"`
|
|
SHA256 string `json:"sha256"`
|
|
Count int64 `json:"count"`
|
|
Bytes int64 `json:"bytes"`
|
|
}
|
|
|
|
type librarySnapshotView struct {
|
|
SnapshotID string `json:"snapshot_id"`
|
|
Scope string `json:"scope"`
|
|
KnowledgeBaseID string `json:"knowledge_base_id"`
|
|
TenantID uint64 `json:"tenant_id"`
|
|
CreatedAt time.Time `json:"created_at"`
|
|
ValidationPolicy string `json:"validation_policy"`
|
|
Format string `json:"format"`
|
|
Compatibility librarysnapshot.Compatibility `json:"compatibility"`
|
|
Parts []librarySnapshotPart `json:"parts"`
|
|
LogicalBytes int64 `json:"logical_bytes"`
|
|
UniqueBytes int64 `json:"unique_bytes"`
|
|
SharedBytes int64 `json:"shared_bytes"`
|
|
Totals map[string]int64 `json:"totals"`
|
|
}
|
|
|
|
type librarySnapshotList struct {
|
|
Available bool `json:"available"`
|
|
Reason string `json:"reason,omitempty"`
|
|
ArchiveRoot string `json:"archive_root,omitempty"`
|
|
Snapshots []librarySnapshotView `json:"snapshots"`
|
|
// 归档级口径：ContentBytes 是**跨版本去重后、被版本引用的内容字节**。
|
|
// 它不是磁盘占用（清单、未引用 blob、块对齐与元数据都不在里面），
|
|
// 更不是可回收空间——清理需要保留策略、依赖保护与在线作业引用。
|
|
Versions int `json:"versions"`
|
|
ContentBytes int64 `json:"content_bytes"`
|
|
BlobFileBytes int64 `json:"blob_file_bytes"`
|
|
BlobAllocatedBytes int64 `json:"blob_allocated_bytes"`
|
|
UnreferencedBlobs int `json:"unreferenced_blobs"`
|
|
UnreferencedBytes int64 `json:"unreferenced_bytes"`
|
|
ManifestFileBytes int64 `json:"manifest_file_bytes"`
|
|
ManifestAllocatedBytes int64 `json:"manifest_allocated_bytes"`
|
|
// 页面必须能说清"封存由谁执行"；能不能从这里发起，取决于主机侧 agent 是否接线。
|
|
CaptureNote string `json:"capture_note"`
|
|
CaptureAvailable bool `json:"capture_available"`
|
|
CaptureReason string `json:"capture_reason,omitempty"`
|
|
}
|
|
|
|
const libraryCaptureNote = "封存由运维命令 scripts/library-capture.sh capture -scope internal|external 在目标主机执行（停写入者→导出→恢复）。页面只读展示已封存的完整版本：它通过了数据完整性检查（身份、双向正文覆盖、生成问题、资源与摘要），这已满足封版所需的完整性口径；授权与生产同步仍是独立步骤。"
|
|
|
|
func librarySnapshotsList(w http.ResponseWriter, r *http.Request, config Config) {
|
|
view, err := readLibrarySnapshots(config.LibraryArchiveRoot)
|
|
view.CaptureAvailable = config.LibraryAgentSocket != ""
|
|
if !view.CaptureAvailable {
|
|
view.CaptureReason = "未接线：主机侧封存 agent 没有挂载，页面只能查看已封存版本"
|
|
}
|
|
if err != nil {
|
|
writeJSON(w, http.StatusServiceUnavailable, librarySnapshotList{Available: false,
|
|
Reason: "归档不可读：" + err.Error(), Snapshots: []librarySnapshotView{}, CaptureNote: libraryCaptureNote})
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, view)
|
|
}
|
|
|
|
func librarySnapshotDetail(w http.ResponseWriter, r *http.Request, config Config) {
|
|
id := strings.TrimSpace(r.PathValue("id"))
|
|
if id == "" {
|
|
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "snapshot id required"})
|
|
return
|
|
}
|
|
if config.LibraryArchiveRoot == "" {
|
|
writeJSON(w, http.StatusOK, map[string]any{"available": false,
|
|
"reason": "未接线：LIBRARY_ARCHIVE_ROOT 未配置，本平台没有整库归档可读"})
|
|
return
|
|
}
|
|
archive, err := librarysnapshot.Open(config.LibraryArchiveRoot, 256*1024*1024)
|
|
if err != nil {
|
|
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"available": false, "reason": "归档不可读"})
|
|
return
|
|
}
|
|
defer archive.Close()
|
|
manifest, err := archive.Load(id)
|
|
if err != nil {
|
|
writeJSON(w, http.StatusNotFound, map[string]any{"available": false, "reason": "快照不存在或校验失败"})
|
|
return
|
|
}
|
|
usage, err := archive.Usage()
|
|
if err != nil {
|
|
writeJSON(w, http.StatusServiceUnavailable, map[string]any{"available": false, "reason": "归档用量不可读"})
|
|
return
|
|
}
|
|
view, ok := librarySnapshotViewOf(id, manifest, usage)
|
|
if !ok {
|
|
view = librarySnapshotViewOfNoUsage(id, manifest)
|
|
}
|
|
writeJSON(w, http.StatusOK, map[string]any{"available": true, "snapshot": view, "capture_note": libraryCaptureNote})
|
|
}
|
|
|
|
func readLibrarySnapshots(root string) (librarySnapshotList, error) {
|
|
view := librarySnapshotList{Snapshots: []librarySnapshotView{}, CaptureNote: libraryCaptureNote}
|
|
_ = root
|
|
if root == "" {
|
|
view.Reason = "未接线：LIBRARY_ARCHIVE_ROOT 未配置，本平台没有整库归档可读"
|
|
return view, nil
|
|
}
|
|
view.ArchiveRoot = root
|
|
archive, err := librarysnapshot.Open(root, 256*1024*1024)
|
|
if err != nil {
|
|
return view, errors.New("归档根不可用")
|
|
}
|
|
defer archive.Close()
|
|
usage, err := archive.Usage()
|
|
if err != nil {
|
|
return view, errors.New("归档用量不可读")
|
|
}
|
|
// Per-version UniqueBytes deduplicates only inside one version, so summing it
|
|
// over versions over-reports shared blobs. The archive-wide accounting below
|
|
// deduplicates across versions and keeps filesystem facts separate.
|
|
total, err := archive.ContentUsage()
|
|
if err != nil {
|
|
return view, errors.New("归档用量不可读")
|
|
}
|
|
view.Versions = total.Versions
|
|
view.ContentBytes = total.ContentBytes
|
|
view.BlobFileBytes = total.BlobFileBytes
|
|
view.BlobAllocatedBytes = total.BlobAllocatedBytes
|
|
view.UnreferencedBlobs = total.UnreferencedBlobs
|
|
view.UnreferencedBytes = total.UnreferencedBytes
|
|
view.ManifestFileBytes = total.ManifestFileBytes
|
|
view.ManifestAllocatedBytes = total.ManifestAllocatedBytes
|
|
byID := map[string]librarysnapshot.Usage{}
|
|
for _, item := range usage {
|
|
byID[item.VersionID] = item
|
|
}
|
|
for id, item := range byID {
|
|
manifest, loadErr := archive.Load(id)
|
|
if loadErr != nil {
|
|
return view, errors.New("归档快照校验失败")
|
|
}
|
|
snapshot, ok := librarySnapshotViewOf(id, manifest, []librarysnapshot.Usage{item})
|
|
if !ok {
|
|
snapshot = librarySnapshotViewOfNoUsage(id, manifest)
|
|
}
|
|
view.Snapshots = append(view.Snapshots, snapshot)
|
|
}
|
|
sort.Slice(view.Snapshots, func(i, j int) bool {
|
|
if view.Snapshots[i].CreatedAt.Equal(view.Snapshots[j].CreatedAt) {
|
|
return view.Snapshots[i].SnapshotID > view.Snapshots[j].SnapshotID
|
|
}
|
|
return view.Snapshots[i].CreatedAt.After(view.Snapshots[j].CreatedAt)
|
|
})
|
|
view.Available = true
|
|
return view, nil
|
|
}
|
|
|
|
func librarySnapshotViewOf(id string, manifest librarysnapshot.Manifest, usage []librarysnapshot.Usage) (librarySnapshotView, bool) {
|
|
snapshot := librarySnapshotViewOfNoUsage(id, manifest)
|
|
for _, item := range usage {
|
|
if item.VersionID == id {
|
|
snapshot.LogicalBytes = item.LogicalBytes
|
|
snapshot.UniqueBytes = item.UniqueBytes
|
|
snapshot.SharedBytes = item.SharedBytes
|
|
return snapshot, true
|
|
}
|
|
}
|
|
return snapshot, false
|
|
}
|
|
|
|
func librarySnapshotViewOfNoUsage(id string, manifest librarysnapshot.Manifest) librarySnapshotView {
|
|
snapshot := librarySnapshotView{SnapshotID: id, Scope: manifest.Scope, KnowledgeBaseID: manifest.KnowledgeBaseID,
|
|
TenantID: manifest.TenantID, CreatedAt: manifest.CreatedAt, ValidationPolicy: manifest.ValidationPolicy,
|
|
Format: manifest.Format, Compatibility: manifest.Compatibility, Parts: make([]librarySnapshotPart, 0, len(manifest.Parts)),
|
|
Totals: map[string]int64{}}
|
|
for _, part := range manifest.Parts {
|
|
snapshot.Parts = append(snapshot.Parts, librarySnapshotPart{Kind: part.Kind, Name: part.Name,
|
|
SHA256: part.SHA256, Count: part.Count, Bytes: part.Bytes})
|
|
snapshot.LogicalBytes += part.Bytes
|
|
// 汇总按"数量字段"给出人看得懂的四个数字：文档/分块/索引/资源/Wiki 页。
|
|
switch part.Name {
|
|
case "knowledges":
|
|
snapshot.Totals["documents"] = part.Count
|
|
case "chunks":
|
|
if part.Kind == "database" {
|
|
snapshot.Totals["chunks"] = part.Count
|
|
} else {
|
|
snapshot.Totals["index_documents"] = part.Count
|
|
}
|
|
case "resources":
|
|
snapshot.Totals["resources"] = part.Count
|
|
case "wiki_pages":
|
|
snapshot.Totals["wiki_pages"] = part.Count
|
|
case "wiki_folders":
|
|
snapshot.Totals["wiki_folders"] = part.Count
|
|
}
|
|
}
|
|
sort.Slice(snapshot.Parts, func(i, j int) bool {
|
|
if snapshot.Parts[i].Kind == snapshot.Parts[j].Kind {
|
|
return snapshot.Parts[i].Name < snapshot.Parts[j].Name
|
|
}
|
|
return snapshot.Parts[i].Kind < snapshot.Parts[j].Kind
|
|
})
|
|
return snapshot
|
|
}
|