姚章浩
0088004cd6
接管分类、基线判定、已知索引映射、同步闸口的创建判据与归档保留的保护标记， 原来都用 LibraryVersions(scope, 50/100) 这个分页窗口再在内存里筛。生效版本每 scope 只有一行、可能很久以前登记，历史一长就被挤出窗口 —— 于是"已生效"被读成 "没有生效"：接管会把已经上线的目标重写一遍，基线入口会给已有权威版本的目标 再建一次基线。 现在统一走两条精确查询：LibraryCurrentVersion（kind='production_effective' AND superseded_at IS NULL，每 scope 至多一行）与 LibraryVersionOfSnapshot（按快照 身份取回"已顶替"证据）。多行、自相矛盾、读不到一律 fail closed；同一含义只留 一份实现（knownIndexMapping 同步与恢复共用）。LibraryVersions 仍是 1..100 的 展示窗口，只用于列表。 验证：真实 PG 上先复现（窗口实现 ⇒ found=false），改后 7 条 store 契约用例与 worker 端到端判定用例全绿；本机 make verify 15 项 success（用例 1264 通过 / 0 失败 / 93 已声明可选跳过）。
224 lines
9.3 KiB
Go
224 lines
9.3 KiB
Go
package httpapi
|
||
|
||
import (
|
||
"net/http"
|
||
"sort"
|
||
"strings"
|
||
"time"
|
||
|
||
"knowledge-governance/internal/governance"
|
||
"knowledge-governance/internal/libraryretention"
|
||
"knowledge-governance/internal/librarysnapshot"
|
||
)
|
||
|
||
// 存储管理接口：容量、保留策略、依赖保护与清理预览。
|
||
//
|
||
// 全部只读：没有删除动作，自动清理恒为关闭。候选版本只是"按策略如果清理会考虑这些"，
|
||
// 受保护版本（生产生效、每库最近封存、每次生产预备份、在飞作业引用）永不进入候选。
|
||
type storageVersionView struct {
|
||
ID string `json:"id"`
|
||
Kind string `json:"kind"`
|
||
Scope string `json:"scope"`
|
||
CreatedAt time.Time `json:"created_at"`
|
||
Protected bool `json:"protected"`
|
||
ProtectedBy []string `json:"protected_by,omitempty"`
|
||
ExclusiveBytes int64 `json:"exclusive_bytes"`
|
||
SharedBytes int64 `json:"shared_bytes"`
|
||
}
|
||
|
||
type libraryStorageResponse struct {
|
||
Available bool `json:"available"`
|
||
Reason string `json:"reason,omitempty"`
|
||
ArchiveRoot string `json:"archive_root,omitempty"`
|
||
Versions []storageVersionView `json:"versions"`
|
||
ContentBytes int64 `json:"content_bytes"`
|
||
BlobFileBytes int64 `json:"blob_file_bytes"`
|
||
BlobAllocatedBytes int64 `json:"blob_allocated_bytes"`
|
||
UnreferencedBlobs int `json:"unreferenced_blobs"`
|
||
UnreferencedBytes int64 `json:"unreferenced_bytes"`
|
||
ManifestFileBytes int64 `json:"manifest_file_bytes"`
|
||
Policy libraryretention.Policy `json:"policy"`
|
||
Plan *libraryretention.Plan `json:"plan,omitempty"`
|
||
Capacity libraryretention.Capacity `json:"capacity"`
|
||
AutomaticCleanup bool `json:"automatic_cleanup"`
|
||
Notes []string `json:"notes"`
|
||
// 容量/保留计数是清单级口径：真实去重、每次重算，但"被引用字节是否完好"
|
||
// 只在实际执行（同步/恢复）时逐部件复核，这两件事不能混成一句"已验证"。
|
||
Verification string `json:"verification"`
|
||
VerificationNote string `json:"verification_note"`
|
||
}
|
||
|
||
func libraryStorageHandler(w http.ResponseWriter, r *http.Request, config Config) {
|
||
response := libraryStorageResponse{Versions: []storageVersionView{}, Notes: []string{}, AutomaticCleanup: false,
|
||
Policy: libraryretention.Policy{KeepVersions: int(config.LibraryRetentionVersions), KeepDays: int(config.LibraryRetentionDays)},
|
||
Verification: libraryVerificationManifest, VerificationNote: libraryVerificationNote}
|
||
if strings.TrimSpace(config.LibraryArchiveRoot) == "" {
|
||
response.Reason = "未接线：LIBRARY_ARCHIVE_ROOT 未配置，没有可统计的归档"
|
||
writeJSON(w, http.StatusServiceUnavailable, response)
|
||
return
|
||
}
|
||
archive, err := librarysnapshot.Open(config.LibraryArchiveRoot, 256*1024*1024)
|
||
if err != nil {
|
||
response.Reason = "归档根不可用"
|
||
writeJSON(w, http.StatusServiceUnavailable, response)
|
||
return
|
||
}
|
||
defer archive.Close()
|
||
total, err := archive.ContentUsage()
|
||
if err != nil {
|
||
response.Reason = "归档用量不可读"
|
||
writeJSON(w, http.StatusServiceUnavailable, response)
|
||
return
|
||
}
|
||
response.Available = true
|
||
response.ArchiveRoot = config.LibraryArchiveRoot
|
||
response.ContentBytes = total.ContentBytes
|
||
response.BlobFileBytes = total.BlobFileBytes
|
||
response.BlobAllocatedBytes = total.BlobAllocatedBytes
|
||
response.UnreferencedBlobs = total.UnreferencedBlobs
|
||
response.UnreferencedBytes = total.UnreferencedBytes
|
||
response.ManifestFileBytes = total.ManifestFileBytes
|
||
|
||
usage, err := archive.UsageFromManifests()
|
||
if err != nil {
|
||
response.Reason = "归档逐版本用量不可读"
|
||
writeJSON(w, http.StatusServiceUnavailable, response)
|
||
return
|
||
}
|
||
usageByID := map[string]librarysnapshot.Usage{}
|
||
ids := make([]string, 0, len(usage))
|
||
for _, item := range usage {
|
||
usageByID[item.VersionID] = item
|
||
ids = append(ids, item.VersionID)
|
||
}
|
||
sort.Strings(ids)
|
||
|
||
// 版本类型与保护依据来自持久登记与在飞作业，不靠文件时间猜。
|
||
kinds := map[string]governance.LibraryVersion{}
|
||
protectedBy := map[string][]string{}
|
||
store, storeOK := libraryStore(config)
|
||
if storeOK {
|
||
for _, scope := range []string{"internal", "external"} {
|
||
versions, listErr := store.LibraryVersions(r.Context(), scope, 100)
|
||
if listErr == nil {
|
||
for _, item := range versions {
|
||
kinds[item.SnapshotID] = item
|
||
if item.Kind == governance.LibraryVersionProductionPrebackup {
|
||
protectedBy[item.SnapshotID] = append(protectedBy[item.SnapshotID], "生产预备份")
|
||
}
|
||
}
|
||
}
|
||
// 生产生效版本 = 唯一一条未顶替的记录，必须精确查：它可能早已掉出上面的
|
||
// 展示窗口，靠列表判"是否受保护"会让当前生效的那份被当成普通版本清掉。
|
||
if current, found, currentErr := store.LibraryCurrentVersion(r.Context(), scope); currentErr == nil && found {
|
||
if _, seen := kinds[current.SnapshotID]; !seen {
|
||
kinds[current.SnapshotID] = current
|
||
}
|
||
protectedBy[current.SnapshotID] = appendUnique(protectedBy[current.SnapshotID], "生产生效版本")
|
||
}
|
||
jobs, jobErr := store.ListLibraryJobs(r.Context(), scope, 20)
|
||
if jobErr == nil {
|
||
for _, job := range jobs {
|
||
if job.State != governance.LibraryJobPending && job.State != governance.LibraryJobRunning {
|
||
continue
|
||
}
|
||
for _, id := range []string{job.SnapshotID, job.TargetSnapshotID} {
|
||
if id != "" {
|
||
protectedBy[id] = append(protectedBy[id], "在飞作业引用")
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
} else {
|
||
response.Notes = append(response.Notes, "版本登记存储不可用：只按归档文件统计，无法识别受保护版本")
|
||
}
|
||
|
||
versions := make([]libraryretention.Version, 0, len(ids))
|
||
latestSealed := map[string]time.Time{}
|
||
var requiredBytes int64
|
||
// 版本元数据走清单级读取：这里只做容量/保留盘点，不执行任何版本。
|
||
// 需要动某个版本时（同步/恢复）仍然整版复核字节。
|
||
manifests, err := libraryManifests(r.Context(), archive, ids)
|
||
if err != nil {
|
||
status, reason := libraryArchiveErrorStatus(err, "归档清单不可读")
|
||
response.Reason = reason
|
||
writeJSON(w, status, response)
|
||
return
|
||
}
|
||
for index, id := range ids {
|
||
manifest := manifests[index]
|
||
kind := "unregistered"
|
||
if record, ok := kinds[id]; ok {
|
||
kind = string(record.Kind)
|
||
} else if manifest.ValidationPolicy != "" {
|
||
kind = "test_sealed"
|
||
}
|
||
if kind == "test_sealed" {
|
||
// 每库最近一次封存受保护：它是最新可发布的测试版本。
|
||
if current, seen := latestSealed[manifest.Scope]; !seen || manifest.CreatedAt.After(current) {
|
||
latestSealed[manifest.Scope] = manifest.CreatedAt
|
||
}
|
||
}
|
||
item := usageByID[id]
|
||
view := storageVersionView{ID: id, Kind: kind, Scope: manifest.Scope, CreatedAt: manifest.CreatedAt,
|
||
ExclusiveBytes: item.ExclusiveBytes, SharedBytes: item.SharedBytes}
|
||
versions = append(versions, libraryretention.Version{ID: id, Kind: kind, CreatedAt: manifest.CreatedAt,
|
||
ExclusiveBytes: item.ExclusiveBytes, SharedBytes: item.SharedBytes})
|
||
response.Versions = append(response.Versions, view)
|
||
}
|
||
// 用与上面相同的清单再定位一次最新测试封存，避免把时间映射回 ID 时出错。
|
||
for index, id := range ids {
|
||
manifest := manifests[index]
|
||
if kindOf := kinds[id].Kind; kindOf == governance.LibraryVersionTestSealed ||
|
||
(kindOf == "" && manifest.ValidationPolicy != "") {
|
||
if manifest.CreatedAt.Equal(latestSealed[manifest.Scope]) {
|
||
protectedBy[id] = append(protectedBy[id], "最近测试封存（"+manifest.Scope+"）")
|
||
}
|
||
}
|
||
}
|
||
|
||
for index := range versions {
|
||
if reasons := protectedBy[versions[index].ID]; len(reasons) > 0 {
|
||
versions[index].Protected = true
|
||
versions[index].ProtectedBy = reasons
|
||
}
|
||
}
|
||
for index := range response.Versions {
|
||
if reasons := protectedBy[response.Versions[index].ID]; len(reasons) > 0 {
|
||
response.Versions[index].Protected = true
|
||
response.Versions[index].ProtectedBy = reasons
|
||
}
|
||
}
|
||
|
||
now := time.Now().UTC()
|
||
plan := libraryretention.Build(versions, total.ContentBytes, response.Policy, now)
|
||
response.Plan = &plan
|
||
response.Notes = append(response.Notes, "候选版本只是清理预览；共享 blob 不会因删除单个版本释放")
|
||
response.Notes = append(response.Notes, "自动清理关闭，页面不提供删除动作；实际回收需要单独授权")
|
||
if !plan.PolicyConfigured {
|
||
response.Notes = append(response.Notes, "保留数量/天数未设置（LIBRARY_RETENTION_VERSIONS / LIBRARY_RETENTION_DAYS）")
|
||
}
|
||
|
||
for index := range response.Versions {
|
||
item := response.Versions[index]
|
||
if item.Kind == "test_sealed" && item.CreatedAt.Equal(latestSealed[item.Scope]) {
|
||
requiredBytes = item.ExclusiveBytes + item.SharedBytes
|
||
break
|
||
}
|
||
}
|
||
response.Capacity = libraryretention.CheckCapacity(total.ContentBytes, requiredBytes, config.LibraryCapacityBudgetBytes)
|
||
writeJSON(w, http.StatusOK, response)
|
||
}
|
||
|
||
// appendUnique 只追加还没出现过的标签：同一条版本可能同时被展示窗口与精确查询命中，
|
||
// 重复标签会让"保护依据"看起来像两条不同理由。
|
||
func appendUnique(labels []string, value string) []string {
|
||
for _, item := range labels {
|
||
if item == value {
|
||
return labels
|
||
}
|
||
}
|
||
return append(labels, value)
|
||
}
|