diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e2a297..301332c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,35 @@ # Changelog +## 开发快照(读接口超时:真实修复) + +### Fixed + +- 整库只读接口不再按版本重读全部 blob:`overview`(internal/external)、`storage`、`snapshots` + 从 10s 超时 503 变为真实返回完整数据(候选归档 79 版,本机实测 <1s)。 + 根因是列举路径用 `Archive.Usage()`(逐版 `Load` 校验全部部件), + 外加 `library_storage.go` 每版两次 `Load`、`snapshots` 每版一次 `Load`。 +- 新增 `Archive.UsageFromManifests` 给只读列举用:与 `Usage` 共用同一份去重实现, + 只把"读清单"换成清单级校验。`Usage`(逐版复核字节)保持不变,仍供同步/恢复与运维 CLI 使用。 +- `LoadManifest` 收紧到与 `Load` 相同的清单校验(严格解码 + 重新编码校验,含必需部件与兼容性契约), + `Load` 改为 `LoadManifest` + 逐部件验字节——同一套清单判据只有一份。 +- wiki 与文档差异只读所需部件(`wiki_pages` / `knowledges`),被读取的那个部件仍由 + `OpenPart` 复核摘要;不再为两页 Wiki 重算整版字节。 +- `libraryWithDeadline` 去掉了"另起 goroutine 到点先回 503"的写法:候选环境实测它会在超时后 + 继续写已结束的响应(日志 `http: superfluous response.WriteHeader call`),并留下仍在扫盘的 goroutine。 + 改为请求期限交给 ctx,读取在版本之间检查期限,响应只有一个写入者。 + +### Changed + +- 列举口径必须看得见:`snapshots`/`storage` 响应新增 `verification=manifest` 与说明, + 每份版本带 `verified=false`;单版详情因为整版复核字节,返回 `verification=content`、`verified=true`。 + 这样"清单可读"不会被读成"完整性已验证/可发布"。 + +### 未做(不在本轮范围) + +- 运维 CLI `libraryctl` 仍走逐版复核(`Usage`/`Load`):它是执行前的操作入口,慢但真校验,本轮不动。 +- 归档规模到上万版本时仍未分页/增量盘点;只保证请求可被期限中止,不承诺增速。 +- 维护协议、Wiki 语义重算、最终部署仍未完成,见后续工作。 + ## 开发快照(读接口超时:根因量化 + 部分修复) ### 实测根因(本机候选归档,真实数字) @@ -22,9 +52,9 @@ ### 仍未修好(本轮没完成) -- `GET /library/overview`、`GET /library/storage` 仍 **503**(10s 超时)。 - 说明还有别的调用点在枚举版本时用 `Load`(候选:最新封存选择、wiki 差异基线解析、保留策略扫描)。 - 需要继续把这些调用点逐个切到 `LoadManifest`,并用同样的分段计时验证。 +- ~~`GET /library/overview`、`GET /library/storage` 仍 **503**(10s 超时)。~~ + 已在上面「读接口超时:真实修复」一节修好:剩余调用点是 `Usage()`、`library_storage.go` 的 + 双 `Load` 与 `snapshots` 的每版 `Load`。 ## 开发快照(目标维护窗口:半成品,未验证) @@ -1127,4 +1157,4 @@ postgres 共 6 个容器;web 发布在 `:8088`。 - **验收口径**:`docs/USER_FLOWS_ACCEPTANCE.md`(C1~C9)与 `docs/OPERATIONS_RUNBOOK.md`。 -⚠️ 上面的端口与容器名前缀是**本机运行栈**的实测值;换环境要按那边的运行栈改,别无脑照抄。 \ No newline at end of file +⚠️ 上面的端口与容器名前缀是**本机运行栈**的实测值;换环境要按那边的运行栈改,别无脑照抄。 diff --git a/internal/httpapi/library_flow.go b/internal/httpapi/library_flow.go index 0db832a..c3175b4 100644 --- a/internal/httpapi/library_flow.go +++ b/internal/httpapi/library_flow.go @@ -111,7 +111,10 @@ func wikiDiffOf(archive *librarysnapshot.Archive, baseID, targetID string) *libr Note: "页面级比对缺失时不能声称 Wiki 已同步"} } rows := func(id string) ([]map[string]any, string) { - manifest, err := archive.Load(id) + // 只读清单定位 wiki_pages 部件:整版 Load 会把"比两页 Wiki"变成 + // 逐部件校验整库字节(79 版归档上单版就 252ms)。被读取的那一个部件 + // 由 OpenPart 自己复核摘要,所以正文仍然是验过的字节。 + manifest, err := archive.LoadManifest(id) if err != nil { return nil, "快照不可读" } @@ -189,7 +192,9 @@ func computeLibraryDiff(root, baseID, targetID string) (libraryDiffView, bool) { } defer archive.Close() read := func(id string) ([]map[string]any, string) { - manifest, loadErr := archive.Load(id) + // 清单级定位 + OpenPart 复核被读取部件:差异只需要 knowledges 一列部件, + // 没有理由为它重算整版所有 blob。 + manifest, loadErr := archive.LoadManifest(id) if loadErr != nil { return nil, "快照不在本地归档或校验失败" } @@ -406,9 +411,9 @@ func libraryOverviewHandler(w http.ResponseWriter, r *http.Request, config Confi } // 测试封存来自归档:它只是测试侧事实。 - snapshots, snapErr := readLibrarySnapshots(config.LibraryArchiveRoot) + snapshots, snapErr := readLibrarySnapshots(r.Context(), config.LibraryArchiveRoot) if snapErr != nil { - overview.TestSealedReason = "归档不可读" + _, overview.TestSealedReason = libraryArchiveErrorStatus(snapErr, "归档不可读") } else if !snapshots.Available { overview.TestSealedReason = snapshots.Reason } else { diff --git a/internal/httpapi/library_read_paths_test.go b/internal/httpapi/library_read_paths_test.go new file mode 100644 index 0000000..0944c5f --- /dev/null +++ b/internal/httpapi/library_read_paths_test.go @@ -0,0 +1,268 @@ +package httpapi + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "knowledge-governance/internal/librarysnapshot" +) + +// 整库只读接口(overview / storage / snapshots)的回归: +// +// 候选归档里 79 个版本共享少量大 blob。按版本重读全部 blob 会让每个接口都 +// 打到请求级超时(实测 10s 后 503),并把"列版本"变成 O(全部版本字节)。 +// 这个夹具把形状缩到最小:多版本共享同一个大部件,枚举必须给出全部版本与真实去重, +// 而被篡改的字节仍然要在执行路径上被拒绝。 + +// sealSharedVersions 造 count 个内容完全一样的版本:它们共享全部 blob。 +func sealSharedVersions(t *testing.T, root string, count, documents int) []string { + t.Helper() + archive, err := librarysnapshot.Open(root, 4*1024*1024) + if err != nil { + t.Fatal(err) + } + rows := func(n int) string { + items := make([]string, n) + for i := range items { + items[i] = fmt.Sprintf(`{"id":"00000000-0000-4000-8000-%012d"}`, i) + } + return "[" + strings.Join(items, ",") + "]" + } + parts := []struct { + kind, name string + count int64 + body string + }{ + {"database", "knowledges", int64(documents), rows(documents)}, + {"database", "chunks", int64(documents) * 8, rows(documents * 8)}, + {"database", "resources", 0, rows(0)}, + {"database", "resource_bindings", 0, rows(0)}, + {"database", "knowledge_tags", 0, rows(0)}, + {"database", "knowledge_tag_relations", 0, rows(0)}, + {"index", "chunks", int64(documents) * 30, rows(documents * 30)}, + {"wiki", "wiki_pages", 3, rows(3)}, + {"wiki", "wiki_folders", 1, rows(1)}, + {"wiki", "wiki_page_issues", 0, rows(0)}, + {"wiki", "wiki_log_entries", 2, rows(2)}, + } + // 每个版本各封一次;内容相同 ⇒ blob 复用,归档里只有一份字节。 + ids := make([]string, 0, count) + for version := 0; version < count; version++ { + manifest := librarysnapshot.Manifest{ValidationPolicy: librarysnapshot.ArtifactAcceptancePolicy, + Format: librarysnapshot.Format, Scope: "internal", TenantID: 10000, + KnowledgeBaseID: "0c84a098-d3ed-4b82-a165-e6de9691d6a8", + CreatedAt: time.Date(2026, 10, 28, version, 0, 0, 0, time.UTC), + Compatibility: librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74, + RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "text-embedding-v4", EmbeddingDimension: 1024, + IndexMappingSHA256: "137cde2965f3b56521f5fb722b4e0617f46d0e509e571b217fe01e090d5e4d01"}} + for _, part := range parts { + stored, err := archive.Put(bytes.NewReader([]byte(part.body))) + if err != nil { + t.Fatal(err) + } + stored.Kind, stored.Name, stored.Count = part.kind, part.name, part.count + manifest.Parts = append(manifest.Parts, stored) + } + id, err := archive.Seal(manifest) + if err != nil { + t.Fatal(err) + } + ids = append(ids, id) + } + if err := archive.Close(); err != nil { + t.Fatal(err) + } + return ids +} + +func libraryFixtureRoot(t *testing.T) string { + t.Helper() + root, err := os.MkdirTemp(os.TempDir(), "kglib-read-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(root) }) + if err = os.Chmod(root, 0700); err != nil { + t.Fatal(err) + } + return root +} + +// 多版本共享大 blob:三个只读接口都必须返回完整正确的数据,不能降级或超时。 +func TestLibraryReadPathsListSharedVersions(t *testing.T) { + root := libraryFixtureRoot(t) + ids := sealSharedVersions(t, root, 12, 200) + config := Config{Store: &fakeLibraryStore{}, LibraryArchiveRoot: root, + LibraryCapacityBudgetBytes: 1 << 30} + + list, err := readLibrarySnapshots(t.Context(), root) + if err != nil { + t.Fatalf("snapshots list: %v", err) + } + if !list.Available || len(list.Snapshots) != len(ids) { + t.Fatalf("list available=%v snapshots=%d want=%d", list.Available, len(list.Snapshots), len(ids)) + } + // 共享字节只算一次:12 个内容相同的版本,跨版本内容字节必须等于"唯一 blob 之和", + // 而不是 12 × 单版逻辑字节(后者会把共享 blob 重复计 12 遍)。 + one := list.Snapshots[0].LogicalBytes + archive, err := librarysnapshot.Open(root, 4*1024*1024) + if err != nil { + t.Fatal(err) + } + manifest, err := archive.LoadManifest(ids[0]) + if err != nil { + t.Fatal(err) + } + archive.Close() + unique := map[string]int64{} + for _, part := range manifest.Parts { + unique[part.SHA256] = part.Bytes + } + var want int64 + for _, size := range unique { + want += size + } + if one <= 0 || list.ContentBytes != want { + t.Fatalf("content bytes must deduplicate shared blobs across %d versions: content=%d want=%d logical_one=%d", + len(ids), list.ContentBytes, want, one) + } + if want >= int64(len(ids))*one { + t.Fatalf("dedup produced no gain: content=%d versions×logical=%d", want, int64(len(ids))*one) + } + for _, snapshot := range list.Snapshots { + if snapshot.Totals["documents"] != 200 || snapshot.Totals["index_documents"] != 6000 { + t.Fatalf("snapshot %s lost totals: %+v", snapshot.SnapshotID, snapshot.Totals) + } + // 枚举只证明清单可读,不能自称完整性已验证。 + if snapshot.Verified { + t.Fatalf("listing must not claim verified integrity: %+v", snapshot) + } + } + if list.Verification != libraryVerificationManifest || list.VerificationNote == "" { + t.Fatalf("listing must state what it verified: %q / %q", list.Verification, list.VerificationNote) + } + + recorder := httptest.NewRecorder() + libraryStorageHandler(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/library/storage", nil), config) + if recorder.Code != http.StatusOK { + t.Fatalf("storage status=%d body=%s", recorder.Code, recorder.Body.String()) + } + var storage struct { + Available bool `json:"available"` + Versions []storageVersionView `json:"versions"` + Plan json.RawMessage `json:"plan"` + Notes []string `json:"notes"` + } + if err = json.Unmarshal(recorder.Body.Bytes(), &storage); err != nil { + t.Fatal(err) + } + if !storage.Available || len(storage.Versions) != len(ids) { + t.Fatalf("storage available=%v versions=%d want=%d", storage.Available, len(storage.Versions), len(ids)) + } + if len(storage.Plan) == 0 { + t.Fatal("storage must still build the retention preview") + } + if strings.Contains(recorder.Body.String(), "校验失败") { + t.Fatalf("storage must not report a verification failure for intact manifests: %s", recorder.Body.String()) + } + + // 单版详情是"要动这个版本"的入口:它仍然逐部件复核字节,所以能给出 content 级结论。 + detail := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, "/api/v1/library/snapshots/"+ids[0], nil) + request.SetPathValue("id", ids[0]) + librarySnapshotDetail(detail, request, config) + if detail.Code != http.StatusOK { + t.Fatalf("detail status=%d body=%s", detail.Code, detail.Body.String()) + } + var body struct { + Snapshot struct { + SnapshotID string `json:"snapshot_id"` + Verified bool `json:"verified"` + } `json:"snapshot"` + Verification string `json:"verification"` + } + if err = json.Unmarshal(detail.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if body.Snapshot.SnapshotID != ids[0] || !body.Snapshot.Verified || body.Verification != libraryVerificationContent { + t.Fatalf("detail must report content-level verification: %+v", body) + } +} + +// 坏 blob:枚举照常(它不读字节,且明确标注自己只做了清单级校验), +// 但详情这个执行入口必须拒绝,不能给出"完整"的结论。 +func TestLibrarySnapshotDetailRejectsTamperedBlob(t *testing.T) { + root := libraryFixtureRoot(t) + ids := sealSharedVersions(t, root, 2, 40) + archive, err := librarysnapshot.Open(root, 4*1024*1024) + if err != nil { + t.Fatal(err) + } + manifest, err := archive.LoadManifest(ids[0]) + if err != nil { + t.Fatal(err) + } + archive.Close() + target := manifest.Parts[0] + for _, part := range manifest.Parts { + if part.Bytes > target.Bytes { + target = part + } + } + path := filepath.Join(root, "blobs", target.SHA256) + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + corrupted := bytes.Repeat([]byte("x"), int(info.Size())) + if err = os.WriteFile(path, corrupted, 0600); err != nil { + t.Fatal(err) + } + + config := Config{Store: &fakeLibraryStore{}, LibraryArchiveRoot: root} + list, err := readLibrarySnapshots(t.Context(), root) + if err != nil || !list.Available || len(list.Snapshots) != len(ids) { + t.Fatalf("listing must still enumerate manifests: %+v err=%v", list, err) + } + + recorder := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, "/api/v1/library/snapshots/"+ids[0], nil) + request.SetPathValue("id", ids[0]) + librarySnapshotDetail(recorder, request, config) + if recorder.Code != http.StatusNotFound { + t.Fatalf("detail must fail closed on a tampered blob: status=%d body=%s", recorder.Code, recorder.Body.String()) + } +} + +// 坏清单:两个接口都必须报错,不能返回一个"看起来正常"的空/残缺列表。 +func TestLibraryReadPathsRejectTamperedManifest(t *testing.T) { + root := libraryFixtureRoot(t) + ids := sealSharedVersions(t, root, 2, 10) + path := filepath.Join(root, "versions", ids[0]+".json") + file, err := os.OpenFile(path, os.O_RDWR, 0600) + if err != nil { + t.Fatal(err) + } + if _, err = file.WriteAt([]byte{' '}, 0); err != nil { + t.Fatal(err) + } + file.Close() + + if list, err := readLibrarySnapshots(t.Context(), root); err == nil { + t.Fatalf("snapshots list must fail closed: %+v", list) + } + recorder := httptest.NewRecorder() + libraryStorageHandler(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/library/storage", nil), + Config{Store: &fakeLibraryStore{}, LibraryArchiveRoot: root}) + if recorder.Code != http.StatusServiceUnavailable { + t.Fatalf("storage must fail closed: status=%d body=%s", recorder.Code, recorder.Body.String()) + } +} diff --git a/internal/httpapi/library_snapshots.go b/internal/httpapi/library_snapshots.go index 7f55132..ded1b11 100644 --- a/internal/httpapi/library_snapshots.go +++ b/internal/httpapi/library_snapshots.go @@ -1,6 +1,7 @@ package httpapi import ( + "context" "errors" "net/http" "sort" @@ -40,13 +41,29 @@ type librarySnapshotView struct { UniqueBytes int64 `json:"unique_bytes"` SharedBytes int64 `json:"shared_bytes"` Totals map[string]int64 `json:"totals"` + // Verified 只在这一份版本的每个部件都被重新哈希并核对通过时为 true。 + // 列表口径下它恒为 false:枚举只证明清单可读,不是"完整性已验证"。 + Verified bool `json:"verified"` } +// 两次读取各自能证明什么,页面必须能看见,不能混成一句"完整性已验证"。 +const ( + libraryVerificationManifest = "manifest" + libraryVerificationContent = "content" + // libraryVerificationNote 说明枚举口径:清单自身(文件名摘要 + 结构)校验通过, + // 被引用字节是否完好要等到执行(同步/恢复)时再逐部件复核。 + libraryVerificationNote = "列表只校验每份清单自身(文件名摘要与结构):被引用的字节在枚举时不做逐部件复核," + + "所以这里的版本不能当作\"完整性已验证\"或\"可发布\"。执行(同步/恢复)前仍会逐部件校验字节,坏 blob 会被拒绝。" +) + type librarySnapshotList struct { Available bool `json:"available"` Reason string `json:"reason,omitempty"` ArchiveRoot string `json:"archive_root,omitempty"` Snapshots []librarySnapshotView `json:"snapshots"` + // Verification 是这份列表的口径(固定 manifest),配合 VerificationNote 说明它证明了什么。 + Verification string `json:"verification"` + VerificationNote string `json:"verification_note"` // 归档级口径:ContentBytes 是**跨版本去重后、被版本引用的内容字节**。 // 它不是磁盘占用(清单、未引用 blob、块对齐与元数据都不在里面), // 更不是可回收空间——清理需要保留策略、依赖保护与在线作业引用。 @@ -67,19 +84,47 @@ type librarySnapshotList struct { const libraryCaptureNote = "封存由运维命令 scripts/library-capture.sh capture -scope internal|external 在目标主机执行(停写入者→导出→恢复)。页面只读展示已封存的完整版本:它通过了数据完整性检查(身份、双向正文覆盖、生成问题、资源与摘要),这已满足封版所需的完整性口径;授权与生产同步仍是独立步骤。" func librarySnapshotsList(w http.ResponseWriter, r *http.Request, config Config) { - view, err := readLibrarySnapshots(config.LibraryArchiveRoot) + view, err := readLibrarySnapshots(r.Context(), config.LibraryArchiveRoot) view.CaptureAvailable = config.LibraryAgentSocket != "" if !view.CaptureAvailable { view.CaptureReason = "未接线:主机侧封存 agent 没有挂载,页面只能查看已封存版本" } if err != nil { - writeJSON(w, http.StatusServiceUnavailable, librarySnapshotList{Available: false, - Reason: "归档不可读:" + err.Error(), Snapshots: []librarySnapshotView{}, CaptureNote: libraryCaptureNote}) + status, reason := libraryArchiveErrorStatus(err, "归档不可读") + writeJSON(w, status, librarySnapshotList{Available: false, Reason: reason, + Snapshots: []librarySnapshotView{}, Verification: libraryVerificationManifest, + VerificationNote: libraryVerificationNote, CaptureNote: libraryCaptureNote}) return } writeJSON(w, http.StatusOK, view) } +// libraryArchiveErrorStatus 把"归档读不动"的两种原因分开:请求被期限中止要说超时, +// 其余(清单坏了、目录结构不对)仍然按不可读报,都不算成功。 +func libraryArchiveErrorStatus(err error, prefix string) (int, string) { + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { + return http.StatusServiceUnavailable, "归档读取超时(超过请求期限),本次没有返回数据" + } + return http.StatusServiceUnavailable, prefix + ":" + err.Error() +} + +// libraryManifests 按版本号逐份读清单,并在版本之间检查请求期限: +// 归档版本多时请求可以被真的中止,而不是把连接挂到读完为止。 +func libraryManifests(ctx context.Context, archive *librarysnapshot.Archive, ids []string) ([]librarysnapshot.Manifest, error) { + manifests := make([]librarysnapshot.Manifest, 0, len(ids)) + for _, id := range ids { + if err := ctx.Err(); err != nil { + return nil, err + } + manifest, err := archive.LoadManifest(id) + if err != nil { + return nil, err + } + manifests = append(manifests, manifest) + } + return manifests, nil +} + func librarySnapshotDetail(w http.ResponseWriter, r *http.Request, config Config) { id := strings.TrimSpace(r.PathValue("id")) if id == "" { @@ -97,12 +142,14 @@ func librarySnapshotDetail(w http.ResponseWriter, r *http.Request, config Config return } defer archive.Close() + // 详情是"要动这个版本"的入口:仍然整版复核字节(单版约 0.25s),失败即 404, + // 不把坏 blob 的版本展示成可用于同步的候选。 manifest, err := archive.Load(id) if err != nil { writeJSON(w, http.StatusNotFound, map[string]any{"available": false, "reason": "快照不存在或校验失败"}) return } - usage, err := archive.Usage() + usage, err := archive.UsageFromManifests() if err != nil { writeJSON(w, http.StatusServiceUnavailable, map[string]any{"available": false, "reason": "归档用量不可读"}) return @@ -111,12 +158,14 @@ func librarySnapshotDetail(w http.ResponseWriter, r *http.Request, config Config if !ok { view = librarySnapshotViewOfNoUsage(id, manifest) } - writeJSON(w, http.StatusOK, map[string]any{"available": true, "snapshot": view, "capture_note": libraryCaptureNote}) + view.Verified = true + writeJSON(w, http.StatusOK, map[string]any{"available": true, "snapshot": view, + "verification": libraryVerificationContent, "capture_note": libraryCaptureNote}) } -func readLibrarySnapshots(root string) (librarySnapshotList, error) { - view := librarySnapshotList{Snapshots: []librarySnapshotView{}, CaptureNote: libraryCaptureNote} - _ = root +func readLibrarySnapshots(ctx context.Context, root string) (librarySnapshotList, error) { + view := librarySnapshotList{Snapshots: []librarySnapshotView{}, CaptureNote: libraryCaptureNote, + Verification: libraryVerificationManifest, VerificationNote: libraryVerificationNote} if root == "" { view.Reason = "未接线:LIBRARY_ARCHIVE_ROOT 未配置,本平台没有整库归档可读" return view, nil @@ -127,13 +176,13 @@ func readLibrarySnapshots(root string) (librarySnapshotList, error) { return view, errors.New("归档根不可用") } defer archive.Close() - usage, err := archive.Usage() + // 列表只做清单级读取:逐版 Load 会把每次枚举放大成 O(全部版本字节) + // (实测 79 版 13.8s → 接口 10s 超时)。执行前仍走 Load/OpenPart。 + usage, err := archive.UsageFromManifests() if err != nil { return view, errors.New("归档用量不可读") } - // Per-version UniqueBytes deduplicates only inside one version, so summing it - // over versions over-reports shared blobs. The archive-wide accounting below - // deduplicates across versions and keeps filesystem facts separate. + // 这段跨版本去重口径与 Usage 共用同一个实现:清单级读取不能改变数字含义。 total, err := archive.ContentUsage() if err != nil { return view, errors.New("归档用量不可读") @@ -147,17 +196,22 @@ func readLibrarySnapshots(root string) (librarySnapshotList, error) { view.ManifestFileBytes = total.ManifestFileBytes view.ManifestAllocatedBytes = total.ManifestAllocatedBytes byID := map[string]librarysnapshot.Usage{} + ids := make([]string, 0, len(usage)) for _, item := range usage { + if _, seen := byID[item.VersionID]; !seen { + ids = append(ids, item.VersionID) + } byID[item.VersionID] = item } - for id, item := range byID { - manifest, loadErr := archive.Load(id) - if loadErr != nil { - return view, errors.New("归档快照校验失败") - } - snapshot, ok := librarySnapshotViewOf(id, manifest, []librarysnapshot.Usage{item}) + sort.Strings(ids) + manifests, err := libraryManifests(ctx, archive, ids) + if err != nil { + return view, err + } + for index, id := range ids { + snapshot, ok := librarySnapshotViewOf(id, manifests[index], []librarysnapshot.Usage{byID[id]}) if !ok { - snapshot = librarySnapshotViewOfNoUsage(id, manifest) + snapshot = librarySnapshotViewOfNoUsage(id, manifests[index]) } view.Snapshots = append(view.Snapshots, snapshot) } diff --git a/internal/httpapi/library_snapshots_test.go b/internal/httpapi/library_snapshots_test.go index d025736..c5da843 100644 --- a/internal/httpapi/library_snapshots_test.go +++ b/internal/httpapi/library_snapshots_test.go @@ -87,7 +87,7 @@ func TestReadLibrarySnapshotsReportsSealedTotals(t *testing.T) { if err = archive.Close(); err != nil { t.Fatal(err) } - list, err := readLibrarySnapshots(root) + list, err := readLibrarySnapshots(t.Context(), root) if err != nil { t.Fatal(err) } @@ -121,7 +121,7 @@ func TestReadLibrarySnapshotsReportsSealedTotals(t *testing.T) { } func TestReadLibrarySnapshotsWithoutArchiveIsExplicitlyUnwired(t *testing.T) { - list, err := readLibrarySnapshots("") + list, err := readLibrarySnapshots(t.Context(), "") if err != nil { t.Fatal(err) } diff --git a/internal/httpapi/library_storage.go b/internal/httpapi/library_storage.go index 3757a48..a802db1 100644 --- a/internal/httpapi/library_storage.go +++ b/internal/httpapi/library_storage.go @@ -42,11 +42,16 @@ type libraryStorageResponse struct { Capacity libraryretention.Capacity `json:"capacity"` AutomaticCleanup bool `json:"automatic_cleanup"` Notes []string `json:"notes"` + // 容量/保留计数是清单级口径:真实去重、每次重算,但"被引用字节是否完好" + // 只在实际执行(同步/恢复)时逐部件复核,这两件事不能混成一句"已验证"。 + Verification string `json:"verification"` + VerificationNote string `json:"verification_note"` } func libraryStorageHandler(w http.ResponseWriter, r *http.Request, config Config) { response := libraryStorageResponse{Versions: []storageVersionView{}, Notes: []string{}, AutomaticCleanup: false, - Policy: libraryretention.Policy{KeepVersions: int(config.LibraryRetentionVersions), KeepDays: int(config.LibraryRetentionDays)}} + Policy: libraryretention.Policy{KeepVersions: int(config.LibraryRetentionVersions), KeepDays: int(config.LibraryRetentionDays)}, + Verification: libraryVerificationManifest, VerificationNote: libraryVerificationNote} if strings.TrimSpace(config.LibraryArchiveRoot) == "" { response.Reason = "未接线:LIBRARY_ARCHIVE_ROOT 未配置,没有可统计的归档" writeJSON(w, http.StatusServiceUnavailable, response) @@ -74,7 +79,7 @@ func libraryStorageHandler(w http.ResponseWriter, r *http.Request, config Config response.UnreferencedBytes = total.UnreferencedBytes response.ManifestFileBytes = total.ManifestFileBytes - usage, err := archive.Usage() + usage, err := archive.UsageFromManifests() if err != nil { response.Reason = "归档逐版本用量不可读" writeJSON(w, http.StatusServiceUnavailable, response) @@ -127,13 +132,17 @@ func libraryStorageHandler(w http.ResponseWriter, r *http.Request, config Config versions := make([]libraryretention.Version, 0, len(ids)) latestSealed := map[string]time.Time{} var requiredBytes int64 - for _, id := range ids { - manifest, loadErr := archive.Load(id) - if loadErr != nil { - response.Reason = "归档快照校验失败" - writeJSON(w, http.StatusServiceUnavailable, response) - return - } + // 版本元数据走清单级读取:这里只做容量/保留盘点,不执行任何版本。 + // 需要动某个版本时(同步/恢复)仍然整版复核字节。 + manifests, err := libraryManifests(r.Context(), archive, ids) + if err != nil { + status, reason := libraryArchiveErrorStatus(err, "归档清单不可读") + response.Reason = reason + writeJSON(w, status, response) + return + } + for index, id := range ids { + manifest := manifests[index] kind := "unregistered" if record, ok := kinds[id]; ok { kind = string(record.Kind) @@ -153,12 +162,9 @@ func libraryStorageHandler(w http.ResponseWriter, r *http.Request, config Config ExclusiveBytes: item.ExclusiveBytes, SharedBytes: item.SharedBytes}) response.Versions = append(response.Versions, view) } - // 用与上面相同的排序再定位一次最新测试封存,避免把时间映射回 ID 时出错。 - for _, id := range ids { - manifest, loadErr := archive.Load(id) - if loadErr != nil { - continue - } + // 用与上面相同的清单再定位一次最新测试封存,避免把时间映射回 ID 时出错。 + for index, id := range ids { + manifest := manifests[index] if kindOf := kinds[id].Kind; kindOf == governance.LibraryVersionTestSealed || (kindOf == "" && manifest.ValidationPolicy != "") { if manifest.CreatedAt.Equal(latestSealed[manifest.Scope]) { diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index 791079e..1784dad 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -103,23 +103,18 @@ type Store interface { RequeueArtifactFetchWithOperation(ctx context.Context, request governance.RequeueArtifactFetchRequest, operationID string) (governance.RequeueArtifactFetchResult, bool, error) } -// libraryWithDeadline 给整库只读接口加请求级超时:超时就返回 503 + 明确原因, -// 而不是让浏览器连接挂住(归档大时读盘可能很慢)。 -func libraryWithDeadline(w http.ResponseWriter, r *http.Request, limit time.Duration, label string, +// libraryWithDeadline 给整库只读接口一个**真实**的请求期限:仓库读取按 ctx 在版本之间 +// 中断,超时由 handler 自己返回 503 并说明原因。 +// +// 为什么不再"另起 goroutine + 到点先回 503":那样主 goroutine 已经把响应结束, +// 被放弃的 handler 十几秒后仍会写同一个 ResponseWriter——候选环境实测日志 +// `http: superfluous response.WriteHeader call`,并且留下仍在扫全部 blob 的 goroutine。 +// 响应只允许一个写入者。 +func libraryWithDeadline(w http.ResponseWriter, r *http.Request, limit time.Duration, handler func(http.ResponseWriter, *http.Request)) { ctx, cancel := context.WithTimeout(r.Context(), limit) defer cancel() - done := make(chan struct{}) - go func() { - defer close(done) - handler(w, r.WithContext(ctx)) - }() - select { - case <-done: - case <-ctx.Done(): - writeJSON(w, http.StatusServiceUnavailable, map[string]any{ - "code": "library_read_timeout", "error": label + " 读取超时(归档较大),请稍后重试"}) - } + handler(w, r.WithContext(ctx)) } type Config struct { @@ -437,13 +432,13 @@ func NewHandler(config Config) (http.Handler, error) { // 整库状态:测试封存 / 生产生效 / 生产预备份 三类版本 + 差异 + 持久作业。 // 归档可能很大:加请求级超时,读不动就明确降级,别让页面请求挂死。 mux.HandleFunc("GET /api/v1/library/overview", func(w http.ResponseWriter, r *http.Request) { - libraryWithDeadline(w, r, 10*time.Second, "overview", func(w http.ResponseWriter, r *http.Request) { + libraryWithDeadline(w, r, 10*time.Second, func(w http.ResponseWriter, r *http.Request) { libraryOverviewHandler(w, r, config) }) }) // 存储管理:容量、保留策略、依赖保护与清理预览(只读,没有删除动作)。 mux.HandleFunc("GET /api/v1/library/storage", func(w http.ResponseWriter, r *http.Request) { - libraryWithDeadline(w, r, 10*time.Second, "storage", func(w http.ResponseWriter, r *http.Request) { + libraryWithDeadline(w, r, 10*time.Second, func(w http.ResponseWriter, r *http.Request) { libraryStorageHandler(w, r, config) }) }) diff --git a/internal/librarysnapshot/archive.go b/internal/librarysnapshot/archive.go index ade011d..d13c21f 100644 --- a/internal/librarysnapshot/archive.go +++ b/internal/librarysnapshot/archive.go @@ -187,34 +187,10 @@ func (a *Archive) Seal(m Manifest) (string, error) { // 用于"盘点/统计/列版本"这类需要遍历全部版本的地方:逐版重算 blob 会把 // 一次读取放大成 O(全部版本字节),真实踩到 79 版 × 55MB = 13.8s 的接口超时。 // 真正要用某个版本(同步/恢复)时仍然走 Load,那里会逐部件校验字节。 +// +// 它证明的只有"清单可读且结构合法",**不证明**这份版本引用的字节当前完好: +// 调用方不得把 LoadManifest 的结果标成"完整性已验证"或"可发布"。 func (a *Archive) LoadManifest(id string) (Manifest, error) { - if !validDigest(id) { - return Manifest{}, errors.New("invalid snapshot ID") - } - file, err := regular(a.versions, id+".json", maxManifestBytes) - if err != nil { - return Manifest{}, err - } - data, err := io.ReadAll(io.LimitReader(file, maxManifestBytes+1)) - file.Close() - if err != nil || int64(len(data)) > maxManifestBytes { - return Manifest{}, errors.New("read snapshot manifest") - } - if digest(data) != id { - return Manifest{}, errors.New("snapshot manifest digest mismatch") - } - var m Manifest - if err := json.Unmarshal(data, &m); err != nil { - return Manifest{}, errors.New("decode snapshot manifest") - } - if m.Format != Format || len(m.Parts) == 0 { - return Manifest{}, errors.New("invalid snapshot manifest") - } - return m, nil -} - -// Load revalidates bytes, not just the manifest's claim of recoverability. -func (a *Archive) Load(id string) (Manifest, error) { if !validDigest(id) { return Manifest{}, errors.New("invalid snapshot ID") } @@ -224,7 +200,7 @@ func (a *Archive) Load(id string) (Manifest, error) { } defer f.Close() data, err := io.ReadAll(io.LimitReader(f, maxManifestBytes+1)) - if err != nil || len(data) > maxManifestBytes || digest(data) != id { + if err != nil || int64(len(data)) > maxManifestBytes || digest(data) != id { return Manifest{}, errors.New("snapshot manifest digest mismatch") } dec := json.NewDecoder(bytes.NewReader(data)) @@ -240,6 +216,15 @@ func (a *Archive) Load(id string) (Manifest, error) { if _, err = encodeManifest(m); err != nil { return Manifest{}, err } + return m, nil +} + +// Load revalidates bytes, not just the manifest's claim of recoverability. +func (a *Archive) Load(id string) (Manifest, error) { + m, err := a.LoadManifest(id) + if err != nil { + return Manifest{}, err + } if err = a.verifyParts(m.Parts); err != nil { return Manifest{}, err } diff --git a/internal/librarysnapshot/usage.go b/internal/librarysnapshot/usage.go index ebc0b76..8e9191f 100644 --- a/internal/librarysnapshot/usage.go +++ b/internal/librarysnapshot/usage.go @@ -45,10 +45,27 @@ func (a *Archive) OpenPart(p Part) (io.ReadCloser, error) { return f, nil } -// Usage reports content bytes, not filesystem allocation or reclaimed space. -// It is deliberately read-only: protection/retention and live-job references -// must be acquired from the coordinator before any future cleanup operation. -func (a *Archive) Usage() ([]Usage, error) { +// Usage reports content bytes per version and is **verified**: every referenced +// blob is re-read and re-hashed through Load. Execution paths (sync/restore) and +// the operator CLI use it, because acting on a version must not trust the +// manifest's own claim about bytes it did not check. +func (a *Archive) Usage() ([]Usage, error) { return a.usage(a.Load) } + +// UsageFromManifests is the read-only listing counterpart: the same accounting, +// but it validates only each manifest's own digest/structure instead of +// re-hashing every blob. +// +// Why it exists: a listing does not execute anything, and re-hashing every blob +// of every version turned a 15 ms inventory into a 13.8 s timeout at 79 versions +// (measured on the candidate archive). Keep the numbers real: they are recomputed +// from the current inventory on every call, under the same "inventory must not +// change while reading" guard, and an unreadable/invalid manifest still fails the +// whole read. What this does *not* prove is that the referenced bytes are intact — +// Load/OpenPart still decide that, and callers must not report this as +// "integrity verified". +func (a *Archive) UsageFromManifests() ([]Usage, error) { return a.usage(a.LoadManifest) } + +func (a *Archive) usage(load func(string) (Manifest, error)) ([]Usage, error) { ids, err := a.versionIDs() if err != nil { return nil, err @@ -60,7 +77,7 @@ func (a *Archive) Usage() ([]Usage, error) { inventories := make([]inventory, 0, len(ids)) refs := map[string]int{} for _, id := range ids { - m, err := a.Load(id) + m, err := load(id) if err != nil { return nil, err } diff --git a/internal/librarysnapshot/usage_listing_test.go b/internal/librarysnapshot/usage_listing_test.go new file mode 100644 index 0000000..14d2c88 --- /dev/null +++ b/internal/librarysnapshot/usage_listing_test.go @@ -0,0 +1,144 @@ +package librarysnapshot + +import ( + "io" + "os" + "path/filepath" + "testing" + "time" +) + +// 枚举/盘点与执行走两条读取路径,这个文件把它们的分工钉死: +// +// - 枚举(UsageFromManifests / ContentUsage)只校验清单自身,**不读 blob 字节** +// —— 否则列版本就退化成 O(全部版本字节),实测 79 版 13.8s 打到接口超时; +// - 执行(Load / OpenPart)仍然逐部件复核字节,坏 blob 必须被拒绝; +// - 坏清单在两条路径上都必须如实报错,不能因为性能修复出现"假完整"。 + +// corruptBlob 把某个 blob 的内容改成长度不变的另一串字节: +// 文件大小与 0600 权限都合法,只有摘要对不上——这正是"被篡改"的形状。 +func corruptBlob(t *testing.T, root, sha string) { + t.Helper() + path := filepath.Join(root, "blobs", sha) + info, err := os.Stat(path) + if err != nil { + t.Fatalf("blob %s: %v", sha, err) + } + body := make([]byte, info.Size()) + for i := range body { + body[i] = 'x' + } + if err = os.WriteFile(path, body, 0600); err != nil { + t.Fatalf("write blob %s: %v", sha, err) + } +} + +// 健康归档上两条路径必须给出同一份数字:清单级盘点的去重口径不能因为不读字节而变形。 +func TestUsageFromManifestsMatchesVerifiedUsage(t *testing.T) { + archive, _ := fixtureArchive(t) + sealFixtureVersion(t, archive, "internal", time.Date(2026, 10, 28, 1, 0, 0, 0, time.UTC), 4) + sealFixtureVersion(t, archive, "internal", time.Date(2026, 10, 28, 2, 0, 0, 0, time.UTC), 4) + verified, err := archive.Usage() + if err != nil { + t.Fatal(err) + } + listed, err := archive.UsageFromManifests() + if err != nil { + t.Fatal(err) + } + if len(verified) != len(listed) || len(listed) != 2 { + t.Fatalf("versions verified=%d listed=%d, want 2", len(verified), len(listed)) + } + for i := range verified { + if verified[i] != listed[i] { + t.Fatalf("version %d: verified %+v != listed %+v", i, verified[i], listed[i]) + } + // 两个版本内容完全一致 ⇒ 每个 blob 都被两版共享,共享字节不得为 0。 + if listed[i].LogicalBytes <= 0 || listed[i].SharedBytes <= 0 || listed[i].UniqueBytes <= 0 { + t.Fatalf("listed usage must keep real accounting: %+v", listed[i]) + } + } +} + +// blob 被篡改后:枚举仍然给出全部版本(证明它没有按版本重读所有 blob), +// 而任何真正要用这个版本的路径都必须失败(证明完整性门槛没被移除)。 +func TestListingSkipsBlobBytesWhileExecutionStillRejects(t *testing.T) { + archive, root := fixtureArchive(t) + first := sealFixtureVersion(t, archive, "internal", time.Date(2026, 10, 28, 1, 0, 0, 0, time.UTC), 4) + second := sealFixtureVersion(t, archive, "internal", time.Date(2026, 10, 28, 2, 0, 0, 0, time.UTC), 4) + + manifest, err := archive.LoadManifest(first) + if err != nil { + t.Fatal(err) + } + // 挑最大的部件来篡改,确保受影响的不是某个空表。 + target := manifest.Parts[0] + for _, part := range manifest.Parts { + if part.Bytes > target.Bytes { + target = part + } + } + corruptBlob(t, root, target.SHA256) + + listed, err := archive.UsageFromManifests() + if err != nil { + t.Fatalf("listing must not depend on blob bytes: %v", err) + } + if len(listed) != 2 || listed[0].VersionID != first || listed[1].VersionID != second { + t.Fatalf("listing lost versions: %+v", listed) + } + total, err := archive.ContentUsage() + if err != nil { + t.Fatalf("content accounting must not depend on blob bytes: %v", err) + } + if total.Versions != 2 || total.ContentBytes <= 0 { + t.Fatalf("content accounting = %+v", total) + } + + // 执行路径:整版 Load、逐部件 OpenPart、逐版 Usage 都必须拒绝被篡改的字节。 + if _, err = archive.Load(first); err == nil { + t.Fatal("Load must reject a tampered blob") + } + if _, err = archive.Load(second); err == nil { + t.Fatal("Load must reject a version sharing the tampered blob") + } + if _, err = archive.OpenPart(target); err == nil { + t.Fatal("OpenPart must reject a tampered blob") + } + if _, err = archive.Usage(); err == nil { + t.Fatal("verified Usage must reject a tampered blob") + } +} + +// 清单被改过(内容与文件名摘要对不上)时,枚举必须整条失败: +// 性能修复不允许把"清单坏了"降级成"看起来还好"。 +func TestListingRejectsTamperedManifest(t *testing.T) { + archive, root := fixtureArchive(t) + id := sealFixtureVersion(t, archive, "internal", time.Date(2026, 10, 28, 1, 0, 0, 0, time.UTC), 4) + path := filepath.Join(root, "versions", id+".json") + file, err := os.OpenFile(path, os.O_RDWR, 0600) + if err != nil { + t.Fatal(err) + } + body, err := io.ReadAll(file) + if err != nil { + t.Fatal(err) + } + // 只改一个字节:JSON 仍然可解析,但内容摘要与文件名不再一致。 + if _, err = file.WriteAt([]byte{' '}, 0); err != nil { + t.Fatal(err) + } + file.Close() + if string(body) == "" { + t.Fatal("empty manifest fixture") + } + if _, err = archive.LoadManifest(id); err == nil { + t.Fatal("LoadManifest must reject a manifest whose digest does not match its name") + } + if _, err = archive.UsageFromManifests(); err == nil { + t.Fatal("listing must fail closed on a tampered manifest") + } + if _, err = archive.ContentUsage(); err == nil { + t.Fatal("content accounting must fail closed on a tampered manifest") + } +} diff --git a/web/src/ReleaseTargets.test.tsx b/web/src/ReleaseTargets.test.tsx index dd78ab5..d2ab4c9 100644 --- a/web/src/ReleaseTargets.test.tsx +++ b/web/src/ReleaseTargets.test.tsx @@ -234,7 +234,7 @@ describe('ReleaseTargetsView(确认稿布局)', () => { const html = render() expect(html).toContain('依据 10/28 11:34 封存的测试库全库快照,不代表实时测试库已就绪') expect(html).toContain('同一份封存快照,不是实时 Wiki 队列状态') - expect(html).toContain('封存时的逐项校验和,重新封存才会更新') + expect(html).toContain('列表只校验清单自身,字节级复核在同步/恢复执行前进行') }) it('版本与快照抽屉按 scope 列出全部历史并区分测试封存/不可直接恢复', () => { diff --git a/web/src/ReleaseTargets.tsx b/web/src/ReleaseTargets.tsx index edf2ddb..1f87cae 100644 --- a/web/src/ReleaseTargets.tsx +++ b/web/src/ReleaseTargets.tsx @@ -1012,8 +1012,10 @@ export function ReleaseTargetsView(props: ViewProps) { {target ? 同一份封存快照,不是实时 Wiki 队列状态 : null}
  • - {target ? `✓ 数据完整性:${target.parts.length} 项摘要逐项一致` : '○ 数据完整性:尚无封存可核验'} - {target ? 封存时的逐项校验和,重新封存才会更新 : null} + {target ? `✓ 清单与摘要登记:${target.parts.length} 项部件摘要已记录` : '○ 数据完整性:尚无封存可核验'} + {target ? ( + 封存时逐项哈希登记;列表只校验清单自身,字节级复核在同步/恢复执行前进行 + ) : null}
  • {gate.reasons.length > 0 ? (