姚章浩
fe6e218d9c
build / build (push) Successful in 1m17s
候选归档 79 版 / 56MB blob 上，overview、storage、snapshots 三个只读接口 都在 10s 请求期限后返回 503。分段计时定位到唯一瓶颈是按版本重读全部 blob： Open 0ms、LoadManifest×79 11~22ms、ContentUsage 15~25ms、 Usage（逐版 Load 验字节）13.75~14.11s；单版 Load 261~290ms。 - Usage 拆成 Usage（逐版复核字节，同步/恢复与运维 CLI 用）与 UsageFromManifests（同一份去重实现，只做清单级校验），列举路径改用后者； - library_storage.go 每版两次 Load、snapshots 每版一次 Load 改为一次清单读取； - wiki/文档差异只读所需部件，被读部件仍由 OpenPart 复核摘要； - LoadManifest 与 Load 共用同一套清单判据（严格解码 + 重新编码校验）； - 列举响应带 verification=manifest / verified=false，详情整版复核后才是 content， 避免把"清单可读"读成"完整性已验证"； - libraryWithDeadline 去掉超时后再写已结束响应的 goroutine 写法 （候选环境实测 http: superfluous response.WriteHeader call）。 回归：多版本共享大 blob 时枚举不读 blob 字节（篡改后枚举仍成功）， 而 Load/OpenPart/Usage 仍拒绝坏 blob；坏清单两条路径都整条报错。 make verify 15 项全绿（local 档案）。
269 lines
9.6 KiB
Go
269 lines
9.6 KiB
Go
package httpapi
|
||
|
||
import (
|
||
"bytes"
|
||
"encoding/json"
|
||
"fmt"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
"testing"
|
||
"time"
|
||
|
||
"knowledge-governance/internal/librarysnapshot"
|
||
)
|
||
|
||
// 整库只读接口（overview / storage / snapshots）的回归：
|
||
//
|
||
// 候选归档里 79 个版本共享少量大 blob。按版本重读全部 blob 会让每个接口都
|
||
// 打到请求级超时（实测 10s 后 503），并把"列版本"变成 O(全部版本字节)。
|
||
// 这个夹具把形状缩到最小：多版本共享同一个大部件，枚举必须给出全部版本与真实去重，
|
||
// 而被篡改的字节仍然要在执行路径上被拒绝。
|
||
|
||
// sealSharedVersions 造 count 个内容完全一样的版本：它们共享全部 blob。
|
||
func sealSharedVersions(t *testing.T, root string, count, documents int) []string {
|
||
t.Helper()
|
||
archive, err := librarysnapshot.Open(root, 4*1024*1024)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
rows := func(n int) string {
|
||
items := make([]string, n)
|
||
for i := range items {
|
||
items[i] = fmt.Sprintf(`{"id":"00000000-0000-4000-8000-%012d"}`, i)
|
||
}
|
||
return "[" + strings.Join(items, ",") + "]"
|
||
}
|
||
parts := []struct {
|
||
kind, name string
|
||
count int64
|
||
body string
|
||
}{
|
||
{"database", "knowledges", int64(documents), rows(documents)},
|
||
{"database", "chunks", int64(documents) * 8, rows(documents * 8)},
|
||
{"database", "resources", 0, rows(0)},
|
||
{"database", "resource_bindings", 0, rows(0)},
|
||
{"database", "knowledge_tags", 0, rows(0)},
|
||
{"database", "knowledge_tag_relations", 0, rows(0)},
|
||
{"index", "chunks", int64(documents) * 30, rows(documents * 30)},
|
||
{"wiki", "wiki_pages", 3, rows(3)},
|
||
{"wiki", "wiki_folders", 1, rows(1)},
|
||
{"wiki", "wiki_page_issues", 0, rows(0)},
|
||
{"wiki", "wiki_log_entries", 2, rows(2)},
|
||
}
|
||
// 每个版本各封一次；内容相同 ⇒ blob 复用，归档里只有一份字节。
|
||
ids := make([]string, 0, count)
|
||
for version := 0; version < count; version++ {
|
||
manifest := librarysnapshot.Manifest{ValidationPolicy: librarysnapshot.ArtifactAcceptancePolicy,
|
||
Format: librarysnapshot.Format, Scope: "internal", TenantID: 10000,
|
||
KnowledgeBaseID: "0c84a098-d3ed-4b82-a165-e6de9691d6a8",
|
||
CreatedAt: time.Date(2026, 10, 28, version, 0, 0, 0, time.UTC),
|
||
Compatibility: librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74,
|
||
RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "text-embedding-v4", EmbeddingDimension: 1024,
|
||
IndexMappingSHA256: "137cde2965f3b56521f5fb722b4e0617f46d0e509e571b217fe01e090d5e4d01"}}
|
||
for _, part := range parts {
|
||
stored, err := archive.Put(bytes.NewReader([]byte(part.body)))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
stored.Kind, stored.Name, stored.Count = part.kind, part.name, part.count
|
||
manifest.Parts = append(manifest.Parts, stored)
|
||
}
|
||
id, err := archive.Seal(manifest)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
ids = append(ids, id)
|
||
}
|
||
if err := archive.Close(); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
return ids
|
||
}
|
||
|
||
func libraryFixtureRoot(t *testing.T) string {
|
||
t.Helper()
|
||
root, err := os.MkdirTemp(os.TempDir(), "kglib-read-")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
t.Cleanup(func() { os.RemoveAll(root) })
|
||
if err = os.Chmod(root, 0700); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
return root
|
||
}
|
||
|
||
// 多版本共享大 blob：三个只读接口都必须返回完整正确的数据，不能降级或超时。
|
||
func TestLibraryReadPathsListSharedVersions(t *testing.T) {
|
||
root := libraryFixtureRoot(t)
|
||
ids := sealSharedVersions(t, root, 12, 200)
|
||
config := Config{Store: &fakeLibraryStore{}, LibraryArchiveRoot: root,
|
||
LibraryCapacityBudgetBytes: 1 << 30}
|
||
|
||
list, err := readLibrarySnapshots(t.Context(), root)
|
||
if err != nil {
|
||
t.Fatalf("snapshots list: %v", err)
|
||
}
|
||
if !list.Available || len(list.Snapshots) != len(ids) {
|
||
t.Fatalf("list available=%v snapshots=%d want=%d", list.Available, len(list.Snapshots), len(ids))
|
||
}
|
||
// 共享字节只算一次：12 个内容相同的版本，跨版本内容字节必须等于"唯一 blob 之和"，
|
||
// 而不是 12 × 单版逻辑字节（后者会把共享 blob 重复计 12 遍）。
|
||
one := list.Snapshots[0].LogicalBytes
|
||
archive, err := librarysnapshot.Open(root, 4*1024*1024)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
manifest, err := archive.LoadManifest(ids[0])
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
archive.Close()
|
||
unique := map[string]int64{}
|
||
for _, part := range manifest.Parts {
|
||
unique[part.SHA256] = part.Bytes
|
||
}
|
||
var want int64
|
||
for _, size := range unique {
|
||
want += size
|
||
}
|
||
if one <= 0 || list.ContentBytes != want {
|
||
t.Fatalf("content bytes must deduplicate shared blobs across %d versions: content=%d want=%d logical_one=%d",
|
||
len(ids), list.ContentBytes, want, one)
|
||
}
|
||
if want >= int64(len(ids))*one {
|
||
t.Fatalf("dedup produced no gain: content=%d versions×logical=%d", want, int64(len(ids))*one)
|
||
}
|
||
for _, snapshot := range list.Snapshots {
|
||
if snapshot.Totals["documents"] != 200 || snapshot.Totals["index_documents"] != 6000 {
|
||
t.Fatalf("snapshot %s lost totals: %+v", snapshot.SnapshotID, snapshot.Totals)
|
||
}
|
||
// 枚举只证明清单可读，不能自称完整性已验证。
|
||
if snapshot.Verified {
|
||
t.Fatalf("listing must not claim verified integrity: %+v", snapshot)
|
||
}
|
||
}
|
||
if list.Verification != libraryVerificationManifest || list.VerificationNote == "" {
|
||
t.Fatalf("listing must state what it verified: %q / %q", list.Verification, list.VerificationNote)
|
||
}
|
||
|
||
recorder := httptest.NewRecorder()
|
||
libraryStorageHandler(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/library/storage", nil), config)
|
||
if recorder.Code != http.StatusOK {
|
||
t.Fatalf("storage status=%d body=%s", recorder.Code, recorder.Body.String())
|
||
}
|
||
var storage struct {
|
||
Available bool `json:"available"`
|
||
Versions []storageVersionView `json:"versions"`
|
||
Plan json.RawMessage `json:"plan"`
|
||
Notes []string `json:"notes"`
|
||
}
|
||
if err = json.Unmarshal(recorder.Body.Bytes(), &storage); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if !storage.Available || len(storage.Versions) != len(ids) {
|
||
t.Fatalf("storage available=%v versions=%d want=%d", storage.Available, len(storage.Versions), len(ids))
|
||
}
|
||
if len(storage.Plan) == 0 {
|
||
t.Fatal("storage must still build the retention preview")
|
||
}
|
||
if strings.Contains(recorder.Body.String(), "校验失败") {
|
||
t.Fatalf("storage must not report a verification failure for intact manifests: %s", recorder.Body.String())
|
||
}
|
||
|
||
// 单版详情是"要动这个版本"的入口：它仍然逐部件复核字节，所以能给出 content 级结论。
|
||
detail := httptest.NewRecorder()
|
||
request := httptest.NewRequest(http.MethodGet, "/api/v1/library/snapshots/"+ids[0], nil)
|
||
request.SetPathValue("id", ids[0])
|
||
librarySnapshotDetail(detail, request, config)
|
||
if detail.Code != http.StatusOK {
|
||
t.Fatalf("detail status=%d body=%s", detail.Code, detail.Body.String())
|
||
}
|
||
var body struct {
|
||
Snapshot struct {
|
||
SnapshotID string `json:"snapshot_id"`
|
||
Verified bool `json:"verified"`
|
||
} `json:"snapshot"`
|
||
Verification string `json:"verification"`
|
||
}
|
||
if err = json.Unmarshal(detail.Body.Bytes(), &body); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if body.Snapshot.SnapshotID != ids[0] || !body.Snapshot.Verified || body.Verification != libraryVerificationContent {
|
||
t.Fatalf("detail must report content-level verification: %+v", body)
|
||
}
|
||
}
|
||
|
||
// 坏 blob：枚举照常（它不读字节，且明确标注自己只做了清单级校验），
|
||
// 但详情这个执行入口必须拒绝，不能给出"完整"的结论。
|
||
func TestLibrarySnapshotDetailRejectsTamperedBlob(t *testing.T) {
|
||
root := libraryFixtureRoot(t)
|
||
ids := sealSharedVersions(t, root, 2, 40)
|
||
archive, err := librarysnapshot.Open(root, 4*1024*1024)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
manifest, err := archive.LoadManifest(ids[0])
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
archive.Close()
|
||
target := manifest.Parts[0]
|
||
for _, part := range manifest.Parts {
|
||
if part.Bytes > target.Bytes {
|
||
target = part
|
||
}
|
||
}
|
||
path := filepath.Join(root, "blobs", target.SHA256)
|
||
info, err := os.Stat(path)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
corrupted := bytes.Repeat([]byte("x"), int(info.Size()))
|
||
if err = os.WriteFile(path, corrupted, 0600); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
|
||
config := Config{Store: &fakeLibraryStore{}, LibraryArchiveRoot: root}
|
||
list, err := readLibrarySnapshots(t.Context(), root)
|
||
if err != nil || !list.Available || len(list.Snapshots) != len(ids) {
|
||
t.Fatalf("listing must still enumerate manifests: %+v err=%v", list, err)
|
||
}
|
||
|
||
recorder := httptest.NewRecorder()
|
||
request := httptest.NewRequest(http.MethodGet, "/api/v1/library/snapshots/"+ids[0], nil)
|
||
request.SetPathValue("id", ids[0])
|
||
librarySnapshotDetail(recorder, request, config)
|
||
if recorder.Code != http.StatusNotFound {
|
||
t.Fatalf("detail must fail closed on a tampered blob: status=%d body=%s", recorder.Code, recorder.Body.String())
|
||
}
|
||
}
|
||
|
||
// 坏清单：两个接口都必须报错，不能返回一个"看起来正常"的空/残缺列表。
|
||
func TestLibraryReadPathsRejectTamperedManifest(t *testing.T) {
|
||
root := libraryFixtureRoot(t)
|
||
ids := sealSharedVersions(t, root, 2, 10)
|
||
path := filepath.Join(root, "versions", ids[0]+".json")
|
||
file, err := os.OpenFile(path, os.O_RDWR, 0600)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if _, err = file.WriteAt([]byte{' '}, 0); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
file.Close()
|
||
|
||
if list, err := readLibrarySnapshots(t.Context(), root); err == nil {
|
||
t.Fatalf("snapshots list must fail closed: %+v", list)
|
||
}
|
||
recorder := httptest.NewRecorder()
|
||
libraryStorageHandler(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/library/storage", nil),
|
||
Config{Store: &fakeLibraryStore{}, LibraryArchiveRoot: root})
|
||
if recorder.Code != http.StatusServiceUnavailable {
|
||
t.Fatalf("storage must fail closed: status=%d body=%s", recorder.Code, recorder.Body.String())
|
||
}
|
||
}
|