Files
knowledge-governance/internal/releaseadapter/v071/library_archive_native_test.go
T
姚章浩 eaa388f76b
build / build (push) Successful in 1m42s
fix(release): 取消封存正文索引与父类型推断规则
按用户明确要求,治理封存不再替 WeKnora 判断正文块“应该”有独立索引:
删除 status/chunk_type/parent/mixed/flat 推导的 needOriginal、mixedDocs
及覆盖门槛;删除 parent 类型搭配限制,仅保留父引用存在与同 knowledge。
capture/preflight/archive/load/replay/writeset 共用校验同步更新,移除已无
用途的 worker 永久失败分类。

保留真实导出范围、PG/ES 原始正文/问题 metadata/vector、tenant/KB/
knowledge/chunk 身份、清单摘要、资源字节、Wiki/问题索引 source_id 及真实
悬空引用等完整性检查;不修改源端,不补数据/索引。旧测试改为原样捕获正例,
新增 status0/2 缺正文索引和父类型不匹配的接受回归,以及 archive/readback
逐行/逐记录比对。

验证:v071、cmd/worker、Go 全量、make verify 17 项。
2026-11-05 18:24:36 +08:00

347 lines
14 KiB
Go

package v071
import (
"context"
"encoding/json"
"fmt"
"io"
"os"
"strconv"
"strings"
"testing"
"time"
"knowledge-governance/internal/librarysnapshot"
)
// Source v0.7.1 chooses which chunks receive an original-text index. R3f
// removes governance's attempt to infer that choice (status/type/parent):
// capture the actual PG and ES sets, while still checking question metadata,
// scopes, digests, resources, Wiki references and real parent identities.
func TestArchiveLibraryCandidateAcceptsNativeMixedParentChild(t *testing.T) {
ArchiveLibraryCandidateAcceptance(t, true)
}
func TestArchiveLibraryCandidateCapturesFlatDocumentWithoutIndex(t *testing.T) {
fixture := nativeMixedFixture(false, true)
if _, _, err := archiveNativeFixture(t, fixture); err != nil {
t.Fatalf("flat document without body index must be captured as-is: %v", err)
}
}
func TestArchiveLibraryCandidateCapturesParentedChildWithoutIndex(t *testing.T) {
fixture := nativeMixedFixture(true, true)
// Remove only the child body index; question/summary/flat indexes stay.
fixture.es.Documents = append([]SearchDocument{}, fixture.es.Documents[1:]...)
fixture.es.DocumentsSHA256, _ = searchDocumentsDigest(fixture.es.Documents)
if _, _, err := archiveNativeFixture(t, fixture); err != nil {
t.Fatalf("parented child without body index must be captured as-is: %v", err)
}
}
func TestArchiveLibraryCandidateCapturesExplicitChunkWithoutIndex(t *testing.T) {
fixture := nativeMixedFixture(true, true)
fixture.pg.Tables = replaceChunkStatusForTest(fixture.pg.Tables, "chunk-standalone", 2)
if _, _, err := archiveNativeFixture(t, fixture); err != nil {
t.Fatalf("status=2 chunk without body index must be captured as-is: %v", err)
}
}
type nativeMixedFixtureState struct {
pg PostgresSnapshot
es SearchSnapshot
}
func nativeMixedFixture(includeFlatIndex, includeChildIndex bool) nativeMixedFixtureState {
knowledges := []json.RawMessage{
json.RawMessage(`{"id":"doc-mixed","tenant_id":10000,"knowledge_base_id":"kb-safe","parse_status":"completed","file_path":"resource://ABCDEFGHIJKLMNOPQRSTUV"}`),
json.RawMessage(`{"id":"doc-flat","tenant_id":10000,"knowledge_base_id":"kb-safe","parse_status":"completed","file_path":"resource://ABCDEFGHIJKLMNOPQRSTUV"}`),
}
chunks := []json.RawMessage{
json.RawMessage(`{"id":"chunk-parent","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_type":"parent_text","content":"parent","status":0}`),
json.RawMessage(`{"id":"chunk-child","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_type":"text","parent_chunk_id":"chunk-parent","content":"child","status":0}`),
json.RawMessage(`{"id":"chunk-standalone","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_type":"text","content":"standalone","status":0,"metadata":{"generated_questions":[{"id":"q-1","question":"Question?"}]}}`),
json.RawMessage(`{"id":"chunk-summary","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_type":"summary","parent_chunk_id":"chunk-child","content":"summary","status":0}`),
json.RawMessage(`{"id":"chunk-flat","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-flat","chunk_type":"text","content":"flat","status":0}`),
}
tables := []TableSnapshot{
newTableSnapshot("knowledges", knowledges),
newTableSnapshot("chunks", chunks),
newTableSnapshot("resources", []json.RawMessage{json.RawMessage(`{"id":"res-safe","tenant_id":10000,"handle":"ABCDEFGHIJKLMNOPQRSTUV","provider":"oss","physical_path":"oss://fixture/source.txt","size":4}`)}),
}
for _, name := range []string{"resource_bindings", "knowledge_tags", "knowledge_tag_relations", "wiki_pages", "wiki_folders", "wiki_page_issues", "wiki_log_entries"} {
tables = append(tables, newTableSnapshot(name, nil))
}
es := SearchSnapshot{MappingSHA256: shaBytes([]byte("mapping"))}
if includeChildIndex {
es.Documents = append(es.Documents, SearchDocument{ID: "es-child", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_id":"chunk-child","content":"child","vector":[0.125,-0.5]}`)})
// The standalone text has a generated-question index but deliberately
// no original-text index; the index must not be counted as body coverage.
es.Documents = append(es.Documents, SearchDocument{ID: "es-question", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_id":"chunk-standalone","source_id":"chunk-standalone-q-1","content":"Question?","vector":[0.25,0.75]}`)})
es.Documents = append(es.Documents, SearchDocument{ID: "es-summary", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_id":"chunk-summary","source_id":"chunk-summary","content":"summary","vector":[0.5,-0.25]}`)})
}
if includeFlatIndex {
es.Documents = append(es.Documents, SearchDocument{ID: "es-flat", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-flat","chunk_id":"chunk-flat","content":"flat","vector":[0.0,1.0]}`)})
}
es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents)
return nativeMixedFixtureState{pg: PostgresSnapshot{Tables: tables}, es: es}
}
func derivedFlatFixture(kind string, keepBody bool) (PostgresSnapshot, SearchSnapshot) {
pg, es, _ := libraryCandidateFixture()
chunks := []json.RawMessage{
json.RawMessage(`{"id":"chunk-safe","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","content":"body","status":0,"chunk_type":"text"}`),
json.RawMessage(`{"id":"derived-safe","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","content":"derived","status":0,"chunk_type":"` + kind + `","parent_chunk_id":"chunk-safe"}`),
}
pg.Tables[1] = newTableSnapshot("chunks", chunks)
if !keepBody {
es.Documents = nil
}
es.Documents = append(es.Documents, SearchDocument{ID: "derived-index", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","chunk_id":"derived-safe","source_id":"derived-safe","content":"derived","vector":[0.5,0.5]}`)})
es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents)
return pg, es
}
func TestDerivedParentRoleIsAccepted(t *testing.T) {
for _, kind := range []string{"summary", "image_ocr", "image_caption"} {
t.Run(kind, func(t *testing.T) {
pg, es := derivedFlatFixture(kind, true)
if err := validateLibraryCandidate(PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}, pg, es); err != nil {
t.Fatalf("native %s -> text link rejected: %v", kind, err)
}
})
}
}
func TestDerivedParentDoesNotAffectFlatBodyCapture(t *testing.T) {
pg, es := derivedFlatFixture("summary", false)
if err := validateLibraryCandidate(PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}, pg, es); err != nil {
t.Fatalf("derived parent must not create an index-coverage gate: %v", err)
}
}
func TestArchiveBodyWithoutOriginalIndexIsAccepted(t *testing.T) {
for _, status := range []int{0, 2} {
t.Run("status="+strconv.Itoa(status), func(t *testing.T) {
pg, es := derivedFlatFixture("summary", false)
pg.Tables = replaceChunkStatusForTest(pg.Tables, "chunk-safe", status)
_, _, err := archiveNativeFixture(t, nativeMixedFixtureState{pg: pg, es: es})
if err != nil {
t.Fatalf("body without original index must be captured as-is: %v", err)
}
})
}
}
func TestArchiveParentTypePairingIsNotEnforced(t *testing.T) {
pg, es := derivedFlatFixture("image_ocr", true)
pg.Tables = replaceChunkTypeForTest(pg.Tables, "chunk-safe", "summary")
_, _, err := archiveNativeFixture(t, nativeMixedFixtureState{pg: pg, es: es})
if err != nil {
t.Fatalf("an existing same-knowledge parent must not be rejected by type: %v", err)
}
}
func archiveNativeFixture(t *testing.T, fixture nativeMixedFixtureState) (string, librarysnapshot.Manifest, error) {
t.Helper()
root := t.TempDir()
if err := os.Chmod(root, 0700); err != nil {
t.Fatal(err)
}
archive, err := librarysnapshot.Open(root, 1024*1024)
if err != nil {
t.Fatal(err)
}
defer archive.Close()
id, err := ArchiveLibraryCandidate(context.Background(), archive, "internal",
PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"},
librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74, RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "fixture-model", EmbeddingDimension: 1024, IndexMappingSHA256: fixture.es.MappingSHA256},
fixture.pg, fixture.es, &libraryCandidateReader{data: []byte("file")},
time.Date(2026, 11, 5, 0, 0, 0, 0, time.UTC))
if err != nil {
return "", librarysnapshot.Manifest{}, err
}
saved, err := archive.Load(id)
if err == nil {
err = validateArchivedLibraryClosure(archive, saved)
}
if err == nil {
err = verifyNativeArchiveReadback(archive, saved, fixture)
}
return id, saved, err
}
func canonicalJSONForTest(raw []byte) (string, error) {
var value any
decoder := json.NewDecoder(strings.NewReader(string(raw)))
decoder.UseNumber()
if err := decoder.Decode(&value); err != nil {
return "", err
}
encoded, err := json.Marshal(value)
return string(encoded), err
}
func verifyNativeArchiveReadback(archive *librarysnapshot.Archive, manifest librarysnapshot.Manifest, fixture nativeMixedFixtureState) error {
pgTables := map[string][]json.RawMessage{}
for _, table := range fixture.pg.Tables {
pgTables[table.Name] = table.Rows
}
for _, part := range manifest.Parts {
if part.Kind != "database" && part.Kind != "wiki" && part.Kind != "index" {
continue
}
reader, err := archive.OpenPart(part)
if err != nil {
return err
}
raw, readErr := io.ReadAll(reader)
_ = reader.Close()
if readErr != nil {
return readErr
}
if part.Kind == "index" {
var got []SearchDocument
if err := json.Unmarshal(raw, &got); err != nil {
return err
}
if len(got) != len(fixture.es.Documents) {
return fmt.Errorf("readback index count=%d want=%d", len(got), len(fixture.es.Documents))
}
for i := range got {
if got[i].ID != fixture.es.Documents[i].ID {
return fmt.Errorf("readback index id changed at %d", i)
}
want, _ := canonicalJSONForTest(fixture.es.Documents[i].Source)
have, err := canonicalJSONForTest(got[i].Source)
if err != nil || have != want {
return fmt.Errorf("readback index source changed at %d", i)
}
}
continue
}
var got []json.RawMessage
if err := json.Unmarshal(raw, &got); err != nil {
return err
}
wantRows, ok := pgTables[part.Name]
if !ok {
continue
}
if len(got) != len(wantRows) {
return fmt.Errorf("readback %s count=%d want=%d", part.Name, len(got), len(wantRows))
}
for i := range got {
have, a := canonicalJSONForTest(got[i])
want, b := canonicalJSONForTest(wantRows[i])
if a != nil || b != nil || have != want {
return fmt.Errorf("readback %s row %d changed", part.Name, i)
}
}
}
return nil
}
func TestArchiveLibraryCandidateRejectsDanglingParentReference(t *testing.T) {
fixture := nativeMixedFixture(true, true)
fixture.pg.Tables = replaceChunkParentForTest(fixture.pg.Tables, "chunk-child", "missing-parent")
_, _, err := archiveNativeFixture(t, fixture)
if err == nil || !strings.Contains(err.Error(), "parent_chunk_id reference") {
t.Fatalf("dangling parent reference must be rejected, got %v", err)
}
}
func replaceChunkParentForTest(tables []TableSnapshot, chunkID, parentID string) []TableSnapshot {
out := make([]TableSnapshot, 0, len(tables))
for _, table := range tables {
if table.Name != "chunks" {
out = append(out, table)
continue
}
rows := make([]json.RawMessage, 0, len(table.Rows))
for _, raw := range table.Rows {
var row map[string]any
_ = json.Unmarshal(raw, &row)
if row["id"] == chunkID {
row["parent_chunk_id"] = parentID
encoded, _ := json.Marshal(row)
rows = append(rows, encoded)
} else {
rows = append(rows, raw)
}
}
out = append(out, newTableSnapshot("chunks", rows))
}
return out
}
func replaceChunkTypeForTest(tables []TableSnapshot, chunkID, chunkType string) []TableSnapshot {
out := make([]TableSnapshot, 0, len(tables))
for _, table := range tables {
if table.Name != "chunks" {
out = append(out, table)
continue
}
rows := make([]json.RawMessage, 0, len(table.Rows))
for _, raw := range table.Rows {
var row map[string]any
_ = json.Unmarshal(raw, &row)
if row["id"] == chunkID {
row["chunk_type"] = chunkType
encoded, _ := json.Marshal(row)
rows = append(rows, encoded)
} else {
rows = append(rows, raw)
}
}
out = append(out, newTableSnapshot("chunks", rows))
}
return out
}
func replaceChunkStatusForTest(tables []TableSnapshot, chunkID string, status int) []TableSnapshot {
out := make([]TableSnapshot, 0, len(tables))
for _, table := range tables {
if table.Name != "chunks" {
out = append(out, table)
continue
}
rows := make([]json.RawMessage, 0, len(table.Rows))
for _, raw := range table.Rows {
var row map[string]any
_ = json.Unmarshal(raw, &row)
if row["id"] == chunkID {
row["status"] = status
encoded, _ := json.Marshal(row)
rows = append(rows, encoded)
} else {
rows = append(rows, raw)
}
}
out = append(out, newTableSnapshot("chunks", rows))
}
return out
}
// ArchiveLibraryCandidateAcceptance is a named helper so the positive case can
// fail with a clear message while the test still calls the real archive entry.
func ArchiveLibraryCandidateAcceptance(t *testing.T, includeFlatIndex bool) {
t.Helper()
_, saved, err := archiveNativeFixture(t, nativeMixedFixture(includeFlatIndex, true))
if err != nil {
t.Fatalf("native mixed parent-child candidate rejected: %v", err)
}
var chunks, index int64
for _, part := range saved.Parts {
if part.Kind == "database" && part.Name == "chunks" {
chunks = part.Count
}
if part.Kind == "index" {
index = part.Count
}
}
if chunks != 5 || index != 4 {
t.Fatalf("archive filtered rows: chunks=%d index=%d", chunks, index)
}
}