eaa388f76b
build / build (push) Successful in 1m42s
按用户明确要求,治理封存不再替 WeKnora 判断正文块“应该”有独立索引: 删除 status/chunk_type/parent/mixed/flat 推导的 needOriginal、mixedDocs 及覆盖门槛;删除 parent 类型搭配限制,仅保留父引用存在与同 knowledge。 capture/preflight/archive/load/replay/writeset 共用校验同步更新,移除已无 用途的 worker 永久失败分类。 保留真实导出范围、PG/ES 原始正文/问题 metadata/vector、tenant/KB/ knowledge/chunk 身份、清单摘要、资源字节、Wiki/问题索引 source_id 及真实 悬空引用等完整性检查;不修改源端,不补数据/索引。旧测试改为原样捕获正例, 新增 status0/2 缺正文索引和父类型不匹配的接受回归,以及 archive/readback 逐行/逐记录比对。 验证:v071、cmd/worker、Go 全量、make verify 17 项。
347 lines
14 KiB
Go
347 lines
14 KiB
Go
package v071
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"knowledge-governance/internal/librarysnapshot"
|
|
)
|
|
|
|
// Source v0.7.1 chooses which chunks receive an original-text index. R3f
|
|
// removes governance's attempt to infer that choice (status/type/parent):
|
|
// capture the actual PG and ES sets, while still checking question metadata,
|
|
// scopes, digests, resources, Wiki references and real parent identities.
|
|
func TestArchiveLibraryCandidateAcceptsNativeMixedParentChild(t *testing.T) {
|
|
ArchiveLibraryCandidateAcceptance(t, true)
|
|
}
|
|
|
|
func TestArchiveLibraryCandidateCapturesFlatDocumentWithoutIndex(t *testing.T) {
|
|
fixture := nativeMixedFixture(false, true)
|
|
if _, _, err := archiveNativeFixture(t, fixture); err != nil {
|
|
t.Fatalf("flat document without body index must be captured as-is: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestArchiveLibraryCandidateCapturesParentedChildWithoutIndex(t *testing.T) {
|
|
fixture := nativeMixedFixture(true, true)
|
|
// Remove only the child body index; question/summary/flat indexes stay.
|
|
fixture.es.Documents = append([]SearchDocument{}, fixture.es.Documents[1:]...)
|
|
fixture.es.DocumentsSHA256, _ = searchDocumentsDigest(fixture.es.Documents)
|
|
if _, _, err := archiveNativeFixture(t, fixture); err != nil {
|
|
t.Fatalf("parented child without body index must be captured as-is: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestArchiveLibraryCandidateCapturesExplicitChunkWithoutIndex(t *testing.T) {
|
|
fixture := nativeMixedFixture(true, true)
|
|
fixture.pg.Tables = replaceChunkStatusForTest(fixture.pg.Tables, "chunk-standalone", 2)
|
|
if _, _, err := archiveNativeFixture(t, fixture); err != nil {
|
|
t.Fatalf("status=2 chunk without body index must be captured as-is: %v", err)
|
|
}
|
|
}
|
|
|
|
type nativeMixedFixtureState struct {
|
|
pg PostgresSnapshot
|
|
es SearchSnapshot
|
|
}
|
|
|
|
func nativeMixedFixture(includeFlatIndex, includeChildIndex bool) nativeMixedFixtureState {
|
|
knowledges := []json.RawMessage{
|
|
json.RawMessage(`{"id":"doc-mixed","tenant_id":10000,"knowledge_base_id":"kb-safe","parse_status":"completed","file_path":"resource://ABCDEFGHIJKLMNOPQRSTUV"}`),
|
|
json.RawMessage(`{"id":"doc-flat","tenant_id":10000,"knowledge_base_id":"kb-safe","parse_status":"completed","file_path":"resource://ABCDEFGHIJKLMNOPQRSTUV"}`),
|
|
}
|
|
chunks := []json.RawMessage{
|
|
json.RawMessage(`{"id":"chunk-parent","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_type":"parent_text","content":"parent","status":0}`),
|
|
json.RawMessage(`{"id":"chunk-child","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_type":"text","parent_chunk_id":"chunk-parent","content":"child","status":0}`),
|
|
json.RawMessage(`{"id":"chunk-standalone","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_type":"text","content":"standalone","status":0,"metadata":{"generated_questions":[{"id":"q-1","question":"Question?"}]}}`),
|
|
json.RawMessage(`{"id":"chunk-summary","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_type":"summary","parent_chunk_id":"chunk-child","content":"summary","status":0}`),
|
|
json.RawMessage(`{"id":"chunk-flat","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-flat","chunk_type":"text","content":"flat","status":0}`),
|
|
}
|
|
tables := []TableSnapshot{
|
|
newTableSnapshot("knowledges", knowledges),
|
|
newTableSnapshot("chunks", chunks),
|
|
newTableSnapshot("resources", []json.RawMessage{json.RawMessage(`{"id":"res-safe","tenant_id":10000,"handle":"ABCDEFGHIJKLMNOPQRSTUV","provider":"oss","physical_path":"oss://fixture/source.txt","size":4}`)}),
|
|
}
|
|
for _, name := range []string{"resource_bindings", "knowledge_tags", "knowledge_tag_relations", "wiki_pages", "wiki_folders", "wiki_page_issues", "wiki_log_entries"} {
|
|
tables = append(tables, newTableSnapshot(name, nil))
|
|
}
|
|
es := SearchSnapshot{MappingSHA256: shaBytes([]byte("mapping"))}
|
|
if includeChildIndex {
|
|
es.Documents = append(es.Documents, SearchDocument{ID: "es-child", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_id":"chunk-child","content":"child","vector":[0.125,-0.5]}`)})
|
|
// The standalone text has a generated-question index but deliberately
|
|
// no original-text index; the index must not be counted as body coverage.
|
|
es.Documents = append(es.Documents, SearchDocument{ID: "es-question", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_id":"chunk-standalone","source_id":"chunk-standalone-q-1","content":"Question?","vector":[0.25,0.75]}`)})
|
|
es.Documents = append(es.Documents, SearchDocument{ID: "es-summary", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_id":"chunk-summary","source_id":"chunk-summary","content":"summary","vector":[0.5,-0.25]}`)})
|
|
}
|
|
if includeFlatIndex {
|
|
es.Documents = append(es.Documents, SearchDocument{ID: "es-flat", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-flat","chunk_id":"chunk-flat","content":"flat","vector":[0.0,1.0]}`)})
|
|
}
|
|
es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents)
|
|
return nativeMixedFixtureState{pg: PostgresSnapshot{Tables: tables}, es: es}
|
|
}
|
|
|
|
func derivedFlatFixture(kind string, keepBody bool) (PostgresSnapshot, SearchSnapshot) {
|
|
pg, es, _ := libraryCandidateFixture()
|
|
chunks := []json.RawMessage{
|
|
json.RawMessage(`{"id":"chunk-safe","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","content":"body","status":0,"chunk_type":"text"}`),
|
|
json.RawMessage(`{"id":"derived-safe","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","content":"derived","status":0,"chunk_type":"` + kind + `","parent_chunk_id":"chunk-safe"}`),
|
|
}
|
|
pg.Tables[1] = newTableSnapshot("chunks", chunks)
|
|
if !keepBody {
|
|
es.Documents = nil
|
|
}
|
|
es.Documents = append(es.Documents, SearchDocument{ID: "derived-index", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","chunk_id":"derived-safe","source_id":"derived-safe","content":"derived","vector":[0.5,0.5]}`)})
|
|
es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents)
|
|
return pg, es
|
|
}
|
|
|
|
func TestDerivedParentRoleIsAccepted(t *testing.T) {
|
|
for _, kind := range []string{"summary", "image_ocr", "image_caption"} {
|
|
t.Run(kind, func(t *testing.T) {
|
|
pg, es := derivedFlatFixture(kind, true)
|
|
if err := validateLibraryCandidate(PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}, pg, es); err != nil {
|
|
t.Fatalf("native %s -> text link rejected: %v", kind, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDerivedParentDoesNotAffectFlatBodyCapture(t *testing.T) {
|
|
pg, es := derivedFlatFixture("summary", false)
|
|
if err := validateLibraryCandidate(PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}, pg, es); err != nil {
|
|
t.Fatalf("derived parent must not create an index-coverage gate: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestArchiveBodyWithoutOriginalIndexIsAccepted(t *testing.T) {
|
|
for _, status := range []int{0, 2} {
|
|
t.Run("status="+strconv.Itoa(status), func(t *testing.T) {
|
|
pg, es := derivedFlatFixture("summary", false)
|
|
pg.Tables = replaceChunkStatusForTest(pg.Tables, "chunk-safe", status)
|
|
_, _, err := archiveNativeFixture(t, nativeMixedFixtureState{pg: pg, es: es})
|
|
if err != nil {
|
|
t.Fatalf("body without original index must be captured as-is: %v", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestArchiveParentTypePairingIsNotEnforced(t *testing.T) {
|
|
pg, es := derivedFlatFixture("image_ocr", true)
|
|
pg.Tables = replaceChunkTypeForTest(pg.Tables, "chunk-safe", "summary")
|
|
_, _, err := archiveNativeFixture(t, nativeMixedFixtureState{pg: pg, es: es})
|
|
if err != nil {
|
|
t.Fatalf("an existing same-knowledge parent must not be rejected by type: %v", err)
|
|
}
|
|
}
|
|
|
|
func archiveNativeFixture(t *testing.T, fixture nativeMixedFixtureState) (string, librarysnapshot.Manifest, error) {
|
|
t.Helper()
|
|
root := t.TempDir()
|
|
if err := os.Chmod(root, 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
archive, err := librarysnapshot.Open(root, 1024*1024)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer archive.Close()
|
|
id, err := ArchiveLibraryCandidate(context.Background(), archive, "internal",
|
|
PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"},
|
|
librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74, RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "fixture-model", EmbeddingDimension: 1024, IndexMappingSHA256: fixture.es.MappingSHA256},
|
|
fixture.pg, fixture.es, &libraryCandidateReader{data: []byte("file")},
|
|
time.Date(2026, 11, 5, 0, 0, 0, 0, time.UTC))
|
|
if err != nil {
|
|
return "", librarysnapshot.Manifest{}, err
|
|
}
|
|
saved, err := archive.Load(id)
|
|
if err == nil {
|
|
err = validateArchivedLibraryClosure(archive, saved)
|
|
}
|
|
if err == nil {
|
|
err = verifyNativeArchiveReadback(archive, saved, fixture)
|
|
}
|
|
return id, saved, err
|
|
}
|
|
|
|
func canonicalJSONForTest(raw []byte) (string, error) {
|
|
var value any
|
|
decoder := json.NewDecoder(strings.NewReader(string(raw)))
|
|
decoder.UseNumber()
|
|
if err := decoder.Decode(&value); err != nil {
|
|
return "", err
|
|
}
|
|
encoded, err := json.Marshal(value)
|
|
return string(encoded), err
|
|
}
|
|
|
|
func verifyNativeArchiveReadback(archive *librarysnapshot.Archive, manifest librarysnapshot.Manifest, fixture nativeMixedFixtureState) error {
|
|
pgTables := map[string][]json.RawMessage{}
|
|
for _, table := range fixture.pg.Tables {
|
|
pgTables[table.Name] = table.Rows
|
|
}
|
|
for _, part := range manifest.Parts {
|
|
if part.Kind != "database" && part.Kind != "wiki" && part.Kind != "index" {
|
|
continue
|
|
}
|
|
reader, err := archive.OpenPart(part)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
raw, readErr := io.ReadAll(reader)
|
|
_ = reader.Close()
|
|
if readErr != nil {
|
|
return readErr
|
|
}
|
|
if part.Kind == "index" {
|
|
var got []SearchDocument
|
|
if err := json.Unmarshal(raw, &got); err != nil {
|
|
return err
|
|
}
|
|
if len(got) != len(fixture.es.Documents) {
|
|
return fmt.Errorf("readback index count=%d want=%d", len(got), len(fixture.es.Documents))
|
|
}
|
|
for i := range got {
|
|
if got[i].ID != fixture.es.Documents[i].ID {
|
|
return fmt.Errorf("readback index id changed at %d", i)
|
|
}
|
|
want, _ := canonicalJSONForTest(fixture.es.Documents[i].Source)
|
|
have, err := canonicalJSONForTest(got[i].Source)
|
|
if err != nil || have != want {
|
|
return fmt.Errorf("readback index source changed at %d", i)
|
|
}
|
|
}
|
|
continue
|
|
}
|
|
var got []json.RawMessage
|
|
if err := json.Unmarshal(raw, &got); err != nil {
|
|
return err
|
|
}
|
|
wantRows, ok := pgTables[part.Name]
|
|
if !ok {
|
|
continue
|
|
}
|
|
if len(got) != len(wantRows) {
|
|
return fmt.Errorf("readback %s count=%d want=%d", part.Name, len(got), len(wantRows))
|
|
}
|
|
for i := range got {
|
|
have, a := canonicalJSONForTest(got[i])
|
|
want, b := canonicalJSONForTest(wantRows[i])
|
|
if a != nil || b != nil || have != want {
|
|
return fmt.Errorf("readback %s row %d changed", part.Name, i)
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func TestArchiveLibraryCandidateRejectsDanglingParentReference(t *testing.T) {
|
|
fixture := nativeMixedFixture(true, true)
|
|
fixture.pg.Tables = replaceChunkParentForTest(fixture.pg.Tables, "chunk-child", "missing-parent")
|
|
_, _, err := archiveNativeFixture(t, fixture)
|
|
if err == nil || !strings.Contains(err.Error(), "parent_chunk_id reference") {
|
|
t.Fatalf("dangling parent reference must be rejected, got %v", err)
|
|
}
|
|
}
|
|
|
|
func replaceChunkParentForTest(tables []TableSnapshot, chunkID, parentID string) []TableSnapshot {
|
|
out := make([]TableSnapshot, 0, len(tables))
|
|
for _, table := range tables {
|
|
if table.Name != "chunks" {
|
|
out = append(out, table)
|
|
continue
|
|
}
|
|
rows := make([]json.RawMessage, 0, len(table.Rows))
|
|
for _, raw := range table.Rows {
|
|
var row map[string]any
|
|
_ = json.Unmarshal(raw, &row)
|
|
if row["id"] == chunkID {
|
|
row["parent_chunk_id"] = parentID
|
|
encoded, _ := json.Marshal(row)
|
|
rows = append(rows, encoded)
|
|
} else {
|
|
rows = append(rows, raw)
|
|
}
|
|
}
|
|
out = append(out, newTableSnapshot("chunks", rows))
|
|
}
|
|
return out
|
|
}
|
|
|
|
func replaceChunkTypeForTest(tables []TableSnapshot, chunkID, chunkType string) []TableSnapshot {
|
|
out := make([]TableSnapshot, 0, len(tables))
|
|
for _, table := range tables {
|
|
if table.Name != "chunks" {
|
|
out = append(out, table)
|
|
continue
|
|
}
|
|
rows := make([]json.RawMessage, 0, len(table.Rows))
|
|
for _, raw := range table.Rows {
|
|
var row map[string]any
|
|
_ = json.Unmarshal(raw, &row)
|
|
if row["id"] == chunkID {
|
|
row["chunk_type"] = chunkType
|
|
encoded, _ := json.Marshal(row)
|
|
rows = append(rows, encoded)
|
|
} else {
|
|
rows = append(rows, raw)
|
|
}
|
|
}
|
|
out = append(out, newTableSnapshot("chunks", rows))
|
|
}
|
|
return out
|
|
}
|
|
|
|
func replaceChunkStatusForTest(tables []TableSnapshot, chunkID string, status int) []TableSnapshot {
|
|
out := make([]TableSnapshot, 0, len(tables))
|
|
for _, table := range tables {
|
|
if table.Name != "chunks" {
|
|
out = append(out, table)
|
|
continue
|
|
}
|
|
rows := make([]json.RawMessage, 0, len(table.Rows))
|
|
for _, raw := range table.Rows {
|
|
var row map[string]any
|
|
_ = json.Unmarshal(raw, &row)
|
|
if row["id"] == chunkID {
|
|
row["status"] = status
|
|
encoded, _ := json.Marshal(row)
|
|
rows = append(rows, encoded)
|
|
} else {
|
|
rows = append(rows, raw)
|
|
}
|
|
}
|
|
out = append(out, newTableSnapshot("chunks", rows))
|
|
}
|
|
return out
|
|
}
|
|
|
|
// ArchiveLibraryCandidateAcceptance is a named helper so the positive case can
|
|
// fail with a clear message while the test still calls the real archive entry.
|
|
func ArchiveLibraryCandidateAcceptance(t *testing.T, includeFlatIndex bool) {
|
|
t.Helper()
|
|
_, saved, err := archiveNativeFixture(t, nativeMixedFixture(includeFlatIndex, true))
|
|
if err != nil {
|
|
t.Fatalf("native mixed parent-child candidate rejected: %v", err)
|
|
}
|
|
var chunks, index int64
|
|
for _, part := range saved.Parts {
|
|
if part.Kind == "database" && part.Name == "chunks" {
|
|
chunks = part.Count
|
|
}
|
|
if part.Kind == "index" {
|
|
index = part.Count
|
|
}
|
|
}
|
|
if chunks != 5 || index != 4 {
|
|
t.Fatalf("archive filtered rows: chunks=%d index=%d", chunks, index)
|
|
}
|
|
}
|