diff --git a/CHANGELOG.md b/CHANGELOG.md index 46023f4..bed3eb5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,26 @@ # Changelog +## 开发快照 0.1.362（原生 parent-child 索引覆盖适配） + +### Fixed + +- **v071 候选封存覆盖校验**：解析 `chunks.parent_chunk_id`，按 knowledge 区分原生 + parent-child 文档与 flat 文档。mixed 文档只要求有 parent 的 text 原文索引；flat + 文档仍要求每段 text 索引；`parent_text` 仍仅作上下文不要求索引；显式 + `status=2` 缺失仍拒绝。未把问题索引当正文索引，未泛化放行所有无 parent/status0 + 行，也未放宽 scope、摘要、资源、Wiki、问题索引和父引用完整性。 +- **父关联完整性**：非空 `parent_chunk_id` 必须指向同一 knowledge 的真实 + `parent_text` 行；standalone 空字段不强制建立关系，不受其他文档 parent 豁免。 +- **读回/重放**：归档和解档共用同一严格校验；合成 mixed 夹具证明 PG 全部 4 个 + chunk 与 2 条原始 ES 记录均保留，未过滤、重写或新增向量。 + +### Tests + +- 非敏感合成夹具的真实 `ArchiveLibraryCandidate` RED→GREEN：mixed native 正例、 + flat 文档缺索引拒绝、parented child 缺索引拒绝、status=2 缺索引拒绝、悬空父引用 + 拒绝；`r3d-native-red.txt` / `r3d-native-green.txt`。 + + ## 开发快照 0.1.361（封存阶段真源、精确验证列表与捕获终态） ### Fixed diff --git a/internal/releaseadapter/v071/library_archive.go b/internal/releaseadapter/v071/library_archive.go index 3feea94..e855119 100644 --- a/internal/releaseadapter/v071/library_archive.go +++ b/internal/releaseadapter/v071/library_archive.go @@ -139,6 +139,7 @@ type libraryRow struct { Content string `json:"content"` Title string `json:"title"` ChunkType string `json:"chunk_type"` + ParentChunkID string `json:"parent_chunk_id"` ChunkStatus int `json:"-"` SourceID string `json:"source_id"` Questions map[string]string `json:"-"` @@ -233,6 +234,20 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es if _, ok := tables["knowledges"][chunk.KnowledgeID]; !ok { return errors.New("orphan library chunk") } + // A parent link must point at a real parent chunk of the same document; + // standalone text has an empty field and is never forced into a relation. + if chunk.ParentChunkID != "" { + parent, ok := tables["chunks"][chunk.ParentChunkID] + if !ok || parent.KnowledgeID != chunk.KnowledgeID || parent.ChunkType != "parent_text" { + return errors.New("invalid library parent chunk reference") + } + } + } + mixedDocs := map[string]bool{} + for _, chunk := range tables["chunks"] { + if chunk.ParentChunkID != "" || chunk.ChunkType == "parent_text" { + mixedDocs[chunk.KnowledgeID] = true + } } for _, relation := range tables["knowledge_tag_relations"] { _, hasKnowledge := tables["knowledges"][relation.KnowledgeID] @@ -328,12 +343,19 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es return errors.New("missing library generated question index") } } - if chunk.ChunkStatus == 0 && chunk.ChunkType != "parent_text" && !indexedChunks[id] { + // Source v0.7.1 indexes (a) every text chunk of a flat document and + // (b) only text chunks with a parent in a mixed parent-child document. + // Standalone text in (b) is persisted and may have question indexes but + // has no original-text ES record; requiring one here rejects native data. + // Explicit status=2 remains required, and parent_text remains context-only. + needOriginal := chunk.ChunkStatus == 2 || (chunk.ChunkStatus == 0 && chunk.ChunkType != "parent_text" && + (chunk.ChunkType != "text" || !mixedDocs[chunk.KnowledgeID] || chunk.ParentChunkID != "")) + if needOriginal && !indexedChunks[id] { + if chunk.ChunkStatus == 2 { + return errors.New("missing indexed library chunk") + } return errors.New("unverified legacy library chunk index coverage") } - if chunk.ChunkStatus == 2 && !indexedChunks[id] { - return errors.New("missing indexed library chunk") - } } return nil } diff --git a/internal/releaseadapter/v071/library_archive_native_test.go b/internal/releaseadapter/v071/library_archive_native_test.go new file mode 100644 index 0000000..0e8a328 --- /dev/null +++ b/internal/releaseadapter/v071/library_archive_native_test.go @@ -0,0 +1,193 @@ +package v071 + +import ( + "context" + "encoding/json" + "os" + "strings" + "testing" + "time" + + "knowledge-governance/internal/librarysnapshot" +) + +// Source v0.7.1 parent-child ingestion indexes only text chunks with a real +// parent, or all text when the whole document is flat. Standalone text in a +// mixed document is persisted and can have question indexes, but has no +// original-text ES document. Governance must mirror that selection per +// knowledge, without weakening explicit/required index checks. +func TestArchiveLibraryCandidateAcceptsNativeMixedParentChild(t *testing.T) { + ArchiveLibraryCandidateAcceptance(t, true) +} + +func TestArchiveLibraryCandidateRejectsFlatDocumentWithoutIndex(t *testing.T) { + fixture := nativeMixedFixture(false, true) + _, _, err := archiveNativeFixture(t, fixture) + if err == nil || !strings.Contains(err.Error(), "unverified legacy library chunk index coverage") { + t.Fatalf("flat document missing index must be rejected, got %v", err) + } +} + +func TestArchiveLibraryCandidateRejectsRequiredParentedChildWithoutIndex(t *testing.T) { + fixture := nativeMixedFixture(true, true) + fixture.es.Documents = fixture.es.Documents[:1] // keep parent? first doc is child + // Remove the child document, leave flat document; recompute digest. + fixture.es.Documents = []SearchDocument{fixture.es.Documents[0]} + fixture.es.Documents[0].Source = json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-flat","chunk_id":"chunk-flat","content":"flat"}`) + fixture.es.DocumentsSHA256, _ = searchDocumentsDigest(fixture.es.Documents) + _, _, err := archiveNativeFixture(t, fixture) + if err == nil || !strings.Contains(err.Error(), "unverified legacy library chunk index coverage") { + t.Fatalf("parented child missing index must be rejected, got %v", err) + } +} + +func TestArchiveLibraryCandidateRejectsExplicitIndexedChunkWithoutIndex(t *testing.T) { + fixture := nativeMixedFixture(true, true) + // Make standalone text explicitly indexed (status 2); it must not be + // exempted by the mixed native rule. + fixture.pg.Tables = replaceChunkStatusForTest(fixture.pg.Tables, "chunk-standalone", 2) + _, _, err := archiveNativeFixture(t, fixture) + if err == nil || !strings.Contains(err.Error(), "missing indexed library chunk") { + t.Fatalf("status=2 chunk without index must be rejected, got %v", err) + } +} + +type nativeMixedFixtureState struct { + pg PostgresSnapshot + es SearchSnapshot +} + +func nativeMixedFixture(includeFlatIndex, includeChildIndex bool) nativeMixedFixtureState { + knowledges := []json.RawMessage{ + json.RawMessage(`{"id":"doc-mixed","tenant_id":10000,"knowledge_base_id":"kb-safe","parse_status":"completed","file_path":"resource://ABCDEFGHIJKLMNOPQRSTUV"}`), + json.RawMessage(`{"id":"doc-flat","tenant_id":10000,"knowledge_base_id":"kb-safe","parse_status":"completed","file_path":"resource://ABCDEFGHIJKLMNOPQRSTUV"}`), + } + chunks := []json.RawMessage{ + json.RawMessage(`{"id":"chunk-parent","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_type":"parent_text","content":"parent","status":0}`), + json.RawMessage(`{"id":"chunk-child","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_type":"text","parent_chunk_id":"chunk-parent","content":"child","status":0}`), + json.RawMessage(`{"id":"chunk-standalone","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_type":"text","content":"standalone","status":0}`), + json.RawMessage(`{"id":"chunk-flat","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-flat","chunk_type":"text","content":"flat","status":0}`), + } + tables := []TableSnapshot{ + newTableSnapshot("knowledges", knowledges), + newTableSnapshot("chunks", chunks), + newTableSnapshot("resources", []json.RawMessage{json.RawMessage(`{"id":"res-safe","tenant_id":10000,"handle":"ABCDEFGHIJKLMNOPQRSTUV","provider":"oss","physical_path":"oss://fixture/source.txt","size":4}`)}), + } + for _, name := range []string{"resource_bindings", "knowledge_tags", "knowledge_tag_relations", "wiki_pages", "wiki_folders", "wiki_page_issues", "wiki_log_entries"} { + tables = append(tables, newTableSnapshot(name, nil)) + } + es := SearchSnapshot{MappingSHA256: shaBytes([]byte("mapping"))} + if includeChildIndex { + es.Documents = append(es.Documents, SearchDocument{ID: "es-child", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_id":"chunk-child","content":"child"}`)}) + } + if includeFlatIndex { + es.Documents = append(es.Documents, SearchDocument{ID: "es-flat", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-flat","chunk_id":"chunk-flat","content":"flat"}`)}) + } + es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents) + return nativeMixedFixtureState{pg: PostgresSnapshot{Tables: tables}, es: es} +} + +func archiveNativeFixture(t *testing.T, fixture nativeMixedFixtureState) (string, librarysnapshot.Manifest, error) { + t.Helper() + root := t.TempDir() + if err := os.Chmod(root, 0700); err != nil { + t.Fatal(err) + } + archive, err := librarysnapshot.Open(root, 1024*1024) + if err != nil { + t.Fatal(err) + } + defer archive.Close() + id, err := ArchiveLibraryCandidate(context.Background(), archive, "internal", + PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}, + librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74, RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "fixture-model", EmbeddingDimension: 1024, IndexMappingSHA256: fixture.es.MappingSHA256}, + fixture.pg, fixture.es, &libraryCandidateReader{data: []byte("file")}, + time.Date(2026, 11, 5, 0, 0, 0, 0, time.UTC)) + if err != nil { + return "", librarysnapshot.Manifest{}, err + } + saved, err := archive.Load(id) + if err == nil { + err = validateArchivedLibraryClosure(archive, saved) + } + return id, saved, err +} + +func TestArchiveLibraryCandidateRejectsDanglingParentReference(t *testing.T) { + fixture := nativeMixedFixture(true, true) + fixture.pg.Tables = replaceChunkParentForTest(fixture.pg.Tables, "chunk-child", "missing-parent") + _, _, err := archiveNativeFixture(t, fixture) + if err == nil || !strings.Contains(err.Error(), "parent_chunk_id reference") { + t.Fatalf("dangling parent reference must be rejected, got %v", err) + } +} + +func replaceChunkParentForTest(tables []TableSnapshot, chunkID, parentID string) []TableSnapshot { + out := make([]TableSnapshot, 0, len(tables)) + for _, table := range tables { + if table.Name != "chunks" { + out = append(out, table) + continue + } + rows := make([]json.RawMessage, 0, len(table.Rows)) + for _, raw := range table.Rows { + var row map[string]any + _ = json.Unmarshal(raw, &row) + if row["id"] == chunkID { + row["parent_chunk_id"] = parentID + encoded, _ := json.Marshal(row) + rows = append(rows, encoded) + } else { + rows = append(rows, raw) + } + } + out = append(out, newTableSnapshot("chunks", rows)) + } + return out +} + +func replaceChunkStatusForTest(tables []TableSnapshot, chunkID string, status int) []TableSnapshot { + out := make([]TableSnapshot, 0, len(tables)) + for _, table := range tables { + if table.Name != "chunks" { + out = append(out, table) + continue + } + rows := make([]json.RawMessage, 0, len(table.Rows)) + for _, raw := range table.Rows { + var row map[string]any + _ = json.Unmarshal(raw, &row) + if row["id"] == chunkID { + row["status"] = status + encoded, _ := json.Marshal(row) + rows = append(rows, encoded) + } else { + rows = append(rows, raw) + } + } + out = append(out, newTableSnapshot("chunks", rows)) + } + return out +} + +// ArchiveLibraryCandidateAcceptance is a named helper so the positive case can +// fail with a clear message while the test still calls the real archive entry. +func ArchiveLibraryCandidateAcceptance(t *testing.T, includeFlatIndex bool) { + t.Helper() + _, saved, err := archiveNativeFixture(t, nativeMixedFixture(includeFlatIndex, true)) + if err != nil { + t.Fatalf("native mixed parent-child candidate rejected: %v", err) + } + var chunks, index int64 + for _, part := range saved.Parts { + if part.Kind == "database" && part.Name == "chunks" { + chunks = part.Count + } + if part.Kind == "index" { + index = part.Count + } + } + if chunks != 4 || index != 2 { + t.Fatalf("archive filtered rows: chunks=%d index=%d", chunks, index) + } +}