fix(release): 适配原生 parent-child 索引覆盖选择
build / build (push) Successful in 2m0s

治理候选封存现在解析 chunks.parent_chunk_id,并按 knowledge 区分源端
v0.7.1 的原生索引选择:mixed 文档只要求有 parent 的 text 原文索引,
flat 文档仍要求全部 text;parent_text 仅作上下文;status=2 缺失仍拒绝。
问题索引仍按 source_id 精确核对,不把问题记录当正文索引。

非空 parent_chunk_id 必须指向同一 knowledge 的真实 parent_text;standalone
空字段不强制关联,不被其他文档的 parent 豁免。归档与解档复用同一校验;
合成夹具证明 PG 全部 4 个 chunk 与 2 条 ES 原始记录读回保留。

验证:真实 ArchiveLibraryCandidate RED→GREEN、v071 全量、Go 全量、
make verify 17 项。源 WeKnora 未修改,不补父关联/索引,不重解析。
This commit is contained in:
姚章浩
2026-11-05 16:18:20 +08:00
parent 7511c59987
commit cf11343cf4
3 changed files with 240 additions and 4 deletions
+21
View File
@@ -1,5 +1,26 @@
# Changelog
## 开发快照 0.1.362(原生 parent-child 索引覆盖适配)
### Fixed
- **v071 候选封存覆盖校验**:解析 `chunks.parent_chunk_id`,按 knowledge 区分原生
parent-child 文档与 flat 文档。mixed 文档只要求有 parent 的 text 原文索引;flat
文档仍要求每段 text 索引;`parent_text` 仍仅作上下文不要求索引;显式
`status=2` 缺失仍拒绝。未把问题索引当正文索引,未泛化放行所有无 parent/status0
行,也未放宽 scope、摘要、资源、Wiki、问题索引和父引用完整性。
- **父关联完整性**:非空 `parent_chunk_id` 必须指向同一 knowledge 的真实
`parent_text` 行;standalone 空字段不强制建立关系,不受其他文档 parent 豁免。
- **读回/重放**:归档和解档共用同一严格校验;合成 mixed 夹具证明 PG 全部 4 个
chunk 与 2 条原始 ES 记录均保留,未过滤、重写或新增向量。
### Tests
- 非敏感合成夹具的真实 `ArchiveLibraryCandidate` RED→GREEN:mixed native 正例、
flat 文档缺索引拒绝、parented child 缺索引拒绝、status=2 缺索引拒绝、悬空父引用
拒绝;`r3d-native-red.txt` / `r3d-native-green.txt`。
## 开发快照 0.1.361(封存阶段真源、精确验证列表与捕获终态)
### Fixed
@@ -139,6 +139,7 @@ type libraryRow struct {
Content string `json:"content"`
Title string `json:"title"`
ChunkType string `json:"chunk_type"`
ParentChunkID string `json:"parent_chunk_id"`
ChunkStatus int `json:"-"`
SourceID string `json:"source_id"`
Questions map[string]string `json:"-"`
@@ -233,6 +234,20 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es
if _, ok := tables["knowledges"][chunk.KnowledgeID]; !ok {
return errors.New("orphan library chunk")
}
// A parent link must point at a real parent chunk of the same document;
// standalone text has an empty field and is never forced into a relation.
if chunk.ParentChunkID != "" {
parent, ok := tables["chunks"][chunk.ParentChunkID]
if !ok || parent.KnowledgeID != chunk.KnowledgeID || parent.ChunkType != "parent_text" {
return errors.New("invalid library parent chunk reference")
}
}
}
mixedDocs := map[string]bool{}
for _, chunk := range tables["chunks"] {
if chunk.ParentChunkID != "" || chunk.ChunkType == "parent_text" {
mixedDocs[chunk.KnowledgeID] = true
}
}
for _, relation := range tables["knowledge_tag_relations"] {
_, hasKnowledge := tables["knowledges"][relation.KnowledgeID]
@@ -328,12 +343,19 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es
return errors.New("missing library generated question index")
}
}
if chunk.ChunkStatus == 0 && chunk.ChunkType != "parent_text" && !indexedChunks[id] {
// Source v0.7.1 indexes (a) every text chunk of a flat document and
// (b) only text chunks with a parent in a mixed parent-child document.
// Standalone text in (b) is persisted and may have question indexes but
// has no original-text ES record; requiring one here rejects native data.
// Explicit status=2 remains required, and parent_text remains context-only.
needOriginal := chunk.ChunkStatus == 2 || (chunk.ChunkStatus == 0 && chunk.ChunkType != "parent_text" &&
(chunk.ChunkType != "text" || !mixedDocs[chunk.KnowledgeID] || chunk.ParentChunkID != ""))
if needOriginal && !indexedChunks[id] {
if chunk.ChunkStatus == 2 {
return errors.New("missing indexed library chunk")
}
return errors.New("unverified legacy library chunk index coverage")
}
if chunk.ChunkStatus == 2 && !indexedChunks[id] {
return errors.New("missing indexed library chunk")
}
}
return nil
}
@@ -0,0 +1,193 @@
package v071
import (
"context"
"encoding/json"
"os"
"strings"
"testing"
"time"
"knowledge-governance/internal/librarysnapshot"
)
// Source v0.7.1 parent-child ingestion indexes only text chunks with a real
// parent, or all text when the whole document is flat. Standalone text in a
// mixed document is persisted and can have question indexes, but has no
// original-text ES document. Governance must mirror that selection per
// knowledge, without weakening explicit/required index checks.
func TestArchiveLibraryCandidateAcceptsNativeMixedParentChild(t *testing.T) {
ArchiveLibraryCandidateAcceptance(t, true)
}
func TestArchiveLibraryCandidateRejectsFlatDocumentWithoutIndex(t *testing.T) {
fixture := nativeMixedFixture(false, true)
_, _, err := archiveNativeFixture(t, fixture)
if err == nil || !strings.Contains(err.Error(), "unverified legacy library chunk index coverage") {
t.Fatalf("flat document missing index must be rejected, got %v", err)
}
}
func TestArchiveLibraryCandidateRejectsRequiredParentedChildWithoutIndex(t *testing.T) {
fixture := nativeMixedFixture(true, true)
fixture.es.Documents = fixture.es.Documents[:1] // keep parent? first doc is child
// Remove the child document, leave flat document; recompute digest.
fixture.es.Documents = []SearchDocument{fixture.es.Documents[0]}
fixture.es.Documents[0].Source = json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-flat","chunk_id":"chunk-flat","content":"flat"}`)
fixture.es.DocumentsSHA256, _ = searchDocumentsDigest(fixture.es.Documents)
_, _, err := archiveNativeFixture(t, fixture)
if err == nil || !strings.Contains(err.Error(), "unverified legacy library chunk index coverage") {
t.Fatalf("parented child missing index must be rejected, got %v", err)
}
}
func TestArchiveLibraryCandidateRejectsExplicitIndexedChunkWithoutIndex(t *testing.T) {
fixture := nativeMixedFixture(true, true)
// Make standalone text explicitly indexed (status 2); it must not be
// exempted by the mixed native rule.
fixture.pg.Tables = replaceChunkStatusForTest(fixture.pg.Tables, "chunk-standalone", 2)
_, _, err := archiveNativeFixture(t, fixture)
if err == nil || !strings.Contains(err.Error(), "missing indexed library chunk") {
t.Fatalf("status=2 chunk without index must be rejected, got %v", err)
}
}
type nativeMixedFixtureState struct {
pg PostgresSnapshot
es SearchSnapshot
}
func nativeMixedFixture(includeFlatIndex, includeChildIndex bool) nativeMixedFixtureState {
knowledges := []json.RawMessage{
json.RawMessage(`{"id":"doc-mixed","tenant_id":10000,"knowledge_base_id":"kb-safe","parse_status":"completed","file_path":"resource://ABCDEFGHIJKLMNOPQRSTUV"}`),
json.RawMessage(`{"id":"doc-flat","tenant_id":10000,"knowledge_base_id":"kb-safe","parse_status":"completed","file_path":"resource://ABCDEFGHIJKLMNOPQRSTUV"}`),
}
chunks := []json.RawMessage{
json.RawMessage(`{"id":"chunk-parent","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_type":"parent_text","content":"parent","status":0}`),
json.RawMessage(`{"id":"chunk-child","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_type":"text","parent_chunk_id":"chunk-parent","content":"child","status":0}`),
json.RawMessage(`{"id":"chunk-standalone","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_type":"text","content":"standalone","status":0}`),
json.RawMessage(`{"id":"chunk-flat","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-flat","chunk_type":"text","content":"flat","status":0}`),
}
tables := []TableSnapshot{
newTableSnapshot("knowledges", knowledges),
newTableSnapshot("chunks", chunks),
newTableSnapshot("resources", []json.RawMessage{json.RawMessage(`{"id":"res-safe","tenant_id":10000,"handle":"ABCDEFGHIJKLMNOPQRSTUV","provider":"oss","physical_path":"oss://fixture/source.txt","size":4}`)}),
}
for _, name := range []string{"resource_bindings", "knowledge_tags", "knowledge_tag_relations", "wiki_pages", "wiki_folders", "wiki_page_issues", "wiki_log_entries"} {
tables = append(tables, newTableSnapshot(name, nil))
}
es := SearchSnapshot{MappingSHA256: shaBytes([]byte("mapping"))}
if includeChildIndex {
es.Documents = append(es.Documents, SearchDocument{ID: "es-child", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_id":"chunk-child","content":"child"}`)})
}
if includeFlatIndex {
es.Documents = append(es.Documents, SearchDocument{ID: "es-flat", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-flat","chunk_id":"chunk-flat","content":"flat"}`)})
}
es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents)
return nativeMixedFixtureState{pg: PostgresSnapshot{Tables: tables}, es: es}
}
func archiveNativeFixture(t *testing.T, fixture nativeMixedFixtureState) (string, librarysnapshot.Manifest, error) {
t.Helper()
root := t.TempDir()
if err := os.Chmod(root, 0700); err != nil {
t.Fatal(err)
}
archive, err := librarysnapshot.Open(root, 1024*1024)
if err != nil {
t.Fatal(err)
}
defer archive.Close()
id, err := ArchiveLibraryCandidate(context.Background(), archive, "internal",
PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"},
librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74, RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "fixture-model", EmbeddingDimension: 1024, IndexMappingSHA256: fixture.es.MappingSHA256},
fixture.pg, fixture.es, &libraryCandidateReader{data: []byte("file")},
time.Date(2026, 11, 5, 0, 0, 0, 0, time.UTC))
if err != nil {
return "", librarysnapshot.Manifest{}, err
}
saved, err := archive.Load(id)
if err == nil {
err = validateArchivedLibraryClosure(archive, saved)
}
return id, saved, err
}
func TestArchiveLibraryCandidateRejectsDanglingParentReference(t *testing.T) {
fixture := nativeMixedFixture(true, true)
fixture.pg.Tables = replaceChunkParentForTest(fixture.pg.Tables, "chunk-child", "missing-parent")
_, _, err := archiveNativeFixture(t, fixture)
if err == nil || !strings.Contains(err.Error(), "parent_chunk_id reference") {
t.Fatalf("dangling parent reference must be rejected, got %v", err)
}
}
func replaceChunkParentForTest(tables []TableSnapshot, chunkID, parentID string) []TableSnapshot {
out := make([]TableSnapshot, 0, len(tables))
for _, table := range tables {
if table.Name != "chunks" {
out = append(out, table)
continue
}
rows := make([]json.RawMessage, 0, len(table.Rows))
for _, raw := range table.Rows {
var row map[string]any
_ = json.Unmarshal(raw, &row)
if row["id"] == chunkID {
row["parent_chunk_id"] = parentID
encoded, _ := json.Marshal(row)
rows = append(rows, encoded)
} else {
rows = append(rows, raw)
}
}
out = append(out, newTableSnapshot("chunks", rows))
}
return out
}
func replaceChunkStatusForTest(tables []TableSnapshot, chunkID string, status int) []TableSnapshot {
out := make([]TableSnapshot, 0, len(tables))
for _, table := range tables {
if table.Name != "chunks" {
out = append(out, table)
continue
}
rows := make([]json.RawMessage, 0, len(table.Rows))
for _, raw := range table.Rows {
var row map[string]any
_ = json.Unmarshal(raw, &row)
if row["id"] == chunkID {
row["status"] = status
encoded, _ := json.Marshal(row)
rows = append(rows, encoded)
} else {
rows = append(rows, raw)
}
}
out = append(out, newTableSnapshot("chunks", rows))
}
return out
}
// ArchiveLibraryCandidateAcceptance is a named helper so the positive case can
// fail with a clear message while the test still calls the real archive entry.
func ArchiveLibraryCandidateAcceptance(t *testing.T, includeFlatIndex bool) {
t.Helper()
_, saved, err := archiveNativeFixture(t, nativeMixedFixture(includeFlatIndex, true))
if err != nil {
t.Fatalf("native mixed parent-child candidate rejected: %v", err)
}
var chunks, index int64
for _, part := range saved.Parts {
if part.Kind == "database" && part.Name == "chunks" {
chunks = part.Count
}
if part.Kind == "index" {
index = part.Count
}
}
if chunks != 4 || index != 2 {
t.Fatalf("archive filtered rows: chunks=%d index=%d", chunks, index)
}
}