治理候选封存现在解析 chunks.parent_chunk_id,并按 knowledge 区分源端 v0.7.1 的原生索引选择:mixed 文档只要求有 parent 的 text 原文索引, flat 文档仍要求全部 text;parent_text 仅作上下文;status=2 缺失仍拒绝。 问题索引仍按 source_id 精确核对,不把问题记录当正文索引。 非空 parent_chunk_id 必须指向同一 knowledge 的真实 parent_text;standalone 空字段不强制关联,不被其他文档的 parent 豁免。归档与解档复用同一校验; 合成夹具证明 PG 全部 4 个 chunk 与 2 条 ES 原始记录读回保留。 验证:真实 ArchiveLibraryCandidate RED→GREEN、v071 全量、Go 全量、 make verify 17 项。源 WeKnora 未修改,不补父关联/索引,不重解析。
This commit is contained in:
@@ -1,5 +1,26 @@
|
||||
# Changelog
|
||||
|
||||
## 开发快照 0.1.362(原生 parent-child 索引覆盖适配)
|
||||
|
||||
### Fixed
|
||||
|
||||
- **v071 候选封存覆盖校验**:解析 `chunks.parent_chunk_id`,按 knowledge 区分原生
|
||||
parent-child 文档与 flat 文档。mixed 文档只要求有 parent 的 text 原文索引;flat
|
||||
文档仍要求每段 text 索引;`parent_text` 仍仅作上下文不要求索引;显式
|
||||
`status=2` 缺失仍拒绝。未把问题索引当正文索引,未泛化放行所有无 parent/status0
|
||||
行,也未放宽 scope、摘要、资源、Wiki、问题索引和父引用完整性。
|
||||
- **父关联完整性**:非空 `parent_chunk_id` 必须指向同一 knowledge 的真实
|
||||
`parent_text` 行;standalone 空字段不强制建立关系,不受其他文档 parent 豁免。
|
||||
- **读回/重放**:归档和解档共用同一严格校验;合成 mixed 夹具证明 PG 全部 4 个
|
||||
chunk 与 2 条原始 ES 记录均保留,未过滤、重写或新增向量。
|
||||
|
||||
### Tests
|
||||
|
||||
- 非敏感合成夹具的真实 `ArchiveLibraryCandidate` RED→GREEN:mixed native 正例、
|
||||
flat 文档缺索引拒绝、parented child 缺索引拒绝、status=2 缺索引拒绝、悬空父引用
|
||||
拒绝;`r3d-native-red.txt` / `r3d-native-green.txt`。
|
||||
|
||||
|
||||
## 开发快照 0.1.361(封存阶段真源、精确验证列表与捕获终态)
|
||||
|
||||
### Fixed
|
||||
|
||||
@@ -139,6 +139,7 @@ type libraryRow struct {
|
||||
Content string `json:"content"`
|
||||
Title string `json:"title"`
|
||||
ChunkType string `json:"chunk_type"`
|
||||
ParentChunkID string `json:"parent_chunk_id"`
|
||||
ChunkStatus int `json:"-"`
|
||||
SourceID string `json:"source_id"`
|
||||
Questions map[string]string `json:"-"`
|
||||
@@ -233,6 +234,20 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es
|
||||
if _, ok := tables["knowledges"][chunk.KnowledgeID]; !ok {
|
||||
return errors.New("orphan library chunk")
|
||||
}
|
||||
// A parent link must point at a real parent chunk of the same document;
|
||||
// standalone text has an empty field and is never forced into a relation.
|
||||
if chunk.ParentChunkID != "" {
|
||||
parent, ok := tables["chunks"][chunk.ParentChunkID]
|
||||
if !ok || parent.KnowledgeID != chunk.KnowledgeID || parent.ChunkType != "parent_text" {
|
||||
return errors.New("invalid library parent chunk reference")
|
||||
}
|
||||
}
|
||||
}
|
||||
mixedDocs := map[string]bool{}
|
||||
for _, chunk := range tables["chunks"] {
|
||||
if chunk.ParentChunkID != "" || chunk.ChunkType == "parent_text" {
|
||||
mixedDocs[chunk.KnowledgeID] = true
|
||||
}
|
||||
}
|
||||
for _, relation := range tables["knowledge_tag_relations"] {
|
||||
_, hasKnowledge := tables["knowledges"][relation.KnowledgeID]
|
||||
@@ -328,12 +343,19 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es
|
||||
return errors.New("missing library generated question index")
|
||||
}
|
||||
}
|
||||
if chunk.ChunkStatus == 0 && chunk.ChunkType != "parent_text" && !indexedChunks[id] {
|
||||
// Source v0.7.1 indexes (a) every text chunk of a flat document and
|
||||
// (b) only text chunks with a parent in a mixed parent-child document.
|
||||
// Standalone text in (b) is persisted and may have question indexes but
|
||||
// has no original-text ES record; requiring one here rejects native data.
|
||||
// Explicit status=2 remains required, and parent_text remains context-only.
|
||||
needOriginal := chunk.ChunkStatus == 2 || (chunk.ChunkStatus == 0 && chunk.ChunkType != "parent_text" &&
|
||||
(chunk.ChunkType != "text" || !mixedDocs[chunk.KnowledgeID] || chunk.ParentChunkID != ""))
|
||||
if needOriginal && !indexedChunks[id] {
|
||||
if chunk.ChunkStatus == 2 {
|
||||
return errors.New("missing indexed library chunk")
|
||||
}
|
||||
return errors.New("unverified legacy library chunk index coverage")
|
||||
}
|
||||
if chunk.ChunkStatus == 2 && !indexedChunks[id] {
|
||||
return errors.New("missing indexed library chunk")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
package v071
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/librarysnapshot"
|
||||
)
|
||||
|
||||
// Source v0.7.1 parent-child ingestion indexes only text chunks with a real
|
||||
// parent, or all text when the whole document is flat. Standalone text in a
|
||||
// mixed document is persisted and can have question indexes, but has no
|
||||
// original-text ES document. Governance must mirror that selection per
|
||||
// knowledge, without weakening explicit/required index checks.
|
||||
func TestArchiveLibraryCandidateAcceptsNativeMixedParentChild(t *testing.T) {
|
||||
ArchiveLibraryCandidateAcceptance(t, true)
|
||||
}
|
||||
|
||||
func TestArchiveLibraryCandidateRejectsFlatDocumentWithoutIndex(t *testing.T) {
|
||||
fixture := nativeMixedFixture(false, true)
|
||||
_, _, err := archiveNativeFixture(t, fixture)
|
||||
if err == nil || !strings.Contains(err.Error(), "unverified legacy library chunk index coverage") {
|
||||
t.Fatalf("flat document missing index must be rejected, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveLibraryCandidateRejectsRequiredParentedChildWithoutIndex(t *testing.T) {
|
||||
fixture := nativeMixedFixture(true, true)
|
||||
fixture.es.Documents = fixture.es.Documents[:1] // keep parent? first doc is child
|
||||
// Remove the child document, leave flat document; recompute digest.
|
||||
fixture.es.Documents = []SearchDocument{fixture.es.Documents[0]}
|
||||
fixture.es.Documents[0].Source = json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-flat","chunk_id":"chunk-flat","content":"flat"}`)
|
||||
fixture.es.DocumentsSHA256, _ = searchDocumentsDigest(fixture.es.Documents)
|
||||
_, _, err := archiveNativeFixture(t, fixture)
|
||||
if err == nil || !strings.Contains(err.Error(), "unverified legacy library chunk index coverage") {
|
||||
t.Fatalf("parented child missing index must be rejected, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveLibraryCandidateRejectsExplicitIndexedChunkWithoutIndex(t *testing.T) {
|
||||
fixture := nativeMixedFixture(true, true)
|
||||
// Make standalone text explicitly indexed (status 2); it must not be
|
||||
// exempted by the mixed native rule.
|
||||
fixture.pg.Tables = replaceChunkStatusForTest(fixture.pg.Tables, "chunk-standalone", 2)
|
||||
_, _, err := archiveNativeFixture(t, fixture)
|
||||
if err == nil || !strings.Contains(err.Error(), "missing indexed library chunk") {
|
||||
t.Fatalf("status=2 chunk without index must be rejected, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
type nativeMixedFixtureState struct {
|
||||
pg PostgresSnapshot
|
||||
es SearchSnapshot
|
||||
}
|
||||
|
||||
func nativeMixedFixture(includeFlatIndex, includeChildIndex bool) nativeMixedFixtureState {
|
||||
knowledges := []json.RawMessage{
|
||||
json.RawMessage(`{"id":"doc-mixed","tenant_id":10000,"knowledge_base_id":"kb-safe","parse_status":"completed","file_path":"resource://ABCDEFGHIJKLMNOPQRSTUV"}`),
|
||||
json.RawMessage(`{"id":"doc-flat","tenant_id":10000,"knowledge_base_id":"kb-safe","parse_status":"completed","file_path":"resource://ABCDEFGHIJKLMNOPQRSTUV"}`),
|
||||
}
|
||||
chunks := []json.RawMessage{
|
||||
json.RawMessage(`{"id":"chunk-parent","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_type":"parent_text","content":"parent","status":0}`),
|
||||
json.RawMessage(`{"id":"chunk-child","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_type":"text","parent_chunk_id":"chunk-parent","content":"child","status":0}`),
|
||||
json.RawMessage(`{"id":"chunk-standalone","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_type":"text","content":"standalone","status":0}`),
|
||||
json.RawMessage(`{"id":"chunk-flat","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-flat","chunk_type":"text","content":"flat","status":0}`),
|
||||
}
|
||||
tables := []TableSnapshot{
|
||||
newTableSnapshot("knowledges", knowledges),
|
||||
newTableSnapshot("chunks", chunks),
|
||||
newTableSnapshot("resources", []json.RawMessage{json.RawMessage(`{"id":"res-safe","tenant_id":10000,"handle":"ABCDEFGHIJKLMNOPQRSTUV","provider":"oss","physical_path":"oss://fixture/source.txt","size":4}`)}),
|
||||
}
|
||||
for _, name := range []string{"resource_bindings", "knowledge_tags", "knowledge_tag_relations", "wiki_pages", "wiki_folders", "wiki_page_issues", "wiki_log_entries"} {
|
||||
tables = append(tables, newTableSnapshot(name, nil))
|
||||
}
|
||||
es := SearchSnapshot{MappingSHA256: shaBytes([]byte("mapping"))}
|
||||
if includeChildIndex {
|
||||
es.Documents = append(es.Documents, SearchDocument{ID: "es-child", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_id":"chunk-child","content":"child"}`)})
|
||||
}
|
||||
if includeFlatIndex {
|
||||
es.Documents = append(es.Documents, SearchDocument{ID: "es-flat", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-flat","chunk_id":"chunk-flat","content":"flat"}`)})
|
||||
}
|
||||
es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents)
|
||||
return nativeMixedFixtureState{pg: PostgresSnapshot{Tables: tables}, es: es}
|
||||
}
|
||||
|
||||
func archiveNativeFixture(t *testing.T, fixture nativeMixedFixtureState) (string, librarysnapshot.Manifest, error) {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
if err := os.Chmod(root, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
archive, err := librarysnapshot.Open(root, 1024*1024)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer archive.Close()
|
||||
id, err := ArchiveLibraryCandidate(context.Background(), archive, "internal",
|
||||
PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"},
|
||||
librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74, RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "fixture-model", EmbeddingDimension: 1024, IndexMappingSHA256: fixture.es.MappingSHA256},
|
||||
fixture.pg, fixture.es, &libraryCandidateReader{data: []byte("file")},
|
||||
time.Date(2026, 11, 5, 0, 0, 0, 0, time.UTC))
|
||||
if err != nil {
|
||||
return "", librarysnapshot.Manifest{}, err
|
||||
}
|
||||
saved, err := archive.Load(id)
|
||||
if err == nil {
|
||||
err = validateArchivedLibraryClosure(archive, saved)
|
||||
}
|
||||
return id, saved, err
|
||||
}
|
||||
|
||||
func TestArchiveLibraryCandidateRejectsDanglingParentReference(t *testing.T) {
|
||||
fixture := nativeMixedFixture(true, true)
|
||||
fixture.pg.Tables = replaceChunkParentForTest(fixture.pg.Tables, "chunk-child", "missing-parent")
|
||||
_, _, err := archiveNativeFixture(t, fixture)
|
||||
if err == nil || !strings.Contains(err.Error(), "parent_chunk_id reference") {
|
||||
t.Fatalf("dangling parent reference must be rejected, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func replaceChunkParentForTest(tables []TableSnapshot, chunkID, parentID string) []TableSnapshot {
|
||||
out := make([]TableSnapshot, 0, len(tables))
|
||||
for _, table := range tables {
|
||||
if table.Name != "chunks" {
|
||||
out = append(out, table)
|
||||
continue
|
||||
}
|
||||
rows := make([]json.RawMessage, 0, len(table.Rows))
|
||||
for _, raw := range table.Rows {
|
||||
var row map[string]any
|
||||
_ = json.Unmarshal(raw, &row)
|
||||
if row["id"] == chunkID {
|
||||
row["parent_chunk_id"] = parentID
|
||||
encoded, _ := json.Marshal(row)
|
||||
rows = append(rows, encoded)
|
||||
} else {
|
||||
rows = append(rows, raw)
|
||||
}
|
||||
}
|
||||
out = append(out, newTableSnapshot("chunks", rows))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func replaceChunkStatusForTest(tables []TableSnapshot, chunkID string, status int) []TableSnapshot {
|
||||
out := make([]TableSnapshot, 0, len(tables))
|
||||
for _, table := range tables {
|
||||
if table.Name != "chunks" {
|
||||
out = append(out, table)
|
||||
continue
|
||||
}
|
||||
rows := make([]json.RawMessage, 0, len(table.Rows))
|
||||
for _, raw := range table.Rows {
|
||||
var row map[string]any
|
||||
_ = json.Unmarshal(raw, &row)
|
||||
if row["id"] == chunkID {
|
||||
row["status"] = status
|
||||
encoded, _ := json.Marshal(row)
|
||||
rows = append(rows, encoded)
|
||||
} else {
|
||||
rows = append(rows, raw)
|
||||
}
|
||||
}
|
||||
out = append(out, newTableSnapshot("chunks", rows))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ArchiveLibraryCandidateAcceptance is a named helper so the positive case can
|
||||
// fail with a clear message while the test still calls the real archive entry.
|
||||
func ArchiveLibraryCandidateAcceptance(t *testing.T, includeFlatIndex bool) {
|
||||
t.Helper()
|
||||
_, saved, err := archiveNativeFixture(t, nativeMixedFixture(includeFlatIndex, true))
|
||||
if err != nil {
|
||||
t.Fatalf("native mixed parent-child candidate rejected: %v", err)
|
||||
}
|
||||
var chunks, index int64
|
||||
for _, part := range saved.Parts {
|
||||
if part.Kind == "database" && part.Name == "chunks" {
|
||||
chunks = part.Count
|
||||
}
|
||||
if part.Kind == "index" {
|
||||
index = part.Count
|
||||
}
|
||||
}
|
||||
if chunks != 4 || index != 2 {
|
||||
t.Fatalf("archive filtered rows: chunks=%d index=%d", chunks, index)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user