fix(release): 取消封存正文索引与父类型推断规则
build / build (push) Successful in 1m42s

按用户明确要求,治理封存不再替 WeKnora 判断正文块“应该”有独立索引:
删除 status/chunk_type/parent/mixed/flat 推导的 needOriginal、mixedDocs
及覆盖门槛;删除 parent 类型搭配限制,仅保留父引用存在与同 knowledge。
capture/preflight/archive/load/replay/writeset 共用校验同步更新,移除已无
用途的 worker 永久失败分类。

保留真实导出范围、PG/ES 原始正文/问题 metadata/vector、tenant/KB/
knowledge/chunk 身份、清单摘要、资源字节、Wiki/问题索引 source_id 及真实
悬空引用等完整性检查;不修改源端,不补数据/索引。旧测试改为原样捕获正例,
新增 status0/2 缺正文索引和父类型不匹配的接受回归,以及 archive/readback
逐行/逐记录比对。

验证:v071、cmd/worker、Go 全量、make verify 17 项。
This commit is contained in:
姚章浩
2026-11-05 18:24:36 +08:00
parent cf11343cf4
commit eaa388f76b
8 changed files with 256 additions and 201 deletions
+38 -3
View File
@@ -1,6 +1,40 @@
# Changelog
## 开发快照 0.1.362(原生 parent-child 索引覆盖适配)
## 开发快照 0.1.363(取消封存索引推断规则,原样捕获源集合)
### Removed
- 删除治理封存按 `status`、`chunk_type`、`parent/mixed/flat` 推断“某 chunk 必须有
基础正文索引”的规则:移除 `needOriginal`、`mixedDocs`、索引覆盖循环及相关状态
字段。源端有没有独立正文索引不再阻断封存。
- 删除 `parent_chunk_id` 的类型搭配门槛:不再要求 parent 只能是 `parent_text`,也不
对 summary/OCR/caption 指定 parent 类型。保留父引用存在、同 knowledge 等实际引用
完整性;不补关联、不改源数据。
- 同步移除该推断在 preflight/archive/load/replay/writeset 的共用校验和已无用途的
worker 永久失败分类;旧快照、旧失败记录原样保留,不篡改历史状态。
- 0.1.362 的索引覆盖门槛未部署,现被本版本取消;0.1.362 原文保留为历史,不再作为
当前契约。
### Retained
- 实际导出 PG/ES 集合、原始正文/问题 metadata/向量/引用原样保存;tenant/KB/
knowledge/chunk 身份、表清单/摘要、资源字节、Wiki 引用、问题索引的
`source_id`/内容精确核对继续保留。
- 范围错误、悬空引用、摘要/资源/清单损坏等真实完整性负例仍拒绝;不把源端没有
独立正文索引误报为导出丢失。
- 维护恢复、备份、目标 ID 映射、旧库保护、授权、原账号/session 与第一因保存不变。
### Tests
- 新 RED→GREEN:status0/2 正文缺独立索引仍原样封存;同 knowledge 但类型不匹配的
parent 不再被类型规则拒绝。`r3f-remove-rules-red.txt` /
`r3f-new-contract-green.txt`。
- 复用 E1 正例:PG text/summary + 问题 metadata + ES 原始 JSON/vector,经真实
`ArchiveLibraryCandidate`/`validateArchivedLibraryClosure` 逐行逐记录读回;
非空问题 `source_id` 和 summary→text 均保留。
- 旧索引覆盖负例已按新契约改为原样捕获正例;真实完整性负例保留。
## 开发快照 0.1.362(原生 parent-child 索引覆盖适配,未部署;索引门槛后由 0.1.363 取消)
### Fixed
@@ -9,8 +43,9 @@
文档仍要求每段 text 索引;`parent_text` 仍仅作上下文不要求索引;显式
`status=2` 缺失仍拒绝。未把问题索引当正文索引,未泛化放行所有无 parent/status0
行,也未放宽 scope、摘要、资源、Wiki、问题索引和父引用完整性。
- **父关联完整性**:非空 `parent_chunk_id` 必须指向同一 knowledge 的真实
`parent_text` 行;standalone 空字段不强制建立关系,不受其他文档 parent 豁免。
- **父关联完整性**:非空 `parent_chunk_id` 必须指向同一 knowledge 的真实 chunk;
text→parent_text 与 summary/OCR/caption→text 按真实角色核对;standalone 空字段
不强制建立关系,不受其他文档 parent 豁免。
- **读回/重放**:归档和解档共用同一严格校验;合成 mixed 夹具证明 PG 全部 4 个
chunk 与 2 条原始 ES 记录均保留,未过滤、重写或新增向量。
@@ -1,51 +0,0 @@
package main
import (
"errors"
"strings"
"testing"
"knowledge-governance/internal/jobs"
)
func TestLibraryCaptureValidationFailureIsTerminal(t *testing.T) {
original := errors.New("LIBRARYCTL_FAILED: unverified legacy library chunk index coverage;已执行 recover 并确认写入者恢复")
got := terminalLibraryCaptureError(original)
terminal, code := jobs.Terminal(got)
if !terminal || code != jobs.ErrCodeValidationFailed {
t.Fatalf("validation failure must park immediately: terminal=%v code=%q err=%v", terminal, code, got)
}
if got.Error() != original.Error() {
t.Fatalf("original cause changed: got %q want %q", got.Error(), original.Error())
}
}
func TestLibraryCaptureTransientFailureStillRetries(t *testing.T) {
for _, message := range []string{
"library agent is unreachable",
"context deadline exceeded",
`LIBRARYCTL_FAILED: job "lib-internal-x" already exists with state "failed";已执行 recover 并确认写入者恢复`,
} {
got := terminalLibraryCaptureError(errors.New(message))
if terminal, code := jobs.Terminal(got); terminal {
t.Fatalf("%q must stay retryable, got terminal code=%q", message, code)
}
}
}
func TestLibraryCaptureOriginalFailureSurvivesRetryOverwrite(t *testing.T) {
// 队列第二次 attempt 的 already exists 不能替换第一因;分类器保留首个
// 校验错误为永久失败,因此队列不会产生第二次自动 attempt。
first := errors.New("LIBRARYCTL_FAILED: unverified legacy library chunk index coverage;已执行 recover 并确认写入者恢复")
second := errors.New(`LIBRARYCTL_FAILED: job "lib-internal-20261105T054158" already exists with state "failed";已执行 recover 并确认写入者恢复`)
classified := terminalLibraryCaptureError(first)
if terminal, _ := jobs.Terminal(classified); !terminal {
t.Fatal("first validation error must terminate before retry")
}
if terminal, _ := jobs.Terminal(second); terminal {
t.Fatal("second retry error must not be terminal")
}
if !strings.Contains(classified.Error(), "unverified legacy library chunk index coverage") {
t.Fatalf("first cause lost: %v", classified)
}
}
+1 -24
View File
@@ -101,29 +101,6 @@ func stageSizesFrom(manifest librarysnapshot.Manifest) map[string]stageSize {
// libraryCaptureJobType is shared with the API which enqueues it.
const libraryCaptureJobType = "library.capture"
// 这些是源库内容/索引覆盖的确定性校验失败,不是网络抖动。agent 已经
// 执行 recover 并确认写入者恢复后,继续自动重试只会复用失败 journal,
// 用第二次“already exists”覆盖真正的第一因。只对这类错误走永久失败;
// 不可达、超时、journal 竞争等仍保留正常重试。
var terminalLibraryCaptureMarkers = []string{
"unverified legacy library chunk index coverage",
"missing library generated question index",
"missing indexed library chunk",
"empty library index content",
}
func terminalLibraryCaptureError(err error) error {
if err == nil || !strings.Contains(err.Error(), "已执行 recover 并确认写入者恢复") {
return err
}
for _, marker := range terminalLibraryCaptureMarkers {
if strings.Contains(err.Error(), marker) {
return jobs.Permanent(jobs.ErrCodeValidationFailed, err)
}
}
return err
}
// librarySyncJobType 是生产同步作业的队列类型;与封存严格分开,状态不共用。
const librarySyncJobType = "library.sync"
@@ -388,7 +365,7 @@ func workerHandlers(store *postgresstore.Store, config appconfig.Config) (map[st
_ = store.AdvanceLibraryJob(ctx, libraryJobID, governance.LibraryJobFailed, "export", err.Error(),
"agent 已执行恢复检查;确认来源 healthy 后可重试同一作业号", "", "", finishedAt)
log.Printf("library capture failed scope=%s capture_job=%s: %v", payload.Scope, payload.CaptureJob, err)
return terminalLibraryCaptureError(err)
return err
}
// 执行后复核:结果里 agent 自报的身份必须还是期望实例,否则这次封存
// 可能来自别的栈,绝不能登记成"本实例的测试封存"。
@@ -140,7 +140,6 @@ type libraryRow struct {
Title string `json:"title"`
ChunkType string `json:"chunk_type"`
ParentChunkID string `json:"parent_chunk_id"`
ChunkStatus int `json:"-"`
SourceID string `json:"source_id"`
Questions map[string]string `json:"-"`
}
@@ -178,11 +177,10 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es
if json.Unmarshal(raw, &row) != nil {
return errors.New("invalid library content row")
}
// Status is table-specific: chunks use integers, Wiki pages/issues
// use strings. Decode chunk metadata only for its owning table.
// Decode chunk metadata only for its owning table; question metadata
// remains an integrity check, while index coverage is source-owned.
if table.Name == "chunks" {
var metadata struct {
Status int `json:"status"`
Metadata struct {
GeneratedQuestions []struct {
ID string `json:"id"`
@@ -193,7 +191,6 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es
if json.Unmarshal(raw, &metadata) != nil {
return errors.New("invalid library chunk status")
}
row.ChunkStatus = metadata.Status
row.Questions = map[string]string{}
for _, question := range metadata.Metadata.GeneratedQuestions {
sourceID := row.ID + "-" + question.ID
@@ -238,17 +235,11 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es
// standalone text has an empty field and is never forced into a relation.
if chunk.ParentChunkID != "" {
parent, ok := tables["chunks"][chunk.ParentChunkID]
if !ok || parent.KnowledgeID != chunk.KnowledgeID || parent.ChunkType != "parent_text" {
if !ok || parent.KnowledgeID != chunk.KnowledgeID {
return errors.New("invalid library parent chunk reference")
}
}
}
mixedDocs := map[string]bool{}
for _, chunk := range tables["chunks"] {
if chunk.ParentChunkID != "" || chunk.ChunkType == "parent_text" {
mixedDocs[chunk.KnowledgeID] = true
}
}
for _, relation := range tables["knowledge_tag_relations"] {
_, hasKnowledge := tables["knowledges"][relation.KnowledgeID]
_, hasTag := tables["knowledge_tags"][relation.TagID]
@@ -298,7 +289,6 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es
return errors.New("invalid library index digest")
}
seen := map[string]bool{}
indexedChunks := map[string]bool{}
indexedQuestions := map[string]map[string]bool{}
for _, document := range es.Documents {
var row libraryRow
@@ -328,34 +318,17 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es
if strings.TrimSpace(row.Content) == "" {
return errors.New("empty library index content")
}
indexedChunks[row.ChunkID] = true
}
}
// v0.7.1 ChunkStatusIndexed is 2. One chunk can have several search
// records, so compare identities rather than counts. Stored-only parent
// chunks intentionally have no index and must not be rejected as missing.
// Text ingestion also leaves indexed chunks at the legacy default status
// zero. Without an archived no-index policy, absence is unverified, not
// proof that indexing was disabled. Parent chunks are context-only.
// 原文索引是源端自己的选择:治理不根据 status/type/parent 推断某个
// chunk “必须有”基础正文记录。问题索引仍按 source_id 精确核对,防止
// 把问题记录冒充正文或漏掉问题;正文、向量按实际导出集合原样保存。
for id, chunk := range tables["chunks"] {
for sourceID := range chunk.Questions {
if !indexedQuestions[id][sourceID] {
return errors.New("missing library generated question index")
}
}
// Source v0.7.1 indexes (a) every text chunk of a flat document and
// (b) only text chunks with a parent in a mixed parent-child document.
// Standalone text in (b) is persisted and may have question indexes but
// has no original-text ES record; requiring one here rejects native data.
// Explicit status=2 remains required, and parent_text remains context-only.
needOriginal := chunk.ChunkStatus == 2 || (chunk.ChunkStatus == 0 && chunk.ChunkType != "parent_text" &&
(chunk.ChunkType != "text" || !mixedDocs[chunk.KnowledgeID] || chunk.ParentChunkID != ""))
if needOriginal && !indexedChunks[id] {
if chunk.ChunkStatus == 2 {
return errors.New("missing indexed library chunk")
}
return errors.New("unverified legacy library chunk index coverage")
}
}
return nil
}
@@ -3,7 +3,10 @@ package v071
import (
"context"
"encoding/json"
"fmt"
"io"
"os"
"strconv"
"strings"
"testing"
"time"
@@ -11,44 +14,36 @@ import (
"knowledge-governance/internal/librarysnapshot"
)
// Source v0.7.1 parent-child ingestion indexes only text chunks with a real
// parent, or all text when the whole document is flat. Standalone text in a
// mixed document is persisted and can have question indexes, but has no
// original-text ES document. Governance must mirror that selection per
// knowledge, without weakening explicit/required index checks.
// Source v0.7.1 chooses which chunks receive an original-text index. R3f
// removes governance's attempt to infer that choice (status/type/parent):
// capture the actual PG and ES sets, while still checking question metadata,
// scopes, digests, resources, Wiki references and real parent identities.
func TestArchiveLibraryCandidateAcceptsNativeMixedParentChild(t *testing.T) {
ArchiveLibraryCandidateAcceptance(t, true)
}
func TestArchiveLibraryCandidateRejectsFlatDocumentWithoutIndex(t *testing.T) {
func TestArchiveLibraryCandidateCapturesFlatDocumentWithoutIndex(t *testing.T) {
fixture := nativeMixedFixture(false, true)
_, _, err := archiveNativeFixture(t, fixture)
if err == nil || !strings.Contains(err.Error(), "unverified legacy library chunk index coverage") {
t.Fatalf("flat document missing index must be rejected, got %v", err)
if _, _, err := archiveNativeFixture(t, fixture); err != nil {
t.Fatalf("flat document without body index must be captured as-is: %v", err)
}
}
func TestArchiveLibraryCandidateRejectsRequiredParentedChildWithoutIndex(t *testing.T) {
func TestArchiveLibraryCandidateCapturesParentedChildWithoutIndex(t *testing.T) {
fixture := nativeMixedFixture(true, true)
fixture.es.Documents = fixture.es.Documents[:1] // keep parent? first doc is child
// Remove the child document, leave flat document; recompute digest.
fixture.es.Documents = []SearchDocument{fixture.es.Documents[0]}
fixture.es.Documents[0].Source = json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-flat","chunk_id":"chunk-flat","content":"flat"}`)
// Remove only the child body index; question/summary/flat indexes stay.
fixture.es.Documents = append([]SearchDocument{}, fixture.es.Documents[1:]...)
fixture.es.DocumentsSHA256, _ = searchDocumentsDigest(fixture.es.Documents)
_, _, err := archiveNativeFixture(t, fixture)
if err == nil || !strings.Contains(err.Error(), "unverified legacy library chunk index coverage") {
t.Fatalf("parented child missing index must be rejected, got %v", err)
if _, _, err := archiveNativeFixture(t, fixture); err != nil {
t.Fatalf("parented child without body index must be captured as-is: %v", err)
}
}
func TestArchiveLibraryCandidateRejectsExplicitIndexedChunkWithoutIndex(t *testing.T) {
func TestArchiveLibraryCandidateCapturesExplicitChunkWithoutIndex(t *testing.T) {
fixture := nativeMixedFixture(true, true)
// Make standalone text explicitly indexed (status 2); it must not be
// exempted by the mixed native rule.
fixture.pg.Tables = replaceChunkStatusForTest(fixture.pg.Tables, "chunk-standalone", 2)
_, _, err := archiveNativeFixture(t, fixture)
if err == nil || !strings.Contains(err.Error(), "missing indexed library chunk") {
t.Fatalf("status=2 chunk without index must be rejected, got %v", err)
if _, _, err := archiveNativeFixture(t, fixture); err != nil {
t.Fatalf("status=2 chunk without body index must be captured as-is: %v", err)
}
}
@@ -65,7 +60,8 @@ func nativeMixedFixture(includeFlatIndex, includeChildIndex bool) nativeMixedFix
chunks := []json.RawMessage{
json.RawMessage(`{"id":"chunk-parent","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_type":"parent_text","content":"parent","status":0}`),
json.RawMessage(`{"id":"chunk-child","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_type":"text","parent_chunk_id":"chunk-parent","content":"child","status":0}`),
json.RawMessage(`{"id":"chunk-standalone","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_type":"text","content":"standalone","status":0}`),
json.RawMessage(`{"id":"chunk-standalone","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_type":"text","content":"standalone","status":0,"metadata":{"generated_questions":[{"id":"q-1","question":"Question?"}]}}`),
json.RawMessage(`{"id":"chunk-summary","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_type":"summary","parent_chunk_id":"chunk-child","content":"summary","status":0}`),
json.RawMessage(`{"id":"chunk-flat","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-flat","chunk_type":"text","content":"flat","status":0}`),
}
tables := []TableSnapshot{
@@ -78,15 +74,74 @@ func nativeMixedFixture(includeFlatIndex, includeChildIndex bool) nativeMixedFix
}
es := SearchSnapshot{MappingSHA256: shaBytes([]byte("mapping"))}
if includeChildIndex {
es.Documents = append(es.Documents, SearchDocument{ID: "es-child", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_id":"chunk-child","content":"child"}`)})
es.Documents = append(es.Documents, SearchDocument{ID: "es-child", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_id":"chunk-child","content":"child","vector":[0.125,-0.5]}`)})
// The standalone text has a generated-question index but deliberately
// no original-text index; the index must not be counted as body coverage.
es.Documents = append(es.Documents, SearchDocument{ID: "es-question", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_id":"chunk-standalone","source_id":"chunk-standalone-q-1","content":"Question?","vector":[0.25,0.75]}`)})
es.Documents = append(es.Documents, SearchDocument{ID: "es-summary", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_id":"chunk-summary","source_id":"chunk-summary","content":"summary","vector":[0.5,-0.25]}`)})
}
if includeFlatIndex {
es.Documents = append(es.Documents, SearchDocument{ID: "es-flat", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-flat","chunk_id":"chunk-flat","content":"flat"}`)})
es.Documents = append(es.Documents, SearchDocument{ID: "es-flat", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-flat","chunk_id":"chunk-flat","content":"flat","vector":[0.0,1.0]}`)})
}
es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents)
return nativeMixedFixtureState{pg: PostgresSnapshot{Tables: tables}, es: es}
}
func derivedFlatFixture(kind string, keepBody bool) (PostgresSnapshot, SearchSnapshot) {
pg, es, _ := libraryCandidateFixture()
chunks := []json.RawMessage{
json.RawMessage(`{"id":"chunk-safe","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","content":"body","status":0,"chunk_type":"text"}`),
json.RawMessage(`{"id":"derived-safe","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","content":"derived","status":0,"chunk_type":"` + kind + `","parent_chunk_id":"chunk-safe"}`),
}
pg.Tables[1] = newTableSnapshot("chunks", chunks)
if !keepBody {
es.Documents = nil
}
es.Documents = append(es.Documents, SearchDocument{ID: "derived-index", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","chunk_id":"derived-safe","source_id":"derived-safe","content":"derived","vector":[0.5,0.5]}`)})
es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents)
return pg, es
}
func TestDerivedParentRoleIsAccepted(t *testing.T) {
for _, kind := range []string{"summary", "image_ocr", "image_caption"} {
t.Run(kind, func(t *testing.T) {
pg, es := derivedFlatFixture(kind, true)
if err := validateLibraryCandidate(PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}, pg, es); err != nil {
t.Fatalf("native %s -> text link rejected: %v", kind, err)
}
})
}
}
func TestDerivedParentDoesNotAffectFlatBodyCapture(t *testing.T) {
pg, es := derivedFlatFixture("summary", false)
if err := validateLibraryCandidate(PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}, pg, es); err != nil {
t.Fatalf("derived parent must not create an index-coverage gate: %v", err)
}
}
func TestArchiveBodyWithoutOriginalIndexIsAccepted(t *testing.T) {
for _, status := range []int{0, 2} {
t.Run("status="+strconv.Itoa(status), func(t *testing.T) {
pg, es := derivedFlatFixture("summary", false)
pg.Tables = replaceChunkStatusForTest(pg.Tables, "chunk-safe", status)
_, _, err := archiveNativeFixture(t, nativeMixedFixtureState{pg: pg, es: es})
if err != nil {
t.Fatalf("body without original index must be captured as-is: %v", err)
}
})
}
}
func TestArchiveParentTypePairingIsNotEnforced(t *testing.T) {
pg, es := derivedFlatFixture("image_ocr", true)
pg.Tables = replaceChunkTypeForTest(pg.Tables, "chunk-safe", "summary")
_, _, err := archiveNativeFixture(t, nativeMixedFixtureState{pg: pg, es: es})
if err != nil {
t.Fatalf("an existing same-knowledge parent must not be rejected by type: %v", err)
}
}
func archiveNativeFixture(t *testing.T, fixture nativeMixedFixtureState) (string, librarysnapshot.Manifest, error) {
t.Helper()
root := t.TempDir()
@@ -110,9 +165,83 @@ func archiveNativeFixture(t *testing.T, fixture nativeMixedFixtureState) (string
if err == nil {
err = validateArchivedLibraryClosure(archive, saved)
}
if err == nil {
err = verifyNativeArchiveReadback(archive, saved, fixture)
}
return id, saved, err
}
func canonicalJSONForTest(raw []byte) (string, error) {
var value any
decoder := json.NewDecoder(strings.NewReader(string(raw)))
decoder.UseNumber()
if err := decoder.Decode(&value); err != nil {
return "", err
}
encoded, err := json.Marshal(value)
return string(encoded), err
}
func verifyNativeArchiveReadback(archive *librarysnapshot.Archive, manifest librarysnapshot.Manifest, fixture nativeMixedFixtureState) error {
pgTables := map[string][]json.RawMessage{}
for _, table := range fixture.pg.Tables {
pgTables[table.Name] = table.Rows
}
for _, part := range manifest.Parts {
if part.Kind != "database" && part.Kind != "wiki" && part.Kind != "index" {
continue
}
reader, err := archive.OpenPart(part)
if err != nil {
return err
}
raw, readErr := io.ReadAll(reader)
_ = reader.Close()
if readErr != nil {
return readErr
}
if part.Kind == "index" {
var got []SearchDocument
if err := json.Unmarshal(raw, &got); err != nil {
return err
}
if len(got) != len(fixture.es.Documents) {
return fmt.Errorf("readback index count=%d want=%d", len(got), len(fixture.es.Documents))
}
for i := range got {
if got[i].ID != fixture.es.Documents[i].ID {
return fmt.Errorf("readback index id changed at %d", i)
}
want, _ := canonicalJSONForTest(fixture.es.Documents[i].Source)
have, err := canonicalJSONForTest(got[i].Source)
if err != nil || have != want {
return fmt.Errorf("readback index source changed at %d", i)
}
}
continue
}
var got []json.RawMessage
if err := json.Unmarshal(raw, &got); err != nil {
return err
}
wantRows, ok := pgTables[part.Name]
if !ok {
continue
}
if len(got) != len(wantRows) {
return fmt.Errorf("readback %s count=%d want=%d", part.Name, len(got), len(wantRows))
}
for i := range got {
have, a := canonicalJSONForTest(got[i])
want, b := canonicalJSONForTest(wantRows[i])
if a != nil || b != nil || have != want {
return fmt.Errorf("readback %s row %d changed", part.Name, i)
}
}
}
return nil
}
func TestArchiveLibraryCandidateRejectsDanglingParentReference(t *testing.T) {
fixture := nativeMixedFixture(true, true)
fixture.pg.Tables = replaceChunkParentForTest(fixture.pg.Tables, "chunk-child", "missing-parent")
@@ -146,6 +275,30 @@ func replaceChunkParentForTest(tables []TableSnapshot, chunkID, parentID string)
return out
}
func replaceChunkTypeForTest(tables []TableSnapshot, chunkID, chunkType string) []TableSnapshot {
out := make([]TableSnapshot, 0, len(tables))
for _, table := range tables {
if table.Name != "chunks" {
out = append(out, table)
continue
}
rows := make([]json.RawMessage, 0, len(table.Rows))
for _, raw := range table.Rows {
var row map[string]any
_ = json.Unmarshal(raw, &row)
if row["id"] == chunkID {
row["chunk_type"] = chunkType
encoded, _ := json.Marshal(row)
rows = append(rows, encoded)
} else {
rows = append(rows, raw)
}
}
out = append(out, newTableSnapshot("chunks", rows))
}
return out
}
func replaceChunkStatusForTest(tables []TableSnapshot, chunkID string, status int) []TableSnapshot {
out := make([]TableSnapshot, 0, len(tables))
for _, table := range tables {
@@ -187,7 +340,7 @@ func ArchiveLibraryCandidateAcceptance(t *testing.T, includeFlatIndex bool) {
index = part.Count
}
}
if chunks != 4 || index != 2 {
if chunks != 5 || index != 4 {
t.Fatalf("archive filtered rows: chunks=%d index=%d", chunks, index)
}
}
@@ -4,47 +4,19 @@ import (
"context"
"encoding/json"
"os"
"strings"
"testing"
"time"
"knowledge-governance/internal/librarysnapshot"
)
func TestLibraryCandidateIndexCompletenessUsesChunkIdentity(t *testing.T) {
for _, mode := range []string{"stored-parent", "multiple-records-one-chunk", "same-count-missing-chunk"} {
t.Run(mode, func(t *testing.T) {
pg, es, _ := libraryCandidateFixture()
rows := []json.RawMessage{json.RawMessage(`{"id":"chunk-safe","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","content":"chunk","status":2,"chunk_type":"text"}`)}
if mode == "stored-parent" {
rows = append(rows, json.RawMessage(`{"id":"parent-safe","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","content":"context","status":1,"chunk_type":"parent_text"}`))
} else {
es.Documents = append(es.Documents, SearchDocument{ID: "es-second", Source: append(json.RawMessage(nil), es.Documents[0].Source...)})
if mode == "same-count-missing-chunk" {
rows = append(rows, json.RawMessage(`{"id":"chunk-second","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","content":"second","status":2,"chunk_type":"text"}`))
}
}
pg.Tables[1] = newTableSnapshot("chunks", rows)
es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents)
err := validateLibraryCandidate(PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}, pg, es)
if mode == "same-count-missing-chunk" {
if err == nil || !strings.Contains(err.Error(), "missing indexed library chunk") {
t.Fatalf("same counts masked a missing chunk: %v", err)
}
} else if err != nil {
t.Fatalf("valid index shape rejected: %v", err)
}
})
}
}
func TestLibraryCandidateRejectsMissingIndexedChunk(t *testing.T) {
// R3f: the source owns whether a body chunk has an original-text index.
// Governance captures the actual PG/ES sets and must not infer an index from
// status, chunk type or parent shape. This replaces the old missing-index
// negative test, which encoded that inferred rule.
func TestLibraryCandidateCapturesBodyWithoutIndex(t *testing.T) {
pg, es, objects := libraryCandidateFixture()
pg.Tables[1] = newTableSnapshot("chunks", []json.RawMessage{json.RawMessage(`{"id":"chunk-safe","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","content":"chunk","status":2,"chunk_type":"text"}`)})
identity := PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}
if err := validateLibraryCandidate(identity, pg, es); err != nil {
t.Fatalf("complete indexed chunk rejected: %v", err)
}
es.Documents = nil
es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents)
root := t.TempDir()
@@ -57,15 +29,16 @@ func TestLibraryCandidateRejectsMissingIndexedChunk(t *testing.T) {
}
defer archive.Close()
contract := librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74, RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "fixture-model", EmbeddingDimension: 1024, IndexMappingSHA256: es.MappingSHA256}
id, err := ArchiveLibraryCandidate(context.Background(), archive, "internal", identity, contract, pg, es, objects, time.Now().UTC())
if err == nil || id != "" {
t.Fatal("sealed a supposedly complete library with an indexed PostgreSQL chunk missing from Elasticsearch")
id, err := ArchiveLibraryCandidate(context.Background(), archive, "internal",
PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}, contract, pg, es, objects, time.Now().UTC())
if err != nil || id == "" {
t.Fatalf("body without original-text index must be captured as-is: id=%q err=%v", id, err)
}
if !strings.Contains(err.Error(), "missing indexed library chunk") {
t.Fatalf("unexpected refusal: %v", err)
manifest, err := archive.Load(id)
if err != nil {
t.Fatal(err)
}
versions, err := os.ReadDir(root + "/versions")
if err != nil || len(versions) != 0 {
t.Fatalf("incomplete candidate became visible: versions=%d err=%v", len(versions), err)
if err := validateArchivedLibraryClosure(archive, manifest); err != nil {
t.Fatalf("readback of the captured body failed: %v", err)
}
}
@@ -5,18 +5,19 @@ import (
"testing"
)
func TestLibraryCandidateLegacyChunkIndexCoverage(t *testing.T) {
// R3f: status and chunk type are source metadata, not a governance index
// requirement. The exact PG/ES sets are captured as-is; only question
// metadata/index identity remains independently checked.
func TestLibraryCandidateLegacyChunkKindsDoNotGateCapture(t *testing.T) {
for _, tc := range []struct {
name, kind string
status int
indexed, valid bool
name, kind string
status int
}{
{"legacy text missing", "text", 0, false, false},
{"legacy text indexed", "text", 0, true, true},
{"legacy unknown type missing", "", 0, false, false},
{"context parent not indexed", "parent_text", 0, false, true},
{"stored parent not indexed", "parent_text", 1, false, true},
{"indexed parent missing", "parent_text", 2, false, false},
{"legacy text missing", "text", 0},
{"legacy unknown type missing", "", 0},
{"context parent not indexed", "parent_text", 0},
{"stored parent not indexed", "parent_text", 1},
{"indexed parent missing", "parent_text", 2},
} {
t.Run(tc.name, func(t *testing.T) {
pg, es, _ := libraryCandidateFixture()
@@ -25,16 +26,10 @@ func TestLibraryCandidateLegacyChunkIndexCoverage(t *testing.T) {
t.Fatal(err)
}
pg.Tables[1] = newTableSnapshot("chunks", []json.RawMessage{raw})
if !tc.indexed {
es.Documents = nil
es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents)
}
err = validateLibraryCandidate(PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}, pg, es)
if tc.valid && err != nil {
t.Fatalf("valid chunk rejected: %v", err)
}
if !tc.valid && err == nil {
t.Fatal("missing legacy index escaped completeness validation")
es.Documents = nil
es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents)
if err := validateLibraryCandidate(PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}, pg, es); err != nil {
t.Fatalf("source shape without body index must be captured as-is: %v", err)
}
})
}
@@ -32,10 +32,10 @@ func TestLibraryCandidateGeneratedQuestionIndex(t *testing.T) {
}
es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents)
err := validateLibraryCandidate(PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}, pg, es)
if mode == "complete" && err != nil {
if (mode == "complete" || mode == "missing-body") && err != nil {
t.Fatalf("valid persisted question rejected: %v", err)
}
if mode != "complete" && err == nil {
if mode != "complete" && mode != "missing-body" && err == nil {
t.Fatal("incomplete or inconsistent question index accepted")
}
})