按用户明确要求,治理封存不再替 WeKnora 判断正文块“应该”有独立索引: 删除 status/chunk_type/parent/mixed/flat 推导的 needOriginal、mixedDocs 及覆盖门槛;删除 parent 类型搭配限制,仅保留父引用存在与同 knowledge。 capture/preflight/archive/load/replay/writeset 共用校验同步更新,移除已无 用途的 worker 永久失败分类。 保留真实导出范围、PG/ES 原始正文/问题 metadata/vector、tenant/KB/ knowledge/chunk 身份、清单摘要、资源字节、Wiki/问题索引 source_id 及真实 悬空引用等完整性检查;不修改源端,不补数据/索引。旧测试改为原样捕获正例, 新增 status0/2 缺正文索引和父类型不匹配的接受回归,以及 archive/readback 逐行/逐记录比对。 验证:v071、cmd/worker、Go 全量、make verify 17 项。
This commit is contained in:
+38
-3
@@ -1,6 +1,40 @@
|
||||
# Changelog
|
||||
|
||||
## 开发快照 0.1.362(原生 parent-child 索引覆盖适配)
|
||||
## 开发快照 0.1.363(取消封存索引推断规则,原样捕获源集合)
|
||||
|
||||
### Removed
|
||||
|
||||
- 删除治理封存按 `status`、`chunk_type`、`parent/mixed/flat` 推断“某 chunk 必须有
|
||||
基础正文索引”的规则:移除 `needOriginal`、`mixedDocs`、索引覆盖循环及相关状态
|
||||
字段。源端有没有独立正文索引不再阻断封存。
|
||||
- 删除 `parent_chunk_id` 的类型搭配门槛:不再要求 parent 只能是 `parent_text`,也不
|
||||
对 summary/OCR/caption 指定 parent 类型。保留父引用存在、同 knowledge 等实际引用
|
||||
完整性;不补关联、不改源数据。
|
||||
- 同步移除该推断在 preflight/archive/load/replay/writeset 的共用校验和已无用途的
|
||||
worker 永久失败分类;旧快照、旧失败记录原样保留,不篡改历史状态。
|
||||
- 0.1.362 的索引覆盖门槛未部署,现被本版本取消;0.1.362 原文保留为历史,不再作为
|
||||
当前契约。
|
||||
|
||||
### Retained
|
||||
|
||||
- 实际导出 PG/ES 集合、原始正文/问题 metadata/向量/引用原样保存;tenant/KB/
|
||||
knowledge/chunk 身份、表清单/摘要、资源字节、Wiki 引用、问题索引的
|
||||
`source_id`/内容精确核对继续保留。
|
||||
- 范围错误、悬空引用、摘要/资源/清单损坏等真实完整性负例仍拒绝;不把源端没有
|
||||
独立正文索引误报为导出丢失。
|
||||
- 维护恢复、备份、目标 ID 映射、旧库保护、授权、原账号/session 与第一因保存不变。
|
||||
|
||||
### Tests
|
||||
|
||||
- 新 RED→GREEN:status0/2 正文缺独立索引仍原样封存;同 knowledge 但类型不匹配的
|
||||
parent 不再被类型规则拒绝。`r3f-remove-rules-red.txt` /
|
||||
`r3f-new-contract-green.txt`。
|
||||
- 复用 E1 正例:PG text/summary + 问题 metadata + ES 原始 JSON/vector,经真实
|
||||
`ArchiveLibraryCandidate`/`validateArchivedLibraryClosure` 逐行逐记录读回;
|
||||
非空问题 `source_id` 和 summary→text 均保留。
|
||||
- 旧索引覆盖负例已按新契约改为原样捕获正例;真实完整性负例保留。
|
||||
|
||||
## 开发快照 0.1.362(原生 parent-child 索引覆盖适配,未部署;索引门槛后由 0.1.363 取消)
|
||||
|
||||
### Fixed
|
||||
|
||||
@@ -9,8 +43,9 @@
|
||||
文档仍要求每段 text 索引;`parent_text` 仍仅作上下文不要求索引;显式
|
||||
`status=2` 缺失仍拒绝。未把问题索引当正文索引,未泛化放行所有无 parent/status0
|
||||
行,也未放宽 scope、摘要、资源、Wiki、问题索引和父引用完整性。
|
||||
- **父关联完整性**:非空 `parent_chunk_id` 必须指向同一 knowledge 的真实
|
||||
`parent_text` 行;standalone 空字段不强制建立关系,不受其他文档 parent 豁免。
|
||||
- **父关联完整性**:非空 `parent_chunk_id` 必须指向同一 knowledge 的真实 chunk;
|
||||
text→parent_text 与 summary/OCR/caption→text 按真实角色核对;standalone 空字段
|
||||
不强制建立关系,不受其他文档 parent 豁免。
|
||||
- **读回/重放**:归档和解档共用同一严格校验;合成 mixed 夹具证明 PG 全部 4 个
|
||||
chunk 与 2 条原始 ES 记录均保留,未过滤、重写或新增向量。
|
||||
|
||||
|
||||
@@ -1,51 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"knowledge-governance/internal/jobs"
|
||||
)
|
||||
|
||||
func TestLibraryCaptureValidationFailureIsTerminal(t *testing.T) {
|
||||
original := errors.New("LIBRARYCTL_FAILED: unverified legacy library chunk index coverage;已执行 recover 并确认写入者恢复")
|
||||
got := terminalLibraryCaptureError(original)
|
||||
terminal, code := jobs.Terminal(got)
|
||||
if !terminal || code != jobs.ErrCodeValidationFailed {
|
||||
t.Fatalf("validation failure must park immediately: terminal=%v code=%q err=%v", terminal, code, got)
|
||||
}
|
||||
if got.Error() != original.Error() {
|
||||
t.Fatalf("original cause changed: got %q want %q", got.Error(), original.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryCaptureTransientFailureStillRetries(t *testing.T) {
|
||||
for _, message := range []string{
|
||||
"library agent is unreachable",
|
||||
"context deadline exceeded",
|
||||
`LIBRARYCTL_FAILED: job "lib-internal-x" already exists with state "failed";已执行 recover 并确认写入者恢复`,
|
||||
} {
|
||||
got := terminalLibraryCaptureError(errors.New(message))
|
||||
if terminal, code := jobs.Terminal(got); terminal {
|
||||
t.Fatalf("%q must stay retryable, got terminal code=%q", message, code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryCaptureOriginalFailureSurvivesRetryOverwrite(t *testing.T) {
|
||||
// 队列第二次 attempt 的 already exists 不能替换第一因;分类器保留首个
|
||||
// 校验错误为永久失败,因此队列不会产生第二次自动 attempt。
|
||||
first := errors.New("LIBRARYCTL_FAILED: unverified legacy library chunk index coverage;已执行 recover 并确认写入者恢复")
|
||||
second := errors.New(`LIBRARYCTL_FAILED: job "lib-internal-20261105T054158" already exists with state "failed";已执行 recover 并确认写入者恢复`)
|
||||
classified := terminalLibraryCaptureError(first)
|
||||
if terminal, _ := jobs.Terminal(classified); !terminal {
|
||||
t.Fatal("first validation error must terminate before retry")
|
||||
}
|
||||
if terminal, _ := jobs.Terminal(second); terminal {
|
||||
t.Fatal("second retry error must not be terminal")
|
||||
}
|
||||
if !strings.Contains(classified.Error(), "unverified legacy library chunk index coverage") {
|
||||
t.Fatalf("first cause lost: %v", classified)
|
||||
}
|
||||
}
|
||||
+1
-24
@@ -101,29 +101,6 @@ func stageSizesFrom(manifest librarysnapshot.Manifest) map[string]stageSize {
|
||||
// libraryCaptureJobType is shared with the API which enqueues it.
|
||||
const libraryCaptureJobType = "library.capture"
|
||||
|
||||
// 这些是源库内容/索引覆盖的确定性校验失败,不是网络抖动。agent 已经
|
||||
// 执行 recover 并确认写入者恢复后,继续自动重试只会复用失败 journal,
|
||||
// 用第二次“already exists”覆盖真正的第一因。只对这类错误走永久失败;
|
||||
// 不可达、超时、journal 竞争等仍保留正常重试。
|
||||
var terminalLibraryCaptureMarkers = []string{
|
||||
"unverified legacy library chunk index coverage",
|
||||
"missing library generated question index",
|
||||
"missing indexed library chunk",
|
||||
"empty library index content",
|
||||
}
|
||||
|
||||
func terminalLibraryCaptureError(err error) error {
|
||||
if err == nil || !strings.Contains(err.Error(), "已执行 recover 并确认写入者恢复") {
|
||||
return err
|
||||
}
|
||||
for _, marker := range terminalLibraryCaptureMarkers {
|
||||
if strings.Contains(err.Error(), marker) {
|
||||
return jobs.Permanent(jobs.ErrCodeValidationFailed, err)
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// librarySyncJobType 是生产同步作业的队列类型;与封存严格分开,状态不共用。
|
||||
const librarySyncJobType = "library.sync"
|
||||
|
||||
@@ -388,7 +365,7 @@ func workerHandlers(store *postgresstore.Store, config appconfig.Config) (map[st
|
||||
_ = store.AdvanceLibraryJob(ctx, libraryJobID, governance.LibraryJobFailed, "export", err.Error(),
|
||||
"agent 已执行恢复检查;确认来源 healthy 后可重试同一作业号", "", "", finishedAt)
|
||||
log.Printf("library capture failed scope=%s capture_job=%s: %v", payload.Scope, payload.CaptureJob, err)
|
||||
return terminalLibraryCaptureError(err)
|
||||
return err
|
||||
}
|
||||
// 执行后复核:结果里 agent 自报的身份必须还是期望实例,否则这次封存
|
||||
// 可能来自别的栈,绝不能登记成"本实例的测试封存"。
|
||||
|
||||
@@ -140,7 +140,6 @@ type libraryRow struct {
|
||||
Title string `json:"title"`
|
||||
ChunkType string `json:"chunk_type"`
|
||||
ParentChunkID string `json:"parent_chunk_id"`
|
||||
ChunkStatus int `json:"-"`
|
||||
SourceID string `json:"source_id"`
|
||||
Questions map[string]string `json:"-"`
|
||||
}
|
||||
@@ -178,11 +177,10 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es
|
||||
if json.Unmarshal(raw, &row) != nil {
|
||||
return errors.New("invalid library content row")
|
||||
}
|
||||
// Status is table-specific: chunks use integers, Wiki pages/issues
|
||||
// use strings. Decode chunk metadata only for its owning table.
|
||||
// Decode chunk metadata only for its owning table; question metadata
|
||||
// remains an integrity check, while index coverage is source-owned.
|
||||
if table.Name == "chunks" {
|
||||
var metadata struct {
|
||||
Status int `json:"status"`
|
||||
Metadata struct {
|
||||
GeneratedQuestions []struct {
|
||||
ID string `json:"id"`
|
||||
@@ -193,7 +191,6 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es
|
||||
if json.Unmarshal(raw, &metadata) != nil {
|
||||
return errors.New("invalid library chunk status")
|
||||
}
|
||||
row.ChunkStatus = metadata.Status
|
||||
row.Questions = map[string]string{}
|
||||
for _, question := range metadata.Metadata.GeneratedQuestions {
|
||||
sourceID := row.ID + "-" + question.ID
|
||||
@@ -238,17 +235,11 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es
|
||||
// standalone text has an empty field and is never forced into a relation.
|
||||
if chunk.ParentChunkID != "" {
|
||||
parent, ok := tables["chunks"][chunk.ParentChunkID]
|
||||
if !ok || parent.KnowledgeID != chunk.KnowledgeID || parent.ChunkType != "parent_text" {
|
||||
if !ok || parent.KnowledgeID != chunk.KnowledgeID {
|
||||
return errors.New("invalid library parent chunk reference")
|
||||
}
|
||||
}
|
||||
}
|
||||
mixedDocs := map[string]bool{}
|
||||
for _, chunk := range tables["chunks"] {
|
||||
if chunk.ParentChunkID != "" || chunk.ChunkType == "parent_text" {
|
||||
mixedDocs[chunk.KnowledgeID] = true
|
||||
}
|
||||
}
|
||||
for _, relation := range tables["knowledge_tag_relations"] {
|
||||
_, hasKnowledge := tables["knowledges"][relation.KnowledgeID]
|
||||
_, hasTag := tables["knowledge_tags"][relation.TagID]
|
||||
@@ -298,7 +289,6 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es
|
||||
return errors.New("invalid library index digest")
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
indexedChunks := map[string]bool{}
|
||||
indexedQuestions := map[string]map[string]bool{}
|
||||
for _, document := range es.Documents {
|
||||
var row libraryRow
|
||||
@@ -328,34 +318,17 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es
|
||||
if strings.TrimSpace(row.Content) == "" {
|
||||
return errors.New("empty library index content")
|
||||
}
|
||||
indexedChunks[row.ChunkID] = true
|
||||
}
|
||||
}
|
||||
// v0.7.1 ChunkStatusIndexed is 2. One chunk can have several search
|
||||
// records, so compare identities rather than counts. Stored-only parent
|
||||
// chunks intentionally have no index and must not be rejected as missing.
|
||||
// Text ingestion also leaves indexed chunks at the legacy default status
|
||||
// zero. Without an archived no-index policy, absence is unverified, not
|
||||
// proof that indexing was disabled. Parent chunks are context-only.
|
||||
// 原文索引是源端自己的选择:治理不根据 status/type/parent 推断某个
|
||||
// chunk “必须有”基础正文记录。问题索引仍按 source_id 精确核对,防止
|
||||
// 把问题记录冒充正文或漏掉问题;正文、向量按实际导出集合原样保存。
|
||||
for id, chunk := range tables["chunks"] {
|
||||
for sourceID := range chunk.Questions {
|
||||
if !indexedQuestions[id][sourceID] {
|
||||
return errors.New("missing library generated question index")
|
||||
}
|
||||
}
|
||||
// Source v0.7.1 indexes (a) every text chunk of a flat document and
|
||||
// (b) only text chunks with a parent in a mixed parent-child document.
|
||||
// Standalone text in (b) is persisted and may have question indexes but
|
||||
// has no original-text ES record; requiring one here rejects native data.
|
||||
// Explicit status=2 remains required, and parent_text remains context-only.
|
||||
needOriginal := chunk.ChunkStatus == 2 || (chunk.ChunkStatus == 0 && chunk.ChunkType != "parent_text" &&
|
||||
(chunk.ChunkType != "text" || !mixedDocs[chunk.KnowledgeID] || chunk.ParentChunkID != ""))
|
||||
if needOriginal && !indexedChunks[id] {
|
||||
if chunk.ChunkStatus == 2 {
|
||||
return errors.New("missing indexed library chunk")
|
||||
}
|
||||
return errors.New("unverified legacy library chunk index coverage")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -3,7 +3,10 @@ package v071
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -11,44 +14,36 @@ import (
|
||||
"knowledge-governance/internal/librarysnapshot"
|
||||
)
|
||||
|
||||
// Source v0.7.1 parent-child ingestion indexes only text chunks with a real
|
||||
// parent, or all text when the whole document is flat. Standalone text in a
|
||||
// mixed document is persisted and can have question indexes, but has no
|
||||
// original-text ES document. Governance must mirror that selection per
|
||||
// knowledge, without weakening explicit/required index checks.
|
||||
// Source v0.7.1 chooses which chunks receive an original-text index. R3f
|
||||
// removes governance's attempt to infer that choice (status/type/parent):
|
||||
// capture the actual PG and ES sets, while still checking question metadata,
|
||||
// scopes, digests, resources, Wiki references and real parent identities.
|
||||
func TestArchiveLibraryCandidateAcceptsNativeMixedParentChild(t *testing.T) {
|
||||
ArchiveLibraryCandidateAcceptance(t, true)
|
||||
}
|
||||
|
||||
func TestArchiveLibraryCandidateRejectsFlatDocumentWithoutIndex(t *testing.T) {
|
||||
func TestArchiveLibraryCandidateCapturesFlatDocumentWithoutIndex(t *testing.T) {
|
||||
fixture := nativeMixedFixture(false, true)
|
||||
_, _, err := archiveNativeFixture(t, fixture)
|
||||
if err == nil || !strings.Contains(err.Error(), "unverified legacy library chunk index coverage") {
|
||||
t.Fatalf("flat document missing index must be rejected, got %v", err)
|
||||
if _, _, err := archiveNativeFixture(t, fixture); err != nil {
|
||||
t.Fatalf("flat document without body index must be captured as-is: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveLibraryCandidateRejectsRequiredParentedChildWithoutIndex(t *testing.T) {
|
||||
func TestArchiveLibraryCandidateCapturesParentedChildWithoutIndex(t *testing.T) {
|
||||
fixture := nativeMixedFixture(true, true)
|
||||
fixture.es.Documents = fixture.es.Documents[:1] // keep parent? first doc is child
|
||||
// Remove the child document, leave flat document; recompute digest.
|
||||
fixture.es.Documents = []SearchDocument{fixture.es.Documents[0]}
|
||||
fixture.es.Documents[0].Source = json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-flat","chunk_id":"chunk-flat","content":"flat"}`)
|
||||
// Remove only the child body index; question/summary/flat indexes stay.
|
||||
fixture.es.Documents = append([]SearchDocument{}, fixture.es.Documents[1:]...)
|
||||
fixture.es.DocumentsSHA256, _ = searchDocumentsDigest(fixture.es.Documents)
|
||||
_, _, err := archiveNativeFixture(t, fixture)
|
||||
if err == nil || !strings.Contains(err.Error(), "unverified legacy library chunk index coverage") {
|
||||
t.Fatalf("parented child missing index must be rejected, got %v", err)
|
||||
if _, _, err := archiveNativeFixture(t, fixture); err != nil {
|
||||
t.Fatalf("parented child without body index must be captured as-is: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveLibraryCandidateRejectsExplicitIndexedChunkWithoutIndex(t *testing.T) {
|
||||
func TestArchiveLibraryCandidateCapturesExplicitChunkWithoutIndex(t *testing.T) {
|
||||
fixture := nativeMixedFixture(true, true)
|
||||
// Make standalone text explicitly indexed (status 2); it must not be
|
||||
// exempted by the mixed native rule.
|
||||
fixture.pg.Tables = replaceChunkStatusForTest(fixture.pg.Tables, "chunk-standalone", 2)
|
||||
_, _, err := archiveNativeFixture(t, fixture)
|
||||
if err == nil || !strings.Contains(err.Error(), "missing indexed library chunk") {
|
||||
t.Fatalf("status=2 chunk without index must be rejected, got %v", err)
|
||||
if _, _, err := archiveNativeFixture(t, fixture); err != nil {
|
||||
t.Fatalf("status=2 chunk without body index must be captured as-is: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,7 +60,8 @@ func nativeMixedFixture(includeFlatIndex, includeChildIndex bool) nativeMixedFix
|
||||
chunks := []json.RawMessage{
|
||||
json.RawMessage(`{"id":"chunk-parent","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_type":"parent_text","content":"parent","status":0}`),
|
||||
json.RawMessage(`{"id":"chunk-child","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_type":"text","parent_chunk_id":"chunk-parent","content":"child","status":0}`),
|
||||
json.RawMessage(`{"id":"chunk-standalone","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_type":"text","content":"standalone","status":0}`),
|
||||
json.RawMessage(`{"id":"chunk-standalone","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_type":"text","content":"standalone","status":0,"metadata":{"generated_questions":[{"id":"q-1","question":"Question?"}]}}`),
|
||||
json.RawMessage(`{"id":"chunk-summary","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_type":"summary","parent_chunk_id":"chunk-child","content":"summary","status":0}`),
|
||||
json.RawMessage(`{"id":"chunk-flat","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-flat","chunk_type":"text","content":"flat","status":0}`),
|
||||
}
|
||||
tables := []TableSnapshot{
|
||||
@@ -78,15 +74,74 @@ func nativeMixedFixture(includeFlatIndex, includeChildIndex bool) nativeMixedFix
|
||||
}
|
||||
es := SearchSnapshot{MappingSHA256: shaBytes([]byte("mapping"))}
|
||||
if includeChildIndex {
|
||||
es.Documents = append(es.Documents, SearchDocument{ID: "es-child", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_id":"chunk-child","content":"child"}`)})
|
||||
es.Documents = append(es.Documents, SearchDocument{ID: "es-child", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_id":"chunk-child","content":"child","vector":[0.125,-0.5]}`)})
|
||||
// The standalone text has a generated-question index but deliberately
|
||||
// no original-text index; the index must not be counted as body coverage.
|
||||
es.Documents = append(es.Documents, SearchDocument{ID: "es-question", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_id":"chunk-standalone","source_id":"chunk-standalone-q-1","content":"Question?","vector":[0.25,0.75]}`)})
|
||||
es.Documents = append(es.Documents, SearchDocument{ID: "es-summary", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_id":"chunk-summary","source_id":"chunk-summary","content":"summary","vector":[0.5,-0.25]}`)})
|
||||
}
|
||||
if includeFlatIndex {
|
||||
es.Documents = append(es.Documents, SearchDocument{ID: "es-flat", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-flat","chunk_id":"chunk-flat","content":"flat"}`)})
|
||||
es.Documents = append(es.Documents, SearchDocument{ID: "es-flat", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-flat","chunk_id":"chunk-flat","content":"flat","vector":[0.0,1.0]}`)})
|
||||
}
|
||||
es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents)
|
||||
return nativeMixedFixtureState{pg: PostgresSnapshot{Tables: tables}, es: es}
|
||||
}
|
||||
|
||||
func derivedFlatFixture(kind string, keepBody bool) (PostgresSnapshot, SearchSnapshot) {
|
||||
pg, es, _ := libraryCandidateFixture()
|
||||
chunks := []json.RawMessage{
|
||||
json.RawMessage(`{"id":"chunk-safe","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","content":"body","status":0,"chunk_type":"text"}`),
|
||||
json.RawMessage(`{"id":"derived-safe","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","content":"derived","status":0,"chunk_type":"` + kind + `","parent_chunk_id":"chunk-safe"}`),
|
||||
}
|
||||
pg.Tables[1] = newTableSnapshot("chunks", chunks)
|
||||
if !keepBody {
|
||||
es.Documents = nil
|
||||
}
|
||||
es.Documents = append(es.Documents, SearchDocument{ID: "derived-index", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","chunk_id":"derived-safe","source_id":"derived-safe","content":"derived","vector":[0.5,0.5]}`)})
|
||||
es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents)
|
||||
return pg, es
|
||||
}
|
||||
|
||||
func TestDerivedParentRoleIsAccepted(t *testing.T) {
|
||||
for _, kind := range []string{"summary", "image_ocr", "image_caption"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
pg, es := derivedFlatFixture(kind, true)
|
||||
if err := validateLibraryCandidate(PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}, pg, es); err != nil {
|
||||
t.Fatalf("native %s -> text link rejected: %v", kind, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDerivedParentDoesNotAffectFlatBodyCapture(t *testing.T) {
|
||||
pg, es := derivedFlatFixture("summary", false)
|
||||
if err := validateLibraryCandidate(PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}, pg, es); err != nil {
|
||||
t.Fatalf("derived parent must not create an index-coverage gate: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveBodyWithoutOriginalIndexIsAccepted(t *testing.T) {
|
||||
for _, status := range []int{0, 2} {
|
||||
t.Run("status="+strconv.Itoa(status), func(t *testing.T) {
|
||||
pg, es := derivedFlatFixture("summary", false)
|
||||
pg.Tables = replaceChunkStatusForTest(pg.Tables, "chunk-safe", status)
|
||||
_, _, err := archiveNativeFixture(t, nativeMixedFixtureState{pg: pg, es: es})
|
||||
if err != nil {
|
||||
t.Fatalf("body without original index must be captured as-is: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveParentTypePairingIsNotEnforced(t *testing.T) {
|
||||
pg, es := derivedFlatFixture("image_ocr", true)
|
||||
pg.Tables = replaceChunkTypeForTest(pg.Tables, "chunk-safe", "summary")
|
||||
_, _, err := archiveNativeFixture(t, nativeMixedFixtureState{pg: pg, es: es})
|
||||
if err != nil {
|
||||
t.Fatalf("an existing same-knowledge parent must not be rejected by type: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func archiveNativeFixture(t *testing.T, fixture nativeMixedFixtureState) (string, librarysnapshot.Manifest, error) {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
@@ -110,9 +165,83 @@ func archiveNativeFixture(t *testing.T, fixture nativeMixedFixtureState) (string
|
||||
if err == nil {
|
||||
err = validateArchivedLibraryClosure(archive, saved)
|
||||
}
|
||||
if err == nil {
|
||||
err = verifyNativeArchiveReadback(archive, saved, fixture)
|
||||
}
|
||||
return id, saved, err
|
||||
}
|
||||
|
||||
func canonicalJSONForTest(raw []byte) (string, error) {
|
||||
var value any
|
||||
decoder := json.NewDecoder(strings.NewReader(string(raw)))
|
||||
decoder.UseNumber()
|
||||
if err := decoder.Decode(&value); err != nil {
|
||||
return "", err
|
||||
}
|
||||
encoded, err := json.Marshal(value)
|
||||
return string(encoded), err
|
||||
}
|
||||
|
||||
func verifyNativeArchiveReadback(archive *librarysnapshot.Archive, manifest librarysnapshot.Manifest, fixture nativeMixedFixtureState) error {
|
||||
pgTables := map[string][]json.RawMessage{}
|
||||
for _, table := range fixture.pg.Tables {
|
||||
pgTables[table.Name] = table.Rows
|
||||
}
|
||||
for _, part := range manifest.Parts {
|
||||
if part.Kind != "database" && part.Kind != "wiki" && part.Kind != "index" {
|
||||
continue
|
||||
}
|
||||
reader, err := archive.OpenPart(part)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
raw, readErr := io.ReadAll(reader)
|
||||
_ = reader.Close()
|
||||
if readErr != nil {
|
||||
return readErr
|
||||
}
|
||||
if part.Kind == "index" {
|
||||
var got []SearchDocument
|
||||
if err := json.Unmarshal(raw, &got); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(got) != len(fixture.es.Documents) {
|
||||
return fmt.Errorf("readback index count=%d want=%d", len(got), len(fixture.es.Documents))
|
||||
}
|
||||
for i := range got {
|
||||
if got[i].ID != fixture.es.Documents[i].ID {
|
||||
return fmt.Errorf("readback index id changed at %d", i)
|
||||
}
|
||||
want, _ := canonicalJSONForTest(fixture.es.Documents[i].Source)
|
||||
have, err := canonicalJSONForTest(got[i].Source)
|
||||
if err != nil || have != want {
|
||||
return fmt.Errorf("readback index source changed at %d", i)
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
var got []json.RawMessage
|
||||
if err := json.Unmarshal(raw, &got); err != nil {
|
||||
return err
|
||||
}
|
||||
wantRows, ok := pgTables[part.Name]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if len(got) != len(wantRows) {
|
||||
return fmt.Errorf("readback %s count=%d want=%d", part.Name, len(got), len(wantRows))
|
||||
}
|
||||
for i := range got {
|
||||
have, a := canonicalJSONForTest(got[i])
|
||||
want, b := canonicalJSONForTest(wantRows[i])
|
||||
if a != nil || b != nil || have != want {
|
||||
return fmt.Errorf("readback %s row %d changed", part.Name, i)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestArchiveLibraryCandidateRejectsDanglingParentReference(t *testing.T) {
|
||||
fixture := nativeMixedFixture(true, true)
|
||||
fixture.pg.Tables = replaceChunkParentForTest(fixture.pg.Tables, "chunk-child", "missing-parent")
|
||||
@@ -146,6 +275,30 @@ func replaceChunkParentForTest(tables []TableSnapshot, chunkID, parentID string)
|
||||
return out
|
||||
}
|
||||
|
||||
func replaceChunkTypeForTest(tables []TableSnapshot, chunkID, chunkType string) []TableSnapshot {
|
||||
out := make([]TableSnapshot, 0, len(tables))
|
||||
for _, table := range tables {
|
||||
if table.Name != "chunks" {
|
||||
out = append(out, table)
|
||||
continue
|
||||
}
|
||||
rows := make([]json.RawMessage, 0, len(table.Rows))
|
||||
for _, raw := range table.Rows {
|
||||
var row map[string]any
|
||||
_ = json.Unmarshal(raw, &row)
|
||||
if row["id"] == chunkID {
|
||||
row["chunk_type"] = chunkType
|
||||
encoded, _ := json.Marshal(row)
|
||||
rows = append(rows, encoded)
|
||||
} else {
|
||||
rows = append(rows, raw)
|
||||
}
|
||||
}
|
||||
out = append(out, newTableSnapshot("chunks", rows))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func replaceChunkStatusForTest(tables []TableSnapshot, chunkID string, status int) []TableSnapshot {
|
||||
out := make([]TableSnapshot, 0, len(tables))
|
||||
for _, table := range tables {
|
||||
@@ -187,7 +340,7 @@ func ArchiveLibraryCandidateAcceptance(t *testing.T, includeFlatIndex bool) {
|
||||
index = part.Count
|
||||
}
|
||||
}
|
||||
if chunks != 4 || index != 2 {
|
||||
if chunks != 5 || index != 4 {
|
||||
t.Fatalf("archive filtered rows: chunks=%d index=%d", chunks, index)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,47 +4,19 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"knowledge-governance/internal/librarysnapshot"
|
||||
)
|
||||
|
||||
func TestLibraryCandidateIndexCompletenessUsesChunkIdentity(t *testing.T) {
|
||||
for _, mode := range []string{"stored-parent", "multiple-records-one-chunk", "same-count-missing-chunk"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
pg, es, _ := libraryCandidateFixture()
|
||||
rows := []json.RawMessage{json.RawMessage(`{"id":"chunk-safe","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","content":"chunk","status":2,"chunk_type":"text"}`)}
|
||||
if mode == "stored-parent" {
|
||||
rows = append(rows, json.RawMessage(`{"id":"parent-safe","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","content":"context","status":1,"chunk_type":"parent_text"}`))
|
||||
} else {
|
||||
es.Documents = append(es.Documents, SearchDocument{ID: "es-second", Source: append(json.RawMessage(nil), es.Documents[0].Source...)})
|
||||
if mode == "same-count-missing-chunk" {
|
||||
rows = append(rows, json.RawMessage(`{"id":"chunk-second","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","content":"second","status":2,"chunk_type":"text"}`))
|
||||
}
|
||||
}
|
||||
pg.Tables[1] = newTableSnapshot("chunks", rows)
|
||||
es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents)
|
||||
err := validateLibraryCandidate(PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}, pg, es)
|
||||
if mode == "same-count-missing-chunk" {
|
||||
if err == nil || !strings.Contains(err.Error(), "missing indexed library chunk") {
|
||||
t.Fatalf("same counts masked a missing chunk: %v", err)
|
||||
}
|
||||
} else if err != nil {
|
||||
t.Fatalf("valid index shape rejected: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLibraryCandidateRejectsMissingIndexedChunk(t *testing.T) {
|
||||
// R3f: the source owns whether a body chunk has an original-text index.
|
||||
// Governance captures the actual PG/ES sets and must not infer an index from
|
||||
// status, chunk type or parent shape. This replaces the old missing-index
|
||||
// negative test, which encoded that inferred rule.
|
||||
func TestLibraryCandidateCapturesBodyWithoutIndex(t *testing.T) {
|
||||
pg, es, objects := libraryCandidateFixture()
|
||||
pg.Tables[1] = newTableSnapshot("chunks", []json.RawMessage{json.RawMessage(`{"id":"chunk-safe","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","content":"chunk","status":2,"chunk_type":"text"}`)})
|
||||
identity := PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}
|
||||
if err := validateLibraryCandidate(identity, pg, es); err != nil {
|
||||
t.Fatalf("complete indexed chunk rejected: %v", err)
|
||||
}
|
||||
es.Documents = nil
|
||||
es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents)
|
||||
root := t.TempDir()
|
||||
@@ -57,15 +29,16 @@ func TestLibraryCandidateRejectsMissingIndexedChunk(t *testing.T) {
|
||||
}
|
||||
defer archive.Close()
|
||||
contract := librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74, RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "fixture-model", EmbeddingDimension: 1024, IndexMappingSHA256: es.MappingSHA256}
|
||||
id, err := ArchiveLibraryCandidate(context.Background(), archive, "internal", identity, contract, pg, es, objects, time.Now().UTC())
|
||||
if err == nil || id != "" {
|
||||
t.Fatal("sealed a supposedly complete library with an indexed PostgreSQL chunk missing from Elasticsearch")
|
||||
id, err := ArchiveLibraryCandidate(context.Background(), archive, "internal",
|
||||
PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}, contract, pg, es, objects, time.Now().UTC())
|
||||
if err != nil || id == "" {
|
||||
t.Fatalf("body without original-text index must be captured as-is: id=%q err=%v", id, err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "missing indexed library chunk") {
|
||||
t.Fatalf("unexpected refusal: %v", err)
|
||||
manifest, err := archive.Load(id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
versions, err := os.ReadDir(root + "/versions")
|
||||
if err != nil || len(versions) != 0 {
|
||||
t.Fatalf("incomplete candidate became visible: versions=%d err=%v", len(versions), err)
|
||||
if err := validateArchivedLibraryClosure(archive, manifest); err != nil {
|
||||
t.Fatalf("readback of the captured body failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,18 +5,19 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLibraryCandidateLegacyChunkIndexCoverage(t *testing.T) {
|
||||
// R3f: status and chunk type are source metadata, not a governance index
|
||||
// requirement. The exact PG/ES sets are captured as-is; only question
|
||||
// metadata/index identity remains independently checked.
|
||||
func TestLibraryCandidateLegacyChunkKindsDoNotGateCapture(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, kind string
|
||||
status int
|
||||
indexed, valid bool
|
||||
name, kind string
|
||||
status int
|
||||
}{
|
||||
{"legacy text missing", "text", 0, false, false},
|
||||
{"legacy text indexed", "text", 0, true, true},
|
||||
{"legacy unknown type missing", "", 0, false, false},
|
||||
{"context parent not indexed", "parent_text", 0, false, true},
|
||||
{"stored parent not indexed", "parent_text", 1, false, true},
|
||||
{"indexed parent missing", "parent_text", 2, false, false},
|
||||
{"legacy text missing", "text", 0},
|
||||
{"legacy unknown type missing", "", 0},
|
||||
{"context parent not indexed", "parent_text", 0},
|
||||
{"stored parent not indexed", "parent_text", 1},
|
||||
{"indexed parent missing", "parent_text", 2},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
pg, es, _ := libraryCandidateFixture()
|
||||
@@ -25,16 +26,10 @@ func TestLibraryCandidateLegacyChunkIndexCoverage(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pg.Tables[1] = newTableSnapshot("chunks", []json.RawMessage{raw})
|
||||
if !tc.indexed {
|
||||
es.Documents = nil
|
||||
es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents)
|
||||
}
|
||||
err = validateLibraryCandidate(PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}, pg, es)
|
||||
if tc.valid && err != nil {
|
||||
t.Fatalf("valid chunk rejected: %v", err)
|
||||
}
|
||||
if !tc.valid && err == nil {
|
||||
t.Fatal("missing legacy index escaped completeness validation")
|
||||
es.Documents = nil
|
||||
es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents)
|
||||
if err := validateLibraryCandidate(PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}, pg, es); err != nil {
|
||||
t.Fatalf("source shape without body index must be captured as-is: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -32,10 +32,10 @@ func TestLibraryCandidateGeneratedQuestionIndex(t *testing.T) {
|
||||
}
|
||||
es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents)
|
||||
err := validateLibraryCandidate(PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}, pg, es)
|
||||
if mode == "complete" && err != nil {
|
||||
if (mode == "complete" || mode == "missing-body") && err != nil {
|
||||
t.Fatalf("valid persisted question rejected: %v", err)
|
||||
}
|
||||
if mode != "complete" && err == nil {
|
||||
if mode != "complete" && mode != "missing-body" && err == nil {
|
||||
t.Fatal("incomplete or inconsistent question index accepted")
|
||||
}
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user