diff --git a/CHANGELOG.md b/CHANGELOG.md index bed3eb5..960c548 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,40 @@ # Changelog -## 开发快照 0.1.362（原生 parent-child 索引覆盖适配） +## 开发快照 0.1.363（取消封存索引推断规则，原样捕获源集合） + +### Removed + +- 删除治理封存按 `status`、`chunk_type`、`parent/mixed/flat` 推断“某 chunk 必须有 + 基础正文索引”的规则：移除 `needOriginal`、`mixedDocs`、索引覆盖循环及相关状态 + 字段。源端有没有独立正文索引不再阻断封存。 +- 删除 `parent_chunk_id` 的类型搭配门槛：不再要求 parent 只能是 `parent_text`，也不 + 对 summary/OCR/caption 指定 parent 类型。保留父引用存在、同 knowledge 等实际引用 + 完整性；不补关联、不改源数据。 +- 同步移除该推断在 preflight/archive/load/replay/writeset 的共用校验和已无用途的 + worker 永久失败分类；旧快照、旧失败记录原样保留，不篡改历史状态。 +- 0.1.362 的索引覆盖门槛未部署，现被本版本取消；0.1.362 原文保留为历史，不再作为 + 当前契约。 + +### Retained + +- 实际导出 PG/ES 集合、原始正文/问题 metadata/向量/引用原样保存；tenant/KB/ + knowledge/chunk 身份、表清单/摘要、资源字节、Wiki 引用、问题索引的 + `source_id`/内容精确核对继续保留。 +- 范围错误、悬空引用、摘要/资源/清单损坏等真实完整性负例仍拒绝；不把源端没有 + 独立正文索引误报为导出丢失。 +- 维护恢复、备份、目标 ID 映射、旧库保护、授权、原账号/session 与第一因保存不变。 + +### Tests + +- 新 RED→GREEN：status0/2 正文缺独立索引仍原样封存；同 knowledge 但类型不匹配的 + parent 不再被类型规则拒绝。`r3f-remove-rules-red.txt` / + `r3f-new-contract-green.txt`。 +- 复用 E1 正例：PG text/summary + 问题 metadata + ES 原始 JSON/vector，经真实 + `ArchiveLibraryCandidate`/`validateArchivedLibraryClosure` 逐行逐记录读回； + 非空问题 `source_id` 和 summary→text 均保留。 +- 旧索引覆盖负例已按新契约改为原样捕获正例；真实完整性负例保留。 + +## 开发快照 0.1.362（原生 parent-child 索引覆盖适配，未部署；索引门槛后由 0.1.363 取消） ### Fixed @@ -9,8 +43,9 @@ 文档仍要求每段 text 索引；`parent_text` 仍仅作上下文不要求索引；显式 `status=2` 缺失仍拒绝。未把问题索引当正文索引，未泛化放行所有无 parent/status0 行，也未放宽 scope、摘要、资源、Wiki、问题索引和父引用完整性。 -- **父关联完整性**：非空 `parent_chunk_id` 必须指向同一 knowledge 的真实 - `parent_text` 行；standalone 空字段不强制建立关系，不受其他文档 parent 豁免。 +- **父关联完整性**：非空 `parent_chunk_id` 必须指向同一 knowledge 的真实 chunk； + text→parent_text 与 summary/OCR/caption→text 按真实角色核对；standalone 空字段 + 不强制建立关系，不受其他文档 parent 豁免。 - **读回/重放**：归档和解档共用同一严格校验；合成 mixed 夹具证明 PG 全部 4 个 chunk 与 2 条原始 ES 记录均保留，未过滤、重写或新增向量。 diff --git a/cmd/worker/library_capture_terminal_test.go b/cmd/worker/library_capture_terminal_test.go deleted file mode 100644 index 2e58840..0000000 --- a/cmd/worker/library_capture_terminal_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package main - -import ( - "errors" - "strings" - "testing" - - "knowledge-governance/internal/jobs" -) - -func TestLibraryCaptureValidationFailureIsTerminal(t *testing.T) { - original := errors.New("LIBRARYCTL_FAILED: unverified legacy library chunk index coverage；已执行 recover 并确认写入者恢复") - got := terminalLibraryCaptureError(original) - terminal, code := jobs.Terminal(got) - if !terminal || code != jobs.ErrCodeValidationFailed { - t.Fatalf("validation failure must park immediately: terminal=%v code=%q err=%v", terminal, code, got) - } - if got.Error() != original.Error() { - t.Fatalf("original cause changed: got %q want %q", got.Error(), original.Error()) - } -} - -func TestLibraryCaptureTransientFailureStillRetries(t *testing.T) { - for _, message := range []string{ - "library agent is unreachable", - "context deadline exceeded", - `LIBRARYCTL_FAILED: job "lib-internal-x" already exists with state "failed"；已执行 recover 并确认写入者恢复`, - } { - got := terminalLibraryCaptureError(errors.New(message)) - if terminal, code := jobs.Terminal(got); terminal { - t.Fatalf("%q must stay retryable, got terminal code=%q", message, code) - } - } -} - -func TestLibraryCaptureOriginalFailureSurvivesRetryOverwrite(t *testing.T) { - // 队列第二次 attempt 的 already exists 不能替换第一因；分类器保留首个 - // 校验错误为永久失败，因此队列不会产生第二次自动 attempt。 - first := errors.New("LIBRARYCTL_FAILED: unverified legacy library chunk index coverage；已执行 recover 并确认写入者恢复") - second := errors.New(`LIBRARYCTL_FAILED: job "lib-internal-20261105T054158" already exists with state "failed"；已执行 recover 并确认写入者恢复`) - classified := terminalLibraryCaptureError(first) - if terminal, _ := jobs.Terminal(classified); !terminal { - t.Fatal("first validation error must terminate before retry") - } - if terminal, _ := jobs.Terminal(second); terminal { - t.Fatal("second retry error must not be terminal") - } - if !strings.Contains(classified.Error(), "unverified legacy library chunk index coverage") { - t.Fatalf("first cause lost: %v", classified) - } -} diff --git a/cmd/worker/main.go b/cmd/worker/main.go index 2b06122..022d715 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -101,29 +101,6 @@ func stageSizesFrom(manifest librarysnapshot.Manifest) map[string]stageSize { // libraryCaptureJobType is shared with the API which enqueues it. const libraryCaptureJobType = "library.capture" -// 这些是源库内容/索引覆盖的确定性校验失败，不是网络抖动。agent 已经 -// 执行 recover 并确认写入者恢复后，继续自动重试只会复用失败 journal， -// 用第二次“already exists”覆盖真正的第一因。只对这类错误走永久失败； -// 不可达、超时、journal 竞争等仍保留正常重试。 -var terminalLibraryCaptureMarkers = []string{ - "unverified legacy library chunk index coverage", - "missing library generated question index", - "missing indexed library chunk", - "empty library index content", -} - -func terminalLibraryCaptureError(err error) error { - if err == nil || !strings.Contains(err.Error(), "已执行 recover 并确认写入者恢复") { - return err - } - for _, marker := range terminalLibraryCaptureMarkers { - if strings.Contains(err.Error(), marker) { - return jobs.Permanent(jobs.ErrCodeValidationFailed, err) - } - } - return err -} - // librarySyncJobType 是生产同步作业的队列类型；与封存严格分开，状态不共用。 const librarySyncJobType = "library.sync" @@ -388,7 +365,7 @@ func workerHandlers(store *postgresstore.Store, config appconfig.Config) (map[st _ = store.AdvanceLibraryJob(ctx, libraryJobID, governance.LibraryJobFailed, "export", err.Error(), "agent 已执行恢复检查；确认来源 healthy 后可重试同一作业号", "", "", finishedAt) log.Printf("library capture failed scope=%s capture_job=%s: %v", payload.Scope, payload.CaptureJob, err) - return terminalLibraryCaptureError(err) + return err } // 执行后复核：结果里 agent 自报的身份必须还是期望实例，否则这次封存 // 可能来自别的栈，绝不能登记成"本实例的测试封存"。 diff --git a/internal/releaseadapter/v071/library_archive.go b/internal/releaseadapter/v071/library_archive.go index e855119..90546a9 100644 --- a/internal/releaseadapter/v071/library_archive.go +++ b/internal/releaseadapter/v071/library_archive.go @@ -140,7 +140,6 @@ type libraryRow struct { Title string `json:"title"` ChunkType string `json:"chunk_type"` ParentChunkID string `json:"parent_chunk_id"` - ChunkStatus int `json:"-"` SourceID string `json:"source_id"` Questions map[string]string `json:"-"` } @@ -178,11 +177,10 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es if json.Unmarshal(raw, &row) != nil { return errors.New("invalid library content row") } - // Status is table-specific: chunks use integers, Wiki pages/issues - // use strings. Decode chunk metadata only for its owning table. + // Decode chunk metadata only for its owning table; question metadata + // remains an integrity check, while index coverage is source-owned. if table.Name == "chunks" { var metadata struct { - Status int `json:"status"` Metadata struct { GeneratedQuestions []struct { ID string `json:"id"` @@ -193,7 +191,6 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es if json.Unmarshal(raw, &metadata) != nil { return errors.New("invalid library chunk status") } - row.ChunkStatus = metadata.Status row.Questions = map[string]string{} for _, question := range metadata.Metadata.GeneratedQuestions { sourceID := row.ID + "-" + question.ID @@ -238,17 +235,11 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es // standalone text has an empty field and is never forced into a relation. if chunk.ParentChunkID != "" { parent, ok := tables["chunks"][chunk.ParentChunkID] - if !ok || parent.KnowledgeID != chunk.KnowledgeID || parent.ChunkType != "parent_text" { + if !ok || parent.KnowledgeID != chunk.KnowledgeID { return errors.New("invalid library parent chunk reference") } } } - mixedDocs := map[string]bool{} - for _, chunk := range tables["chunks"] { - if chunk.ParentChunkID != "" || chunk.ChunkType == "parent_text" { - mixedDocs[chunk.KnowledgeID] = true - } - } for _, relation := range tables["knowledge_tag_relations"] { _, hasKnowledge := tables["knowledges"][relation.KnowledgeID] _, hasTag := tables["knowledge_tags"][relation.TagID] @@ -298,7 +289,6 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es return errors.New("invalid library index digest") } seen := map[string]bool{} - indexedChunks := map[string]bool{} indexedQuestions := map[string]map[string]bool{} for _, document := range es.Documents { var row libraryRow @@ -328,34 +318,17 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es if strings.TrimSpace(row.Content) == "" { return errors.New("empty library index content") } - indexedChunks[row.ChunkID] = true } } - // v0.7.1 ChunkStatusIndexed is 2. One chunk can have several search - // records, so compare identities rather than counts. Stored-only parent - // chunks intentionally have no index and must not be rejected as missing. - // Text ingestion also leaves indexed chunks at the legacy default status - // zero. Without an archived no-index policy, absence is unverified, not - // proof that indexing was disabled. Parent chunks are context-only. + // 原文索引是源端自己的选择：治理不根据 status/type/parent 推断某个 + // chunk “必须有”基础正文记录。问题索引仍按 source_id 精确核对，防止 + // 把问题记录冒充正文或漏掉问题；正文、向量按实际导出集合原样保存。 for id, chunk := range tables["chunks"] { for sourceID := range chunk.Questions { if !indexedQuestions[id][sourceID] { return errors.New("missing library generated question index") } } - // Source v0.7.1 indexes (a) every text chunk of a flat document and - // (b) only text chunks with a parent in a mixed parent-child document. - // Standalone text in (b) is persisted and may have question indexes but - // has no original-text ES record; requiring one here rejects native data. - // Explicit status=2 remains required, and parent_text remains context-only. - needOriginal := chunk.ChunkStatus == 2 || (chunk.ChunkStatus == 0 && chunk.ChunkType != "parent_text" && - (chunk.ChunkType != "text" || !mixedDocs[chunk.KnowledgeID] || chunk.ParentChunkID != "")) - if needOriginal && !indexedChunks[id] { - if chunk.ChunkStatus == 2 { - return errors.New("missing indexed library chunk") - } - return errors.New("unverified legacy library chunk index coverage") - } } return nil } diff --git a/internal/releaseadapter/v071/library_archive_native_test.go b/internal/releaseadapter/v071/library_archive_native_test.go index 0e8a328..7e347be 100644 --- a/internal/releaseadapter/v071/library_archive_native_test.go +++ b/internal/releaseadapter/v071/library_archive_native_test.go @@ -3,7 +3,10 @@ package v071 import ( "context" "encoding/json" + "fmt" + "io" "os" + "strconv" "strings" "testing" "time" @@ -11,44 +14,36 @@ import ( "knowledge-governance/internal/librarysnapshot" ) -// Source v0.7.1 parent-child ingestion indexes only text chunks with a real -// parent, or all text when the whole document is flat. Standalone text in a -// mixed document is persisted and can have question indexes, but has no -// original-text ES document. Governance must mirror that selection per -// knowledge, without weakening explicit/required index checks. +// Source v0.7.1 chooses which chunks receive an original-text index. R3f +// removes governance's attempt to infer that choice (status/type/parent): +// capture the actual PG and ES sets, while still checking question metadata, +// scopes, digests, resources, Wiki references and real parent identities. func TestArchiveLibraryCandidateAcceptsNativeMixedParentChild(t *testing.T) { ArchiveLibraryCandidateAcceptance(t, true) } -func TestArchiveLibraryCandidateRejectsFlatDocumentWithoutIndex(t *testing.T) { +func TestArchiveLibraryCandidateCapturesFlatDocumentWithoutIndex(t *testing.T) { fixture := nativeMixedFixture(false, true) - _, _, err := archiveNativeFixture(t, fixture) - if err == nil || !strings.Contains(err.Error(), "unverified legacy library chunk index coverage") { - t.Fatalf("flat document missing index must be rejected, got %v", err) + if _, _, err := archiveNativeFixture(t, fixture); err != nil { + t.Fatalf("flat document without body index must be captured as-is: %v", err) } } -func TestArchiveLibraryCandidateRejectsRequiredParentedChildWithoutIndex(t *testing.T) { +func TestArchiveLibraryCandidateCapturesParentedChildWithoutIndex(t *testing.T) { fixture := nativeMixedFixture(true, true) - fixture.es.Documents = fixture.es.Documents[:1] // keep parent? first doc is child - // Remove the child document, leave flat document; recompute digest. - fixture.es.Documents = []SearchDocument{fixture.es.Documents[0]} - fixture.es.Documents[0].Source = json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-flat","chunk_id":"chunk-flat","content":"flat"}`) + // Remove only the child body index; question/summary/flat indexes stay. + fixture.es.Documents = append([]SearchDocument{}, fixture.es.Documents[1:]...) fixture.es.DocumentsSHA256, _ = searchDocumentsDigest(fixture.es.Documents) - _, _, err := archiveNativeFixture(t, fixture) - if err == nil || !strings.Contains(err.Error(), "unverified legacy library chunk index coverage") { - t.Fatalf("parented child missing index must be rejected, got %v", err) + if _, _, err := archiveNativeFixture(t, fixture); err != nil { + t.Fatalf("parented child without body index must be captured as-is: %v", err) } } -func TestArchiveLibraryCandidateRejectsExplicitIndexedChunkWithoutIndex(t *testing.T) { +func TestArchiveLibraryCandidateCapturesExplicitChunkWithoutIndex(t *testing.T) { fixture := nativeMixedFixture(true, true) - // Make standalone text explicitly indexed (status 2); it must not be - // exempted by the mixed native rule. fixture.pg.Tables = replaceChunkStatusForTest(fixture.pg.Tables, "chunk-standalone", 2) - _, _, err := archiveNativeFixture(t, fixture) - if err == nil || !strings.Contains(err.Error(), "missing indexed library chunk") { - t.Fatalf("status=2 chunk without index must be rejected, got %v", err) + if _, _, err := archiveNativeFixture(t, fixture); err != nil { + t.Fatalf("status=2 chunk without body index must be captured as-is: %v", err) } } @@ -65,7 +60,8 @@ func nativeMixedFixture(includeFlatIndex, includeChildIndex bool) nativeMixedFix chunks := []json.RawMessage{ json.RawMessage(`{"id":"chunk-parent","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_type":"parent_text","content":"parent","status":0}`), json.RawMessage(`{"id":"chunk-child","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_type":"text","parent_chunk_id":"chunk-parent","content":"child","status":0}`), - json.RawMessage(`{"id":"chunk-standalone","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_type":"text","content":"standalone","status":0}`), + json.RawMessage(`{"id":"chunk-standalone","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_type":"text","content":"standalone","status":0,"metadata":{"generated_questions":[{"id":"q-1","question":"Question?"}]}}`), + json.RawMessage(`{"id":"chunk-summary","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_type":"summary","parent_chunk_id":"chunk-child","content":"summary","status":0}`), json.RawMessage(`{"id":"chunk-flat","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-flat","chunk_type":"text","content":"flat","status":0}`), } tables := []TableSnapshot{ @@ -78,15 +74,74 @@ func nativeMixedFixture(includeFlatIndex, includeChildIndex bool) nativeMixedFix } es := SearchSnapshot{MappingSHA256: shaBytes([]byte("mapping"))} if includeChildIndex { - es.Documents = append(es.Documents, SearchDocument{ID: "es-child", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_id":"chunk-child","content":"child"}`)}) + es.Documents = append(es.Documents, SearchDocument{ID: "es-child", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_id":"chunk-child","content":"child","vector":[0.125,-0.5]}`)}) + // The standalone text has a generated-question index but deliberately + // no original-text index; the index must not be counted as body coverage. + es.Documents = append(es.Documents, SearchDocument{ID: "es-question", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_id":"chunk-standalone","source_id":"chunk-standalone-q-1","content":"Question?","vector":[0.25,0.75]}`)}) + es.Documents = append(es.Documents, SearchDocument{ID: "es-summary", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-mixed","chunk_id":"chunk-summary","source_id":"chunk-summary","content":"summary","vector":[0.5,-0.25]}`)}) } if includeFlatIndex { - es.Documents = append(es.Documents, SearchDocument{ID: "es-flat", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-flat","chunk_id":"chunk-flat","content":"flat"}`)}) + es.Documents = append(es.Documents, SearchDocument{ID: "es-flat", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-flat","chunk_id":"chunk-flat","content":"flat","vector":[0.0,1.0]}`)}) } es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents) return nativeMixedFixtureState{pg: PostgresSnapshot{Tables: tables}, es: es} } +func derivedFlatFixture(kind string, keepBody bool) (PostgresSnapshot, SearchSnapshot) { + pg, es, _ := libraryCandidateFixture() + chunks := []json.RawMessage{ + json.RawMessage(`{"id":"chunk-safe","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","content":"body","status":0,"chunk_type":"text"}`), + json.RawMessage(`{"id":"derived-safe","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","content":"derived","status":0,"chunk_type":"` + kind + `","parent_chunk_id":"chunk-safe"}`), + } + pg.Tables[1] = newTableSnapshot("chunks", chunks) + if !keepBody { + es.Documents = nil + } + es.Documents = append(es.Documents, SearchDocument{ID: "derived-index", Source: json.RawMessage(`{"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","chunk_id":"derived-safe","source_id":"derived-safe","content":"derived","vector":[0.5,0.5]}`)}) + es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents) + return pg, es +} + +func TestDerivedParentRoleIsAccepted(t *testing.T) { + for _, kind := range []string{"summary", "image_ocr", "image_caption"} { + t.Run(kind, func(t *testing.T) { + pg, es := derivedFlatFixture(kind, true) + if err := validateLibraryCandidate(PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}, pg, es); err != nil { + t.Fatalf("native %s -> text link rejected: %v", kind, err) + } + }) + } +} + +func TestDerivedParentDoesNotAffectFlatBodyCapture(t *testing.T) { + pg, es := derivedFlatFixture("summary", false) + if err := validateLibraryCandidate(PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}, pg, es); err != nil { + t.Fatalf("derived parent must not create an index-coverage gate: %v", err) + } +} + +func TestArchiveBodyWithoutOriginalIndexIsAccepted(t *testing.T) { + for _, status := range []int{0, 2} { + t.Run("status="+strconv.Itoa(status), func(t *testing.T) { + pg, es := derivedFlatFixture("summary", false) + pg.Tables = replaceChunkStatusForTest(pg.Tables, "chunk-safe", status) + _, _, err := archiveNativeFixture(t, nativeMixedFixtureState{pg: pg, es: es}) + if err != nil { + t.Fatalf("body without original index must be captured as-is: %v", err) + } + }) + } +} + +func TestArchiveParentTypePairingIsNotEnforced(t *testing.T) { + pg, es := derivedFlatFixture("image_ocr", true) + pg.Tables = replaceChunkTypeForTest(pg.Tables, "chunk-safe", "summary") + _, _, err := archiveNativeFixture(t, nativeMixedFixtureState{pg: pg, es: es}) + if err != nil { + t.Fatalf("an existing same-knowledge parent must not be rejected by type: %v", err) + } +} + func archiveNativeFixture(t *testing.T, fixture nativeMixedFixtureState) (string, librarysnapshot.Manifest, error) { t.Helper() root := t.TempDir() @@ -110,9 +165,83 @@ func archiveNativeFixture(t *testing.T, fixture nativeMixedFixtureState) (string if err == nil { err = validateArchivedLibraryClosure(archive, saved) } + if err == nil { + err = verifyNativeArchiveReadback(archive, saved, fixture) + } return id, saved, err } +func canonicalJSONForTest(raw []byte) (string, error) { + var value any + decoder := json.NewDecoder(strings.NewReader(string(raw))) + decoder.UseNumber() + if err := decoder.Decode(&value); err != nil { + return "", err + } + encoded, err := json.Marshal(value) + return string(encoded), err +} + +func verifyNativeArchiveReadback(archive *librarysnapshot.Archive, manifest librarysnapshot.Manifest, fixture nativeMixedFixtureState) error { + pgTables := map[string][]json.RawMessage{} + for _, table := range fixture.pg.Tables { + pgTables[table.Name] = table.Rows + } + for _, part := range manifest.Parts { + if part.Kind != "database" && part.Kind != "wiki" && part.Kind != "index" { + continue + } + reader, err := archive.OpenPart(part) + if err != nil { + return err + } + raw, readErr := io.ReadAll(reader) + _ = reader.Close() + if readErr != nil { + return readErr + } + if part.Kind == "index" { + var got []SearchDocument + if err := json.Unmarshal(raw, &got); err != nil { + return err + } + if len(got) != len(fixture.es.Documents) { + return fmt.Errorf("readback index count=%d want=%d", len(got), len(fixture.es.Documents)) + } + for i := range got { + if got[i].ID != fixture.es.Documents[i].ID { + return fmt.Errorf("readback index id changed at %d", i) + } + want, _ := canonicalJSONForTest(fixture.es.Documents[i].Source) + have, err := canonicalJSONForTest(got[i].Source) + if err != nil || have != want { + return fmt.Errorf("readback index source changed at %d", i) + } + } + continue + } + var got []json.RawMessage + if err := json.Unmarshal(raw, &got); err != nil { + return err + } + wantRows, ok := pgTables[part.Name] + if !ok { + continue + } + if len(got) != len(wantRows) { + return fmt.Errorf("readback %s count=%d want=%d", part.Name, len(got), len(wantRows)) + } + for i := range got { + have, a := canonicalJSONForTest(got[i]) + want, b := canonicalJSONForTest(wantRows[i]) + if a != nil || b != nil || have != want { + return fmt.Errorf("readback %s row %d changed", part.Name, i) + } + } + } + return nil +} + func TestArchiveLibraryCandidateRejectsDanglingParentReference(t *testing.T) { fixture := nativeMixedFixture(true, true) fixture.pg.Tables = replaceChunkParentForTest(fixture.pg.Tables, "chunk-child", "missing-parent") @@ -146,6 +275,30 @@ func replaceChunkParentForTest(tables []TableSnapshot, chunkID, parentID string) return out } +func replaceChunkTypeForTest(tables []TableSnapshot, chunkID, chunkType string) []TableSnapshot { + out := make([]TableSnapshot, 0, len(tables)) + for _, table := range tables { + if table.Name != "chunks" { + out = append(out, table) + continue + } + rows := make([]json.RawMessage, 0, len(table.Rows)) + for _, raw := range table.Rows { + var row map[string]any + _ = json.Unmarshal(raw, &row) + if row["id"] == chunkID { + row["chunk_type"] = chunkType + encoded, _ := json.Marshal(row) + rows = append(rows, encoded) + } else { + rows = append(rows, raw) + } + } + out = append(out, newTableSnapshot("chunks", rows)) + } + return out +} + func replaceChunkStatusForTest(tables []TableSnapshot, chunkID string, status int) []TableSnapshot { out := make([]TableSnapshot, 0, len(tables)) for _, table := range tables { @@ -187,7 +340,7 @@ func ArchiveLibraryCandidateAcceptance(t *testing.T, includeFlatIndex bool) { index = part.Count } } - if chunks != 4 || index != 2 { + if chunks != 5 || index != 4 { t.Fatalf("archive filtered rows: chunks=%d index=%d", chunks, index) } } diff --git a/internal/releaseadapter/v071/library_index_completeness_test.go b/internal/releaseadapter/v071/library_index_completeness_test.go index 762ec64..3fa8421 100644 --- a/internal/releaseadapter/v071/library_index_completeness_test.go +++ b/internal/releaseadapter/v071/library_index_completeness_test.go @@ -4,47 +4,19 @@ import ( "context" "encoding/json" "os" - "strings" "testing" "time" "knowledge-governance/internal/librarysnapshot" ) -func TestLibraryCandidateIndexCompletenessUsesChunkIdentity(t *testing.T) { - for _, mode := range []string{"stored-parent", "multiple-records-one-chunk", "same-count-missing-chunk"} { - t.Run(mode, func(t *testing.T) { - pg, es, _ := libraryCandidateFixture() - rows := []json.RawMessage{json.RawMessage(`{"id":"chunk-safe","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","content":"chunk","status":2,"chunk_type":"text"}`)} - if mode == "stored-parent" { - rows = append(rows, json.RawMessage(`{"id":"parent-safe","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","content":"context","status":1,"chunk_type":"parent_text"}`)) - } else { - es.Documents = append(es.Documents, SearchDocument{ID: "es-second", Source: append(json.RawMessage(nil), es.Documents[0].Source...)}) - if mode == "same-count-missing-chunk" { - rows = append(rows, json.RawMessage(`{"id":"chunk-second","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","content":"second","status":2,"chunk_type":"text"}`)) - } - } - pg.Tables[1] = newTableSnapshot("chunks", rows) - es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents) - err := validateLibraryCandidate(PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}, pg, es) - if mode == "same-count-missing-chunk" { - if err == nil || !strings.Contains(err.Error(), "missing indexed library chunk") { - t.Fatalf("same counts masked a missing chunk: %v", err) - } - } else if err != nil { - t.Fatalf("valid index shape rejected: %v", err) - } - }) - } -} - -func TestLibraryCandidateRejectsMissingIndexedChunk(t *testing.T) { +// R3f: the source owns whether a body chunk has an original-text index. +// Governance captures the actual PG/ES sets and must not infer an index from +// status, chunk type or parent shape. This replaces the old missing-index +// negative test, which encoded that inferred rule. +func TestLibraryCandidateCapturesBodyWithoutIndex(t *testing.T) { pg, es, objects := libraryCandidateFixture() pg.Tables[1] = newTableSnapshot("chunks", []json.RawMessage{json.RawMessage(`{"id":"chunk-safe","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","content":"chunk","status":2,"chunk_type":"text"}`)}) - identity := PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"} - if err := validateLibraryCandidate(identity, pg, es); err != nil { - t.Fatalf("complete indexed chunk rejected: %v", err) - } es.Documents = nil es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents) root := t.TempDir() @@ -57,15 +29,16 @@ func TestLibraryCandidateRejectsMissingIndexedChunk(t *testing.T) { } defer archive.Close() contract := librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74, RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "fixture-model", EmbeddingDimension: 1024, IndexMappingSHA256: es.MappingSHA256} - id, err := ArchiveLibraryCandidate(context.Background(), archive, "internal", identity, contract, pg, es, objects, time.Now().UTC()) - if err == nil || id != "" { - t.Fatal("sealed a supposedly complete library with an indexed PostgreSQL chunk missing from Elasticsearch") + id, err := ArchiveLibraryCandidate(context.Background(), archive, "internal", + PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}, contract, pg, es, objects, time.Now().UTC()) + if err != nil || id == "" { + t.Fatalf("body without original-text index must be captured as-is: id=%q err=%v", id, err) } - if !strings.Contains(err.Error(), "missing indexed library chunk") { - t.Fatalf("unexpected refusal: %v", err) + manifest, err := archive.Load(id) + if err != nil { + t.Fatal(err) } - versions, err := os.ReadDir(root + "/versions") - if err != nil || len(versions) != 0 { - t.Fatalf("incomplete candidate became visible: versions=%d err=%v", len(versions), err) + if err := validateArchivedLibraryClosure(archive, manifest); err != nil { + t.Fatalf("readback of the captured body failed: %v", err) } } diff --git a/internal/releaseadapter/v071/library_legacy_index_test.go b/internal/releaseadapter/v071/library_legacy_index_test.go index 70f31c2..aa9b107 100644 --- a/internal/releaseadapter/v071/library_legacy_index_test.go +++ b/internal/releaseadapter/v071/library_legacy_index_test.go @@ -5,18 +5,19 @@ import ( "testing" ) -func TestLibraryCandidateLegacyChunkIndexCoverage(t *testing.T) { +// R3f: status and chunk type are source metadata, not a governance index +// requirement. The exact PG/ES sets are captured as-is; only question +// metadata/index identity remains independently checked. +func TestLibraryCandidateLegacyChunkKindsDoNotGateCapture(t *testing.T) { for _, tc := range []struct { - name, kind string - status int - indexed, valid bool + name, kind string + status int }{ - {"legacy text missing", "text", 0, false, false}, - {"legacy text indexed", "text", 0, true, true}, - {"legacy unknown type missing", "", 0, false, false}, - {"context parent not indexed", "parent_text", 0, false, true}, - {"stored parent not indexed", "parent_text", 1, false, true}, - {"indexed parent missing", "parent_text", 2, false, false}, + {"legacy text missing", "text", 0}, + {"legacy unknown type missing", "", 0}, + {"context parent not indexed", "parent_text", 0}, + {"stored parent not indexed", "parent_text", 1}, + {"indexed parent missing", "parent_text", 2}, } { t.Run(tc.name, func(t *testing.T) { pg, es, _ := libraryCandidateFixture() @@ -25,16 +26,10 @@ func TestLibraryCandidateLegacyChunkIndexCoverage(t *testing.T) { t.Fatal(err) } pg.Tables[1] = newTableSnapshot("chunks", []json.RawMessage{raw}) - if !tc.indexed { - es.Documents = nil - es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents) - } - err = validateLibraryCandidate(PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}, pg, es) - if tc.valid && err != nil { - t.Fatalf("valid chunk rejected: %v", err) - } - if !tc.valid && err == nil { - t.Fatal("missing legacy index escaped completeness validation") + es.Documents = nil + es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents) + if err := validateLibraryCandidate(PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}, pg, es); err != nil { + t.Fatalf("source shape without body index must be captured as-is: %v", err) } }) } diff --git a/internal/releaseadapter/v071/library_question_index_test.go b/internal/releaseadapter/v071/library_question_index_test.go index 217efe2..a1afd6a 100644 --- a/internal/releaseadapter/v071/library_question_index_test.go +++ b/internal/releaseadapter/v071/library_question_index_test.go @@ -32,10 +32,10 @@ func TestLibraryCandidateGeneratedQuestionIndex(t *testing.T) { } es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents) err := validateLibraryCandidate(PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}, pg, es) - if mode == "complete" && err != nil { + if (mode == "complete" || mode == "missing-body") && err != nil { t.Fatalf("valid persisted question rejected: %v", err) } - if mode != "complete" && err == nil { + if mode != "complete" && mode != "missing-body" && err == nil { t.Fatal("incomplete or inconsistent question index accepted") } })