diff --git a/CHANGELOG.md b/CHANGELOG.md index c363104..68022d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## 开发快照 snapshot/v0.1.103 - 2026-10-28 + +### Fixed + +- 整库索引校验支持从分块持久元数据核验生成问题:严格匹配分块、问题 source_id 与问题原文,拒绝未知、过期或内容不一致的索引。 +- 分开核验正文与生成问题的覆盖,问题记录不能掩盖正文索引缺失;元数据中已记录的问题缺少索引同样阻断。未放宽临时章节上下文或未知索引变换的检查。 + ## 开发快照 snapshot/v0.1.102 - 2026-10-28 ### Fixed diff --git a/internal/releaseadapter/v071/library_archive.go b/internal/releaseadapter/v071/library_archive.go index e7dc615..b641db8 100644 --- a/internal/releaseadapter/v071/library_archive.go +++ b/internal/releaseadapter/v071/library_archive.go @@ -105,20 +105,22 @@ func ArchiveLibraryCandidate(ctx context.Context, archive *librarysnapshot.Archi } type libraryRow struct { - ID string `json:"id"` - TenantID uint64 `json:"tenant_id"` - KnowledgeBaseID string `json:"knowledge_base_id"` - KnowledgeID string `json:"knowledge_id"` - ChunkID string `json:"chunk_id"` - TagID string `json:"tag_id"` - ResourceID string `json:"resource_id"` - Handle string `json:"handle"` - OwnerType string `json:"owner_type"` - OwnerID string `json:"owner_id"` - Content string `json:"content"` - Title string `json:"title"` - ChunkType string `json:"chunk_type"` - ChunkStatus int `json:"-"` + ID string `json:"id"` + TenantID uint64 `json:"tenant_id"` + KnowledgeBaseID string `json:"knowledge_base_id"` + KnowledgeID string `json:"knowledge_id"` + ChunkID string `json:"chunk_id"` + TagID string `json:"tag_id"` + ResourceID string `json:"resource_id"` + Handle string `json:"handle"` + OwnerType string `json:"owner_type"` + OwnerID string `json:"owner_id"` + Content string `json:"content"` + Title string `json:"title"` + ChunkType string `json:"chunk_type"` + ChunkStatus int `json:"-"` + SourceID string `json:"source_id"` + Questions map[string]string `json:"-"` } func libraryIndexContentMatches(indexContent string, chunk, knowledge libraryRow) bool { @@ -176,12 +178,26 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es // use strings. Decode chunk metadata only for its owning table. if table.Name == "chunks" { var metadata struct { - Status int `json:"status"` + Status int `json:"status"` + Metadata struct { + GeneratedQuestions []struct { + ID string `json:"id"` + Question string `json:"question"` + } `json:"generated_questions"` + } `json:"metadata"` } if json.Unmarshal(raw, &metadata) != nil { return errors.New("invalid library chunk status") } row.ChunkStatus = metadata.Status + row.Questions = map[string]string{} + for _, question := range metadata.Metadata.GeneratedQuestions { + sourceID := row.ID + "-" + question.ID + if row.ChunkType != "text" || !validSearchIdentifier(question.ID) || strings.TrimSpace(question.Question) == "" || row.Questions[sourceID] != "" { + return errors.New("invalid library generated question metadata") + } + row.Questions[sourceID] = question.Question + } } if table.Name == "knowledge_tag_relations" { if !validSearchIdentifier(row.KnowledgeID) || !validSearchIdentifier(row.TagID) { @@ -253,6 +269,7 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es } seen := map[string]bool{} indexedChunks := map[string]bool{} + indexedQuestions := map[string]map[string]bool{} for _, document := range es.Documents { var row libraryRow if json.Unmarshal(document.Source, &row) != nil || !validSearchIdentifier(document.ID) || seen[document.ID] { @@ -260,13 +277,27 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es } seen[document.ID] = true chunk, ok := tables["chunks"][row.ChunkID] - if !ok || row.KnowledgeID != chunk.KnowledgeID || !libraryIndexContentMatches(row.Content, chunk, tables["knowledges"][chunk.KnowledgeID]) { + if !ok || row.KnowledgeID != chunk.KnowledgeID { return errors.New("library index does not match database chunks") } if !searchHitMatches(document.Source, SearchIdentity{TenantID: identity.TenantID, KnowledgeBaseID: identity.KnowledgeBaseID, KnowledgeID: row.KnowledgeID}) { return errors.New("cross-scope library index") } - indexedChunks[row.ChunkID] = true + if row.SourceID != "" && row.SourceID != row.ChunkID { + question, exists := chunk.Questions[row.SourceID] + if !exists || question != row.Content { + return errors.New("library question index does not match persisted metadata") + } + if indexedQuestions[row.ChunkID] == nil { + indexedQuestions[row.ChunkID] = map[string]bool{} + } + indexedQuestions[row.ChunkID][row.SourceID] = true + } else { + if !libraryIndexContentMatches(row.Content, chunk, tables["knowledges"][chunk.KnowledgeID]) { + return errors.New("library index does not match database chunks") + } + indexedChunks[row.ChunkID] = true + } } // v0.7.1 ChunkStatusIndexed is 2. One chunk can have several search // records, so compare identities rather than counts. Stored-only parent @@ -275,6 +306,11 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es // zero. Without an archived no-index policy, absence is unverified, not // proof that indexing was disabled. Parent chunks are context-only. for id, chunk := range tables["chunks"] { + for sourceID := range chunk.Questions { + if !indexedQuestions[id][sourceID] { + return errors.New("missing library generated question index") + } + } if chunk.ChunkStatus == 0 && chunk.ChunkType != "parent_text" && !indexedChunks[id] { return errors.New("unverified legacy library chunk index coverage") } diff --git a/internal/releaseadapter/v071/library_question_index_test.go b/internal/releaseadapter/v071/library_question_index_test.go new file mode 100644 index 0000000..217efe2 --- /dev/null +++ b/internal/releaseadapter/v071/library_question_index_test.go @@ -0,0 +1,43 @@ +package v071 + +import ( + "encoding/json" + "testing" +) + +func TestLibraryCandidateGeneratedQuestionIndex(t *testing.T) { + for _, mode := range []string{"complete", "wrong-id", "wrong-content", "missing-question", "missing-body", "foreign-source", "question-as-body"} { + t.Run(mode, func(t *testing.T) { + pg, es, _ := libraryCandidateFixture() + pg.Tables[1] = newTableSnapshot("chunks", []json.RawMessage{json.RawMessage(`{"id":"chunk-safe","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","content":"chunk","chunk_type":"text","status":0,"metadata":{"generated_questions":[{"id":"q1","question":"How does it work?"}]}}`)}) + questionID, content := "chunk-safe-q1", "How does it work?" + if mode == "wrong-id" { + questionID = "chunk-safe-q2" + } + if mode == "wrong-content" { + content = "Different question" + } + if mode == "foreign-source" { + questionID = "foreign-q1" + } + if mode == "question-as-body" { + content = "chunk" + } + raw, _ := json.Marshal(map[string]any{"chunk_id": "chunk-safe", "knowledge_id": "doc-safe", "knowledge_base_id": "kb-safe", "source_id": questionID, "content": content}) + q := SearchDocument{ID: "question-index", Source: raw} + if mode == "missing-body" { + es.Documents = []SearchDocument{q} + } else if mode != "missing-question" { + es.Documents = append(es.Documents, q) + } + es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents) + err := validateLibraryCandidate(PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}, pg, es) + if mode == "complete" && err != nil { + t.Fatalf("valid persisted question rejected: %v", err) + } + if mode != "complete" && err == nil { + t.Fatal("incomplete or inconsistent question index accepted") + } + }) + } +}