fix(snapshot): 按持久元数据核验生成问题索引
build / build (push) Successful in 1m11s

This commit is contained in:
姚章浩
2026-10-28 01:24:32 +08:00
parent fe2e7ac4f4
commit 2fc6dc068b
3 changed files with 103 additions and 17 deletions
+7
View File
@@ -1,5 +1,12 @@
# Changelog
## 开发快照 snapshot/v0.1.103 - 2026-10-28
### Fixed
- 整库索引校验支持从分块持久元数据核验生成问题:严格匹配分块、问题 source_id 与问题原文,拒绝未知、过期或内容不一致的索引。
- 分开核验正文与生成问题的覆盖,问题记录不能掩盖正文索引缺失;元数据中已记录的问题缺少索引同样阻断。未放宽临时章节上下文或未知索引变换的检查。
## 开发快照 snapshot/v0.1.102 - 2026-10-28
### Fixed
+53 -17
View File
@@ -105,20 +105,22 @@ func ArchiveLibraryCandidate(ctx context.Context, archive *librarysnapshot.Archi
}
type libraryRow struct {
ID string `json:"id"`
TenantID uint64 `json:"tenant_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
KnowledgeID string `json:"knowledge_id"`
ChunkID string `json:"chunk_id"`
TagID string `json:"tag_id"`
ResourceID string `json:"resource_id"`
Handle string `json:"handle"`
OwnerType string `json:"owner_type"`
OwnerID string `json:"owner_id"`
Content string `json:"content"`
Title string `json:"title"`
ChunkType string `json:"chunk_type"`
ChunkStatus int `json:"-"`
ID string `json:"id"`
TenantID uint64 `json:"tenant_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
KnowledgeID string `json:"knowledge_id"`
ChunkID string `json:"chunk_id"`
TagID string `json:"tag_id"`
ResourceID string `json:"resource_id"`
Handle string `json:"handle"`
OwnerType string `json:"owner_type"`
OwnerID string `json:"owner_id"`
Content string `json:"content"`
Title string `json:"title"`
ChunkType string `json:"chunk_type"`
ChunkStatus int `json:"-"`
SourceID string `json:"source_id"`
Questions map[string]string `json:"-"`
}
func libraryIndexContentMatches(indexContent string, chunk, knowledge libraryRow) bool {
@@ -176,12 +178,26 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es
// use strings. Decode chunk metadata only for its owning table.
if table.Name == "chunks" {
var metadata struct {
Status int `json:"status"`
Status int `json:"status"`
Metadata struct {
GeneratedQuestions []struct {
ID string `json:"id"`
Question string `json:"question"`
} `json:"generated_questions"`
} `json:"metadata"`
}
if json.Unmarshal(raw, &metadata) != nil {
return errors.New("invalid library chunk status")
}
row.ChunkStatus = metadata.Status
row.Questions = map[string]string{}
for _, question := range metadata.Metadata.GeneratedQuestions {
sourceID := row.ID + "-" + question.ID
if row.ChunkType != "text" || !validSearchIdentifier(question.ID) || strings.TrimSpace(question.Question) == "" || row.Questions[sourceID] != "" {
return errors.New("invalid library generated question metadata")
}
row.Questions[sourceID] = question.Question
}
}
if table.Name == "knowledge_tag_relations" {
if !validSearchIdentifier(row.KnowledgeID) || !validSearchIdentifier(row.TagID) {
@@ -253,6 +269,7 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es
}
seen := map[string]bool{}
indexedChunks := map[string]bool{}
indexedQuestions := map[string]map[string]bool{}
for _, document := range es.Documents {
var row libraryRow
if json.Unmarshal(document.Source, &row) != nil || !validSearchIdentifier(document.ID) || seen[document.ID] {
@@ -260,13 +277,27 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es
}
seen[document.ID] = true
chunk, ok := tables["chunks"][row.ChunkID]
if !ok || row.KnowledgeID != chunk.KnowledgeID || !libraryIndexContentMatches(row.Content, chunk, tables["knowledges"][chunk.KnowledgeID]) {
if !ok || row.KnowledgeID != chunk.KnowledgeID {
return errors.New("library index does not match database chunks")
}
if !searchHitMatches(document.Source, SearchIdentity{TenantID: identity.TenantID, KnowledgeBaseID: identity.KnowledgeBaseID, KnowledgeID: row.KnowledgeID}) {
return errors.New("cross-scope library index")
}
indexedChunks[row.ChunkID] = true
if row.SourceID != "" && row.SourceID != row.ChunkID {
question, exists := chunk.Questions[row.SourceID]
if !exists || question != row.Content {
return errors.New("library question index does not match persisted metadata")
}
if indexedQuestions[row.ChunkID] == nil {
indexedQuestions[row.ChunkID] = map[string]bool{}
}
indexedQuestions[row.ChunkID][row.SourceID] = true
} else {
if !libraryIndexContentMatches(row.Content, chunk, tables["knowledges"][chunk.KnowledgeID]) {
return errors.New("library index does not match database chunks")
}
indexedChunks[row.ChunkID] = true
}
}
// v0.7.1 ChunkStatusIndexed is 2. One chunk can have several search
// records, so compare identities rather than counts. Stored-only parent
@@ -275,6 +306,11 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es
// zero. Without an archived no-index policy, absence is unverified, not
// proof that indexing was disabled. Parent chunks are context-only.
for id, chunk := range tables["chunks"] {
for sourceID := range chunk.Questions {
if !indexedQuestions[id][sourceID] {
return errors.New("missing library generated question index")
}
}
if chunk.ChunkStatus == 0 && chunk.ChunkType != "parent_text" && !indexedChunks[id] {
return errors.New("unverified legacy library chunk index coverage")
}
@@ -0,0 +1,43 @@
package v071
import (
"encoding/json"
"testing"
)
func TestLibraryCandidateGeneratedQuestionIndex(t *testing.T) {
for _, mode := range []string{"complete", "wrong-id", "wrong-content", "missing-question", "missing-body", "foreign-source", "question-as-body"} {
t.Run(mode, func(t *testing.T) {
pg, es, _ := libraryCandidateFixture()
pg.Tables[1] = newTableSnapshot("chunks", []json.RawMessage{json.RawMessage(`{"id":"chunk-safe","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","content":"chunk","chunk_type":"text","status":0,"metadata":{"generated_questions":[{"id":"q1","question":"How does it work?"}]}}`)})
questionID, content := "chunk-safe-q1", "How does it work?"
if mode == "wrong-id" {
questionID = "chunk-safe-q2"
}
if mode == "wrong-content" {
content = "Different question"
}
if mode == "foreign-source" {
questionID = "foreign-q1"
}
if mode == "question-as-body" {
content = "chunk"
}
raw, _ := json.Marshal(map[string]any{"chunk_id": "chunk-safe", "knowledge_id": "doc-safe", "knowledge_base_id": "kb-safe", "source_id": questionID, "content": content})
q := SearchDocument{ID: "question-index", Source: raw}
if mode == "missing-body" {
es.Documents = []SearchDocument{q}
} else if mode != "missing-question" {
es.Documents = append(es.Documents, q)
}
es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents)
err := validateLibraryCandidate(PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}, pg, es)
if mode == "complete" && err != nil {
t.Fatalf("valid persisted question rejected: %v", err)
}
if mode != "complete" && err == nil {
t.Fatal("incomplete or inconsistent question index accepted")
}
})
}
}