fix(snapshot): 校验正文索引的确定性标题变换
build / build (push) Successful in 1m13s

This commit is contained in:
姚章浩
2026-10-28 01:13:16 +08:00
parent bf049cee1f
commit c4759bb61c
3 changed files with 68 additions and 1 deletions
+7
View File
@@ -1,5 +1,12 @@
# Changelog
## 开发快照 snapshot/v0.1.101 - 2026-10-28
### Fixed
- 修复正常正文索引因标题前缀或正文首尾空白处理而被整库归档误拒的问题;仅按已持久化的标题与正文重建确定性内容,保持逐字核验,不使用后缀包含匹配。
- 无法由持久数据核验的临时章节上下文、非正文类型的变换及未知内容仍阻断;不将本次兼容修复视为完整索引或整库业务验收通过。
## 开发快照 snapshot/v0.1.100 - 2026-10-28
### Fixed
@@ -116,9 +116,29 @@ type libraryRow struct {
OwnerType string `json:"owner_type"`
OwnerID string `json:"owner_id"`
Content string `json:"content"`
Title string `json:"title"`
ChunkType string `json:"chunk_type"`
ChunkStatus int `json:"-"`
}
func libraryIndexContentMatches(indexContent string, chunk, knowledge libraryRow) bool {
if indexContent == chunk.Content {
return true
}
if chunk.ChunkType != "text" {
return false
}
// v0.7.1 text indexing trims the body and may prepend the title. Only
// reconstruct transformations backed by archived fields. ContextHeader is
// transient upstream; suffix matching would silently accept stale content.
body := strings.TrimSpace(chunk.Content)
if indexContent == body {
return true
}
title := strings.TrimSpace(knowledge.Title)
return title != "" && indexContent == title+"\n"+body
}
func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es SearchSnapshot) error {
required := map[string]bool{"resources": true}
for _, entry := range libraryContentQueries {
@@ -240,7 +260,7 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es
}
seen[document.ID] = true
chunk, ok := tables["chunks"][row.ChunkID]
if !ok || row.KnowledgeID != chunk.KnowledgeID || row.Content != chunk.Content {
if !ok || row.KnowledgeID != chunk.KnowledgeID || !libraryIndexContentMatches(row.Content, chunk, tables["knowledges"][chunk.KnowledgeID]) {
return errors.New("library index does not match database chunks")
}
if !searchHitMatches(document.Source, SearchIdentity{TenantID: identity.TenantID, KnowledgeBaseID: identity.KnowledgeBaseID, KnowledgeID: row.KnowledgeID}) {
@@ -0,0 +1,40 @@
package v071
import (
"encoding/json"
"testing"
)
func TestLibraryCandidateValidatesPersistedTextIndexTransform(t *testing.T) {
for _, tc := range []struct {
name, kind, body, index string
valid bool
}{
{"current title prefix", "text", "\n chunk \t", "Document title\nchunk", true},
{"legacy raw body", "text", "\n chunk \t", "\n chunk \t", true},
{"trimmed body", "text", "\n chunk \t", "chunk", true},
{"wrong title", "text", "chunk", "Other title\nchunk", false},
{"missing body", "text", "chunk", "Document title\n", false},
{"different body", "text", "chunk", "Document title\nchanged", false},
{"unverifiable context header", "text", "chunk", "Document title\nTransient heading\n\nchunk", false},
{"nontext transform not assumed", "faq", "chunk", "Document title\nchunk", false},
} {
t.Run(tc.name, func(t *testing.T) {
pg, es, _ := libraryCandidateFixture()
k, _ := json.Marshal(map[string]any{"id": "doc-safe", "tenant_id": 10000, "knowledge_base_id": "kb-safe", "title": " Document title\n"})
c, _ := json.Marshal(map[string]any{"id": "chunk-safe", "tenant_id": 10000, "knowledge_base_id": "kb-safe", "knowledge_id": "doc-safe", "content": tc.body, "chunk_type": tc.kind, "status": 2})
x, _ := json.Marshal(map[string]any{"knowledge_base_id": "kb-safe", "knowledge_id": "doc-safe", "chunk_id": "chunk-safe", "content": tc.index})
pg.Tables[0] = newTableSnapshot("knowledges", []json.RawMessage{k})
pg.Tables[1] = newTableSnapshot("chunks", []json.RawMessage{c})
es.Documents[0].Source = x
es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents)
err := validateLibraryCandidate(PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}, pg, es)
if tc.valid && err != nil {
t.Fatalf("valid indexed text refused: %v", err)
}
if !tc.valid && err == nil {
t.Fatal("unverifiable index content accepted")
}
})
}
}