fix(snapshot): 阻止已索引分块缺失时封存整库
build / build (push) Successful in 1m12s

This commit is contained in:
姚章浩
2026-10-28 00:43:31 +08:00
parent 2c5ecdf0f3
commit ce2f90c1c5
3 changed files with 89 additions and 0 deletions
+7
View File
@@ -1,5 +1,12 @@
# Changelog
## 开发快照 snapshot/v0.1.98 - 2026-10-28
### Fixed
- 修复整库候选封存只单向校验搜索索引的问题:数据库标记为已索引的分块必须在导出索引中存在,否则拒绝封存;按分块身份而非总数量核对,避免重复索引掩盖缺失。仅存储的父分块仍允许不建索引。
- 本修复增强候选完整性,不代表整库业务入口、生产同步恢复或语义验收已经完成。
## 开发快照 snapshot/v0.1.97 - 2026-10-28
### Fixed
@@ -116,6 +116,7 @@ type libraryRow struct {
OwnerType string `json:"owner_type"`
OwnerID string `json:"owner_id"`
Content string `json:"content"`
Status int `json:"status"`
}
func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es SearchSnapshot) error {
@@ -220,6 +221,7 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es
return errors.New("invalid library index digest")
}
seen := map[string]bool{}
indexedChunks := map[string]bool{}
for _, document := range es.Documents {
var row libraryRow
if json.Unmarshal(document.Source, &row) != nil || !validSearchIdentifier(document.ID) || seen[document.ID] {
@@ -233,6 +235,15 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es
if !searchHitMatches(document.Source, SearchIdentity{TenantID: identity.TenantID, KnowledgeBaseID: identity.KnowledgeBaseID, KnowledgeID: row.KnowledgeID}) {
return errors.New("cross-scope library index")
}
indexedChunks[row.ChunkID] = true
}
// v0.7.1 ChunkStatusIndexed is 2. One chunk can have several search
// records, so compare identities rather than counts. Stored-only parent
// chunks intentionally have no index and must not be rejected as missing.
for id, chunk := range tables["chunks"] {
if chunk.Status == 2 && !indexedChunks[id] {
return errors.New("missing indexed library chunk")
}
}
return nil
}
@@ -0,0 +1,71 @@
package v071
import (
"context"
"encoding/json"
"os"
"strings"
"testing"
"time"
"knowledge-governance/internal/librarysnapshot"
)
func TestLibraryCandidateIndexCompletenessUsesChunkIdentity(t *testing.T) {
for _, mode := range []string{"stored-parent", "multiple-records-one-chunk", "same-count-missing-chunk"} {
t.Run(mode, func(t *testing.T) {
pg, es, _ := libraryCandidateFixture()
rows := []json.RawMessage{json.RawMessage(`{"id":"chunk-safe","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","content":"chunk","status":2,"chunk_type":"text"}`)}
if mode == "stored-parent" {
rows = append(rows, json.RawMessage(`{"id":"parent-safe","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","content":"context","status":1,"chunk_type":"parent_text"}`))
} else {
es.Documents = append(es.Documents, SearchDocument{ID: "es-second", Source: append(json.RawMessage(nil), es.Documents[0].Source...)})
if mode == "same-count-missing-chunk" {
rows = append(rows, json.RawMessage(`{"id":"chunk-second","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","content":"second","status":2,"chunk_type":"text"}`))
}
}
pg.Tables[1] = newTableSnapshot("chunks", rows)
es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents)
err := validateLibraryCandidate(PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}, pg, es)
if mode == "same-count-missing-chunk" {
if err == nil || !strings.Contains(err.Error(), "missing indexed library chunk") {
t.Fatalf("same counts masked a missing chunk: %v", err)
}
} else if err != nil {
t.Fatalf("valid index shape rejected: %v", err)
}
})
}
}
func TestLibraryCandidateRejectsMissingIndexedChunk(t *testing.T) {
pg, es, objects := libraryCandidateFixture()
pg.Tables[1] = newTableSnapshot("chunks", []json.RawMessage{json.RawMessage(`{"id":"chunk-safe","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","content":"chunk","status":2,"chunk_type":"text"}`)})
identity := PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}
if err := validateLibraryCandidate(identity, pg, es); err != nil {
t.Fatalf("complete indexed chunk rejected: %v", err)
}
es.Documents = nil
es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents)
root := t.TempDir()
if err := os.Chmod(root, 0700); err != nil {
t.Fatal(err)
}
archive, err := librarysnapshot.Open(root, 1024*1024)
if err != nil {
t.Fatal(err)
}
defer archive.Close()
contract := librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74, RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "fixture-model", EmbeddingDimension: 1024, IndexMappingSHA256: es.MappingSHA256}
id, err := ArchiveLibraryCandidate(context.Background(), archive, "internal", identity, contract, pg, es, objects, time.Now().UTC())
if err == nil || id != "" {
t.Fatal("sealed a supposedly complete library with an indexed PostgreSQL chunk missing from Elasticsearch")
}
if !strings.Contains(err.Error(), "missing indexed library chunk") {
t.Fatalf("unexpected refusal: %v", err)
}
versions, err := os.ReadDir(root + "/versions")
if err != nil || len(versions) != 0 {
t.Fatalf("incomplete candidate became visible: versions=%d err=%v", len(versions), err)
}
}