|
|
|
@@ -0,0 +1,71 @@
|
|
|
|
|
package v071
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
"context"
|
|
|
|
|
"encoding/json"
|
|
|
|
|
"os"
|
|
|
|
|
"strings"
|
|
|
|
|
"testing"
|
|
|
|
|
"time"
|
|
|
|
|
|
|
|
|
|
"knowledge-governance/internal/librarysnapshot"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
func TestLibraryCandidateIndexCompletenessUsesChunkIdentity(t *testing.T) {
|
|
|
|
|
for _, mode := range []string{"stored-parent", "multiple-records-one-chunk", "same-count-missing-chunk"} {
|
|
|
|
|
t.Run(mode, func(t *testing.T) {
|
|
|
|
|
pg, es, _ := libraryCandidateFixture()
|
|
|
|
|
rows := []json.RawMessage{json.RawMessage(`{"id":"chunk-safe","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","content":"chunk","status":2,"chunk_type":"text"}`)}
|
|
|
|
|
if mode == "stored-parent" {
|
|
|
|
|
rows = append(rows, json.RawMessage(`{"id":"parent-safe","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","content":"context","status":1,"chunk_type":"parent_text"}`))
|
|
|
|
|
} else {
|
|
|
|
|
es.Documents = append(es.Documents, SearchDocument{ID: "es-second", Source: append(json.RawMessage(nil), es.Documents[0].Source...)})
|
|
|
|
|
if mode == "same-count-missing-chunk" {
|
|
|
|
|
rows = append(rows, json.RawMessage(`{"id":"chunk-second","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","content":"second","status":2,"chunk_type":"text"}`))
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
pg.Tables[1] = newTableSnapshot("chunks", rows)
|
|
|
|
|
es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents)
|
|
|
|
|
err := validateLibraryCandidate(PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}, pg, es)
|
|
|
|
|
if mode == "same-count-missing-chunk" {
|
|
|
|
|
if err == nil || !strings.Contains(err.Error(), "missing indexed library chunk") {
|
|
|
|
|
t.Fatalf("same counts masked a missing chunk: %v", err)
|
|
|
|
|
}
|
|
|
|
|
} else if err != nil {
|
|
|
|
|
t.Fatalf("valid index shape rejected: %v", err)
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func TestLibraryCandidateRejectsMissingIndexedChunk(t *testing.T) {
|
|
|
|
|
pg, es, objects := libraryCandidateFixture()
|
|
|
|
|
pg.Tables[1] = newTableSnapshot("chunks", []json.RawMessage{json.RawMessage(`{"id":"chunk-safe","tenant_id":10000,"knowledge_base_id":"kb-safe","knowledge_id":"doc-safe","content":"chunk","status":2,"chunk_type":"text"}`)})
|
|
|
|
|
identity := PostgresIdentity{TenantID: 10000, KnowledgeBaseID: "kb-safe"}
|
|
|
|
|
if err := validateLibraryCandidate(identity, pg, es); err != nil {
|
|
|
|
|
t.Fatalf("complete indexed chunk rejected: %v", err)
|
|
|
|
|
}
|
|
|
|
|
es.Documents = nil
|
|
|
|
|
es.DocumentsSHA256, _ = searchDocumentsDigest(es.Documents)
|
|
|
|
|
root := t.TempDir()
|
|
|
|
|
if err := os.Chmod(root, 0700); err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
archive, err := librarysnapshot.Open(root, 1024*1024)
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
}
|
|
|
|
|
defer archive.Close()
|
|
|
|
|
contract := librarysnapshot.Compatibility{WeKnoraVersion: "v0.7.1", SchemaVersion: 74, RetrieverEngine: "elasticsearch_v8", EmbeddingModel: "fixture-model", EmbeddingDimension: 1024, IndexMappingSHA256: es.MappingSHA256}
|
|
|
|
|
id, err := ArchiveLibraryCandidate(context.Background(), archive, "internal", identity, contract, pg, es, objects, time.Now().UTC())
|
|
|
|
|
if err == nil || id != "" {
|
|
|
|
|
t.Fatal("sealed a supposedly complete library with an indexed PostgreSQL chunk missing from Elasticsearch")
|
|
|
|
|
}
|
|
|
|
|
if !strings.Contains(err.Error(), "missing indexed library chunk") {
|
|
|
|
|
t.Fatalf("unexpected refusal: %v", err)
|
|
|
|
|
}
|
|
|
|
|
versions, err := os.ReadDir(root + "/versions")
|
|
|
|
|
if err != nil || len(versions) != 0 {
|
|
|
|
|
t.Fatalf("incomplete candidate became visible: versions=%d err=%v", len(versions), err)
|
|
|
|
|
}
|
|
|
|
|
}
|