|
|
|
@@ -105,20 +105,22 @@ func ArchiveLibraryCandidate(ctx context.Context, archive *librarysnapshot.Archi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type libraryRow struct {
|
|
|
|
|
ID string `json:"id"`
|
|
|
|
|
TenantID uint64 `json:"tenant_id"`
|
|
|
|
|
KnowledgeBaseID string `json:"knowledge_base_id"`
|
|
|
|
|
KnowledgeID string `json:"knowledge_id"`
|
|
|
|
|
ChunkID string `json:"chunk_id"`
|
|
|
|
|
TagID string `json:"tag_id"`
|
|
|
|
|
ResourceID string `json:"resource_id"`
|
|
|
|
|
Handle string `json:"handle"`
|
|
|
|
|
OwnerType string `json:"owner_type"`
|
|
|
|
|
OwnerID string `json:"owner_id"`
|
|
|
|
|
Content string `json:"content"`
|
|
|
|
|
Title string `json:"title"`
|
|
|
|
|
ChunkType string `json:"chunk_type"`
|
|
|
|
|
ChunkStatus int `json:"-"`
|
|
|
|
|
ID string `json:"id"`
|
|
|
|
|
TenantID uint64 `json:"tenant_id"`
|
|
|
|
|
KnowledgeBaseID string `json:"knowledge_base_id"`
|
|
|
|
|
KnowledgeID string `json:"knowledge_id"`
|
|
|
|
|
ChunkID string `json:"chunk_id"`
|
|
|
|
|
TagID string `json:"tag_id"`
|
|
|
|
|
ResourceID string `json:"resource_id"`
|
|
|
|
|
Handle string `json:"handle"`
|
|
|
|
|
OwnerType string `json:"owner_type"`
|
|
|
|
|
OwnerID string `json:"owner_id"`
|
|
|
|
|
Content string `json:"content"`
|
|
|
|
|
Title string `json:"title"`
|
|
|
|
|
ChunkType string `json:"chunk_type"`
|
|
|
|
|
ChunkStatus int `json:"-"`
|
|
|
|
|
SourceID string `json:"source_id"`
|
|
|
|
|
Questions map[string]string `json:"-"`
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func libraryIndexContentMatches(indexContent string, chunk, knowledge libraryRow) bool {
|
|
|
|
@@ -176,12 +178,26 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es
|
|
|
|
|
// use strings. Decode chunk metadata only for its owning table.
|
|
|
|
|
if table.Name == "chunks" {
|
|
|
|
|
var metadata struct {
|
|
|
|
|
Status int `json:"status"`
|
|
|
|
|
Status int `json:"status"`
|
|
|
|
|
Metadata struct {
|
|
|
|
|
GeneratedQuestions []struct {
|
|
|
|
|
ID string `json:"id"`
|
|
|
|
|
Question string `json:"question"`
|
|
|
|
|
} `json:"generated_questions"`
|
|
|
|
|
} `json:"metadata"`
|
|
|
|
|
}
|
|
|
|
|
if json.Unmarshal(raw, &metadata) != nil {
|
|
|
|
|
return errors.New("invalid library chunk status")
|
|
|
|
|
}
|
|
|
|
|
row.ChunkStatus = metadata.Status
|
|
|
|
|
row.Questions = map[string]string{}
|
|
|
|
|
for _, question := range metadata.Metadata.GeneratedQuestions {
|
|
|
|
|
sourceID := row.ID + "-" + question.ID
|
|
|
|
|
if row.ChunkType != "text" || !validSearchIdentifier(question.ID) || strings.TrimSpace(question.Question) == "" || row.Questions[sourceID] != "" {
|
|
|
|
|
return errors.New("invalid library generated question metadata")
|
|
|
|
|
}
|
|
|
|
|
row.Questions[sourceID] = question.Question
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if table.Name == "knowledge_tag_relations" {
|
|
|
|
|
if !validSearchIdentifier(row.KnowledgeID) || !validSearchIdentifier(row.TagID) {
|
|
|
|
@@ -253,6 +269,7 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es
|
|
|
|
|
}
|
|
|
|
|
seen := map[string]bool{}
|
|
|
|
|
indexedChunks := map[string]bool{}
|
|
|
|
|
indexedQuestions := map[string]map[string]bool{}
|
|
|
|
|
for _, document := range es.Documents {
|
|
|
|
|
var row libraryRow
|
|
|
|
|
if json.Unmarshal(document.Source, &row) != nil || !validSearchIdentifier(document.ID) || seen[document.ID] {
|
|
|
|
@@ -260,13 +277,27 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es
|
|
|
|
|
}
|
|
|
|
|
seen[document.ID] = true
|
|
|
|
|
chunk, ok := tables["chunks"][row.ChunkID]
|
|
|
|
|
if !ok || row.KnowledgeID != chunk.KnowledgeID || !libraryIndexContentMatches(row.Content, chunk, tables["knowledges"][chunk.KnowledgeID]) {
|
|
|
|
|
if !ok || row.KnowledgeID != chunk.KnowledgeID {
|
|
|
|
|
return errors.New("library index does not match database chunks")
|
|
|
|
|
}
|
|
|
|
|
if !searchHitMatches(document.Source, SearchIdentity{TenantID: identity.TenantID, KnowledgeBaseID: identity.KnowledgeBaseID, KnowledgeID: row.KnowledgeID}) {
|
|
|
|
|
return errors.New("cross-scope library index")
|
|
|
|
|
}
|
|
|
|
|
indexedChunks[row.ChunkID] = true
|
|
|
|
|
if row.SourceID != "" && row.SourceID != row.ChunkID {
|
|
|
|
|
question, exists := chunk.Questions[row.SourceID]
|
|
|
|
|
if !exists || question != row.Content {
|
|
|
|
|
return errors.New("library question index does not match persisted metadata")
|
|
|
|
|
}
|
|
|
|
|
if indexedQuestions[row.ChunkID] == nil {
|
|
|
|
|
indexedQuestions[row.ChunkID] = map[string]bool{}
|
|
|
|
|
}
|
|
|
|
|
indexedQuestions[row.ChunkID][row.SourceID] = true
|
|
|
|
|
} else {
|
|
|
|
|
if !libraryIndexContentMatches(row.Content, chunk, tables["knowledges"][chunk.KnowledgeID]) {
|
|
|
|
|
return errors.New("library index does not match database chunks")
|
|
|
|
|
}
|
|
|
|
|
indexedChunks[row.ChunkID] = true
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
// v0.7.1 ChunkStatusIndexed is 2. One chunk can have several search
|
|
|
|
|
// records, so compare identities rather than counts. Stored-only parent
|
|
|
|
@@ -275,6 +306,11 @@ func validateLibraryCandidate(identity PostgresIdentity, pg PostgresSnapshot, es
|
|
|
|
|
// zero. Without an archived no-index policy, absence is unverified, not
|
|
|
|
|
// proof that indexing was disabled. Parent chunks are context-only.
|
|
|
|
|
for id, chunk := range tables["chunks"] {
|
|
|
|
|
for sourceID := range chunk.Questions {
|
|
|
|
|
if !indexedQuestions[id][sourceID] {
|
|
|
|
|
return errors.New("missing library generated question index")
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if chunk.ChunkStatus == 0 && chunk.ChunkType != "parent_text" && !indexedChunks[id] {
|
|
|
|
|
return errors.New("unverified legacy library chunk index coverage")
|
|
|
|
|
}
|
|
|
|
|