refactor: Optimize wiki ingestion process by consolidating entity and concept deduplication
- Introduced a new `WikiBatchContext` to share data across Map and Reduce phases, reducing redundant database queries. - Replaced separate deduplication calls with a single `deduplicateExtractedBatch` method, improving efficiency by leveraging pre-loaded page data. - Enhanced JSON handling by implementing a `cleanLLMJSON` function to sanitize LLM-generated output, ensuring safe parsing. - Updated relevant methods to utilize the new context and deduplication logic, streamlining the overall ingestion workflow. These changes enhance the performance and clarity of the wiki ingestion service, leading to more efficient content processing.
This commit is contained in:
@@ -292,8 +292,9 @@ type docIngestResult struct {
|
||||
Pages []string // affected page slugs
|
||||
}
|
||||
|
||||
// WikiBatchContext holds data needed during the Reduce phase
|
||||
// WikiBatchContext holds shared data across Map and Reduce phases
|
||||
type WikiBatchContext struct {
|
||||
AllPages []*types.WikiPage
|
||||
SlugTitleMap map[string]string
|
||||
SummaryContentByKnowledgeID map[string]string
|
||||
}
|
||||
@@ -773,96 +774,88 @@ func (s *wikiIngestService) publishDraftPages(ctx context.Context, kbID string,
|
||||
}
|
||||
}
|
||||
|
||||
// deduplicateItems uses LLM to identify new items that refer to the same entity/concept
|
||||
// as existing wiki pages, and remaps their slugs so upsertExtractedPages merges them.
|
||||
func (s *wikiIngestService) deduplicateItems(
|
||||
// deduplicateExtractedBatch deduplicates both entities and concepts against
|
||||
// existing wiki pages in a single LLM call. Uses pre-loaded allPages to avoid
|
||||
// redundant DB queries. This replaces the two separate deduplicateItems calls
|
||||
// that each queried ListAllPages + made a separate LLM call.
|
||||
func (s *wikiIngestService) deduplicateExtractedBatch(
|
||||
ctx context.Context,
|
||||
chatModel chat.Chat,
|
||||
items []extractedItem,
|
||||
pageType string,
|
||||
kbID string,
|
||||
) []extractedItem {
|
||||
if len(items) == 0 {
|
||||
return items
|
||||
entities, concepts []extractedItem,
|
||||
allPages []*types.WikiPage,
|
||||
) ([]extractedItem, []extractedItem) {
|
||||
if len(entities) == 0 && len(concepts) == 0 {
|
||||
return entities, concepts
|
||||
}
|
||||
|
||||
// Get existing pages of the same type
|
||||
allPages, err := s.wikiService.ListAllPages(ctx, kbID)
|
||||
if err != nil || len(allPages) == 0 {
|
||||
return items
|
||||
}
|
||||
|
||||
// Filter to the target type
|
||||
var typedPages []*types.WikiPage
|
||||
for _, p := range allPages {
|
||||
if p.PageType == pageType {
|
||||
typedPages = append(typedPages, p)
|
||||
}
|
||||
}
|
||||
if len(typedPages) == 0 {
|
||||
return items // No existing pages of this type → nothing to deduplicate against
|
||||
}
|
||||
|
||||
// Build existing pages listing
|
||||
var existingBuf strings.Builder
|
||||
for _, p := range typedPages {
|
||||
for _, p := range allPages {
|
||||
if p.PageType != types.WikiPageTypeEntity && p.PageType != types.WikiPageTypeConcept {
|
||||
continue
|
||||
}
|
||||
aliases := ""
|
||||
if len(p.Aliases) > 0 {
|
||||
aliases = fmt.Sprintf(" | aliases: %s", strings.Join(p.Aliases, ", "))
|
||||
}
|
||||
fmt.Fprintf(&existingBuf, "- slug: %s | title: %s%s\n", p.Slug, p.Title, aliases)
|
||||
fmt.Fprintf(&existingBuf, "- slug: %s | title: %s | type: %s%s\n", p.Slug, p.Title, p.PageType, aliases)
|
||||
}
|
||||
if existingBuf.Len() == 0 {
|
||||
return entities, concepts
|
||||
}
|
||||
|
||||
// Build new items listing
|
||||
var newBuf strings.Builder
|
||||
for _, item := range items {
|
||||
for _, item := range entities {
|
||||
aliases := ""
|
||||
if len(item.Aliases) > 0 {
|
||||
aliases = fmt.Sprintf(" | aliases: %s", strings.Join(item.Aliases, ", "))
|
||||
}
|
||||
fmt.Fprintf(&newBuf, "- slug: %s | name: %s%s\n", item.Slug, item.Name, aliases)
|
||||
fmt.Fprintf(&newBuf, "- slug: %s | name: %s | type: entity%s\n", item.Slug, item.Name, aliases)
|
||||
}
|
||||
for _, item := range concepts {
|
||||
aliases := ""
|
||||
if len(item.Aliases) > 0 {
|
||||
aliases = fmt.Sprintf(" | aliases: %s", strings.Join(item.Aliases, ", "))
|
||||
}
|
||||
fmt.Fprintf(&newBuf, "- slug: %s | name: %s | type: concept%s\n", item.Slug, item.Name, aliases)
|
||||
}
|
||||
|
||||
// Call LLM for deduplication
|
||||
dedupeJSON, err := s.generateWithTemplate(ctx, chatModel, agent.WikiDeduplicationPrompt, map[string]string{
|
||||
"NewItems": newBuf.String(),
|
||||
"ExistingPages": existingBuf.String(),
|
||||
})
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: deduplication LLM call failed: %v", err)
|
||||
return items
|
||||
return entities, concepts
|
||||
}
|
||||
|
||||
// Parse response
|
||||
dedupeJSON = strings.TrimSpace(dedupeJSON)
|
||||
dedupeJSON = strings.TrimPrefix(dedupeJSON, "```json")
|
||||
dedupeJSON = strings.TrimPrefix(dedupeJSON, "```")
|
||||
dedupeJSON = strings.TrimSuffix(dedupeJSON, "```")
|
||||
dedupeJSON = strings.TrimSpace(dedupeJSON)
|
||||
|
||||
dedupeJSON = sanitizeJSONString(dedupeJSON)
|
||||
dedupeJSON = cleanLLMJSON(dedupeJSON)
|
||||
|
||||
var dedupeResult struct {
|
||||
Merges map[string]string `json:"merges"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(dedupeJSON), &dedupeResult); err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: failed to parse dedup JSON: %v\nRaw: %s", err, dedupeJSON)
|
||||
return items
|
||||
return entities, concepts
|
||||
}
|
||||
|
||||
if len(dedupeResult.Merges) == 0 {
|
||||
return items
|
||||
return entities, concepts
|
||||
}
|
||||
|
||||
// Remap slugs for matched items
|
||||
for i, item := range items {
|
||||
for i, item := range entities {
|
||||
if existingSlug, ok := dedupeResult.Merges[item.Slug]; ok {
|
||||
logger.Infof(ctx, "wiki ingest: dedup merge %s → %s", item.Slug, existingSlug)
|
||||
items[i].Slug = existingSlug
|
||||
entities[i].Slug = existingSlug
|
||||
}
|
||||
}
|
||||
for i, item := range concepts {
|
||||
if existingSlug, ok := dedupeResult.Merges[item.Slug]; ok {
|
||||
logger.Infof(ctx, "wiki ingest: dedup merge %s → %s", item.Slug, existingSlug)
|
||||
concepts[i].Slug = existingSlug
|
||||
}
|
||||
}
|
||||
|
||||
return items
|
||||
return entities, concepts
|
||||
}
|
||||
|
||||
// generateWithTemplate executes a prompt template and calls the LLM
|
||||
@@ -1035,6 +1028,17 @@ func appendUnique(arr types.StringArray, s string) types.StringArray {
|
||||
return append(arr, s)
|
||||
}
|
||||
|
||||
// cleanLLMJSON strips markdown code-fence wrappers and sanitizes control characters
|
||||
// from LLM-generated JSON output so it can be safely unmarshalled.
|
||||
func cleanLLMJSON(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
s = strings.TrimPrefix(s, "```json")
|
||||
s = strings.TrimPrefix(s, "```")
|
||||
s = strings.TrimSuffix(s, "```")
|
||||
s = strings.TrimSpace(s)
|
||||
return sanitizeJSONString(s)
|
||||
}
|
||||
|
||||
// sanitizeJSONString sanitizes a string that is intended to be parsed as JSON,
|
||||
// by properly escaping unescaped control characters (like newlines) inside string literals.
|
||||
func sanitizeJSONString(s string) string {
|
||||
|
||||
@@ -187,9 +187,10 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
|
||||
logger.Infof(ctx, "wiki ingest: batch processing %d ops for KB %s", len(pendingOps), payload.KnowledgeBaseID)
|
||||
|
||||
// Fetch all existing pages to pass to the Map-Reduce phases
|
||||
// Fetch all existing pages once, shared across Map and Reduce phases
|
||||
allPages, _ := s.wikiService.ListAllPages(ctx, payload.KnowledgeBaseID)
|
||||
batchCtx := &WikiBatchContext{
|
||||
AllPages: allPages,
|
||||
SlugTitleMap: make(map[string]string),
|
||||
SummaryContentByKnowledgeID: make(map[string]string),
|
||||
}
|
||||
@@ -411,7 +412,7 @@ func (s *wikiIngestService) mapOneDocument(
|
||||
oldPageSlugs := s.getExistingPageSlugsForKnowledge(ctx, payload.KnowledgeBaseID, knowledgeID)
|
||||
|
||||
logger.Infof(ctx, "wiki ingest: extracting entities and concepts for %s", knowledgeID)
|
||||
extractedEntities, extractedConcepts, slugItems, err := s.extractEntitiesAndConceptsNoUpsert(ctx, chatModel, content, docTitle, lang, payload, oldPageSlugs)
|
||||
extractedEntities, extractedConcepts, slugItems, err := s.extractEntitiesAndConceptsNoUpsert(ctx, chatModel, content, docTitle, lang, oldPageSlugs, batchCtx)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: knowledge extraction failed for %s: %v", knowledgeID, err)
|
||||
return nil, nil, err
|
||||
@@ -538,8 +539,8 @@ func (s *wikiIngestService) extractEntitiesAndConceptsNoUpsert(
|
||||
ctx context.Context,
|
||||
chatModel chat.Chat,
|
||||
content, docTitle, lang string,
|
||||
payload WikiIngestPayload,
|
||||
oldPageSlugs map[string]bool,
|
||||
batchCtx *WikiBatchContext,
|
||||
) ([]extractedItem, []extractedItem, map[string]extractedItem, error) {
|
||||
var prevSlugsText string
|
||||
if len(oldPageSlugs) > 0 {
|
||||
@@ -562,12 +563,7 @@ func (s *wikiIngestService) extractEntitiesAndConceptsNoUpsert(
|
||||
return nil, nil, nil, fmt.Errorf("combined extraction failed: %w", err)
|
||||
}
|
||||
|
||||
extractionJSON = strings.TrimSpace(extractionJSON)
|
||||
extractionJSON = strings.TrimPrefix(extractionJSON, "```json")
|
||||
extractionJSON = strings.TrimPrefix(extractionJSON, "```")
|
||||
extractionJSON = strings.TrimSuffix(extractionJSON, "```")
|
||||
extractionJSON = strings.TrimSpace(extractionJSON)
|
||||
extractionJSON = sanitizeJSONString(extractionJSON)
|
||||
extractionJSON = cleanLLMJSON(extractionJSON)
|
||||
|
||||
var result combinedExtraction
|
||||
if err := json.Unmarshal([]byte(extractionJSON), &result); err != nil {
|
||||
@@ -575,8 +571,9 @@ func (s *wikiIngestService) extractEntitiesAndConceptsNoUpsert(
|
||||
return nil, nil, nil, fmt.Errorf("parse combined extraction JSON: %w", err)
|
||||
}
|
||||
|
||||
result.Entities = s.deduplicateItems(ctx, chatModel, result.Entities, types.WikiPageTypeEntity, payload.KnowledgeBaseID)
|
||||
result.Concepts = s.deduplicateItems(ctx, chatModel, result.Concepts, types.WikiPageTypeConcept, payload.KnowledgeBaseID)
|
||||
result.Entities, result.Concepts = s.deduplicateExtractedBatch(
|
||||
ctx, chatModel, result.Entities, result.Concepts, batchCtx.AllPages,
|
||||
)
|
||||
|
||||
slugItems := make(map[string]extractedItem)
|
||||
for _, item := range result.Entities {
|
||||
|
||||
Reference in New Issue
Block a user