refactor: Optimize wiki ingestion process by consolidating entity and concept deduplication

- Introduced a new `WikiBatchContext` to share data across Map and Reduce phases, reducing redundant database queries.
- Replaced separate deduplication calls with a single `deduplicateExtractedBatch` method, improving efficiency by leveraging pre-loaded page data.
- Enhanced JSON handling by implementing a `cleanLLMJSON` function to sanitize LLM-generated output, ensuring safe parsing.
- Updated relevant methods to utilize the new context and deduplication logic, streamlining the overall ingestion workflow.

These changes enhance the performance and clarity of the wiki ingestion service, leading to more efficient content processing.
This commit is contained in:
wizardchen
2026-04-09 18:01:43 +08:00
parent 8dd50de8d4
commit 28781785f7
2 changed files with 61 additions and 60 deletions
+53 -49
View File
@@ -292,8 +292,9 @@ type docIngestResult struct {
Pages []string // affected page slugs
}
// WikiBatchContext holds data needed during the Reduce phase
// WikiBatchContext holds shared data across Map and Reduce phases
type WikiBatchContext struct {
AllPages []*types.WikiPage
SlugTitleMap map[string]string
SummaryContentByKnowledgeID map[string]string
}
@@ -773,96 +774,88 @@ func (s *wikiIngestService) publishDraftPages(ctx context.Context, kbID string,
}
}
// deduplicateItems uses LLM to identify new items that refer to the same entity/concept
// as existing wiki pages, and remaps their slugs so upsertExtractedPages merges them.
func (s *wikiIngestService) deduplicateItems(
// deduplicateExtractedBatch deduplicates both entities and concepts against
// existing wiki pages in a single LLM call. Uses pre-loaded allPages to avoid
// redundant DB queries. This replaces the two separate deduplicateItems calls
// that each queried ListAllPages + made a separate LLM call.
func (s *wikiIngestService) deduplicateExtractedBatch(
ctx context.Context,
chatModel chat.Chat,
items []extractedItem,
pageType string,
kbID string,
) []extractedItem {
if len(items) == 0 {
return items
entities, concepts []extractedItem,
allPages []*types.WikiPage,
) ([]extractedItem, []extractedItem) {
if len(entities) == 0 && len(concepts) == 0 {
return entities, concepts
}
// Get existing pages of the same type
allPages, err := s.wikiService.ListAllPages(ctx, kbID)
if err != nil || len(allPages) == 0 {
return items
}
// Filter to the target type
var typedPages []*types.WikiPage
for _, p := range allPages {
if p.PageType == pageType {
typedPages = append(typedPages, p)
}
}
if len(typedPages) == 0 {
return items // No existing pages of this type → nothing to deduplicate against
}
// Build existing pages listing
var existingBuf strings.Builder
for _, p := range typedPages {
for _, p := range allPages {
if p.PageType != types.WikiPageTypeEntity && p.PageType != types.WikiPageTypeConcept {
continue
}
aliases := ""
if len(p.Aliases) > 0 {
aliases = fmt.Sprintf(" | aliases: %s", strings.Join(p.Aliases, ", "))
}
fmt.Fprintf(&existingBuf, "- slug: %s | title: %s%s\n", p.Slug, p.Title, aliases)
fmt.Fprintf(&existingBuf, "- slug: %s | title: %s | type: %s%s\n", p.Slug, p.Title, p.PageType, aliases)
}
if existingBuf.Len() == 0 {
return entities, concepts
}
// Build new items listing
var newBuf strings.Builder
for _, item := range items {
for _, item := range entities {
aliases := ""
if len(item.Aliases) > 0 {
aliases = fmt.Sprintf(" | aliases: %s", strings.Join(item.Aliases, ", "))
}
fmt.Fprintf(&newBuf, "- slug: %s | name: %s%s\n", item.Slug, item.Name, aliases)
fmt.Fprintf(&newBuf, "- slug: %s | name: %s | type: entity%s\n", item.Slug, item.Name, aliases)
}
for _, item := range concepts {
aliases := ""
if len(item.Aliases) > 0 {
aliases = fmt.Sprintf(" | aliases: %s", strings.Join(item.Aliases, ", "))
}
fmt.Fprintf(&newBuf, "- slug: %s | name: %s | type: concept%s\n", item.Slug, item.Name, aliases)
}
// Call LLM for deduplication
dedupeJSON, err := s.generateWithTemplate(ctx, chatModel, agent.WikiDeduplicationPrompt, map[string]string{
"NewItems": newBuf.String(),
"ExistingPages": existingBuf.String(),
})
if err != nil {
logger.Warnf(ctx, "wiki ingest: deduplication LLM call failed: %v", err)
return items
return entities, concepts
}
// Parse response
dedupeJSON = strings.TrimSpace(dedupeJSON)
dedupeJSON = strings.TrimPrefix(dedupeJSON, "```json")
dedupeJSON = strings.TrimPrefix(dedupeJSON, "```")
dedupeJSON = strings.TrimSuffix(dedupeJSON, "```")
dedupeJSON = strings.TrimSpace(dedupeJSON)
dedupeJSON = sanitizeJSONString(dedupeJSON)
dedupeJSON = cleanLLMJSON(dedupeJSON)
var dedupeResult struct {
Merges map[string]string `json:"merges"`
}
if err := json.Unmarshal([]byte(dedupeJSON), &dedupeResult); err != nil {
logger.Warnf(ctx, "wiki ingest: failed to parse dedup JSON: %v\nRaw: %s", err, dedupeJSON)
return items
return entities, concepts
}
if len(dedupeResult.Merges) == 0 {
return items
return entities, concepts
}
// Remap slugs for matched items
for i, item := range items {
for i, item := range entities {
if existingSlug, ok := dedupeResult.Merges[item.Slug]; ok {
logger.Infof(ctx, "wiki ingest: dedup merge %s → %s", item.Slug, existingSlug)
items[i].Slug = existingSlug
entities[i].Slug = existingSlug
}
}
for i, item := range concepts {
if existingSlug, ok := dedupeResult.Merges[item.Slug]; ok {
logger.Infof(ctx, "wiki ingest: dedup merge %s → %s", item.Slug, existingSlug)
concepts[i].Slug = existingSlug
}
}
return items
return entities, concepts
}
// generateWithTemplate executes a prompt template and calls the LLM
@@ -1035,6 +1028,17 @@ func appendUnique(arr types.StringArray, s string) types.StringArray {
return append(arr, s)
}
// cleanLLMJSON strips markdown code-fence wrappers and sanitizes control characters
// from LLM-generated JSON output so it can be safely unmarshalled.
func cleanLLMJSON(s string) string {
s = strings.TrimSpace(s)
s = strings.TrimPrefix(s, "```json")
s = strings.TrimPrefix(s, "```")
s = strings.TrimSuffix(s, "```")
s = strings.TrimSpace(s)
return sanitizeJSONString(s)
}
// sanitizeJSONString sanitizes a string that is intended to be parsed as JSON,
// by properly escaping unescaped control characters (like newlines) inside string literals.
func sanitizeJSONString(s string) string {
@@ -187,9 +187,10 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
logger.Infof(ctx, "wiki ingest: batch processing %d ops for KB %s", len(pendingOps), payload.KnowledgeBaseID)
// Fetch all existing pages to pass to the Map-Reduce phases
// Fetch all existing pages once, shared across Map and Reduce phases
allPages, _ := s.wikiService.ListAllPages(ctx, payload.KnowledgeBaseID)
batchCtx := &WikiBatchContext{
AllPages: allPages,
SlugTitleMap: make(map[string]string),
SummaryContentByKnowledgeID: make(map[string]string),
}
@@ -411,7 +412,7 @@ func (s *wikiIngestService) mapOneDocument(
oldPageSlugs := s.getExistingPageSlugsForKnowledge(ctx, payload.KnowledgeBaseID, knowledgeID)
logger.Infof(ctx, "wiki ingest: extracting entities and concepts for %s", knowledgeID)
extractedEntities, extractedConcepts, slugItems, err := s.extractEntitiesAndConceptsNoUpsert(ctx, chatModel, content, docTitle, lang, payload, oldPageSlugs)
extractedEntities, extractedConcepts, slugItems, err := s.extractEntitiesAndConceptsNoUpsert(ctx, chatModel, content, docTitle, lang, oldPageSlugs, batchCtx)
if err != nil {
logger.Warnf(ctx, "wiki ingest: knowledge extraction failed for %s: %v", knowledgeID, err)
return nil, nil, err
@@ -538,8 +539,8 @@ func (s *wikiIngestService) extractEntitiesAndConceptsNoUpsert(
ctx context.Context,
chatModel chat.Chat,
content, docTitle, lang string,
payload WikiIngestPayload,
oldPageSlugs map[string]bool,
batchCtx *WikiBatchContext,
) ([]extractedItem, []extractedItem, map[string]extractedItem, error) {
var prevSlugsText string
if len(oldPageSlugs) > 0 {
@@ -562,12 +563,7 @@ func (s *wikiIngestService) extractEntitiesAndConceptsNoUpsert(
return nil, nil, nil, fmt.Errorf("combined extraction failed: %w", err)
}
extractionJSON = strings.TrimSpace(extractionJSON)
extractionJSON = strings.TrimPrefix(extractionJSON, "```json")
extractionJSON = strings.TrimPrefix(extractionJSON, "```")
extractionJSON = strings.TrimSuffix(extractionJSON, "```")
extractionJSON = strings.TrimSpace(extractionJSON)
extractionJSON = sanitizeJSONString(extractionJSON)
extractionJSON = cleanLLMJSON(extractionJSON)
var result combinedExtraction
if err := json.Unmarshal([]byte(extractionJSON), &result); err != nil {
@@ -575,8 +571,9 @@ func (s *wikiIngestService) extractEntitiesAndConceptsNoUpsert(
return nil, nil, nil, fmt.Errorf("parse combined extraction JSON: %w", err)
}
result.Entities = s.deduplicateItems(ctx, chatModel, result.Entities, types.WikiPageTypeEntity, payload.KnowledgeBaseID)
result.Concepts = s.deduplicateItems(ctx, chatModel, result.Concepts, types.WikiPageTypeConcept, payload.KnowledgeBaseID)
result.Entities, result.Concepts = s.deduplicateExtractedBatch(
ctx, chatModel, result.Entities, result.Concepts, batchCtx.AllPages,
)
slugItems := make(map[string]extractedItem)
for _, item := range result.Entities {