From 28781785f72fec9ac0ce920f3d97b2b072e815c5 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Thu, 9 Apr 2026 18:01:43 +0800 Subject: [PATCH] refactor: Optimize wiki ingestion process by consolidating entity and concept deduplication - Introduced a new `WikiBatchContext` to share data across Map and Reduce phases, reducing redundant database queries. - Replaced separate deduplication calls with a single `deduplicateExtractedBatch` method, improving efficiency by leveraging pre-loaded page data. - Enhanced JSON handling by implementing a `cleanLLMJSON` function to sanitize LLM-generated output, ensuring safe parsing. - Updated relevant methods to utilize the new context and deduplication logic, streamlining the overall ingestion workflow. These changes enhance the performance and clarity of the wiki ingestion service, leading to more efficient content processing. --- internal/application/service/wiki_ingest.go | 102 +++++++++--------- .../application/service/wiki_ingest_batch.go | 19 ++-- 2 files changed, 61 insertions(+), 60 deletions(-) diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go index 95132d73..74af5e1c 100644 --- a/internal/application/service/wiki_ingest.go +++ b/internal/application/service/wiki_ingest.go @@ -292,8 +292,9 @@ type docIngestResult struct { Pages []string // affected page slugs } -// WikiBatchContext holds data needed during the Reduce phase +// WikiBatchContext holds shared data across Map and Reduce phases type WikiBatchContext struct { + AllPages []*types.WikiPage SlugTitleMap map[string]string SummaryContentByKnowledgeID map[string]string } @@ -773,96 +774,88 @@ func (s *wikiIngestService) publishDraftPages(ctx context.Context, kbID string, } } -// deduplicateItems uses LLM to identify new items that refer to the same entity/concept -// as existing wiki pages, and remaps their slugs so upsertExtractedPages merges them. -func (s *wikiIngestService) deduplicateItems( +// deduplicateExtractedBatch deduplicates both entities and concepts against +// existing wiki pages in a single LLM call. Uses pre-loaded allPages to avoid +// redundant DB queries. This replaces the two separate deduplicateItems calls +// that each queried ListAllPages + made a separate LLM call. +func (s *wikiIngestService) deduplicateExtractedBatch( ctx context.Context, chatModel chat.Chat, - items []extractedItem, - pageType string, - kbID string, -) []extractedItem { - if len(items) == 0 { - return items + entities, concepts []extractedItem, + allPages []*types.WikiPage, +) ([]extractedItem, []extractedItem) { + if len(entities) == 0 && len(concepts) == 0 { + return entities, concepts } - // Get existing pages of the same type - allPages, err := s.wikiService.ListAllPages(ctx, kbID) - if err != nil || len(allPages) == 0 { - return items - } - - // Filter to the target type - var typedPages []*types.WikiPage - for _, p := range allPages { - if p.PageType == pageType { - typedPages = append(typedPages, p) - } - } - if len(typedPages) == 0 { - return items // No existing pages of this type → nothing to deduplicate against - } - - // Build existing pages listing var existingBuf strings.Builder - for _, p := range typedPages { + for _, p := range allPages { + if p.PageType != types.WikiPageTypeEntity && p.PageType != types.WikiPageTypeConcept { + continue + } aliases := "" if len(p.Aliases) > 0 { aliases = fmt.Sprintf(" | aliases: %s", strings.Join(p.Aliases, ", ")) } - fmt.Fprintf(&existingBuf, "- slug: %s | title: %s%s\n", p.Slug, p.Title, aliases) + fmt.Fprintf(&existingBuf, "- slug: %s | title: %s | type: %s%s\n", p.Slug, p.Title, p.PageType, aliases) + } + if existingBuf.Len() == 0 { + return entities, concepts } - // Build new items listing var newBuf strings.Builder - for _, item := range items { + for _, item := range entities { aliases := "" if len(item.Aliases) > 0 { aliases = fmt.Sprintf(" | aliases: %s", strings.Join(item.Aliases, ", ")) } - fmt.Fprintf(&newBuf, "- slug: %s | name: %s%s\n", item.Slug, item.Name, aliases) + fmt.Fprintf(&newBuf, "- slug: %s | name: %s | type: entity%s\n", item.Slug, item.Name, aliases) + } + for _, item := range concepts { + aliases := "" + if len(item.Aliases) > 0 { + aliases = fmt.Sprintf(" | aliases: %s", strings.Join(item.Aliases, ", ")) + } + fmt.Fprintf(&newBuf, "- slug: %s | name: %s | type: concept%s\n", item.Slug, item.Name, aliases) } - // Call LLM for deduplication dedupeJSON, err := s.generateWithTemplate(ctx, chatModel, agent.WikiDeduplicationPrompt, map[string]string{ "NewItems": newBuf.String(), "ExistingPages": existingBuf.String(), }) if err != nil { logger.Warnf(ctx, "wiki ingest: deduplication LLM call failed: %v", err) - return items + return entities, concepts } - // Parse response - dedupeJSON = strings.TrimSpace(dedupeJSON) - dedupeJSON = strings.TrimPrefix(dedupeJSON, "```json") - dedupeJSON = strings.TrimPrefix(dedupeJSON, "```") - dedupeJSON = strings.TrimSuffix(dedupeJSON, "```") - dedupeJSON = strings.TrimSpace(dedupeJSON) - - dedupeJSON = sanitizeJSONString(dedupeJSON) + dedupeJSON = cleanLLMJSON(dedupeJSON) var dedupeResult struct { Merges map[string]string `json:"merges"` } if err := json.Unmarshal([]byte(dedupeJSON), &dedupeResult); err != nil { logger.Warnf(ctx, "wiki ingest: failed to parse dedup JSON: %v\nRaw: %s", err, dedupeJSON) - return items + return entities, concepts } if len(dedupeResult.Merges) == 0 { - return items + return entities, concepts } - // Remap slugs for matched items - for i, item := range items { + for i, item := range entities { if existingSlug, ok := dedupeResult.Merges[item.Slug]; ok { logger.Infof(ctx, "wiki ingest: dedup merge %s → %s", item.Slug, existingSlug) - items[i].Slug = existingSlug + entities[i].Slug = existingSlug + } + } + for i, item := range concepts { + if existingSlug, ok := dedupeResult.Merges[item.Slug]; ok { + logger.Infof(ctx, "wiki ingest: dedup merge %s → %s", item.Slug, existingSlug) + concepts[i].Slug = existingSlug } } - return items + return entities, concepts } // generateWithTemplate executes a prompt template and calls the LLM @@ -1035,6 +1028,17 @@ func appendUnique(arr types.StringArray, s string) types.StringArray { return append(arr, s) } +// cleanLLMJSON strips markdown code-fence wrappers and sanitizes control characters +// from LLM-generated JSON output so it can be safely unmarshalled. +func cleanLLMJSON(s string) string { + s = strings.TrimSpace(s) + s = strings.TrimPrefix(s, "```json") + s = strings.TrimPrefix(s, "```") + s = strings.TrimSuffix(s, "```") + s = strings.TrimSpace(s) + return sanitizeJSONString(s) +} + // sanitizeJSONString sanitizes a string that is intended to be parsed as JSON, // by properly escaping unescaped control characters (like newlines) inside string literals. func sanitizeJSONString(s string) string { diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go index ccdae79c..15033c7c 100644 --- a/internal/application/service/wiki_ingest_batch.go +++ b/internal/application/service/wiki_ingest_batch.go @@ -187,9 +187,10 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task logger.Infof(ctx, "wiki ingest: batch processing %d ops for KB %s", len(pendingOps), payload.KnowledgeBaseID) - // Fetch all existing pages to pass to the Map-Reduce phases + // Fetch all existing pages once, shared across Map and Reduce phases allPages, _ := s.wikiService.ListAllPages(ctx, payload.KnowledgeBaseID) batchCtx := &WikiBatchContext{ + AllPages: allPages, SlugTitleMap: make(map[string]string), SummaryContentByKnowledgeID: make(map[string]string), } @@ -411,7 +412,7 @@ func (s *wikiIngestService) mapOneDocument( oldPageSlugs := s.getExistingPageSlugsForKnowledge(ctx, payload.KnowledgeBaseID, knowledgeID) logger.Infof(ctx, "wiki ingest: extracting entities and concepts for %s", knowledgeID) - extractedEntities, extractedConcepts, slugItems, err := s.extractEntitiesAndConceptsNoUpsert(ctx, chatModel, content, docTitle, lang, payload, oldPageSlugs) + extractedEntities, extractedConcepts, slugItems, err := s.extractEntitiesAndConceptsNoUpsert(ctx, chatModel, content, docTitle, lang, oldPageSlugs, batchCtx) if err != nil { logger.Warnf(ctx, "wiki ingest: knowledge extraction failed for %s: %v", knowledgeID, err) return nil, nil, err @@ -538,8 +539,8 @@ func (s *wikiIngestService) extractEntitiesAndConceptsNoUpsert( ctx context.Context, chatModel chat.Chat, content, docTitle, lang string, - payload WikiIngestPayload, oldPageSlugs map[string]bool, + batchCtx *WikiBatchContext, ) ([]extractedItem, []extractedItem, map[string]extractedItem, error) { var prevSlugsText string if len(oldPageSlugs) > 0 { @@ -562,12 +563,7 @@ func (s *wikiIngestService) extractEntitiesAndConceptsNoUpsert( return nil, nil, nil, fmt.Errorf("combined extraction failed: %w", err) } - extractionJSON = strings.TrimSpace(extractionJSON) - extractionJSON = strings.TrimPrefix(extractionJSON, "```json") - extractionJSON = strings.TrimPrefix(extractionJSON, "```") - extractionJSON = strings.TrimSuffix(extractionJSON, "```") - extractionJSON = strings.TrimSpace(extractionJSON) - extractionJSON = sanitizeJSONString(extractionJSON) + extractionJSON = cleanLLMJSON(extractionJSON) var result combinedExtraction if err := json.Unmarshal([]byte(extractionJSON), &result); err != nil { @@ -575,8 +571,9 @@ func (s *wikiIngestService) extractEntitiesAndConceptsNoUpsert( return nil, nil, nil, fmt.Errorf("parse combined extraction JSON: %w", err) } - result.Entities = s.deduplicateItems(ctx, chatModel, result.Entities, types.WikiPageTypeEntity, payload.KnowledgeBaseID) - result.Concepts = s.deduplicateItems(ctx, chatModel, result.Concepts, types.WikiPageTypeConcept, payload.KnowledgeBaseID) + result.Entities, result.Concepts = s.deduplicateExtractedBatch( + ctx, chatModel, result.Entities, result.Concepts, batchCtx.AllPages, + ) slugItems := make(map[string]extractedItem) for _, item := range result.Entities {