diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go index 95132d73..74af5e1c 100644 --- a/internal/application/service/wiki_ingest.go +++ b/internal/application/service/wiki_ingest.go @@ -292,8 +292,9 @@ type docIngestResult struct { Pages []string // affected page slugs } -// WikiBatchContext holds data needed during the Reduce phase +// WikiBatchContext holds shared data across Map and Reduce phases type WikiBatchContext struct { + AllPages []*types.WikiPage SlugTitleMap map[string]string SummaryContentByKnowledgeID map[string]string } @@ -773,96 +774,88 @@ func (s *wikiIngestService) publishDraftPages(ctx context.Context, kbID string, } } -// deduplicateItems uses LLM to identify new items that refer to the same entity/concept -// as existing wiki pages, and remaps their slugs so upsertExtractedPages merges them. -func (s *wikiIngestService) deduplicateItems( +// deduplicateExtractedBatch deduplicates both entities and concepts against +// existing wiki pages in a single LLM call. Uses pre-loaded allPages to avoid +// redundant DB queries. This replaces the two separate deduplicateItems calls +// that each queried ListAllPages + made a separate LLM call. +func (s *wikiIngestService) deduplicateExtractedBatch( ctx context.Context, chatModel chat.Chat, - items []extractedItem, - pageType string, - kbID string, -) []extractedItem { - if len(items) == 0 { - return items + entities, concepts []extractedItem, + allPages []*types.WikiPage, +) ([]extractedItem, []extractedItem) { + if len(entities) == 0 && len(concepts) == 0 { + return entities, concepts } - // Get existing pages of the same type - allPages, err := s.wikiService.ListAllPages(ctx, kbID) - if err != nil || len(allPages) == 0 { - return items - } - - // Filter to the target type - var typedPages []*types.WikiPage - for _, p := range allPages { - if p.PageType == pageType { - typedPages = append(typedPages, p) - } - } - if len(typedPages) == 0 { - return items // No existing pages of this type → nothing to deduplicate against - } - - // Build existing pages listing var existingBuf strings.Builder - for _, p := range typedPages { + for _, p := range allPages { + if p.PageType != types.WikiPageTypeEntity && p.PageType != types.WikiPageTypeConcept { + continue + } aliases := "" if len(p.Aliases) > 0 { aliases = fmt.Sprintf(" | aliases: %s", strings.Join(p.Aliases, ", ")) } - fmt.Fprintf(&existingBuf, "- slug: %s | title: %s%s\n", p.Slug, p.Title, aliases) + fmt.Fprintf(&existingBuf, "- slug: %s | title: %s | type: %s%s\n", p.Slug, p.Title, p.PageType, aliases) + } + if existingBuf.Len() == 0 { + return entities, concepts } - // Build new items listing var newBuf strings.Builder - for _, item := range items { + for _, item := range entities { aliases := "" if len(item.Aliases) > 0 { aliases = fmt.Sprintf(" | aliases: %s", strings.Join(item.Aliases, ", ")) } - fmt.Fprintf(&newBuf, "- slug: %s | name: %s%s\n", item.Slug, item.Name, aliases) + fmt.Fprintf(&newBuf, "- slug: %s | name: %s | type: entity%s\n", item.Slug, item.Name, aliases) + } + for _, item := range concepts { + aliases := "" + if len(item.Aliases) > 0 { + aliases = fmt.Sprintf(" | aliases: %s", strings.Join(item.Aliases, ", ")) + } + fmt.Fprintf(&newBuf, "- slug: %s | name: %s | type: concept%s\n", item.Slug, item.Name, aliases) } - // Call LLM for deduplication dedupeJSON, err := s.generateWithTemplate(ctx, chatModel, agent.WikiDeduplicationPrompt, map[string]string{ "NewItems": newBuf.String(), "ExistingPages": existingBuf.String(), }) if err != nil { logger.Warnf(ctx, "wiki ingest: deduplication LLM call failed: %v", err) - return items + return entities, concepts } - // Parse response - dedupeJSON = strings.TrimSpace(dedupeJSON) - dedupeJSON = strings.TrimPrefix(dedupeJSON, "```json") - dedupeJSON = strings.TrimPrefix(dedupeJSON, "```") - dedupeJSON = strings.TrimSuffix(dedupeJSON, "```") - dedupeJSON = strings.TrimSpace(dedupeJSON) - - dedupeJSON = sanitizeJSONString(dedupeJSON) + dedupeJSON = cleanLLMJSON(dedupeJSON) var dedupeResult struct { Merges map[string]string `json:"merges"` } if err := json.Unmarshal([]byte(dedupeJSON), &dedupeResult); err != nil { logger.Warnf(ctx, "wiki ingest: failed to parse dedup JSON: %v\nRaw: %s", err, dedupeJSON) - return items + return entities, concepts } if len(dedupeResult.Merges) == 0 { - return items + return entities, concepts } - // Remap slugs for matched items - for i, item := range items { + for i, item := range entities { if existingSlug, ok := dedupeResult.Merges[item.Slug]; ok { logger.Infof(ctx, "wiki ingest: dedup merge %s → %s", item.Slug, existingSlug) - items[i].Slug = existingSlug + entities[i].Slug = existingSlug + } + } + for i, item := range concepts { + if existingSlug, ok := dedupeResult.Merges[item.Slug]; ok { + logger.Infof(ctx, "wiki ingest: dedup merge %s → %s", item.Slug, existingSlug) + concepts[i].Slug = existingSlug } } - return items + return entities, concepts } // generateWithTemplate executes a prompt template and calls the LLM @@ -1035,6 +1028,17 @@ func appendUnique(arr types.StringArray, s string) types.StringArray { return append(arr, s) } +// cleanLLMJSON strips markdown code-fence wrappers and sanitizes control characters +// from LLM-generated JSON output so it can be safely unmarshalled. +func cleanLLMJSON(s string) string { + s = strings.TrimSpace(s) + s = strings.TrimPrefix(s, "```json") + s = strings.TrimPrefix(s, "```") + s = strings.TrimSuffix(s, "```") + s = strings.TrimSpace(s) + return sanitizeJSONString(s) +} + // sanitizeJSONString sanitizes a string that is intended to be parsed as JSON, // by properly escaping unescaped control characters (like newlines) inside string literals. func sanitizeJSONString(s string) string { diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go index ccdae79c..15033c7c 100644 --- a/internal/application/service/wiki_ingest_batch.go +++ b/internal/application/service/wiki_ingest_batch.go @@ -187,9 +187,10 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task logger.Infof(ctx, "wiki ingest: batch processing %d ops for KB %s", len(pendingOps), payload.KnowledgeBaseID) - // Fetch all existing pages to pass to the Map-Reduce phases + // Fetch all existing pages once, shared across Map and Reduce phases allPages, _ := s.wikiService.ListAllPages(ctx, payload.KnowledgeBaseID) batchCtx := &WikiBatchContext{ + AllPages: allPages, SlugTitleMap: make(map[string]string), SummaryContentByKnowledgeID: make(map[string]string), } @@ -411,7 +412,7 @@ func (s *wikiIngestService) mapOneDocument( oldPageSlugs := s.getExistingPageSlugsForKnowledge(ctx, payload.KnowledgeBaseID, knowledgeID) logger.Infof(ctx, "wiki ingest: extracting entities and concepts for %s", knowledgeID) - extractedEntities, extractedConcepts, slugItems, err := s.extractEntitiesAndConceptsNoUpsert(ctx, chatModel, content, docTitle, lang, payload, oldPageSlugs) + extractedEntities, extractedConcepts, slugItems, err := s.extractEntitiesAndConceptsNoUpsert(ctx, chatModel, content, docTitle, lang, oldPageSlugs, batchCtx) if err != nil { logger.Warnf(ctx, "wiki ingest: knowledge extraction failed for %s: %v", knowledgeID, err) return nil, nil, err @@ -538,8 +539,8 @@ func (s *wikiIngestService) extractEntitiesAndConceptsNoUpsert( ctx context.Context, chatModel chat.Chat, content, docTitle, lang string, - payload WikiIngestPayload, oldPageSlugs map[string]bool, + batchCtx *WikiBatchContext, ) ([]extractedItem, []extractedItem, map[string]extractedItem, error) { var prevSlugsText string if len(oldPageSlugs) > 0 { @@ -562,12 +563,7 @@ func (s *wikiIngestService) extractEntitiesAndConceptsNoUpsert( return nil, nil, nil, fmt.Errorf("combined extraction failed: %w", err) } - extractionJSON = strings.TrimSpace(extractionJSON) - extractionJSON = strings.TrimPrefix(extractionJSON, "```json") - extractionJSON = strings.TrimPrefix(extractionJSON, "```") - extractionJSON = strings.TrimSuffix(extractionJSON, "```") - extractionJSON = strings.TrimSpace(extractionJSON) - extractionJSON = sanitizeJSONString(extractionJSON) + extractionJSON = cleanLLMJSON(extractionJSON) var result combinedExtraction if err := json.Unmarshal([]byte(extractionJSON), &result); err != nil { @@ -575,8 +571,9 @@ func (s *wikiIngestService) extractEntitiesAndConceptsNoUpsert( return nil, nil, nil, fmt.Errorf("parse combined extraction JSON: %w", err) } - result.Entities = s.deduplicateItems(ctx, chatModel, result.Entities, types.WikiPageTypeEntity, payload.KnowledgeBaseID) - result.Concepts = s.deduplicateItems(ctx, chatModel, result.Concepts, types.WikiPageTypeConcept, payload.KnowledgeBaseID) + result.Entities, result.Concepts = s.deduplicateExtractedBatch( + ctx, chatModel, result.Entities, result.Concepts, batchCtx.AllPages, + ) slugItems := make(map[string]extractedItem) for _, item := range result.Entities {