From a75233c90f10a4e166ddc224d77c9c49e37a7db6 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Thu, 9 Apr 2026 19:15:58 +0800 Subject: [PATCH] refactor: Enhance deduplication logic and logging in wiki ingestion service - Updated the `WikiDeduplicationPrompt` to clarify the criteria for identifying exact duplicates, emphasizing strict matching of real-world entities and concepts. - Improved the `appendLogEntry` method to include a summary parameter for better context in log entries, enhancing traceability of operations. - Consolidated log entry creation for both retraction and ingestion operations, ensuring chronological clarity in the log. - Implemented validation for slug merges to prevent mismatches based on entity types, improving the accuracy of deduplication. These changes enhance the clarity and reliability of the wiki ingestion process, leading to more accurate content management. --- internal/agent/prompts_wiki.go | 27 +++++++++---- internal/application/service/wiki_ingest.go | 38 ++++++++++++++----- .../application/service/wiki_ingest_batch.go | 16 +++----- 3 files changed, 54 insertions(+), 27 deletions(-) diff --git a/internal/agent/prompts_wiki.go b/internal/agent/prompts_wiki.go index 79c264d6..54a6cfa9 100644 --- a/internal/agent/prompts_wiki.go +++ b/internal/agent/prompts_wiki.go @@ -248,7 +248,7 @@ The wiki has a log page (slug: "log") that records all ingest and update activit // WikiDeduplicationPrompt asks the LLM to identify duplicate entities/concepts // between newly extracted items and existing wiki pages. -const WikiDeduplicationPrompt = `You are a deduplication system. Given a list of newly extracted items and a list of existing wiki pages, determine which new items refer to the same entity or concept as an existing page. +const WikiDeduplicationPrompt = `You are a strict deduplication system. Given a list of newly extracted items and a list of existing wiki pages, determine which new items refer to the **exact same** real-world entity or concept as an existing page. {{.NewItems}} @@ -259,13 +259,26 @@ const WikiDeduplicationPrompt = `You are a deduplication system. Given a list of -For each newly extracted item, check if it refers to the same real-world entity or concept as any existing page. Consider: -- Name variations (e.g. "Acme Corp" vs "Acme Corporation", "RAG" vs "Retrieval-Augmented Generation") -- Abbreviations and full names -- Translations (e.g. "苹果公司" vs "Apple Inc.") -- Minor spelling/formatting differences +### Merge criteria — ALL must be true: +1. The new item and the existing page refer to the **same real-world thing** (same person, same organization, same specific concept). +2. The match is a **name variation**: abbreviation ↔ full name, translation, or minor spelling difference. +3. The types are compatible: entities merge with entities, concepts merge with concepts. **Never merge an entity into a concept or vice versa.** -Return a JSON object with a "merges" map. The key is the NEW item's slug, the value is the EXISTING page's slug that it should merge into. Only include items that have a match. +### Examples of CORRECT merges: +- "Acme Corp" → "Acme Corporation" (same company, abbreviation) +- "RAG" → "Retrieval-Augmented Generation" (same concept, acronym) +- "苹果公司" → "Apple Inc." (same entity, translation) + +### Examples of INCORRECT merges — do NOT merge these: +- "AI Safety" → "Content Review Mechanism" (related topics, but different concepts) +- "Athlete Registration" → "Degree Verification" (both involve verification, but completely different domains) +- "Competition Categories" → "Age Groups" (age groups are one aspect of categories, not the same concept) +- "Performance Standard" → "Competition Rounds" (both relate to competitions, but are different concepts) +- "Machine Learning" → "Neural Networks" (neural networks are a subset of ML, not the same concept) + +### Key principle: **related ≠ same**. When in doubt, do NOT merge. It is far better to have two separate pages for the same thing than to wrongly merge two different things. + +Return a JSON object with a "merges" map. The key is the NEW item's slug, the value is the EXISTING page's slug that it should merge into. Only include items where you are highly confident they are the same thing. If no items match any existing pages, return: {"merges": {}} diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go index 74af5e1c..1b8a2083 100644 --- a/internal/application/service/wiki_ingest.go +++ b/internal/application/service/wiki_ingest.go @@ -725,31 +725,30 @@ func splitSummaryLine(raw string) (summary string, content string) { return "", raw } -// appendLogEntry appends an entry to the log page, including any synthesis suggestions // appendLogEntry appends a structured, grep-parseable entry to the log page. // Format: ## [2026-04-07 19:50:02] action | title // Followed by key-value metadata lines. No sub-headings — keeps `grep "^## \[" log.md` clean. -func (s *wikiIngestService) appendLogEntry(ctx context.Context, payload WikiIngestPayload, action, knowledgeID, docTitle string, pagesAffected []string, extra string) { - logPage, _ := s.wikiService.GetLog(ctx, payload.KnowledgeBaseID) +func (s *wikiIngestService) appendLogEntry(ctx context.Context, kbID string, action, knowledgeID, docTitle, summary string, pagesAffected []string) { + logPage, _ := s.wikiService.GetLog(ctx, kbID) if logPage == nil { return } var sb strings.Builder fmt.Fprintf(&sb, "\n## [%s] %s | %s\n", - time.Now().UTC().Format("2006-01-02 15:04:05"), + time.Now().Format("2006-01-02 15:04:05"), action, docTitle, ) if knowledgeID != "" { - fmt.Fprintf(&sb, "- **Source**: knowledge/%s\n", knowledgeID) + fmt.Fprintf(&sb, "- **KnowledgeID**: %s\n", knowledgeID) + } + if summary != "" { + fmt.Fprintf(&sb, "- **Summary**: %s\n", summary) } if len(pagesAffected) > 0 { fmt.Fprintf(&sb, "- **Pages affected**: %d (%s)\n", len(pagesAffected), strings.Join(pagesAffected, ", ")) } - if extra != "" { - sb.WriteString(extra) - } logPage.Content = logPage.Content + sb.String() if _, err := s.wikiService.UpdatePage(ctx, logPage); err != nil { @@ -842,14 +841,33 @@ func (s *wikiIngestService) deduplicateExtractedBatch( return entities, concepts } + existingSlugs := make(map[string]bool, len(allPages)) + for _, p := range allPages { + existingSlugs[p.Slug] = true + } + + validMerge := func(srcSlug, dstSlug string) bool { + if !existingSlugs[dstSlug] { + logger.Warnf(ctx, "wiki ingest: dedup rejected %s → %s (target slug does not exist)", srcSlug, dstSlug) + return false + } + srcPrefix := srcSlug[:strings.Index(srcSlug, "/")+1] + dstPrefix := dstSlug[:strings.Index(dstSlug, "/")+1] + if srcPrefix != dstPrefix { + logger.Warnf(ctx, "wiki ingest: dedup rejected %s → %s (type mismatch: %s vs %s)", srcSlug, dstSlug, srcPrefix, dstPrefix) + return false + } + return true + } + for i, item := range entities { - if existingSlug, ok := dedupeResult.Merges[item.Slug]; ok { + if existingSlug, ok := dedupeResult.Merges[item.Slug]; ok && validMerge(item.Slug, existingSlug) { logger.Infof(ctx, "wiki ingest: dedup merge %s → %s", item.Slug, existingSlug) entities[i].Slug = existingSlug } } for i, item := range concepts { - if existingSlug, ok := dedupeResult.Merges[item.Slug]; ok { + if existingSlug, ok := dedupeResult.Merges[item.Slug]; ok && validMerge(item.Slug, existingSlug) { logger.Infof(ctx, "wiki ingest: dedup merge %s → %s", item.Slug, existingSlug) concepts[i].Slug = existingSlug } diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go index 15033c7c..ea1ccbcf 100644 --- a/internal/application/service/wiki_ingest_batch.go +++ b/internal/application/service/wiki_ingest_batch.go @@ -306,15 +306,15 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task totalPagesAffected = len(allPagesAffected) - // Append log entry for retracts (since retracts aren't in docResults) + // Append log entries — one per operation for chronological traceability for _, op := range pendingOps { if op.Op == WikiOpRetract { - s.appendLogEntry(ctx, WikiIngestPayload{ - TenantID: payload.TenantID, - KnowledgeBaseID: payload.KnowledgeBaseID, - }, "retract", op.KnowledgeID, op.DocTitle, op.PageSlugs, "") + s.appendLogEntry(ctx, payload.KnowledgeBaseID, "retract", op.KnowledgeID, op.DocTitle, op.DocSummary, op.PageSlugs) } } + for _, r := range docResults { + s.appendLogEntry(ctx, payload.KnowledgeBaseID, "ingest", r.KnowledgeID, r.DocTitle, r.Summary, r.Pages) + } // Build change description for the Index Intro LLM prompt var changeDesc strings.Builder @@ -340,11 +340,6 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task } } - // Append log entry for ingests - if len(docResults) > 0 { - s.appendLogEntry(ctx, payload, "ingest", "", fmt.Sprintf("%d documents", len(docResults)), ingestPagesAffected, "") - } - if len(retractPagesAffected) > 0 { logger.Infof(ctx, "wiki ingest: cleaning dead links") s.cleanDeadLinks(ctx, payload.KnowledgeBaseID) @@ -532,6 +527,7 @@ func (s *wikiIngestService) mapOneDocument( KnowledgeID: knowledgeID, DocTitle: docTitle, Summary: docSummaryLine, + Pages: extractedPages, }, updates, nil }