refactor: Enhance deduplication logic and logging in wiki ingestion service

- Updated the `WikiDeduplicationPrompt` to clarify the criteria for identifying exact duplicates, emphasizing strict matching of real-world entities and concepts.
- Improved the `appendLogEntry` method to include a summary parameter for better context in log entries, enhancing traceability of operations.
- Consolidated log entry creation for both retraction and ingestion operations, ensuring chronological clarity in the log.
- Implemented validation for slug merges to prevent mismatches based on entity types, improving the accuracy of deduplication.

These changes enhance the clarity and reliability of the wiki ingestion process, leading to more accurate content management.
This commit is contained in:
wizardchen
2026-04-09 19:15:58 +08:00
parent 28781785f7
commit a75233c90f
3 changed files with 54 additions and 27 deletions
+28 -10
View File
@@ -725,31 +725,30 @@ func splitSummaryLine(raw string) (summary string, content string) {
return "", raw
}
// appendLogEntry appends an entry to the log page, including any synthesis suggestions
// appendLogEntry appends a structured, grep-parseable entry to the log page.
// Format: ## [2026-04-07 19:50:02] action | title
// Followed by key-value metadata lines. No sub-headings — keeps `grep "^## \[" log.md` clean.
func (s *wikiIngestService) appendLogEntry(ctx context.Context, payload WikiIngestPayload, action, knowledgeID, docTitle string, pagesAffected []string, extra string) {
logPage, _ := s.wikiService.GetLog(ctx, payload.KnowledgeBaseID)
func (s *wikiIngestService) appendLogEntry(ctx context.Context, kbID string, action, knowledgeID, docTitle, summary string, pagesAffected []string) {
logPage, _ := s.wikiService.GetLog(ctx, kbID)
if logPage == nil {
return
}
var sb strings.Builder
fmt.Fprintf(&sb, "\n## [%s] %s | %s\n",
time.Now().UTC().Format("2006-01-02 15:04:05"),
time.Now().Format("2006-01-02 15:04:05"),
action,
docTitle,
)
if knowledgeID != "" {
fmt.Fprintf(&sb, "- **Source**: knowledge/%s\n", knowledgeID)
fmt.Fprintf(&sb, "- **KnowledgeID**: %s\n", knowledgeID)
}
if summary != "" {
fmt.Fprintf(&sb, "- **Summary**: %s\n", summary)
}
if len(pagesAffected) > 0 {
fmt.Fprintf(&sb, "- **Pages affected**: %d (%s)\n", len(pagesAffected), strings.Join(pagesAffected, ", "))
}
if extra != "" {
sb.WriteString(extra)
}
logPage.Content = logPage.Content + sb.String()
if _, err := s.wikiService.UpdatePage(ctx, logPage); err != nil {
@@ -842,14 +841,33 @@ func (s *wikiIngestService) deduplicateExtractedBatch(
return entities, concepts
}
existingSlugs := make(map[string]bool, len(allPages))
for _, p := range allPages {
existingSlugs[p.Slug] = true
}
validMerge := func(srcSlug, dstSlug string) bool {
if !existingSlugs[dstSlug] {
logger.Warnf(ctx, "wiki ingest: dedup rejected %s → %s (target slug does not exist)", srcSlug, dstSlug)
return false
}
srcPrefix := srcSlug[:strings.Index(srcSlug, "/")+1]
dstPrefix := dstSlug[:strings.Index(dstSlug, "/")+1]
if srcPrefix != dstPrefix {
logger.Warnf(ctx, "wiki ingest: dedup rejected %s → %s (type mismatch: %s vs %s)", srcSlug, dstSlug, srcPrefix, dstPrefix)
return false
}
return true
}
for i, item := range entities {
if existingSlug, ok := dedupeResult.Merges[item.Slug]; ok {
if existingSlug, ok := dedupeResult.Merges[item.Slug]; ok && validMerge(item.Slug, existingSlug) {
logger.Infof(ctx, "wiki ingest: dedup merge %s → %s", item.Slug, existingSlug)
entities[i].Slug = existingSlug
}
}
for i, item := range concepts {
if existingSlug, ok := dedupeResult.Merges[item.Slug]; ok {
if existingSlug, ok := dedupeResult.Merges[item.Slug]; ok && validMerge(item.Slug, existingSlug) {
logger.Infof(ctx, "wiki ingest: dedup merge %s → %s", item.Slug, existingSlug)
concepts[i].Slug = existingSlug
}
@@ -306,15 +306,15 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
totalPagesAffected = len(allPagesAffected)
// Append log entry for retracts (since retracts aren't in docResults)
// Append log entries — one per operation for chronological traceability
for _, op := range pendingOps {
if op.Op == WikiOpRetract {
s.appendLogEntry(ctx, WikiIngestPayload{
TenantID: payload.TenantID,
KnowledgeBaseID: payload.KnowledgeBaseID,
}, "retract", op.KnowledgeID, op.DocTitle, op.PageSlugs, "")
s.appendLogEntry(ctx, payload.KnowledgeBaseID, "retract", op.KnowledgeID, op.DocTitle, op.DocSummary, op.PageSlugs)
}
}
for _, r := range docResults {
s.appendLogEntry(ctx, payload.KnowledgeBaseID, "ingest", r.KnowledgeID, r.DocTitle, r.Summary, r.Pages)
}
// Build change description for the Index Intro LLM prompt
var changeDesc strings.Builder
@@ -340,11 +340,6 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
}
}
// Append log entry for ingests
if len(docResults) > 0 {
s.appendLogEntry(ctx, payload, "ingest", "", fmt.Sprintf("%d documents", len(docResults)), ingestPagesAffected, "")
}
if len(retractPagesAffected) > 0 {
logger.Infof(ctx, "wiki ingest: cleaning dead links")
s.cleanDeadLinks(ctx, payload.KnowledgeBaseID)
@@ -532,6 +527,7 @@ func (s *wikiIngestService) mapOneDocument(
KnowledgeID: knowledgeID,
DocTitle: docTitle,
Summary: docSummaryLine,
Pages: extractedPages,
}, updates, nil
}