refactor: Update wiki ingest service to support aliases in slug mapping

- Modified the `extractEntitiesAndConcepts` method to return a map of slugs to extracted items, including aliases for improved wiki-link generation.
- Updated the `processOneDocument` method to handle slug items with aliases, enhancing the clarity of generated summaries.
- Adjusted the `injectCrossLinks` method to utilize match text for cross-linking, ensuring accurate references in content.

These changes improve the accuracy and usability of wiki links and summaries by incorporating alias management.
This commit is contained in:
wizardchen
2026-04-08 22:38:25 +08:00
parent f07561f6b6
commit 2b495c8232
2 changed files with 25 additions and 16 deletions
+1 -1
View File
@@ -25,7 +25,7 @@ const WikiSummaryPrompt = `You are a wiki editor. Given the following document c
2. After the SUMMARY line, write a comprehensive summary of the document in Markdown format.
3. Include the key facts, arguments, and conclusions.
4. Use proper heading hierarchy (## for sections, ### for subsections).
5. **Wiki-link rule**: The available_wiki_pages list above maps slugs to display names (format: "[[slug]] = display name"). Whenever you mention a name that matches a listed entry, you MUST write it as [[slug|display name]] (e.g. [[entity/zhong-guo|中国]]), NOT as bold (**name**) or bare [[slug]]. Use the EXACT slugs provided — do NOT invent new slugs.
5. **Wiki-link rule**: The available_wiki_pages list above maps slugs to display names and their aliases (format: "[[slug]] = display name (Aliases: a, b)"). Whenever you mention a name or alias that matches a listed entry, you MUST write it as [[slug|display name]] (e.g. [[entity/zhong-guo|中国]]), NOT as bold (**name**) or bare [[slug]]. Use the EXACT slugs provided — do NOT invent new slugs.
6. **Image rule**: If the document contains <images> tags with <image> elements, you SHOULD include the relevant images in your summary using the Markdown syntax: ![caption](url). Place the images where they are contextually relevant to the text.
7. At the end, include a "## Key Takeaways" section with bullet points.
8. Write in {{.Language}}.
+24 -15
View File
@@ -569,7 +569,7 @@ func (s *wikiIngestService) processOneDocument(
}
// Step 1: Extract entities and concepts
extractedPages, slugNames, err := s.extractEntitiesAndConcepts(ctx, chatModel, content, docTitle, lang, docPayload, sourceRef, oldPageSlugs)
extractedPages, slugItems, err := s.extractEntitiesAndConcepts(ctx, chatModel, content, docTitle, lang, docPayload, sourceRef, oldPageSlugs)
if err != nil {
logger.Warnf(ctx, "wiki ingest: knowledge extraction failed for %s: %v", knowledgeID, err)
} else {
@@ -580,8 +580,12 @@ func (s *wikiIngestService) processOneDocument(
summarySlug := fmt.Sprintf("summary/%s", slugify(docTitle))
var slugListing string
for _, slug := range extractedPages {
if name, ok := slugNames[slug]; ok {
slugListing += fmt.Sprintf("- [[%s]] = %s\n", slug, name)
if item, ok := slugItems[slug]; ok {
aliases := ""
if len(item.Aliases) > 0 {
aliases = fmt.Sprintf(" (Aliases: %s)", strings.Join(item.Aliases, ", "))
}
slugListing += fmt.Sprintf("- [[%s]] = %s%s\n", slug, item.Name, aliases)
} else {
slugListing += fmt.Sprintf("- [[%s]]\n", slug)
}
@@ -708,8 +712,8 @@ func (s *wikiIngestService) injectCrossLinks(ctx context.Context, kbID string, a
}
type pageRef struct {
slug string
title string
slug string
matchText string
}
var allRefs []pageRef
for _, p := range allPages {
@@ -717,7 +721,12 @@ func (s *wikiIngestService) injectCrossLinks(ctx context.Context, kbID string, a
continue
}
if p.Title != "" {
allRefs = append(allRefs, pageRef{slug: p.Slug, title: p.Title})
allRefs = append(allRefs, pageRef{slug: p.Slug, matchText: p.Title})
}
for _, alias := range p.Aliases {
if alias != "" {
allRefs = append(allRefs, pageRef{slug: p.Slug, matchText: alias})
}
}
}
if len(allRefs) == 0 {
@@ -728,7 +737,7 @@ func (s *wikiIngestService) injectCrossLinks(ctx context.Context, kbID string, a
// partial matches (e.g. "北京邮电大学" before "北京")
for i := 0; i < len(allRefs); i++ {
for j := i + 1; j < len(allRefs); j++ {
if len([]rune(allRefs[j].title)) > len([]rune(allRefs[i].title)) {
if len([]rune(allRefs[j].matchText)) > len([]rune(allRefs[i].matchText)) {
allRefs[i], allRefs[j] = allRefs[j], allRefs[i]
}
}
@@ -761,8 +770,8 @@ func (s *wikiIngestService) injectCrossLinks(ctx context.Context, kbID string, a
if strings.Contains(content, "[["+ref.slug+"|") || strings.Contains(content, "[["+ref.slug+"]]") {
continue
}
if strings.Contains(content, ref.title) {
content = replaceFirstOutsideLinks(content, ref.title, "[["+ref.slug+"|"+ref.title+"]]")
if strings.Contains(content, ref.matchText) {
content = replaceFirstOutsideLinks(content, ref.matchText, "[["+ref.slug+"|"+ref.matchText+"]]")
changed = true
}
}
@@ -940,7 +949,7 @@ func (s *wikiIngestService) extractEntitiesAndConcepts(
payload WikiIngestPayload,
sourceRef string,
oldPageSlugs map[string]bool,
) ([]string, map[string]string, error) {
) ([]string, map[string]extractedItem, error) {
// Build previous slugs listing for the prompt
var prevSlugsText string
if len(oldPageSlugs) > 0 {
@@ -985,16 +994,16 @@ func (s *wikiIngestService) extractEntitiesAndConcepts(
// Deduplicate concepts against existing wiki pages (LLM-based)
result.Concepts = s.deduplicateItems(ctx, chatModel, result.Concepts, types.WikiPageTypeConcept, payload.KnowledgeBaseID)
// Build slug→name map for wiki-link generation in summary pages
slugNames := make(map[string]string)
// Build slug→item map for wiki-link generation in summary pages
slugItems := make(map[string]extractedItem)
for _, item := range result.Entities {
if item.Slug != "" && item.Name != "" {
slugNames[item.Slug] = item.Name
slugItems[item.Slug] = item
}
}
for _, item := range result.Concepts {
if item.Slug != "" && item.Name != "" {
slugNames[item.Slug] = item.Name
slugItems[item.Slug] = item
}
}
@@ -1014,7 +1023,7 @@ func (s *wikiIngestService) extractEntitiesAndConcepts(
affected = append(affected, conceptSlugs...)
}
return affected, slugNames, nil
return affected, slugItems, nil
}
// upsertExtractedPages creates or updates wiki pages from pre-extracted items.