refactor: Enhance wiki page management and index rebuilding

- Introduced a new method to list all wiki pages without pagination, facilitating index rebuilding and cross-linking.
- Updated the wiki ingest and retract processes to include document summaries, improving clarity in log entries and index updates.
- Enhanced the cleanup process for deleted knowledge, ensuring accurate summary retention and index regeneration.
- Streamlined the handling of page updates and deletions, optimizing the overall wiki management workflow.

These changes improve the efficiency and reliability of wiki content management and indexing.
This commit is contained in:
wizardchen
2026-04-08 16:50:38 +08:00
parent f00b2a176b
commit ec8a5fc392
5 changed files with 479 additions and 286 deletions
+122 -66
View File
@@ -7,48 +7,54 @@ package agent
// WikiSummaryPrompt generates a summary page for a newly ingested document.
const WikiSummaryPrompt = `You are a wiki editor. Given the following document content, create a structured wiki summary page in Markdown format.
## Document Info
- Title: {{.Title}}
- File Name: {{.FileName}}
- File Type: {{.FileType}}
## Document Content
<document>
<title>{{.Title}}</title>
<file_name>{{.FileName}}</file_name>
<file_type>{{.FileType}}</file_type>
<content>
{{.Content}}
</content>
</document>
## Available Wiki Pages (use these exact slugs for links)
<available_wiki_pages>
{{.ExtractedSlugs}}
</available_wiki_pages>
## Instructions
1. Write a comprehensive summary of the document in Markdown format.
2. Include the key facts, arguments, and conclusions.
3. Use proper heading hierarchy (## for sections, ### for subsections).
4. Use [[wiki-link]] syntax to reference the available wiki pages listed above. Use the EXACT slugs provided — do NOT invent new slugs.
5. At the end, include a "## Key Takeaways" section with bullet points.
6. Write in {{.Language}}.
7. Keep the summary concise but thorough (500-1500 words depending on document length).
<instructions>
1. The FIRST line of your output MUST be: SUMMARY: {one sentence, 15-40 words, describing what this document is about — for wiki index listing}
2. After the SUMMARY line, write a comprehensive summary of the document in Markdown format.
3. Include the key facts, arguments, and conclusions.
4. Use proper heading hierarchy (## for sections, ### for subsections).
5. Use [[wiki-link]] syntax to reference the available wiki pages listed above. Use the EXACT slugs provided — do NOT invent new slugs.
6. At the end, include a "## Key Takeaways" section with bullet points.
7. Write in {{.Language}}.
8. Keep the summary concise but thorough (500-1500 words depending on document length).
</instructions>
Output ONLY the Markdown content for the wiki page. Do not include any preamble or explanation.`
Output the SUMMARY line first, then the Markdown content. Do not include any other preamble.`
// WikiKnowledgeExtractPrompt extracts both entities and concepts in a single LLM call.
// Returns a JSON object with "entities" and "concepts" arrays.
// This replaces the former separate WikiEntityExtractPrompt and WikiConceptExtractPrompt.
const WikiKnowledgeExtractPrompt = `You are a knowledge extraction system. Analyze the following document and extract all significant entities AND key concepts.
## Document Info
- Title: {{.Title}}
## Document Content
<document>
<title>{{.Title}}</title>
<content>
{{.Content}}
</content>
</document>
## Previously Extracted Slugs (from earlier version of this document)
<previous_slugs>
{{.PreviousSlugs}}
</previous_slugs>
## Instructions
<instructions>
Return a JSON object with two arrays: "entities" and "concepts".
**IMPORTANT: Write ALL names, descriptions, and details in {{.Language}}.**
### Slug Continuity Rules
If a "Previously Extracted Slugs" list is provided above, you MUST follow these rules:
If previous slugs are provided above, you MUST follow these rules:
- If an entity or concept from the previous extraction still exists in the current document, **reuse its exact slug** from the previous list. Do NOT generate a new slug for the same thing.
- If an entity or concept no longer appears in the document, **do NOT include it** in the output.
- Only generate new slugs for entities/concepts that are genuinely new (not present in the previous list).
@@ -58,7 +64,7 @@ If a "Previously Extracted Slugs" list is provided above, you MUST follow these
Each entity should have:
- "name": The entity name in {{.Language}} (human-readable)
- "slug": URL-friendly slug, format "entity/<lowercase-hyphenated-name>" (use romanized/pinyin form for non-Latin names). **Reuse previous slug if the entity was extracted before.**
- "description": A one-sentence description in {{.Language}} based on what the document says
- "description": **Index listing summary** — one sentence, 15-40 words, in {{.Language}}. Describes WHAT this entity IS and its role in the document. Must be self-contained (understandable without reading the full page). This will be displayed in the wiki index.
- "details": A 2-5 sentence summary in {{.Language}} of key facts from the document
Only include entities that are substantively discussed (mentioned at least twice or described in detail). Do NOT include generic terms.
@@ -67,7 +73,7 @@ Only include entities that are substantively discussed (mentioned at least twice
Each concept should have:
- "name": The concept name in {{.Language}} (human-readable)
- "slug": URL-friendly slug, format "concept/<lowercase-hyphenated-name>" (use romanized/pinyin form for non-Latin names). **Reuse previous slug if the concept was extracted before.**
- "description": A one-sentence definition or description in {{.Language}}
- "description": **Index listing summary** — one sentence, 15-40 words, in {{.Language}}. Defines WHAT this concept IS. Must be self-contained (understandable without reading the full page). This will be displayed in the wiki index.
- "details": A 2-5 sentence explanation in {{.Language}} as discussed in the document
Only include concepts that are substantively discussed. Skip trivial or overly generic concepts.
@@ -76,6 +82,7 @@ Only include concepts that are substantively discussed. Skip trivial or overly g
- If something is a specific named thing (person, company, product, place), put it ONLY in "entities".
- If something is an abstract idea, methodology, or theory, put it ONLY in "concepts".
- Never duplicate items across the two arrays.
</instructions>
Output ONLY valid JSON. Example:
{
@@ -100,63 +107,109 @@ Output ONLY valid JSON. Example:
// WikiPageUpdatePrompt incrementally updates an existing wiki page with new information.
const WikiPageUpdatePrompt = `You are a wiki editor tasked with updating an existing wiki page with new information from a recently ingested document.
## Existing Page Content
<existing_page_content>
{{.ExistingContent}}
</existing_page_content>
## New Information from Document "{{.NewDocTitle}}"
<new_information>
<source_document>{{.NewDocTitle}}</source_document>
<content>
{{.NewContent}}
</content>
</new_information>
## Instructions
1. Merge the new information into the existing page content.
2. Preserve all existing information that is still valid.
3. If the new information contradicts existing content, prefer the newer information and silently replace the old claim.
4. Add new facts, details, and context from the new document.
5. Update cross-references: add new [[wiki-link]] references where appropriate.
6. Maintain the existing page structure and formatting style.
7. Add a source reference to the new document at the bottom.
8. Write in {{.Language}}.
<instructions>
1. The FIRST line of your output MUST be: SUMMARY: {one sentence, 15-40 words, describing what this page is about after the update — for wiki index listing}
2. Merge the new information into the existing page content.
3. Preserve all existing information that is still valid.
4. If the new information contradicts existing content, prefer the newer information and silently replace the old claim.
5. Add new facts, details, and context from the new document.
6. Preserve any existing [[wiki-link]] references in the content. Do NOT invent new wiki-link slugs.
7. Maintain the existing page structure and formatting style.
8. Add a source reference to the new document at the bottom.
9. Write in {{.Language}}.
</instructions>
Output ONLY the updated Markdown content. Do not include any preamble or explanation.`
Output the SUMMARY line first, then the updated Markdown content. Do not include any other preamble.`
// WikiPageRetractPrompt removes information contributed by a deleted document from an existing wiki page.
const WikiPageRetractPrompt = `You are a wiki editor. A source document has been DELETED from the knowledge base. You must update the existing wiki page to remove any information that came exclusively from this deleted document.
## Existing Page Content
<existing_page_content>
{{.ExistingContent}}
</existing_page_content>
## Deleted Document Title
{{.DeletedDocTitle}}
<deleted_document>
<title>{{.DeletedDocTitle}}</title>
<content>
{{.DeletedDocContent}}
</content>
</deleted_document>
## Remaining Source Documents
<remaining_sources>
{{.RemainingSources}}
</remaining_sources>
## Instructions
1. Carefully review the existing page content.
2. Remove any facts, claims, or details that were ONLY sourced from "{{.DeletedDocTitle}}".
3. If a fact is also supported by the remaining source documents, KEEP it.
4. If you are unsure whether a fact came from the deleted document, keep it as-is — do NOT add any review notes or annotations.
5. Update or remove the "Source: {{.DeletedDocTitle}}" reference line if present.
6. Remove any [[wiki-link]] references that point to pages that no longer exist, if you can identify them.
7. Maintain the existing page structure, formatting style, and language.
8. If after removing the deleted document's contributions the page becomes nearly empty, output just: "# [Title]\n\n*This page's primary source document was removed.*"
9. Write in {{.Language}}.
<valid_wiki_links>
{{.AvailableSlugs}}
</valid_wiki_links>
Output ONLY the updated Markdown content. Do not include any preamble or explanation.`
<instructions>
1. The FIRST line of your output MUST be: SUMMARY: {one sentence, 15-40 words, describing what this page is about after retraction — for wiki index listing}
2. Carefully review the existing page content and compare it with the deleted document content above.
3. Remove any facts, claims, or details that were ONLY sourced from the deleted document.
4. If a fact is also supported by the remaining source documents, KEEP it.
5. If you are unsure whether a fact came from the deleted document, keep it as-is — do NOT add any review notes or annotations.
6. Update or remove the "Source: {{.DeletedDocTitle}}" reference line if present.
7. Keep [[wiki-link]] references ONLY if the slug appears in the <valid_wiki_links> list above. Remove any [[wiki-link]] whose slug is NOT in that list.
8. Maintain the existing page structure, formatting style, and language.
9. If after removing the deleted document's contributions the page becomes nearly empty, output just: "SUMMARY: (empty page)\n# [Title]\n\n*This page's primary source document was removed.*"
10. Write in {{.Language}}.
</instructions>
// WikiIndexRebuildPrompt generates the index page content from a list of all pages.
const WikiIndexRebuildPrompt = `You are a wiki editor. Generate an index page (table of contents) for a wiki based on the following page listing.
Output the SUMMARY line first, then the updated Markdown content. Do not include any other preamble.`
## Pages
{{.PageListing}}
// WikiIndexIntroPrompt generates the introduction for a NEW index page (first time only).
const WikiIndexIntroPrompt = `You are a wiki editor. Write a brief introduction for a wiki knowledge base index page.
## Instructions
1. Organize pages by category (Summaries, Entities, Concepts, Analyses, etc.).
2. For each page, include: [[slug]] — one-line summary
3. Within each category, sort alphabetically.
4. Include a brief introduction at the top explaining what this wiki covers.
5. Write in {{.Language}}.
<document_summaries>
{{.DocumentSummaries}}
</document_summaries>
Output ONLY the Markdown content for the index page.`
<instructions>
1. Write a title line starting with "# " that reflects the knowledge domain.
2. Follow with 2-3 sentences describing what this wiki covers, based on the document summaries above.
3. Keep it concise — this is just the header section, the directory listing will be added separately below.
4. Write in {{.Language}}.
</instructions>
Output ONLY the title and introduction paragraph. Do NOT generate any directory listings or page links.`
// WikiIndexIntroUpdatePrompt incrementally updates an existing index introduction.
const WikiIndexIntroUpdatePrompt = `You are a wiki editor. Update the introduction section of a wiki index page to reflect recent changes.
<current_introduction>
{{.ExistingIntro}}
</current_introduction>
<changes>
{{.ChangeDescription}}
</changes>
<document_summaries>
{{.DocumentSummaries}}
</document_summaries>
<instructions>
1. Update the introduction to accurately reflect the current state of the wiki.
2. If documents were added, mention the new topics if they significantly change the wiki's scope.
3. If documents were removed, remove references to those topics if they no longer apply.
4. Keep the same tone, style, and title format as the existing introduction.
5. Keep it concise — 1 title line + 2-3 sentences.
6. Write in {{.Language}}.
</instructions>
Output ONLY the updated title and introduction paragraph. Do NOT generate any directory listings or page links.`
// WikiLogEntryTemplate is a simple template for log entries (not LLM-generated).
const WikiLogEntryTemplate = `## [{{.Date}}] {{.Operation}} | {{.Title}}
@@ -207,13 +260,15 @@ The wiki has a log page (slug: "log") that records all ingest and update activit
// between newly extracted items and existing wiki pages.
const WikiDeduplicationPrompt = `You are a deduplication system. Given a list of newly extracted items and a list of existing wiki pages, determine which new items refer to the same entity or concept as an existing page.
## Newly Extracted Items
<new_items>
{{.NewItems}}
</new_items>
## Existing Wiki Pages
<existing_pages>
{{.ExistingPages}}
</existing_pages>
## Instructions
<instructions>
For each newly extracted item, check if it refers to the same real-world entity or concept as any existing page. Consider:
- Name variations (e.g. "Acme Corp" vs "Acme Corporation", "RAG" vs "Retrieval-Augmented Generation")
- Abbreviations and full names
@@ -223,6 +278,7 @@ For each newly extracted item, check if it refers to the same real-world entity
Return a JSON object with a "merges" map. The key is the NEW item's slug, the value is the EXISTING page's slug that it should merge into. Only include items that have a match.
If no items match any existing pages, return: {"merges": {}}
</instructions>
Output ONLY valid JSON. Example:
{"merges": {"entity/acme-corporation": "entity/acme-corp", "concept/rag": "concept/retrieval-augmented-generation"}}`
+53 -28
View File
@@ -1284,6 +1284,15 @@ func (s *knowledgeService) cleanupWikiOnKnowledgeDelete(ctx context.Context, kbI
}
}
// Capture the summary page's description before deleting (for retract changeDesc)
var docSummary string
for _, page := range pages {
if page.PageType == types.WikiPageTypeSummary {
docSummary = page.Summary
break
}
}
var deletedSlugs []string
var retractSlugs []string
for _, page := range pages {
@@ -1327,14 +1336,17 @@ func (s *knowledgeService) cleanupWikiOnKnowledgeDelete(ctx context.Context, kbI
KnowledgeBaseID: kbID,
KnowledgeID: knowledgeID,
DocTitle: docTitle,
DocSummary: docSummary,
Language: lang,
PageSlugs: retractSlugs,
})
logger.Infof(ctx, "wiki cleanup: enqueued retract task for %d pages: %v", len(retractSlugs), retractSlugs)
}
// Rebuild index page to reflect deleted pages
if len(deletedSlugs) > 0 {
// If pages were deleted but no retract task was enqueued,
// we still need to update the index and log synchronously.
// (When retractSlugs > 0, ProcessWikiRetract handles index/log.)
if len(deletedSlugs) > 0 && len(retractSlugs) == 0 {
s.rebuildWikiIndexSimple(ctx, kbID, knowledgeID, docTitle, deletedSlugs)
}
}
@@ -1355,47 +1367,60 @@ func removeSourceRef(refs types.StringArray, knowledgeID string) types.StringArr
// rebuildWikiIndexSimple regenerates the index page from current page listing
// and appends a deletion log entry. Does NOT use LLM — generates a simple
// structured listing. Used during document deletion when LLM is not available.
// structured listing while preserving the LLM-generated intro from Summary field.
func (s *knowledgeService) rebuildWikiIndexSimple(ctx context.Context, kbID, knowledgeID, docTitle string, deletedSlugs []string) {
// Rebuild index
resp, err := s.wikiService.ListPages(ctx, &types.WikiPageListRequest{
KnowledgeBaseID: kbID,
PageSize: 500,
SortBy: "page_type",
SortOrder: "asc",
})
if err != nil || resp == nil {
allPages, err := s.wikiService.ListAllPages(ctx, kbID)
if err != nil {
logger.Warnf(ctx, "wiki cleanup: failed to list pages: %v", err)
return
}
var indexContent strings.Builder
indexContent.WriteString("# Wiki Index\n\n")
indexPage, _ := s.wikiService.GetIndex(ctx, kbID)
if indexPage == nil {
return
}
currentType := ""
for _, p := range resp.Pages {
// Preserve LLM-generated intro from Summary field
var indexContent strings.Builder
if indexPage.Summary != "" && indexPage.Summary != "Index" {
indexContent.WriteString(indexPage.Summary)
indexContent.WriteString("\n")
} else {
indexContent.WriteString("# Wiki Index\n\n")
}
// Group pages by type
typeOrder := []string{types.WikiPageTypeSummary, types.WikiPageTypeEntity, types.WikiPageTypeConcept, types.WikiPageTypeSynthesis, types.WikiPageTypeComparison}
grouped := make(map[string][]*types.WikiPage)
for _, p := range allPages {
if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog {
continue
}
if p.Status == types.WikiPageStatusArchived {
continue
}
if p.PageType != currentType {
currentType = p.PageType
label := currentType
if len(label) > 0 {
label = strings.ToUpper(label[:1]) + label[1:]
}
fmt.Fprintf(&indexContent, "\n## %s\n\n", label)
}
fmt.Fprintf(&indexContent, "- [[%s]] — %s\n", p.Slug, p.Summary)
grouped[p.PageType] = append(grouped[p.PageType], p)
}
indexPage, _ := s.wikiService.GetIndex(ctx, kbID)
if indexPage != nil {
indexPage.Content = indexContent.String()
if _, err := s.wikiService.UpdatePage(ctx, indexPage); err != nil {
logger.Warnf(ctx, "wiki cleanup: failed to rebuild index: %v", err)
for _, pt := range typeOrder {
pages := grouped[pt]
if len(pages) == 0 {
continue
}
label := pt
if len(label) > 0 {
label = strings.ToUpper(label[:1]) + label[1:]
}
fmt.Fprintf(&indexContent, "\n## %s\n\n", label)
for _, p := range pages {
fmt.Fprintf(&indexContent, "- [[%s]] — %s\n", p.Slug, p.Summary)
}
}
indexPage.Content = indexContent.String()
if _, err := s.wikiService.UpdatePage(ctx, indexPage); err != nil {
logger.Warnf(ctx, "wiki cleanup: failed to rebuild index: %v", err)
}
// Append log entry
+292 -152
View File
@@ -63,8 +63,9 @@ type WikiRetractPayload struct {
KnowledgeBaseID string `json:"knowledge_base_id"`
KnowledgeID string `json:"knowledge_id"`
DocTitle string `json:"doc_title"`
DocSummary string `json:"doc_summary,omitempty"` // one-line summary of the deleted document
Language string `json:"language,omitempty"`
PageSlugs []string `json:"page_slugs"` // pages to retract content from
PageSlugs []string `json:"page_slugs"`
}
// wikiIngestService handles the LLM-powered wiki generation pipeline
@@ -205,51 +206,14 @@ func (s *wikiIngestService) ProcessWikiRetract(ctx context.Context, t *asynq.Tas
lang := types.LanguageNameFromContext(ctx)
for _, slug := range payload.PageSlugs {
page, err := s.wikiService.GetPageBySlug(ctx, payload.KnowledgeBaseID, slug)
if err != nil || page == nil {
continue
}
// Build remaining sources list
var remainingSources string
for _, ref := range page.SourceRefs {
pipeIdx := strings.Index(ref, "|")
if pipeIdx > 0 {
remainingSources += "- " + ref[pipeIdx+1:] + "\n"
} else {
remainingSources += "- " + ref + "\n"
}
}
if remainingSources == "" {
remainingSources = "(no other sources)"
}
// Call LLM to retract content
updatedContent, err := s.generateWithTemplate(ctx, chatModel, agent.WikiPageRetractPrompt, map[string]string{
"ExistingContent": page.Content,
"DeletedDocTitle": payload.DocTitle,
"RemainingSources": remainingSources,
"Language": lang,
})
if err != nil {
logger.Warnf(ctx, "wiki retract: LLM call failed for page %s: %v", slug, err)
continue
}
page.Content = updatedContent
if _, err := s.wikiService.UpdatePage(ctx, page); err != nil {
logger.Warnf(ctx, "wiki retract: failed to update page %s: %v", slug, err)
} else {
logger.Infof(ctx, "wiki retract: updated page %s after removing content from '%s'", slug, payload.DocTitle)
}
}
s.retractPagesContent(ctx, chatModel, payload.KnowledgeBaseID, payload.DocTitle, payload.DocSummary, payload.PageSlugs, lang)
// Rebuild index page to reflect changes
retractChangeDesc := fmt.Sprintf("Removed document '%s': %s", payload.DocTitle, payload.DocSummary)
if err := s.rebuildIndexPage(ctx, chatModel, WikiIngestPayload{
TenantID: payload.TenantID,
KnowledgeBaseID: payload.KnowledgeBaseID,
}, lang); err != nil {
}, retractChangeDesc, lang); err != nil {
logger.Warnf(ctx, "wiki retract: rebuild index failed: %v", err)
}
@@ -266,6 +230,81 @@ func (s *wikiIngestService) ProcessWikiRetract(ctx context.Context, t *asynq.Tas
return nil
}
// retractPagesContent uses LLM to remove a deleted/stale document's contributions
// from the given wiki pages. docContent is the deleted document's content (or summary)
// so the LLM can accurately identify which parts to remove. It does NOT rebuild the
// index or append log entries — callers are responsible for those post-processing steps.
func (s *wikiIngestService) retractPagesContent(
ctx context.Context,
chatModel chat.Chat,
kbID, docTitle, docContent string,
pageSlugs []string,
lang string,
) {
allPages, _ := s.wikiService.ListAllPages(ctx, kbID)
slugTitleMap := make(map[string]string)
for _, p := range allPages {
if p.PageType != types.WikiPageTypeIndex && p.PageType != types.WikiPageTypeLog && p.Status != types.WikiPageStatusArchived {
slugTitleMap[p.Slug] = p.Title
}
}
for _, slug := range pageSlugs {
page, err := s.wikiService.GetPageBySlug(ctx, kbID, slug)
if err != nil || page == nil {
continue
}
var remainingSources string
for _, ref := range page.SourceRefs {
pipeIdx := strings.Index(ref, "|")
if pipeIdx > 0 {
remainingSources += "- " + ref[pipeIdx+1:] + "\n"
} else {
remainingSources += "- " + ref + "\n"
}
}
if remainingSources == "" {
remainingSources = "(no other sources)"
}
var relatedSlugs strings.Builder
for _, outSlug := range page.OutLinks {
if title, ok := slugTitleMap[outSlug]; ok {
fmt.Fprintf(&relatedSlugs, "- %s (%s)\n", outSlug, title)
}
}
updatedContent, err := s.generateWithTemplate(ctx, chatModel, agent.WikiPageRetractPrompt, map[string]string{
"ExistingContent": page.Content,
"DeletedDocTitle": docTitle,
"DeletedDocContent": docContent,
"RemainingSources": remainingSources,
"AvailableSlugs": relatedSlugs.String(),
"Language": lang,
})
if err != nil {
logger.Warnf(ctx, "wiki retract: LLM call failed for page %s: %v", slug, err)
continue
}
retractedSummary, retractedBody := splitSummaryLine(updatedContent)
if retractedBody != "" {
page.Content = retractedBody
} else {
page.Content = updatedContent
}
if retractedSummary != "" {
page.Summary = retractedSummary
}
if _, err := s.wikiService.UpdatePage(ctx, page); err != nil {
logger.Warnf(ctx, "wiki retract: failed to update page %s: %v", slug, err)
} else {
logger.Infof(ctx, "wiki retract: updated page %s after removing content from '%s'", slug, docTitle)
}
}
}
// ProcessWikiIngest processes a batch wiki ingest task.
//
// Concurrency model (Redis mode):
@@ -356,19 +395,30 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
// Process each document
var allPagesAffected []string
var docResults []*docIngestResult
for _, knowledgeID := range knowledgeIDs {
pages, err := s.processOneDocument(ctx, chatModel, payload, knowledgeID, lang)
result, err := s.processOneDocument(ctx, chatModel, payload, knowledgeID, lang)
if err != nil {
logger.Warnf(ctx, "wiki ingest: failed to process knowledge %s: %v", knowledgeID, err)
continue
}
allPagesAffected = append(allPagesAffected, pages...)
if result != nil {
allPagesAffected = append(allPagesAffected, result.Pages...)
docResults = append(docResults, result)
}
}
// Batch post-processing (once for the whole batch, not per-doc)
// Build change description from processed documents
var changeDesc strings.Builder
fmt.Fprintf(&changeDesc, "Added %d documents:\n", len(docResults))
for _, r := range docResults {
fmt.Fprintf(&changeDesc, "- %s: %s\n", r.DocTitle, r.Summary)
}
// Rebuild index page
if err := s.rebuildIndexPage(ctx, chatModel, payload, lang); err != nil {
if err := s.rebuildIndexPage(ctx, chatModel, payload, changeDesc.String(), lang); err != nil {
logger.Warnf(ctx, "wiki ingest: rebuild index failed: %v", err)
}
@@ -453,15 +503,22 @@ func (s *wikiIngestService) drainPendingList(ctx context.Context, kbID string) [
return unique
}
// docIngestResult captures per-document info for batch post-processing.
type docIngestResult struct {
KnowledgeID string
DocTitle string
Summary string // one-line summary of the document (from summary page)
Pages []string // affected page slugs
}
// processOneDocument handles wiki ingest for a single document.
// Returns the list of affected page slugs.
func (s *wikiIngestService) processOneDocument(
ctx context.Context,
chatModel chat.Chat,
payload WikiIngestPayload,
knowledgeID string,
lang string,
) ([]string, error) {
) (*docIngestResult, error) {
// Get document chunks and reconstruct content
chunks, err := s.chunkRepo.ListChunksByKnowledgeID(ctx, payload.TenantID, knowledgeID)
if err != nil {
@@ -489,6 +546,7 @@ func (s *wikiIngestService) processOneDocument(
}
var pagesAffected []string
var docSummaryLine string
sourceRef := fmt.Sprintf("%s|%s", knowledgeID, docTitle)
// Snapshot existing page slugs for stale detection
@@ -503,7 +561,7 @@ func (s *wikiIngestService) processOneDocument(
}
// Step 1: Extract entities and concepts
extractedPages, extractedSlugs, err := s.extractEntitiesAndConcepts(ctx, chatModel, content, docTitle, lang, docPayload, sourceRef, oldPageSlugs)
extractedPages, err := s.extractEntitiesAndConcepts(ctx, chatModel, content, docTitle, lang, docPayload, sourceRef, oldPageSlugs)
if err != nil {
logger.Warnf(ctx, "wiki ingest: knowledge extraction failed for %s: %v", knowledgeID, err)
} else {
@@ -513,7 +571,7 @@ func (s *wikiIngestService) processOneDocument(
// Step 2: Generate summary page
summarySlug := fmt.Sprintf("summary/%s", slugify(docTitle))
var slugListing string
for _, slug := range extractedSlugs {
for _, slug := range extractedPages {
slugListing += fmt.Sprintf("- [[%s]]\n", slug)
}
summaryContent, err := s.generateWithTemplate(ctx, chatModel, agent.WikiSummaryPrompt, map[string]string{
@@ -527,6 +585,14 @@ func (s *wikiIngestService) processOneDocument(
if err != nil {
logger.Errorf(ctx, "wiki ingest: generate summary failed for %s: %v", knowledgeID, err)
} else {
sumLine, sumBody := splitSummaryLine(summaryContent)
if sumBody == "" {
sumBody = summaryContent
}
if sumLine == "" {
sumLine = docTitle
}
docSummaryLine = sumLine
_, err := s.wikiService.CreatePage(ctx, &types.WikiPage{
ID: uuid.New().String(),
TenantID: payload.TenantID,
@@ -535,8 +601,8 @@ func (s *wikiIngestService) processOneDocument(
Title: docTitle + " - Summary",
PageType: types.WikiPageTypeSummary,
Status: types.WikiPageStatusDraft,
Content: summaryContent,
Summary: truncateString(summaryContent, 200),
Content: sumBody,
Summary: sumLine,
SourceRefs: types.StringArray{sourceRef},
})
if err != nil {
@@ -547,34 +613,36 @@ func (s *wikiIngestService) processOneDocument(
}
// Retract stale pages (pages this doc previously contributed to but no longer does)
s.retractStalePages(ctx, docPayload, oldPageSlugs, pagesAffected, docTitle, lang)
s.retractStalePages(ctx, chatModel, docPayload, oldPageSlugs, pagesAffected, docTitle, content, lang)
logger.Infof(ctx, "wiki ingest: processed knowledge %s, %d pages affected", knowledgeID, len(pagesAffected))
return pagesAffected, nil
return &docIngestResult{
KnowledgeID: knowledgeID,
DocTitle: docTitle,
Summary: docSummaryLine,
Pages: pagesAffected,
}, nil
}
// cleanDeadLinks removes [[wiki-links]] that point to archived or deleted pages.
// Scans all published pages, checks each out_link, and removes references to
// pages that no longer exist or are archived. No LLM call — pure text cleanup.
func (s *wikiIngestService) cleanDeadLinks(ctx context.Context, kbID string) {
resp, err := s.wikiService.ListPages(ctx, &types.WikiPageListRequest{
KnowledgeBaseID: kbID,
PageSize: 500,
})
if err != nil || resp == nil {
allPages, err := s.wikiService.ListAllPages(ctx, kbID)
if err != nil || len(allPages) == 0 {
return
}
// Build set of live (non-archived, non-system) slugs
liveSlugs := make(map[string]bool)
for _, p := range resp.Pages {
for _, p := range allPages {
if p.Status != types.WikiPageStatusArchived {
liveSlugs[p.Slug] = true
}
}
var cleaned int
for _, p := range resp.Pages {
for _, p := range allPages {
if p.Status == types.WikiPageStatusArchived {
continue
}
@@ -622,11 +690,8 @@ func (s *wikiIngestService) cleanDeadLinks(ctx context.Context, kbID string) {
// Only processes entity/concept/synthesis/comparison pages (not index/log/summary).
func (s *wikiIngestService) injectCrossLinks(ctx context.Context, kbID string, affectedSlugs []string) {
// Build a title→slug lookup from ALL pages in this KB
resp, err := s.wikiService.ListPages(ctx, &types.WikiPageListRequest{
KnowledgeBaseID: kbID,
PageSize: 500,
})
if err != nil || resp == nil || len(resp.Pages) < 2 {
allPages, err := s.wikiService.ListAllPages(ctx, kbID)
if err != nil || len(allPages) < 2 {
return
}
@@ -635,7 +700,7 @@ func (s *wikiIngestService) injectCrossLinks(ctx context.Context, kbID string, a
title string
}
var allRefs []pageRef
for _, p := range resp.Pages {
for _, p := range allPages {
if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog {
continue
}
@@ -664,7 +729,7 @@ func (s *wikiIngestService) injectCrossLinks(ctx context.Context, kbID string, a
}
var updated int
for _, p := range resp.Pages {
for _, p := range allPages {
if !affectedSet[p.Slug] {
continue
}
@@ -745,17 +810,14 @@ func replaceFirstOutsideLinks(s, old, newStr string) string {
// getExistingPageSlugsForKnowledge returns all page slugs that currently reference
// a given knowledge ID in their source_refs. Used to snapshot state before re-ingest.
func (s *wikiIngestService) getExistingPageSlugsForKnowledge(ctx context.Context, kbID, knowledgeID string) map[string]bool {
resp, err := s.wikiService.ListPages(ctx, &types.WikiPageListRequest{
KnowledgeBaseID: kbID,
PageSize: 500,
})
if err != nil || resp == nil {
allPages, err := s.wikiService.ListAllPages(ctx, kbID)
if err != nil || len(allPages) == 0 {
return nil
}
slugs := make(map[string]bool)
prefix := knowledgeID + "|"
for _, p := range resp.Pages {
for _, p := range allPages {
if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog {
continue
}
@@ -771,14 +833,15 @@ func (s *wikiIngestService) getExistingPageSlugsForKnowledge(ctx context.Context
// retractStalePages handles pages that were previously linked to this document
// but are no longer produced by the updated extraction.
// - Single-source stale pages → archived
// - Multi-source stale pages → enqueue LLM retract to clean content
// - Single-source stale pages → deleted
// - Multi-source stale pages → LLM retract to clean content synchronously
func (s *wikiIngestService) retractStalePages(
ctx context.Context,
chatModel chat.Chat,
payload WikiIngestPayload,
oldSlugs map[string]bool,
newSlugs []string,
docTitle, lang string,
docTitle, docContent, lang string,
) {
if len(oldSlugs) == 0 {
return
@@ -838,16 +901,8 @@ func (s *wikiIngestService) retractStalePages(
}
}
// Enqueue LLM retraction for multi-source stale pages
if len(retractSlugs) > 0 {
EnqueueWikiRetract(ctx, s.task, WikiRetractPayload{
TenantID: payload.TenantID,
KnowledgeBaseID: payload.KnowledgeBaseID,
KnowledgeID: payload.KnowledgeID,
DocTitle: docTitle,
Language: payload.Language,
PageSlugs: retractSlugs,
})
s.retractPagesContent(ctx, chatModel, payload.KnowledgeBaseID, docTitle, docContent, retractSlugs, lang)
}
}
@@ -866,7 +921,7 @@ type combinedExtraction struct {
}
// extractEntitiesAndConcepts performs a single LLM call to extract both entities and concepts,
// then upserts pages for each. Returns the list of affected page slugs and all extracted slugs (for linking).
// then upserts pages for each. Returns the list of successfully upserted page slugs.
// oldPageSlugs contains slugs from the previous version of this document — passed to LLM for slug stability.
func (s *wikiIngestService) extractEntitiesAndConcepts(
ctx context.Context,
@@ -875,7 +930,7 @@ func (s *wikiIngestService) extractEntitiesAndConcepts(
payload WikiIngestPayload,
sourceRef string,
oldPageSlugs map[string]bool,
) ([]string, []string, error) {
) ([]string, error) {
// Build previous slugs listing for the prompt
var prevSlugsText string
if len(oldPageSlugs) > 0 {
@@ -896,7 +951,7 @@ func (s *wikiIngestService) extractEntitiesAndConcepts(
"PreviousSlugs": prevSlugsText,
})
if err != nil {
return nil, nil, fmt.Errorf("combined extraction failed: %w", err)
return nil, fmt.Errorf("combined extraction failed: %w", err)
}
// Clean JSON - strip markdown code blocks if present
@@ -909,20 +964,7 @@ func (s *wikiIngestService) extractEntitiesAndConcepts(
var result combinedExtraction
if err := json.Unmarshal([]byte(extractionJSON), &result); err != nil {
logger.Warnf(ctx, "wiki ingest: failed to parse combined extraction JSON: %v\nRaw: %s", err, truncateString(extractionJSON, 500))
return nil, nil, fmt.Errorf("parse combined extraction JSON: %w", err)
}
// Collect all extracted slugs (before dedup remapping) for summary linking
var allSlugs []string
for _, item := range result.Entities {
if item.Slug != "" {
allSlugs = append(allSlugs, item.Slug)
}
}
for _, item := range result.Concepts {
if item.Slug != "" {
allSlugs = append(allSlugs, item.Slug)
}
return nil, fmt.Errorf("parse combined extraction JSON: %w", err)
}
var affected []string
@@ -933,19 +975,6 @@ func (s *wikiIngestService) extractEntitiesAndConcepts(
// Deduplicate concepts against existing wiki pages (LLM-based)
result.Concepts = s.deduplicateItems(ctx, chatModel, result.Concepts, types.WikiPageTypeConcept, payload.KnowledgeBaseID)
// Update allSlugs after dedup (use final slugs)
allSlugs = allSlugs[:0]
for _, item := range result.Entities {
if item.Slug != "" {
allSlugs = append(allSlugs, item.Slug)
}
}
for _, item := range result.Concepts {
if item.Slug != "" {
allSlugs = append(allSlugs, item.Slug)
}
}
// Upsert entity pages
entitySlugs, err := s.upsertExtractedPages(ctx, chatModel, result.Entities, types.WikiPageTypeEntity, docTitle, lang, payload, sourceRef)
if err != nil {
@@ -962,7 +991,7 @@ func (s *wikiIngestService) extractEntitiesAndConcepts(
affected = append(affected, conceptSlugs...)
}
return affected, allSlugs, nil
return affected, nil
}
// upsertExtractedPages creates or updates wiki pages from pre-extracted items.
@@ -996,8 +1025,16 @@ func (s *wikiIngestService) upsertExtractedPages(
continue
}
existing.Content = updatedContent
existing.Summary = item.Description
// LLM returns "SUMMARY: ..." on first line — use it as the authoritative summary
updatedSummary, updatedBody := splitSummaryLine(updatedContent)
if updatedBody != "" {
existing.Content = updatedBody
} else {
existing.Content = updatedContent
}
if updatedSummary != "" {
existing.Summary = updatedSummary
}
existing.SourceRefs = appendUnique(existing.SourceRefs, sourceRef)
if _, err := s.wikiService.UpdatePage(ctx, existing); err != nil {
@@ -1032,47 +1069,143 @@ func (s *wikiIngestService) upsertExtractedPages(
return affected, nil
}
// rebuildIndexPage regenerates the index page from all existing pages
func (s *wikiIngestService) rebuildIndexPage(ctx context.Context, chatModel chat.Chat, payload WikiIngestPayload, lang string) error {
// List all pages
resp, err := s.wikiService.ListPages(ctx, &types.WikiPageListRequest{
KnowledgeBaseID: payload.KnowledgeBaseID,
PageSize: 500,
SortBy: "page_type",
SortOrder: "asc",
})
// rebuildIndexPage regenerates the index page.
//
// Strategy: Index = LLM-generated intro (stored in Summary field) + code-generated directory.
// - Intro: stored in indexPage.Summary. First time: generated from document summaries.
// Subsequent: incrementally updated with changeDescription.
// - Directory: pure code, rebuilt every time. O(N) string concat, no LLM.
func (s *wikiIngestService) rebuildIndexPage(ctx context.Context, chatModel chat.Chat, payload WikiIngestPayload, changeDesc, lang string) error {
indexPage, _ := s.wikiService.GetIndex(ctx, payload.KnowledgeBaseID)
if indexPage == nil {
return nil
}
// List all live pages
allPages, err := s.wikiService.ListAllPages(ctx, payload.KnowledgeBaseID)
if err != nil {
return err
}
// Build page listing
var listing strings.Builder
for _, p := range resp.Pages {
typeOrder := []string{
types.WikiPageTypeSummary, types.WikiPageTypeEntity, types.WikiPageTypeConcept,
types.WikiPageTypeSynthesis, types.WikiPageTypeComparison,
}
typeLabels := map[string]string{
types.WikiPageTypeSummary: "Summary", types.WikiPageTypeEntity: "Entity",
types.WikiPageTypeConcept: "Concept", types.WikiPageTypeSynthesis: "Synthesis",
types.WikiPageTypeComparison: "Comparison",
}
grouped := make(map[string][]*types.WikiPage)
totalPages := 0
for _, p := range allPages {
if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog {
continue
}
listing.WriteString(fmt.Sprintf("- [%s] [[%s]] | %s — %s\n", p.PageType, p.Slug, p.Title, p.Summary))
if p.Status == types.WikiPageStatusArchived {
continue
}
grouped[p.PageType] = append(grouped[p.PageType], p)
totalPages++
}
indexContent, err := s.generateWithTemplate(ctx, chatModel, agent.WikiIndexRebuildPrompt, map[string]string{
"PageListing": listing.String(),
"Language": lang,
})
if err != nil {
return fmt.Errorf("generate index: %w", err)
// Build document summaries listing (only summary-type pages — they represent documents)
var docSummaries strings.Builder
for _, p := range grouped[types.WikiPageTypeSummary] {
fmt.Fprintf(&docSummaries, "- %s: %s\n", p.Title, p.Summary)
}
if docSummaries.Len() == 0 {
docSummaries.WriteString("(no documents yet)")
}
// Get or create index page
indexPage, _ := s.wikiService.GetIndex(ctx, payload.KnowledgeBaseID)
if indexPage != nil {
indexPage.Content = indexContent
indexPage.Summary = "Wiki index - table of contents"
if _, err := s.wikiService.UpdatePage(ctx, indexPage); err != nil {
return fmt.Errorf("update index page: %w", err)
// Generate or update intro
existingIntro := indexPage.Summary
var intro string
if existingIntro == "" || existingIntro == "Wiki index - table of contents" {
// First time — generate intro from scratch
generatedIntro, genErr := s.generateWithTemplate(ctx, chatModel, agent.WikiIndexIntroPrompt, map[string]string{
"DocumentSummaries": docSummaries.String(),
"Language": lang,
})
if genErr != nil {
intro = "# Wiki Index\n\nThis wiki contains knowledge extracted from uploaded documents.\n"
} else {
intro = strings.TrimSpace(generatedIntro)
}
} else if changeDesc != "" {
// Incremental update — tell LLM what changed
updatedIntro, genErr := s.generateWithTemplate(ctx, chatModel, agent.WikiIndexIntroUpdatePrompt, map[string]string{
"ExistingIntro": existingIntro,
"ChangeDescription": changeDesc,
"DocumentSummaries": docSummaries.String(),
"Language": lang,
})
if genErr != nil {
intro = existingIntro // keep existing on error
} else {
intro = strings.TrimSpace(updatedIntro)
}
} else {
intro = existingIntro // no change description, keep as-is
}
// Build directory (pure code, no LLM)
var dir strings.Builder
for _, pt := range typeOrder {
pages := grouped[pt]
if len(pages) == 0 {
continue
}
fmt.Fprintf(&dir, "\n## %s (%d)\n\n", typeLabels[pt], len(pages))
for _, p := range pages {
summary := p.Summary
fmt.Fprintf(&dir, "[[%s]] — %s\n", p.Slug, summary)
}
}
for pt, pages := range grouped {
inOrder := false
for _, o := range typeOrder {
if o == pt {
inOrder = true
break
}
}
if inOrder || len(pages) == 0 {
continue
}
fmt.Fprintf(&dir, "\n## %s (%d)\n\n", pt, len(pages))
for _, p := range pages {
fmt.Fprintf(&dir, "[[%s]] — %s\n", p.Slug, p.Summary)
}
}
if totalPages == 0 {
dir.WriteString("\n*No wiki pages yet. Upload documents to get started.*\n")
}
return nil
indexPage.Content = intro + "\n" + dir.String()
indexPage.Summary = intro // persist intro for next incremental update
_, err = s.wikiService.UpdatePage(ctx, indexPage)
return err
}
// splitSummaryLine extracts the "SUMMARY: ..." line from LLM output.
// Returns (summary, content). If no SUMMARY line found, summary is empty.
func splitSummaryLine(raw string) (summary string, content string) {
raw = strings.TrimSpace(raw)
if strings.HasPrefix(raw, "SUMMARY:") || strings.HasPrefix(raw, "SUMMARY:") {
idx := strings.IndexByte(raw, '\n')
if idx < 0 {
// Only one line
return strings.TrimSpace(strings.TrimPrefix(strings.TrimPrefix(raw, "SUMMARY:"), "SUMMARY:")), ""
}
summaryLine := raw[:idx]
summaryLine = strings.TrimPrefix(summaryLine, "SUMMARY:")
summaryLine = strings.TrimPrefix(summaryLine, "SUMMARY:")
return strings.TrimSpace(summaryLine), strings.TrimSpace(raw[idx+1:])
}
return "", raw
}
// appendLogEntry appends an entry to the log page, including any synthesis suggestions
@@ -1136,18 +1269,25 @@ func (s *wikiIngestService) deduplicateItems(
}
// Get existing pages of the same type
resp, err := s.wikiService.ListPages(ctx, &types.WikiPageListRequest{
KnowledgeBaseID: kbID,
PageType: pageType,
PageSize: 2000,
})
if err != nil || resp == nil || len(resp.Pages) == 0 {
return items // No existing pages → nothing to deduplicate against
allPages, err := s.wikiService.ListAllPages(ctx, kbID)
if err != nil || len(allPages) == 0 {
return items
}
// Filter to the target type
var typedPages []*types.WikiPage
for _, p := range allPages {
if p.PageType == pageType {
typedPages = append(typedPages, p)
}
}
if len(typedPages) == 0 {
return items // No existing pages of this type → nothing to deduplicate against
}
// Build existing pages listing
var existingBuf strings.Builder
for _, p := range resp.Pages {
for _, p := range typedPages {
fmt.Fprintf(&existingBuf, "- slug: %s | title: %s\n", p.Slug, p.Title)
}
+8 -40
View File
@@ -20,9 +20,9 @@ var wikiLinkRegex = regexp.MustCompile(`\[\[([^\]]+)\]\]`)
// wikiPageService implements the WikiPageService interface
type wikiPageService struct {
repo interfaces.WikiPageRepository
chunkRepo interfaces.ChunkRepository
kbService interfaces.KnowledgeBaseService
repo interfaces.WikiPageRepository
chunkRepo interfaces.ChunkRepository
kbService interfaces.KnowledgeBaseService
}
// NewWikiPageService creates a new wiki page service
@@ -70,11 +70,6 @@ func (s *wikiPageService) CreatePage(ctx context.Context, page *types.WikiPage)
// Update inbound links on target pages
s.updateInLinks(ctx, page.KnowledgeBaseID, page.Slug, page.OutLinks)
// Sync to chunks for retrieval pipeline integration
if err := s.syncToChunk(ctx, page); err != nil {
logger.Warnf(ctx, "wiki: failed to sync page %s to chunks: %v", page.Slug, err)
}
return page, nil
}
@@ -108,11 +103,6 @@ func (s *wikiPageService) UpdatePage(ctx context.Context, page *types.WikiPage)
s.removeInLinks(ctx, existing.KnowledgeBaseID, existing.Slug, oldOutLinks)
s.updateInLinks(ctx, existing.KnowledgeBaseID, existing.Slug, existing.OutLinks)
// Re-sync to chunks
if err := s.syncToChunk(ctx, existing); err != nil {
logger.Warnf(ctx, "wiki: failed to sync page %s to chunks: %v", existing.Slug, err)
}
return existing, nil
}
@@ -341,6 +331,11 @@ func (s *wikiPageService) RebuildLinks(ctx context.Context, kbID string) error {
return nil
}
// ListAllPages retrieves all wiki pages without pagination.
func (s *wikiPageService) ListAllPages(ctx context.Context, kbID string) ([]*types.WikiPage, error) {
return s.repo.ListAll(ctx, kbID)
}
// SearchPages performs full-text search over wiki pages
func (s *wikiPageService) SearchPages(ctx context.Context, kbID string, query string, limit int) ([]*types.WikiPage, error) {
return s.repo.Search(ctx, kbID, query, limit)
@@ -413,33 +408,6 @@ func (s *wikiPageService) removeInLinks(ctx context.Context, kbID string, source
}
}
// syncToChunk syncs a wiki page to the chunk table for retrieval pipeline integration.
// Each wiki page maps to a single chunk with ChunkType = "wiki_page".
// The chunk's KnowledgeID is set to the wiki page ID for tracking.
func (s *wikiPageService) syncToChunk(ctx context.Context, page *types.WikiPage) error {
// Build content for the chunk: title + summary + content
chunkContent := fmt.Sprintf("# %s\n\n%s\n\n%s", page.Title, page.Summary, page.Content)
chunk := &types.Chunk{
ID: "wp-" + page.ID, // prefix to avoid collision with regular chunks
TenantID: page.TenantID,
KnowledgeID: page.ID, // link back to wiki page
KnowledgeBaseID: page.KnowledgeBaseID,
Content: chunkContent,
ChunkType: types.ChunkTypeWikiPage,
IsEnabled: page.Status == types.WikiPageStatusPublished,
Status: int(types.ChunkStatusStored),
}
// Try to create; if it already exists, update
err := s.chunkRepo.CreateChunks(ctx, []*types.Chunk{chunk})
if err != nil {
// Fallback: update existing chunk
return s.chunkRepo.UpdateChunk(ctx, chunk)
}
return nil
}
// deleteChunkForPage removes the synced chunk for a wiki page
func (s *wikiPageService) deleteChunkForPage(ctx context.Context, page *types.WikiPage) {
chunkID := "wp-" + page.ID
+4
View File
@@ -52,6 +52,10 @@ type WikiPageService interface {
// RebuildLinks re-parses all pages and rebuilds bidirectional link references.
RebuildLinks(ctx context.Context, kbID string) error
// ListAllPages retrieves all wiki pages in a knowledge base without pagination.
// Used for index rebuild, graph generation, cross-link injection, etc.
ListAllPages(ctx context.Context, kbID string) ([]*types.WikiPage, error)
// SearchPages performs full-text search over wiki pages.
SearchPages(ctx context.Context, kbID string, query string, limit int) ([]*types.WikiPage, error)
}