From ec8a5fc392678c042eb023a1a8211de427ba4c8f Mon Sep 17 00:00:00 2001 From: wizardchen Date: Wed, 8 Apr 2026 16:50:38 +0800 Subject: [PATCH] refactor: Enhance wiki page management and index rebuilding - Introduced a new method to list all wiki pages without pagination, facilitating index rebuilding and cross-linking. - Updated the wiki ingest and retract processes to include document summaries, improving clarity in log entries and index updates. - Enhanced the cleanup process for deleted knowledge, ensuring accurate summary retention and index regeneration. - Streamlined the handling of page updates and deletions, optimizing the overall wiki management workflow. These changes improve the efficiency and reliability of wiki content management and indexing. --- internal/agent/prompts_wiki.go | 188 ++++++--- internal/application/service/knowledge.go | 81 ++-- internal/application/service/wiki_ingest.go | 444 +++++++++++++------- internal/application/service/wiki_page.go | 48 +-- internal/types/interfaces/wiki_page.go | 4 + 5 files changed, 479 insertions(+), 286 deletions(-) diff --git a/internal/agent/prompts_wiki.go b/internal/agent/prompts_wiki.go index 6d741a8c..1febdd7d 100644 --- a/internal/agent/prompts_wiki.go +++ b/internal/agent/prompts_wiki.go @@ -7,48 +7,54 @@ package agent // WikiSummaryPrompt generates a summary page for a newly ingested document. const WikiSummaryPrompt = `You are a wiki editor. Given the following document content, create a structured wiki summary page in Markdown format. -## Document Info -- Title: {{.Title}} -- File Name: {{.FileName}} -- File Type: {{.FileType}} - -## Document Content + +{{.Title}} +{{.FileName}} +{{.FileType}} + {{.Content}} + + -## Available Wiki Pages (use these exact slugs for links) + {{.ExtractedSlugs}} + -## Instructions -1. Write a comprehensive summary of the document in Markdown format. -2. Include the key facts, arguments, and conclusions. -3. Use proper heading hierarchy (## for sections, ### for subsections). -4. Use [[wiki-link]] syntax to reference the available wiki pages listed above. Use the EXACT slugs provided — do NOT invent new slugs. -5. At the end, include a "## Key Takeaways" section with bullet points. -6. Write in {{.Language}}. -7. Keep the summary concise but thorough (500-1500 words depending on document length). + +1. The FIRST line of your output MUST be: SUMMARY: {one sentence, 15-40 words, describing what this document is about — for wiki index listing} +2. After the SUMMARY line, write a comprehensive summary of the document in Markdown format. +3. Include the key facts, arguments, and conclusions. +4. Use proper heading hierarchy (## for sections, ### for subsections). +5. Use [[wiki-link]] syntax to reference the available wiki pages listed above. Use the EXACT slugs provided — do NOT invent new slugs. +6. At the end, include a "## Key Takeaways" section with bullet points. +7. Write in {{.Language}}. +8. Keep the summary concise but thorough (500-1500 words depending on document length). + -Output ONLY the Markdown content for the wiki page. Do not include any preamble or explanation.` +Output the SUMMARY line first, then the Markdown content. Do not include any other preamble.` // WikiKnowledgeExtractPrompt extracts both entities and concepts in a single LLM call. // Returns a JSON object with "entities" and "concepts" arrays. // This replaces the former separate WikiEntityExtractPrompt and WikiConceptExtractPrompt. const WikiKnowledgeExtractPrompt = `You are a knowledge extraction system. Analyze the following document and extract all significant entities AND key concepts. -## Document Info -- Title: {{.Title}} - -## Document Content + +{{.Title}} + {{.Content}} + + -## Previously Extracted Slugs (from earlier version of this document) + {{.PreviousSlugs}} + -## Instructions + Return a JSON object with two arrays: "entities" and "concepts". **IMPORTANT: Write ALL names, descriptions, and details in {{.Language}}.** ### Slug Continuity Rules -If a "Previously Extracted Slugs" list is provided above, you MUST follow these rules: +If previous slugs are provided above, you MUST follow these rules: - If an entity or concept from the previous extraction still exists in the current document, **reuse its exact slug** from the previous list. Do NOT generate a new slug for the same thing. - If an entity or concept no longer appears in the document, **do NOT include it** in the output. - Only generate new slugs for entities/concepts that are genuinely new (not present in the previous list). @@ -58,7 +64,7 @@ If a "Previously Extracted Slugs" list is provided above, you MUST follow these Each entity should have: - "name": The entity name in {{.Language}} (human-readable) - "slug": URL-friendly slug, format "entity/" (use romanized/pinyin form for non-Latin names). **Reuse previous slug if the entity was extracted before.** -- "description": A one-sentence description in {{.Language}} based on what the document says +- "description": **Index listing summary** — one sentence, 15-40 words, in {{.Language}}. Describes WHAT this entity IS and its role in the document. Must be self-contained (understandable without reading the full page). This will be displayed in the wiki index. - "details": A 2-5 sentence summary in {{.Language}} of key facts from the document Only include entities that are substantively discussed (mentioned at least twice or described in detail). Do NOT include generic terms. @@ -67,7 +73,7 @@ Only include entities that are substantively discussed (mentioned at least twice Each concept should have: - "name": The concept name in {{.Language}} (human-readable) - "slug": URL-friendly slug, format "concept/" (use romanized/pinyin form for non-Latin names). **Reuse previous slug if the concept was extracted before.** -- "description": A one-sentence definition or description in {{.Language}} +- "description": **Index listing summary** — one sentence, 15-40 words, in {{.Language}}. Defines WHAT this concept IS. Must be self-contained (understandable without reading the full page). This will be displayed in the wiki index. - "details": A 2-5 sentence explanation in {{.Language}} as discussed in the document Only include concepts that are substantively discussed. Skip trivial or overly generic concepts. @@ -76,6 +82,7 @@ Only include concepts that are substantively discussed. Skip trivial or overly g - If something is a specific named thing (person, company, product, place), put it ONLY in "entities". - If something is an abstract idea, methodology, or theory, put it ONLY in "concepts". - Never duplicate items across the two arrays. + Output ONLY valid JSON. Example: { @@ -100,63 +107,109 @@ Output ONLY valid JSON. Example: // WikiPageUpdatePrompt incrementally updates an existing wiki page with new information. const WikiPageUpdatePrompt = `You are a wiki editor tasked with updating an existing wiki page with new information from a recently ingested document. -## Existing Page Content + {{.ExistingContent}} + -## New Information from Document "{{.NewDocTitle}}" + +{{.NewDocTitle}} + {{.NewContent}} + + -## Instructions -1. Merge the new information into the existing page content. -2. Preserve all existing information that is still valid. -3. If the new information contradicts existing content, prefer the newer information and silently replace the old claim. -4. Add new facts, details, and context from the new document. -5. Update cross-references: add new [[wiki-link]] references where appropriate. -6. Maintain the existing page structure and formatting style. -7. Add a source reference to the new document at the bottom. -8. Write in {{.Language}}. + +1. The FIRST line of your output MUST be: SUMMARY: {one sentence, 15-40 words, describing what this page is about after the update — for wiki index listing} +2. Merge the new information into the existing page content. +3. Preserve all existing information that is still valid. +4. If the new information contradicts existing content, prefer the newer information and silently replace the old claim. +5. Add new facts, details, and context from the new document. +6. Preserve any existing [[wiki-link]] references in the content. Do NOT invent new wiki-link slugs. +7. Maintain the existing page structure and formatting style. +8. Add a source reference to the new document at the bottom. +9. Write in {{.Language}}. + -Output ONLY the updated Markdown content. Do not include any preamble or explanation.` +Output the SUMMARY line first, then the updated Markdown content. Do not include any other preamble.` // WikiPageRetractPrompt removes information contributed by a deleted document from an existing wiki page. const WikiPageRetractPrompt = `You are a wiki editor. A source document has been DELETED from the knowledge base. You must update the existing wiki page to remove any information that came exclusively from this deleted document. -## Existing Page Content + {{.ExistingContent}} + -## Deleted Document Title -{{.DeletedDocTitle}} + +{{.DeletedDocTitle}} + +{{.DeletedDocContent}} + + -## Remaining Source Documents + {{.RemainingSources}} + -## Instructions -1. Carefully review the existing page content. -2. Remove any facts, claims, or details that were ONLY sourced from "{{.DeletedDocTitle}}". -3. If a fact is also supported by the remaining source documents, KEEP it. -4. If you are unsure whether a fact came from the deleted document, keep it as-is — do NOT add any review notes or annotations. -5. Update or remove the "Source: {{.DeletedDocTitle}}" reference line if present. -6. Remove any [[wiki-link]] references that point to pages that no longer exist, if you can identify them. -7. Maintain the existing page structure, formatting style, and language. -8. If after removing the deleted document's contributions the page becomes nearly empty, output just: "# [Title]\n\n*This page's primary source document was removed.*" -9. Write in {{.Language}}. + +{{.AvailableSlugs}} + -Output ONLY the updated Markdown content. Do not include any preamble or explanation.` + +1. The FIRST line of your output MUST be: SUMMARY: {one sentence, 15-40 words, describing what this page is about after retraction — for wiki index listing} +2. Carefully review the existing page content and compare it with the deleted document content above. +3. Remove any facts, claims, or details that were ONLY sourced from the deleted document. +4. If a fact is also supported by the remaining source documents, KEEP it. +5. If you are unsure whether a fact came from the deleted document, keep it as-is — do NOT add any review notes or annotations. +6. Update or remove the "Source: {{.DeletedDocTitle}}" reference line if present. +7. Keep [[wiki-link]] references ONLY if the slug appears in the list above. Remove any [[wiki-link]] whose slug is NOT in that list. +8. Maintain the existing page structure, formatting style, and language. +9. If after removing the deleted document's contributions the page becomes nearly empty, output just: "SUMMARY: (empty page)\n# [Title]\n\n*This page's primary source document was removed.*" +10. Write in {{.Language}}. + -// WikiIndexRebuildPrompt generates the index page content from a list of all pages. -const WikiIndexRebuildPrompt = `You are a wiki editor. Generate an index page (table of contents) for a wiki based on the following page listing. +Output the SUMMARY line first, then the updated Markdown content. Do not include any other preamble.` -## Pages -{{.PageListing}} +// WikiIndexIntroPrompt generates the introduction for a NEW index page (first time only). +const WikiIndexIntroPrompt = `You are a wiki editor. Write a brief introduction for a wiki knowledge base index page. -## Instructions -1. Organize pages by category (Summaries, Entities, Concepts, Analyses, etc.). -2. For each page, include: [[slug]] — one-line summary -3. Within each category, sort alphabetically. -4. Include a brief introduction at the top explaining what this wiki covers. -5. Write in {{.Language}}. + +{{.DocumentSummaries}} + -Output ONLY the Markdown content for the index page.` + +1. Write a title line starting with "# " that reflects the knowledge domain. +2. Follow with 2-3 sentences describing what this wiki covers, based on the document summaries above. +3. Keep it concise — this is just the header section, the directory listing will be added separately below. +4. Write in {{.Language}}. + + +Output ONLY the title and introduction paragraph. Do NOT generate any directory listings or page links.` + +// WikiIndexIntroUpdatePrompt incrementally updates an existing index introduction. +const WikiIndexIntroUpdatePrompt = `You are a wiki editor. Update the introduction section of a wiki index page to reflect recent changes. + + +{{.ExistingIntro}} + + + +{{.ChangeDescription}} + + + +{{.DocumentSummaries}} + + + +1. Update the introduction to accurately reflect the current state of the wiki. +2. If documents were added, mention the new topics if they significantly change the wiki's scope. +3. If documents were removed, remove references to those topics if they no longer apply. +4. Keep the same tone, style, and title format as the existing introduction. +5. Keep it concise — 1 title line + 2-3 sentences. +6. Write in {{.Language}}. + + +Output ONLY the updated title and introduction paragraph. Do NOT generate any directory listings or page links.` // WikiLogEntryTemplate is a simple template for log entries (not LLM-generated). const WikiLogEntryTemplate = `## [{{.Date}}] {{.Operation}} | {{.Title}} @@ -207,13 +260,15 @@ The wiki has a log page (slug: "log") that records all ingest and update activit // between newly extracted items and existing wiki pages. const WikiDeduplicationPrompt = `You are a deduplication system. Given a list of newly extracted items and a list of existing wiki pages, determine which new items refer to the same entity or concept as an existing page. -## Newly Extracted Items + {{.NewItems}} + -## Existing Wiki Pages + {{.ExistingPages}} + -## Instructions + For each newly extracted item, check if it refers to the same real-world entity or concept as any existing page. Consider: - Name variations (e.g. "Acme Corp" vs "Acme Corporation", "RAG" vs "Retrieval-Augmented Generation") - Abbreviations and full names @@ -223,6 +278,7 @@ For each newly extracted item, check if it refers to the same real-world entity Return a JSON object with a "merges" map. The key is the NEW item's slug, the value is the EXISTING page's slug that it should merge into. Only include items that have a match. If no items match any existing pages, return: {"merges": {}} + Output ONLY valid JSON. Example: {"merges": {"entity/acme-corporation": "entity/acme-corp", "concept/rag": "concept/retrieval-augmented-generation"}}` diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index 0acf7e44..eca241f4 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -1284,6 +1284,15 @@ func (s *knowledgeService) cleanupWikiOnKnowledgeDelete(ctx context.Context, kbI } } + // Capture the summary page's description before deleting (for retract changeDesc) + var docSummary string + for _, page := range pages { + if page.PageType == types.WikiPageTypeSummary { + docSummary = page.Summary + break + } + } + var deletedSlugs []string var retractSlugs []string for _, page := range pages { @@ -1327,14 +1336,17 @@ func (s *knowledgeService) cleanupWikiOnKnowledgeDelete(ctx context.Context, kbI KnowledgeBaseID: kbID, KnowledgeID: knowledgeID, DocTitle: docTitle, + DocSummary: docSummary, Language: lang, PageSlugs: retractSlugs, }) logger.Infof(ctx, "wiki cleanup: enqueued retract task for %d pages: %v", len(retractSlugs), retractSlugs) } - // Rebuild index page to reflect deleted pages - if len(deletedSlugs) > 0 { + // If pages were deleted but no retract task was enqueued, + // we still need to update the index and log synchronously. + // (When retractSlugs > 0, ProcessWikiRetract handles index/log.) + if len(deletedSlugs) > 0 && len(retractSlugs) == 0 { s.rebuildWikiIndexSimple(ctx, kbID, knowledgeID, docTitle, deletedSlugs) } } @@ -1355,47 +1367,60 @@ func removeSourceRef(refs types.StringArray, knowledgeID string) types.StringArr // rebuildWikiIndexSimple regenerates the index page from current page listing // and appends a deletion log entry. Does NOT use LLM — generates a simple -// structured listing. Used during document deletion when LLM is not available. +// structured listing while preserving the LLM-generated intro from Summary field. func (s *knowledgeService) rebuildWikiIndexSimple(ctx context.Context, kbID, knowledgeID, docTitle string, deletedSlugs []string) { // Rebuild index - resp, err := s.wikiService.ListPages(ctx, &types.WikiPageListRequest{ - KnowledgeBaseID: kbID, - PageSize: 500, - SortBy: "page_type", - SortOrder: "asc", - }) - if err != nil || resp == nil { + allPages, err := s.wikiService.ListAllPages(ctx, kbID) + if err != nil { + logger.Warnf(ctx, "wiki cleanup: failed to list pages: %v", err) return } - var indexContent strings.Builder - indexContent.WriteString("# Wiki Index\n\n") + indexPage, _ := s.wikiService.GetIndex(ctx, kbID) + if indexPage == nil { + return + } - currentType := "" - for _, p := range resp.Pages { + // Preserve LLM-generated intro from Summary field + var indexContent strings.Builder + if indexPage.Summary != "" && indexPage.Summary != "Index" { + indexContent.WriteString(indexPage.Summary) + indexContent.WriteString("\n") + } else { + indexContent.WriteString("# Wiki Index\n\n") + } + + // Group pages by type + typeOrder := []string{types.WikiPageTypeSummary, types.WikiPageTypeEntity, types.WikiPageTypeConcept, types.WikiPageTypeSynthesis, types.WikiPageTypeComparison} + grouped := make(map[string][]*types.WikiPage) + for _, p := range allPages { if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog { continue } if p.Status == types.WikiPageStatusArchived { continue } - if p.PageType != currentType { - currentType = p.PageType - label := currentType - if len(label) > 0 { - label = strings.ToUpper(label[:1]) + label[1:] - } - fmt.Fprintf(&indexContent, "\n## %s\n\n", label) - } - fmt.Fprintf(&indexContent, "- [[%s]] — %s\n", p.Slug, p.Summary) + grouped[p.PageType] = append(grouped[p.PageType], p) } - indexPage, _ := s.wikiService.GetIndex(ctx, kbID) - if indexPage != nil { - indexPage.Content = indexContent.String() - if _, err := s.wikiService.UpdatePage(ctx, indexPage); err != nil { - logger.Warnf(ctx, "wiki cleanup: failed to rebuild index: %v", err) + for _, pt := range typeOrder { + pages := grouped[pt] + if len(pages) == 0 { + continue } + label := pt + if len(label) > 0 { + label = strings.ToUpper(label[:1]) + label[1:] + } + fmt.Fprintf(&indexContent, "\n## %s\n\n", label) + for _, p := range pages { + fmt.Fprintf(&indexContent, "- [[%s]] — %s\n", p.Slug, p.Summary) + } + } + + indexPage.Content = indexContent.String() + if _, err := s.wikiService.UpdatePage(ctx, indexPage); err != nil { + logger.Warnf(ctx, "wiki cleanup: failed to rebuild index: %v", err) } // Append log entry diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go index 90eadb77..81396cd5 100644 --- a/internal/application/service/wiki_ingest.go +++ b/internal/application/service/wiki_ingest.go @@ -63,8 +63,9 @@ type WikiRetractPayload struct { KnowledgeBaseID string `json:"knowledge_base_id"` KnowledgeID string `json:"knowledge_id"` DocTitle string `json:"doc_title"` + DocSummary string `json:"doc_summary,omitempty"` // one-line summary of the deleted document Language string `json:"language,omitempty"` - PageSlugs []string `json:"page_slugs"` // pages to retract content from + PageSlugs []string `json:"page_slugs"` } // wikiIngestService handles the LLM-powered wiki generation pipeline @@ -205,51 +206,14 @@ func (s *wikiIngestService) ProcessWikiRetract(ctx context.Context, t *asynq.Tas lang := types.LanguageNameFromContext(ctx) - for _, slug := range payload.PageSlugs { - page, err := s.wikiService.GetPageBySlug(ctx, payload.KnowledgeBaseID, slug) - if err != nil || page == nil { - continue - } - - // Build remaining sources list - var remainingSources string - for _, ref := range page.SourceRefs { - pipeIdx := strings.Index(ref, "|") - if pipeIdx > 0 { - remainingSources += "- " + ref[pipeIdx+1:] + "\n" - } else { - remainingSources += "- " + ref + "\n" - } - } - if remainingSources == "" { - remainingSources = "(no other sources)" - } - - // Call LLM to retract content - updatedContent, err := s.generateWithTemplate(ctx, chatModel, agent.WikiPageRetractPrompt, map[string]string{ - "ExistingContent": page.Content, - "DeletedDocTitle": payload.DocTitle, - "RemainingSources": remainingSources, - "Language": lang, - }) - if err != nil { - logger.Warnf(ctx, "wiki retract: LLM call failed for page %s: %v", slug, err) - continue - } - - page.Content = updatedContent - if _, err := s.wikiService.UpdatePage(ctx, page); err != nil { - logger.Warnf(ctx, "wiki retract: failed to update page %s: %v", slug, err) - } else { - logger.Infof(ctx, "wiki retract: updated page %s after removing content from '%s'", slug, payload.DocTitle) - } - } + s.retractPagesContent(ctx, chatModel, payload.KnowledgeBaseID, payload.DocTitle, payload.DocSummary, payload.PageSlugs, lang) // Rebuild index page to reflect changes + retractChangeDesc := fmt.Sprintf("Removed document '%s': %s", payload.DocTitle, payload.DocSummary) if err := s.rebuildIndexPage(ctx, chatModel, WikiIngestPayload{ TenantID: payload.TenantID, KnowledgeBaseID: payload.KnowledgeBaseID, - }, lang); err != nil { + }, retractChangeDesc, lang); err != nil { logger.Warnf(ctx, "wiki retract: rebuild index failed: %v", err) } @@ -266,6 +230,81 @@ func (s *wikiIngestService) ProcessWikiRetract(ctx context.Context, t *asynq.Tas return nil } +// retractPagesContent uses LLM to remove a deleted/stale document's contributions +// from the given wiki pages. docContent is the deleted document's content (or summary) +// so the LLM can accurately identify which parts to remove. It does NOT rebuild the +// index or append log entries — callers are responsible for those post-processing steps. +func (s *wikiIngestService) retractPagesContent( + ctx context.Context, + chatModel chat.Chat, + kbID, docTitle, docContent string, + pageSlugs []string, + lang string, +) { + allPages, _ := s.wikiService.ListAllPages(ctx, kbID) + slugTitleMap := make(map[string]string) + for _, p := range allPages { + if p.PageType != types.WikiPageTypeIndex && p.PageType != types.WikiPageTypeLog && p.Status != types.WikiPageStatusArchived { + slugTitleMap[p.Slug] = p.Title + } + } + + for _, slug := range pageSlugs { + page, err := s.wikiService.GetPageBySlug(ctx, kbID, slug) + if err != nil || page == nil { + continue + } + + var remainingSources string + for _, ref := range page.SourceRefs { + pipeIdx := strings.Index(ref, "|") + if pipeIdx > 0 { + remainingSources += "- " + ref[pipeIdx+1:] + "\n" + } else { + remainingSources += "- " + ref + "\n" + } + } + if remainingSources == "" { + remainingSources = "(no other sources)" + } + + var relatedSlugs strings.Builder + for _, outSlug := range page.OutLinks { + if title, ok := slugTitleMap[outSlug]; ok { + fmt.Fprintf(&relatedSlugs, "- %s (%s)\n", outSlug, title) + } + } + + updatedContent, err := s.generateWithTemplate(ctx, chatModel, agent.WikiPageRetractPrompt, map[string]string{ + "ExistingContent": page.Content, + "DeletedDocTitle": docTitle, + "DeletedDocContent": docContent, + "RemainingSources": remainingSources, + "AvailableSlugs": relatedSlugs.String(), + "Language": lang, + }) + if err != nil { + logger.Warnf(ctx, "wiki retract: LLM call failed for page %s: %v", slug, err) + continue + } + + retractedSummary, retractedBody := splitSummaryLine(updatedContent) + if retractedBody != "" { + page.Content = retractedBody + } else { + page.Content = updatedContent + } + if retractedSummary != "" { + page.Summary = retractedSummary + } + if _, err := s.wikiService.UpdatePage(ctx, page); err != nil { + logger.Warnf(ctx, "wiki retract: failed to update page %s: %v", slug, err) + } else { + logger.Infof(ctx, "wiki retract: updated page %s after removing content from '%s'", slug, docTitle) + } + } +} + // ProcessWikiIngest processes a batch wiki ingest task. // // Concurrency model (Redis mode): @@ -356,19 +395,30 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task // Process each document var allPagesAffected []string + var docResults []*docIngestResult for _, knowledgeID := range knowledgeIDs { - pages, err := s.processOneDocument(ctx, chatModel, payload, knowledgeID, lang) + result, err := s.processOneDocument(ctx, chatModel, payload, knowledgeID, lang) if err != nil { logger.Warnf(ctx, "wiki ingest: failed to process knowledge %s: %v", knowledgeID, err) continue } - allPagesAffected = append(allPagesAffected, pages...) + if result != nil { + allPagesAffected = append(allPagesAffected, result.Pages...) + docResults = append(docResults, result) + } } // Batch post-processing (once for the whole batch, not per-doc) + // Build change description from processed documents + var changeDesc strings.Builder + fmt.Fprintf(&changeDesc, "Added %d documents:\n", len(docResults)) + for _, r := range docResults { + fmt.Fprintf(&changeDesc, "- %s: %s\n", r.DocTitle, r.Summary) + } + // Rebuild index page - if err := s.rebuildIndexPage(ctx, chatModel, payload, lang); err != nil { + if err := s.rebuildIndexPage(ctx, chatModel, payload, changeDesc.String(), lang); err != nil { logger.Warnf(ctx, "wiki ingest: rebuild index failed: %v", err) } @@ -453,15 +503,22 @@ func (s *wikiIngestService) drainPendingList(ctx context.Context, kbID string) [ return unique } +// docIngestResult captures per-document info for batch post-processing. +type docIngestResult struct { + KnowledgeID string + DocTitle string + Summary string // one-line summary of the document (from summary page) + Pages []string // affected page slugs +} + // processOneDocument handles wiki ingest for a single document. -// Returns the list of affected page slugs. func (s *wikiIngestService) processOneDocument( ctx context.Context, chatModel chat.Chat, payload WikiIngestPayload, knowledgeID string, lang string, -) ([]string, error) { +) (*docIngestResult, error) { // Get document chunks and reconstruct content chunks, err := s.chunkRepo.ListChunksByKnowledgeID(ctx, payload.TenantID, knowledgeID) if err != nil { @@ -489,6 +546,7 @@ func (s *wikiIngestService) processOneDocument( } var pagesAffected []string + var docSummaryLine string sourceRef := fmt.Sprintf("%s|%s", knowledgeID, docTitle) // Snapshot existing page slugs for stale detection @@ -503,7 +561,7 @@ func (s *wikiIngestService) processOneDocument( } // Step 1: Extract entities and concepts - extractedPages, extractedSlugs, err := s.extractEntitiesAndConcepts(ctx, chatModel, content, docTitle, lang, docPayload, sourceRef, oldPageSlugs) + extractedPages, err := s.extractEntitiesAndConcepts(ctx, chatModel, content, docTitle, lang, docPayload, sourceRef, oldPageSlugs) if err != nil { logger.Warnf(ctx, "wiki ingest: knowledge extraction failed for %s: %v", knowledgeID, err) } else { @@ -513,7 +571,7 @@ func (s *wikiIngestService) processOneDocument( // Step 2: Generate summary page summarySlug := fmt.Sprintf("summary/%s", slugify(docTitle)) var slugListing string - for _, slug := range extractedSlugs { + for _, slug := range extractedPages { slugListing += fmt.Sprintf("- [[%s]]\n", slug) } summaryContent, err := s.generateWithTemplate(ctx, chatModel, agent.WikiSummaryPrompt, map[string]string{ @@ -527,6 +585,14 @@ func (s *wikiIngestService) processOneDocument( if err != nil { logger.Errorf(ctx, "wiki ingest: generate summary failed for %s: %v", knowledgeID, err) } else { + sumLine, sumBody := splitSummaryLine(summaryContent) + if sumBody == "" { + sumBody = summaryContent + } + if sumLine == "" { + sumLine = docTitle + } + docSummaryLine = sumLine _, err := s.wikiService.CreatePage(ctx, &types.WikiPage{ ID: uuid.New().String(), TenantID: payload.TenantID, @@ -535,8 +601,8 @@ func (s *wikiIngestService) processOneDocument( Title: docTitle + " - Summary", PageType: types.WikiPageTypeSummary, Status: types.WikiPageStatusDraft, - Content: summaryContent, - Summary: truncateString(summaryContent, 200), + Content: sumBody, + Summary: sumLine, SourceRefs: types.StringArray{sourceRef}, }) if err != nil { @@ -547,34 +613,36 @@ func (s *wikiIngestService) processOneDocument( } // Retract stale pages (pages this doc previously contributed to but no longer does) - s.retractStalePages(ctx, docPayload, oldPageSlugs, pagesAffected, docTitle, lang) + s.retractStalePages(ctx, chatModel, docPayload, oldPageSlugs, pagesAffected, docTitle, content, lang) logger.Infof(ctx, "wiki ingest: processed knowledge %s, %d pages affected", knowledgeID, len(pagesAffected)) - return pagesAffected, nil + return &docIngestResult{ + KnowledgeID: knowledgeID, + DocTitle: docTitle, + Summary: docSummaryLine, + Pages: pagesAffected, + }, nil } // cleanDeadLinks removes [[wiki-links]] that point to archived or deleted pages. // Scans all published pages, checks each out_link, and removes references to // pages that no longer exist or are archived. No LLM call — pure text cleanup. func (s *wikiIngestService) cleanDeadLinks(ctx context.Context, kbID string) { - resp, err := s.wikiService.ListPages(ctx, &types.WikiPageListRequest{ - KnowledgeBaseID: kbID, - PageSize: 500, - }) - if err != nil || resp == nil { + allPages, err := s.wikiService.ListAllPages(ctx, kbID) + if err != nil || len(allPages) == 0 { return } // Build set of live (non-archived, non-system) slugs liveSlugs := make(map[string]bool) - for _, p := range resp.Pages { + for _, p := range allPages { if p.Status != types.WikiPageStatusArchived { liveSlugs[p.Slug] = true } } var cleaned int - for _, p := range resp.Pages { + for _, p := range allPages { if p.Status == types.WikiPageStatusArchived { continue } @@ -622,11 +690,8 @@ func (s *wikiIngestService) cleanDeadLinks(ctx context.Context, kbID string) { // Only processes entity/concept/synthesis/comparison pages (not index/log/summary). func (s *wikiIngestService) injectCrossLinks(ctx context.Context, kbID string, affectedSlugs []string) { // Build a title→slug lookup from ALL pages in this KB - resp, err := s.wikiService.ListPages(ctx, &types.WikiPageListRequest{ - KnowledgeBaseID: kbID, - PageSize: 500, - }) - if err != nil || resp == nil || len(resp.Pages) < 2 { + allPages, err := s.wikiService.ListAllPages(ctx, kbID) + if err != nil || len(allPages) < 2 { return } @@ -635,7 +700,7 @@ func (s *wikiIngestService) injectCrossLinks(ctx context.Context, kbID string, a title string } var allRefs []pageRef - for _, p := range resp.Pages { + for _, p := range allPages { if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog { continue } @@ -664,7 +729,7 @@ func (s *wikiIngestService) injectCrossLinks(ctx context.Context, kbID string, a } var updated int - for _, p := range resp.Pages { + for _, p := range allPages { if !affectedSet[p.Slug] { continue } @@ -745,17 +810,14 @@ func replaceFirstOutsideLinks(s, old, newStr string) string { // getExistingPageSlugsForKnowledge returns all page slugs that currently reference // a given knowledge ID in their source_refs. Used to snapshot state before re-ingest. func (s *wikiIngestService) getExistingPageSlugsForKnowledge(ctx context.Context, kbID, knowledgeID string) map[string]bool { - resp, err := s.wikiService.ListPages(ctx, &types.WikiPageListRequest{ - KnowledgeBaseID: kbID, - PageSize: 500, - }) - if err != nil || resp == nil { + allPages, err := s.wikiService.ListAllPages(ctx, kbID) + if err != nil || len(allPages) == 0 { return nil } slugs := make(map[string]bool) prefix := knowledgeID + "|" - for _, p := range resp.Pages { + for _, p := range allPages { if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog { continue } @@ -771,14 +833,15 @@ func (s *wikiIngestService) getExistingPageSlugsForKnowledge(ctx context.Context // retractStalePages handles pages that were previously linked to this document // but are no longer produced by the updated extraction. -// - Single-source stale pages → archived -// - Multi-source stale pages → enqueue LLM retract to clean content +// - Single-source stale pages → deleted +// - Multi-source stale pages → LLM retract to clean content synchronously func (s *wikiIngestService) retractStalePages( ctx context.Context, + chatModel chat.Chat, payload WikiIngestPayload, oldSlugs map[string]bool, newSlugs []string, - docTitle, lang string, + docTitle, docContent, lang string, ) { if len(oldSlugs) == 0 { return @@ -838,16 +901,8 @@ func (s *wikiIngestService) retractStalePages( } } - // Enqueue LLM retraction for multi-source stale pages if len(retractSlugs) > 0 { - EnqueueWikiRetract(ctx, s.task, WikiRetractPayload{ - TenantID: payload.TenantID, - KnowledgeBaseID: payload.KnowledgeBaseID, - KnowledgeID: payload.KnowledgeID, - DocTitle: docTitle, - Language: payload.Language, - PageSlugs: retractSlugs, - }) + s.retractPagesContent(ctx, chatModel, payload.KnowledgeBaseID, docTitle, docContent, retractSlugs, lang) } } @@ -866,7 +921,7 @@ type combinedExtraction struct { } // extractEntitiesAndConcepts performs a single LLM call to extract both entities and concepts, -// then upserts pages for each. Returns the list of affected page slugs and all extracted slugs (for linking). +// then upserts pages for each. Returns the list of successfully upserted page slugs. // oldPageSlugs contains slugs from the previous version of this document — passed to LLM for slug stability. func (s *wikiIngestService) extractEntitiesAndConcepts( ctx context.Context, @@ -875,7 +930,7 @@ func (s *wikiIngestService) extractEntitiesAndConcepts( payload WikiIngestPayload, sourceRef string, oldPageSlugs map[string]bool, -) ([]string, []string, error) { +) ([]string, error) { // Build previous slugs listing for the prompt var prevSlugsText string if len(oldPageSlugs) > 0 { @@ -896,7 +951,7 @@ func (s *wikiIngestService) extractEntitiesAndConcepts( "PreviousSlugs": prevSlugsText, }) if err != nil { - return nil, nil, fmt.Errorf("combined extraction failed: %w", err) + return nil, fmt.Errorf("combined extraction failed: %w", err) } // Clean JSON - strip markdown code blocks if present @@ -909,20 +964,7 @@ func (s *wikiIngestService) extractEntitiesAndConcepts( var result combinedExtraction if err := json.Unmarshal([]byte(extractionJSON), &result); err != nil { logger.Warnf(ctx, "wiki ingest: failed to parse combined extraction JSON: %v\nRaw: %s", err, truncateString(extractionJSON, 500)) - return nil, nil, fmt.Errorf("parse combined extraction JSON: %w", err) - } - - // Collect all extracted slugs (before dedup remapping) for summary linking - var allSlugs []string - for _, item := range result.Entities { - if item.Slug != "" { - allSlugs = append(allSlugs, item.Slug) - } - } - for _, item := range result.Concepts { - if item.Slug != "" { - allSlugs = append(allSlugs, item.Slug) - } + return nil, fmt.Errorf("parse combined extraction JSON: %w", err) } var affected []string @@ -933,19 +975,6 @@ func (s *wikiIngestService) extractEntitiesAndConcepts( // Deduplicate concepts against existing wiki pages (LLM-based) result.Concepts = s.deduplicateItems(ctx, chatModel, result.Concepts, types.WikiPageTypeConcept, payload.KnowledgeBaseID) - // Update allSlugs after dedup (use final slugs) - allSlugs = allSlugs[:0] - for _, item := range result.Entities { - if item.Slug != "" { - allSlugs = append(allSlugs, item.Slug) - } - } - for _, item := range result.Concepts { - if item.Slug != "" { - allSlugs = append(allSlugs, item.Slug) - } - } - // Upsert entity pages entitySlugs, err := s.upsertExtractedPages(ctx, chatModel, result.Entities, types.WikiPageTypeEntity, docTitle, lang, payload, sourceRef) if err != nil { @@ -962,7 +991,7 @@ func (s *wikiIngestService) extractEntitiesAndConcepts( affected = append(affected, conceptSlugs...) } - return affected, allSlugs, nil + return affected, nil } // upsertExtractedPages creates or updates wiki pages from pre-extracted items. @@ -996,8 +1025,16 @@ func (s *wikiIngestService) upsertExtractedPages( continue } - existing.Content = updatedContent - existing.Summary = item.Description + // LLM returns "SUMMARY: ..." on first line — use it as the authoritative summary + updatedSummary, updatedBody := splitSummaryLine(updatedContent) + if updatedBody != "" { + existing.Content = updatedBody + } else { + existing.Content = updatedContent + } + if updatedSummary != "" { + existing.Summary = updatedSummary + } existing.SourceRefs = appendUnique(existing.SourceRefs, sourceRef) if _, err := s.wikiService.UpdatePage(ctx, existing); err != nil { @@ -1032,47 +1069,143 @@ func (s *wikiIngestService) upsertExtractedPages( return affected, nil } -// rebuildIndexPage regenerates the index page from all existing pages -func (s *wikiIngestService) rebuildIndexPage(ctx context.Context, chatModel chat.Chat, payload WikiIngestPayload, lang string) error { - // List all pages - resp, err := s.wikiService.ListPages(ctx, &types.WikiPageListRequest{ - KnowledgeBaseID: payload.KnowledgeBaseID, - PageSize: 500, - SortBy: "page_type", - SortOrder: "asc", - }) +// rebuildIndexPage regenerates the index page. +// +// Strategy: Index = LLM-generated intro (stored in Summary field) + code-generated directory. +// - Intro: stored in indexPage.Summary. First time: generated from document summaries. +// Subsequent: incrementally updated with changeDescription. +// - Directory: pure code, rebuilt every time. O(N) string concat, no LLM. +func (s *wikiIngestService) rebuildIndexPage(ctx context.Context, chatModel chat.Chat, payload WikiIngestPayload, changeDesc, lang string) error { + indexPage, _ := s.wikiService.GetIndex(ctx, payload.KnowledgeBaseID) + if indexPage == nil { + return nil + } + + // List all live pages + allPages, err := s.wikiService.ListAllPages(ctx, payload.KnowledgeBaseID) if err != nil { return err } - // Build page listing - var listing strings.Builder - for _, p := range resp.Pages { + typeOrder := []string{ + types.WikiPageTypeSummary, types.WikiPageTypeEntity, types.WikiPageTypeConcept, + types.WikiPageTypeSynthesis, types.WikiPageTypeComparison, + } + typeLabels := map[string]string{ + types.WikiPageTypeSummary: "Summary", types.WikiPageTypeEntity: "Entity", + types.WikiPageTypeConcept: "Concept", types.WikiPageTypeSynthesis: "Synthesis", + types.WikiPageTypeComparison: "Comparison", + } + + grouped := make(map[string][]*types.WikiPage) + totalPages := 0 + for _, p := range allPages { if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog { continue } - listing.WriteString(fmt.Sprintf("- [%s] [[%s]] | %s — %s\n", p.PageType, p.Slug, p.Title, p.Summary)) + if p.Status == types.WikiPageStatusArchived { + continue + } + grouped[p.PageType] = append(grouped[p.PageType], p) + totalPages++ } - indexContent, err := s.generateWithTemplate(ctx, chatModel, agent.WikiIndexRebuildPrompt, map[string]string{ - "PageListing": listing.String(), - "Language": lang, - }) - if err != nil { - return fmt.Errorf("generate index: %w", err) + // Build document summaries listing (only summary-type pages — they represent documents) + var docSummaries strings.Builder + for _, p := range grouped[types.WikiPageTypeSummary] { + fmt.Fprintf(&docSummaries, "- %s: %s\n", p.Title, p.Summary) + } + if docSummaries.Len() == 0 { + docSummaries.WriteString("(no documents yet)") } - // Get or create index page - indexPage, _ := s.wikiService.GetIndex(ctx, payload.KnowledgeBaseID) - if indexPage != nil { - indexPage.Content = indexContent - indexPage.Summary = "Wiki index - table of contents" - if _, err := s.wikiService.UpdatePage(ctx, indexPage); err != nil { - return fmt.Errorf("update index page: %w", err) + // Generate or update intro + existingIntro := indexPage.Summary + var intro string + + if existingIntro == "" || existingIntro == "Wiki index - table of contents" { + // First time — generate intro from scratch + generatedIntro, genErr := s.generateWithTemplate(ctx, chatModel, agent.WikiIndexIntroPrompt, map[string]string{ + "DocumentSummaries": docSummaries.String(), + "Language": lang, + }) + if genErr != nil { + intro = "# Wiki Index\n\nThis wiki contains knowledge extracted from uploaded documents.\n" + } else { + intro = strings.TrimSpace(generatedIntro) + } + } else if changeDesc != "" { + // Incremental update — tell LLM what changed + updatedIntro, genErr := s.generateWithTemplate(ctx, chatModel, agent.WikiIndexIntroUpdatePrompt, map[string]string{ + "ExistingIntro": existingIntro, + "ChangeDescription": changeDesc, + "DocumentSummaries": docSummaries.String(), + "Language": lang, + }) + if genErr != nil { + intro = existingIntro // keep existing on error + } else { + intro = strings.TrimSpace(updatedIntro) + } + } else { + intro = existingIntro // no change description, keep as-is + } + + // Build directory (pure code, no LLM) + var dir strings.Builder + for _, pt := range typeOrder { + pages := grouped[pt] + if len(pages) == 0 { + continue + } + fmt.Fprintf(&dir, "\n## %s (%d)\n\n", typeLabels[pt], len(pages)) + for _, p := range pages { + summary := p.Summary + fmt.Fprintf(&dir, "[[%s]] — %s\n", p.Slug, summary) } } + for pt, pages := range grouped { + inOrder := false + for _, o := range typeOrder { + if o == pt { + inOrder = true + break + } + } + if inOrder || len(pages) == 0 { + continue + } + fmt.Fprintf(&dir, "\n## %s (%d)\n\n", pt, len(pages)) + for _, p := range pages { + fmt.Fprintf(&dir, "[[%s]] — %s\n", p.Slug, p.Summary) + } + } + if totalPages == 0 { + dir.WriteString("\n*No wiki pages yet. Upload documents to get started.*\n") + } - return nil + indexPage.Content = intro + "\n" + dir.String() + indexPage.Summary = intro // persist intro for next incremental update + _, err = s.wikiService.UpdatePage(ctx, indexPage) + return err +} + +// splitSummaryLine extracts the "SUMMARY: ..." line from LLM output. +// Returns (summary, content). If no SUMMARY line found, summary is empty. +func splitSummaryLine(raw string) (summary string, content string) { + raw = strings.TrimSpace(raw) + if strings.HasPrefix(raw, "SUMMARY:") || strings.HasPrefix(raw, "SUMMARY:") { + idx := strings.IndexByte(raw, '\n') + if idx < 0 { + // Only one line + return strings.TrimSpace(strings.TrimPrefix(strings.TrimPrefix(raw, "SUMMARY:"), "SUMMARY:")), "" + } + summaryLine := raw[:idx] + summaryLine = strings.TrimPrefix(summaryLine, "SUMMARY:") + summaryLine = strings.TrimPrefix(summaryLine, "SUMMARY:") + return strings.TrimSpace(summaryLine), strings.TrimSpace(raw[idx+1:]) + } + return "", raw } // appendLogEntry appends an entry to the log page, including any synthesis suggestions @@ -1136,18 +1269,25 @@ func (s *wikiIngestService) deduplicateItems( } // Get existing pages of the same type - resp, err := s.wikiService.ListPages(ctx, &types.WikiPageListRequest{ - KnowledgeBaseID: kbID, - PageType: pageType, - PageSize: 2000, - }) - if err != nil || resp == nil || len(resp.Pages) == 0 { - return items // No existing pages → nothing to deduplicate against + allPages, err := s.wikiService.ListAllPages(ctx, kbID) + if err != nil || len(allPages) == 0 { + return items + } + + // Filter to the target type + var typedPages []*types.WikiPage + for _, p := range allPages { + if p.PageType == pageType { + typedPages = append(typedPages, p) + } + } + if len(typedPages) == 0 { + return items // No existing pages of this type → nothing to deduplicate against } // Build existing pages listing var existingBuf strings.Builder - for _, p := range resp.Pages { + for _, p := range typedPages { fmt.Fprintf(&existingBuf, "- slug: %s | title: %s\n", p.Slug, p.Title) } diff --git a/internal/application/service/wiki_page.go b/internal/application/service/wiki_page.go index bd36dc1c..1816b2a5 100644 --- a/internal/application/service/wiki_page.go +++ b/internal/application/service/wiki_page.go @@ -20,9 +20,9 @@ var wikiLinkRegex = regexp.MustCompile(`\[\[([^\]]+)\]\]`) // wikiPageService implements the WikiPageService interface type wikiPageService struct { - repo interfaces.WikiPageRepository - chunkRepo interfaces.ChunkRepository - kbService interfaces.KnowledgeBaseService + repo interfaces.WikiPageRepository + chunkRepo interfaces.ChunkRepository + kbService interfaces.KnowledgeBaseService } // NewWikiPageService creates a new wiki page service @@ -70,11 +70,6 @@ func (s *wikiPageService) CreatePage(ctx context.Context, page *types.WikiPage) // Update inbound links on target pages s.updateInLinks(ctx, page.KnowledgeBaseID, page.Slug, page.OutLinks) - // Sync to chunks for retrieval pipeline integration - if err := s.syncToChunk(ctx, page); err != nil { - logger.Warnf(ctx, "wiki: failed to sync page %s to chunks: %v", page.Slug, err) - } - return page, nil } @@ -108,11 +103,6 @@ func (s *wikiPageService) UpdatePage(ctx context.Context, page *types.WikiPage) s.removeInLinks(ctx, existing.KnowledgeBaseID, existing.Slug, oldOutLinks) s.updateInLinks(ctx, existing.KnowledgeBaseID, existing.Slug, existing.OutLinks) - // Re-sync to chunks - if err := s.syncToChunk(ctx, existing); err != nil { - logger.Warnf(ctx, "wiki: failed to sync page %s to chunks: %v", existing.Slug, err) - } - return existing, nil } @@ -341,6 +331,11 @@ func (s *wikiPageService) RebuildLinks(ctx context.Context, kbID string) error { return nil } +// ListAllPages retrieves all wiki pages without pagination. +func (s *wikiPageService) ListAllPages(ctx context.Context, kbID string) ([]*types.WikiPage, error) { + return s.repo.ListAll(ctx, kbID) +} + // SearchPages performs full-text search over wiki pages func (s *wikiPageService) SearchPages(ctx context.Context, kbID string, query string, limit int) ([]*types.WikiPage, error) { return s.repo.Search(ctx, kbID, query, limit) @@ -413,33 +408,6 @@ func (s *wikiPageService) removeInLinks(ctx context.Context, kbID string, source } } -// syncToChunk syncs a wiki page to the chunk table for retrieval pipeline integration. -// Each wiki page maps to a single chunk with ChunkType = "wiki_page". -// The chunk's KnowledgeID is set to the wiki page ID for tracking. -func (s *wikiPageService) syncToChunk(ctx context.Context, page *types.WikiPage) error { - // Build content for the chunk: title + summary + content - chunkContent := fmt.Sprintf("# %s\n\n%s\n\n%s", page.Title, page.Summary, page.Content) - - chunk := &types.Chunk{ - ID: "wp-" + page.ID, // prefix to avoid collision with regular chunks - TenantID: page.TenantID, - KnowledgeID: page.ID, // link back to wiki page - KnowledgeBaseID: page.KnowledgeBaseID, - Content: chunkContent, - ChunkType: types.ChunkTypeWikiPage, - IsEnabled: page.Status == types.WikiPageStatusPublished, - Status: int(types.ChunkStatusStored), - } - - // Try to create; if it already exists, update - err := s.chunkRepo.CreateChunks(ctx, []*types.Chunk{chunk}) - if err != nil { - // Fallback: update existing chunk - return s.chunkRepo.UpdateChunk(ctx, chunk) - } - return nil -} - // deleteChunkForPage removes the synced chunk for a wiki page func (s *wikiPageService) deleteChunkForPage(ctx context.Context, page *types.WikiPage) { chunkID := "wp-" + page.ID diff --git a/internal/types/interfaces/wiki_page.go b/internal/types/interfaces/wiki_page.go index 0cad8e76..3982bf7a 100644 --- a/internal/types/interfaces/wiki_page.go +++ b/internal/types/interfaces/wiki_page.go @@ -52,6 +52,10 @@ type WikiPageService interface { // RebuildLinks re-parses all pages and rebuilds bidirectional link references. RebuildLinks(ctx context.Context, kbID string) error + // ListAllPages retrieves all wiki pages in a knowledge base without pagination. + // Used for index rebuild, graph generation, cross-link injection, etc. + ListAllPages(ctx context.Context, kbID string) ([]*types.WikiPage, error) + // SearchPages performs full-text search over wiki pages. SearchPages(ctx context.Context, kbID string, query string, limit int) ([]*types.WikiPage, error) }