refactor: Enhance wiki page management and index rebuilding
- Introduced a new method to list all wiki pages without pagination, facilitating index rebuilding and cross-linking. - Updated the wiki ingest and retract processes to include document summaries, improving clarity in log entries and index updates. - Enhanced the cleanup process for deleted knowledge, ensuring accurate summary retention and index regeneration. - Streamlined the handling of page updates and deletions, optimizing the overall wiki management workflow. These changes improve the efficiency and reliability of wiki content management and indexing.
This commit is contained in:
+122
-66
@@ -7,48 +7,54 @@ package agent
|
||||
// WikiSummaryPrompt generates a summary page for a newly ingested document.
|
||||
const WikiSummaryPrompt = `You are a wiki editor. Given the following document content, create a structured wiki summary page in Markdown format.
|
||||
|
||||
## Document Info
|
||||
- Title: {{.Title}}
|
||||
- File Name: {{.FileName}}
|
||||
- File Type: {{.FileType}}
|
||||
|
||||
## Document Content
|
||||
<document>
|
||||
<title>{{.Title}}</title>
|
||||
<file_name>{{.FileName}}</file_name>
|
||||
<file_type>{{.FileType}}</file_type>
|
||||
<content>
|
||||
{{.Content}}
|
||||
</content>
|
||||
</document>
|
||||
|
||||
## Available Wiki Pages (use these exact slugs for links)
|
||||
<available_wiki_pages>
|
||||
{{.ExtractedSlugs}}
|
||||
</available_wiki_pages>
|
||||
|
||||
## Instructions
|
||||
1. Write a comprehensive summary of the document in Markdown format.
|
||||
2. Include the key facts, arguments, and conclusions.
|
||||
3. Use proper heading hierarchy (## for sections, ### for subsections).
|
||||
4. Use [[wiki-link]] syntax to reference the available wiki pages listed above. Use the EXACT slugs provided — do NOT invent new slugs.
|
||||
5. At the end, include a "## Key Takeaways" section with bullet points.
|
||||
6. Write in {{.Language}}.
|
||||
7. Keep the summary concise but thorough (500-1500 words depending on document length).
|
||||
<instructions>
|
||||
1. The FIRST line of your output MUST be: SUMMARY: {one sentence, 15-40 words, describing what this document is about — for wiki index listing}
|
||||
2. After the SUMMARY line, write a comprehensive summary of the document in Markdown format.
|
||||
3. Include the key facts, arguments, and conclusions.
|
||||
4. Use proper heading hierarchy (## for sections, ### for subsections).
|
||||
5. Use [[wiki-link]] syntax to reference the available wiki pages listed above. Use the EXACT slugs provided — do NOT invent new slugs.
|
||||
6. At the end, include a "## Key Takeaways" section with bullet points.
|
||||
7. Write in {{.Language}}.
|
||||
8. Keep the summary concise but thorough (500-1500 words depending on document length).
|
||||
</instructions>
|
||||
|
||||
Output ONLY the Markdown content for the wiki page. Do not include any preamble or explanation.`
|
||||
Output the SUMMARY line first, then the Markdown content. Do not include any other preamble.`
|
||||
|
||||
// WikiKnowledgeExtractPrompt extracts both entities and concepts in a single LLM call.
|
||||
// Returns a JSON object with "entities" and "concepts" arrays.
|
||||
// This replaces the former separate WikiEntityExtractPrompt and WikiConceptExtractPrompt.
|
||||
const WikiKnowledgeExtractPrompt = `You are a knowledge extraction system. Analyze the following document and extract all significant entities AND key concepts.
|
||||
|
||||
## Document Info
|
||||
- Title: {{.Title}}
|
||||
|
||||
## Document Content
|
||||
<document>
|
||||
<title>{{.Title}}</title>
|
||||
<content>
|
||||
{{.Content}}
|
||||
</content>
|
||||
</document>
|
||||
|
||||
## Previously Extracted Slugs (from earlier version of this document)
|
||||
<previous_slugs>
|
||||
{{.PreviousSlugs}}
|
||||
</previous_slugs>
|
||||
|
||||
## Instructions
|
||||
<instructions>
|
||||
Return a JSON object with two arrays: "entities" and "concepts".
|
||||
**IMPORTANT: Write ALL names, descriptions, and details in {{.Language}}.**
|
||||
|
||||
### Slug Continuity Rules
|
||||
If a "Previously Extracted Slugs" list is provided above, you MUST follow these rules:
|
||||
If previous slugs are provided above, you MUST follow these rules:
|
||||
- If an entity or concept from the previous extraction still exists in the current document, **reuse its exact slug** from the previous list. Do NOT generate a new slug for the same thing.
|
||||
- If an entity or concept no longer appears in the document, **do NOT include it** in the output.
|
||||
- Only generate new slugs for entities/concepts that are genuinely new (not present in the previous list).
|
||||
@@ -58,7 +64,7 @@ If a "Previously Extracted Slugs" list is provided above, you MUST follow these
|
||||
Each entity should have:
|
||||
- "name": The entity name in {{.Language}} (human-readable)
|
||||
- "slug": URL-friendly slug, format "entity/<lowercase-hyphenated-name>" (use romanized/pinyin form for non-Latin names). **Reuse previous slug if the entity was extracted before.**
|
||||
- "description": A one-sentence description in {{.Language}} based on what the document says
|
||||
- "description": **Index listing summary** — one sentence, 15-40 words, in {{.Language}}. Describes WHAT this entity IS and its role in the document. Must be self-contained (understandable without reading the full page). This will be displayed in the wiki index.
|
||||
- "details": A 2-5 sentence summary in {{.Language}} of key facts from the document
|
||||
|
||||
Only include entities that are substantively discussed (mentioned at least twice or described in detail). Do NOT include generic terms.
|
||||
@@ -67,7 +73,7 @@ Only include entities that are substantively discussed (mentioned at least twice
|
||||
Each concept should have:
|
||||
- "name": The concept name in {{.Language}} (human-readable)
|
||||
- "slug": URL-friendly slug, format "concept/<lowercase-hyphenated-name>" (use romanized/pinyin form for non-Latin names). **Reuse previous slug if the concept was extracted before.**
|
||||
- "description": A one-sentence definition or description in {{.Language}}
|
||||
- "description": **Index listing summary** — one sentence, 15-40 words, in {{.Language}}. Defines WHAT this concept IS. Must be self-contained (understandable without reading the full page). This will be displayed in the wiki index.
|
||||
- "details": A 2-5 sentence explanation in {{.Language}} as discussed in the document
|
||||
|
||||
Only include concepts that are substantively discussed. Skip trivial or overly generic concepts.
|
||||
@@ -76,6 +82,7 @@ Only include concepts that are substantively discussed. Skip trivial or overly g
|
||||
- If something is a specific named thing (person, company, product, place), put it ONLY in "entities".
|
||||
- If something is an abstract idea, methodology, or theory, put it ONLY in "concepts".
|
||||
- Never duplicate items across the two arrays.
|
||||
</instructions>
|
||||
|
||||
Output ONLY valid JSON. Example:
|
||||
{
|
||||
@@ -100,63 +107,109 @@ Output ONLY valid JSON. Example:
|
||||
// WikiPageUpdatePrompt incrementally updates an existing wiki page with new information.
|
||||
const WikiPageUpdatePrompt = `You are a wiki editor tasked with updating an existing wiki page with new information from a recently ingested document.
|
||||
|
||||
## Existing Page Content
|
||||
<existing_page_content>
|
||||
{{.ExistingContent}}
|
||||
</existing_page_content>
|
||||
|
||||
## New Information from Document "{{.NewDocTitle}}"
|
||||
<new_information>
|
||||
<source_document>{{.NewDocTitle}}</source_document>
|
||||
<content>
|
||||
{{.NewContent}}
|
||||
</content>
|
||||
</new_information>
|
||||
|
||||
## Instructions
|
||||
1. Merge the new information into the existing page content.
|
||||
2. Preserve all existing information that is still valid.
|
||||
3. If the new information contradicts existing content, prefer the newer information and silently replace the old claim.
|
||||
4. Add new facts, details, and context from the new document.
|
||||
5. Update cross-references: add new [[wiki-link]] references where appropriate.
|
||||
6. Maintain the existing page structure and formatting style.
|
||||
7. Add a source reference to the new document at the bottom.
|
||||
8. Write in {{.Language}}.
|
||||
<instructions>
|
||||
1. The FIRST line of your output MUST be: SUMMARY: {one sentence, 15-40 words, describing what this page is about after the update — for wiki index listing}
|
||||
2. Merge the new information into the existing page content.
|
||||
3. Preserve all existing information that is still valid.
|
||||
4. If the new information contradicts existing content, prefer the newer information and silently replace the old claim.
|
||||
5. Add new facts, details, and context from the new document.
|
||||
6. Preserve any existing [[wiki-link]] references in the content. Do NOT invent new wiki-link slugs.
|
||||
7. Maintain the existing page structure and formatting style.
|
||||
8. Add a source reference to the new document at the bottom.
|
||||
9. Write in {{.Language}}.
|
||||
</instructions>
|
||||
|
||||
Output ONLY the updated Markdown content. Do not include any preamble or explanation.`
|
||||
Output the SUMMARY line first, then the updated Markdown content. Do not include any other preamble.`
|
||||
|
||||
// WikiPageRetractPrompt removes information contributed by a deleted document from an existing wiki page.
|
||||
const WikiPageRetractPrompt = `You are a wiki editor. A source document has been DELETED from the knowledge base. You must update the existing wiki page to remove any information that came exclusively from this deleted document.
|
||||
|
||||
## Existing Page Content
|
||||
<existing_page_content>
|
||||
{{.ExistingContent}}
|
||||
</existing_page_content>
|
||||
|
||||
## Deleted Document Title
|
||||
{{.DeletedDocTitle}}
|
||||
<deleted_document>
|
||||
<title>{{.DeletedDocTitle}}</title>
|
||||
<content>
|
||||
{{.DeletedDocContent}}
|
||||
</content>
|
||||
</deleted_document>
|
||||
|
||||
## Remaining Source Documents
|
||||
<remaining_sources>
|
||||
{{.RemainingSources}}
|
||||
</remaining_sources>
|
||||
|
||||
## Instructions
|
||||
1. Carefully review the existing page content.
|
||||
2. Remove any facts, claims, or details that were ONLY sourced from "{{.DeletedDocTitle}}".
|
||||
3. If a fact is also supported by the remaining source documents, KEEP it.
|
||||
4. If you are unsure whether a fact came from the deleted document, keep it as-is — do NOT add any review notes or annotations.
|
||||
5. Update or remove the "Source: {{.DeletedDocTitle}}" reference line if present.
|
||||
6. Remove any [[wiki-link]] references that point to pages that no longer exist, if you can identify them.
|
||||
7. Maintain the existing page structure, formatting style, and language.
|
||||
8. If after removing the deleted document's contributions the page becomes nearly empty, output just: "# [Title]\n\n*This page's primary source document was removed.*"
|
||||
9. Write in {{.Language}}.
|
||||
<valid_wiki_links>
|
||||
{{.AvailableSlugs}}
|
||||
</valid_wiki_links>
|
||||
|
||||
Output ONLY the updated Markdown content. Do not include any preamble or explanation.`
|
||||
<instructions>
|
||||
1. The FIRST line of your output MUST be: SUMMARY: {one sentence, 15-40 words, describing what this page is about after retraction — for wiki index listing}
|
||||
2. Carefully review the existing page content and compare it with the deleted document content above.
|
||||
3. Remove any facts, claims, or details that were ONLY sourced from the deleted document.
|
||||
4. If a fact is also supported by the remaining source documents, KEEP it.
|
||||
5. If you are unsure whether a fact came from the deleted document, keep it as-is — do NOT add any review notes or annotations.
|
||||
6. Update or remove the "Source: {{.DeletedDocTitle}}" reference line if present.
|
||||
7. Keep [[wiki-link]] references ONLY if the slug appears in the <valid_wiki_links> list above. Remove any [[wiki-link]] whose slug is NOT in that list.
|
||||
8. Maintain the existing page structure, formatting style, and language.
|
||||
9. If after removing the deleted document's contributions the page becomes nearly empty, output just: "SUMMARY: (empty page)\n# [Title]\n\n*This page's primary source document was removed.*"
|
||||
10. Write in {{.Language}}.
|
||||
</instructions>
|
||||
|
||||
// WikiIndexRebuildPrompt generates the index page content from a list of all pages.
|
||||
const WikiIndexRebuildPrompt = `You are a wiki editor. Generate an index page (table of contents) for a wiki based on the following page listing.
|
||||
Output the SUMMARY line first, then the updated Markdown content. Do not include any other preamble.`
|
||||
|
||||
## Pages
|
||||
{{.PageListing}}
|
||||
// WikiIndexIntroPrompt generates the introduction for a NEW index page (first time only).
|
||||
const WikiIndexIntroPrompt = `You are a wiki editor. Write a brief introduction for a wiki knowledge base index page.
|
||||
|
||||
## Instructions
|
||||
1. Organize pages by category (Summaries, Entities, Concepts, Analyses, etc.).
|
||||
2. For each page, include: [[slug]] — one-line summary
|
||||
3. Within each category, sort alphabetically.
|
||||
4. Include a brief introduction at the top explaining what this wiki covers.
|
||||
5. Write in {{.Language}}.
|
||||
<document_summaries>
|
||||
{{.DocumentSummaries}}
|
||||
</document_summaries>
|
||||
|
||||
Output ONLY the Markdown content for the index page.`
|
||||
<instructions>
|
||||
1. Write a title line starting with "# " that reflects the knowledge domain.
|
||||
2. Follow with 2-3 sentences describing what this wiki covers, based on the document summaries above.
|
||||
3. Keep it concise — this is just the header section, the directory listing will be added separately below.
|
||||
4. Write in {{.Language}}.
|
||||
</instructions>
|
||||
|
||||
Output ONLY the title and introduction paragraph. Do NOT generate any directory listings or page links.`
|
||||
|
||||
// WikiIndexIntroUpdatePrompt incrementally updates an existing index introduction.
|
||||
const WikiIndexIntroUpdatePrompt = `You are a wiki editor. Update the introduction section of a wiki index page to reflect recent changes.
|
||||
|
||||
<current_introduction>
|
||||
{{.ExistingIntro}}
|
||||
</current_introduction>
|
||||
|
||||
<changes>
|
||||
{{.ChangeDescription}}
|
||||
</changes>
|
||||
|
||||
<document_summaries>
|
||||
{{.DocumentSummaries}}
|
||||
</document_summaries>
|
||||
|
||||
<instructions>
|
||||
1. Update the introduction to accurately reflect the current state of the wiki.
|
||||
2. If documents were added, mention the new topics if they significantly change the wiki's scope.
|
||||
3. If documents were removed, remove references to those topics if they no longer apply.
|
||||
4. Keep the same tone, style, and title format as the existing introduction.
|
||||
5. Keep it concise — 1 title line + 2-3 sentences.
|
||||
6. Write in {{.Language}}.
|
||||
</instructions>
|
||||
|
||||
Output ONLY the updated title and introduction paragraph. Do NOT generate any directory listings or page links.`
|
||||
|
||||
// WikiLogEntryTemplate is a simple template for log entries (not LLM-generated).
|
||||
const WikiLogEntryTemplate = `## [{{.Date}}] {{.Operation}} | {{.Title}}
|
||||
@@ -207,13 +260,15 @@ The wiki has a log page (slug: "log") that records all ingest and update activit
|
||||
// between newly extracted items and existing wiki pages.
|
||||
const WikiDeduplicationPrompt = `You are a deduplication system. Given a list of newly extracted items and a list of existing wiki pages, determine which new items refer to the same entity or concept as an existing page.
|
||||
|
||||
## Newly Extracted Items
|
||||
<new_items>
|
||||
{{.NewItems}}
|
||||
</new_items>
|
||||
|
||||
## Existing Wiki Pages
|
||||
<existing_pages>
|
||||
{{.ExistingPages}}
|
||||
</existing_pages>
|
||||
|
||||
## Instructions
|
||||
<instructions>
|
||||
For each newly extracted item, check if it refers to the same real-world entity or concept as any existing page. Consider:
|
||||
- Name variations (e.g. "Acme Corp" vs "Acme Corporation", "RAG" vs "Retrieval-Augmented Generation")
|
||||
- Abbreviations and full names
|
||||
@@ -223,6 +278,7 @@ For each newly extracted item, check if it refers to the same real-world entity
|
||||
Return a JSON object with a "merges" map. The key is the NEW item's slug, the value is the EXISTING page's slug that it should merge into. Only include items that have a match.
|
||||
|
||||
If no items match any existing pages, return: {"merges": {}}
|
||||
</instructions>
|
||||
|
||||
Output ONLY valid JSON. Example:
|
||||
{"merges": {"entity/acme-corporation": "entity/acme-corp", "concept/rag": "concept/retrieval-augmented-generation"}}`
|
||||
|
||||
@@ -1284,6 +1284,15 @@ func (s *knowledgeService) cleanupWikiOnKnowledgeDelete(ctx context.Context, kbI
|
||||
}
|
||||
}
|
||||
|
||||
// Capture the summary page's description before deleting (for retract changeDesc)
|
||||
var docSummary string
|
||||
for _, page := range pages {
|
||||
if page.PageType == types.WikiPageTypeSummary {
|
||||
docSummary = page.Summary
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
var deletedSlugs []string
|
||||
var retractSlugs []string
|
||||
for _, page := range pages {
|
||||
@@ -1327,14 +1336,17 @@ func (s *knowledgeService) cleanupWikiOnKnowledgeDelete(ctx context.Context, kbI
|
||||
KnowledgeBaseID: kbID,
|
||||
KnowledgeID: knowledgeID,
|
||||
DocTitle: docTitle,
|
||||
DocSummary: docSummary,
|
||||
Language: lang,
|
||||
PageSlugs: retractSlugs,
|
||||
})
|
||||
logger.Infof(ctx, "wiki cleanup: enqueued retract task for %d pages: %v", len(retractSlugs), retractSlugs)
|
||||
}
|
||||
|
||||
// Rebuild index page to reflect deleted pages
|
||||
if len(deletedSlugs) > 0 {
|
||||
// If pages were deleted but no retract task was enqueued,
|
||||
// we still need to update the index and log synchronously.
|
||||
// (When retractSlugs > 0, ProcessWikiRetract handles index/log.)
|
||||
if len(deletedSlugs) > 0 && len(retractSlugs) == 0 {
|
||||
s.rebuildWikiIndexSimple(ctx, kbID, knowledgeID, docTitle, deletedSlugs)
|
||||
}
|
||||
}
|
||||
@@ -1355,47 +1367,60 @@ func removeSourceRef(refs types.StringArray, knowledgeID string) types.StringArr
|
||||
|
||||
// rebuildWikiIndexSimple regenerates the index page from current page listing
|
||||
// and appends a deletion log entry. Does NOT use LLM — generates a simple
|
||||
// structured listing. Used during document deletion when LLM is not available.
|
||||
// structured listing while preserving the LLM-generated intro from Summary field.
|
||||
func (s *knowledgeService) rebuildWikiIndexSimple(ctx context.Context, kbID, knowledgeID, docTitle string, deletedSlugs []string) {
|
||||
// Rebuild index
|
||||
resp, err := s.wikiService.ListPages(ctx, &types.WikiPageListRequest{
|
||||
KnowledgeBaseID: kbID,
|
||||
PageSize: 500,
|
||||
SortBy: "page_type",
|
||||
SortOrder: "asc",
|
||||
})
|
||||
if err != nil || resp == nil {
|
||||
allPages, err := s.wikiService.ListAllPages(ctx, kbID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "wiki cleanup: failed to list pages: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
var indexContent strings.Builder
|
||||
indexContent.WriteString("# Wiki Index\n\n")
|
||||
indexPage, _ := s.wikiService.GetIndex(ctx, kbID)
|
||||
if indexPage == nil {
|
||||
return
|
||||
}
|
||||
|
||||
currentType := ""
|
||||
for _, p := range resp.Pages {
|
||||
// Preserve LLM-generated intro from Summary field
|
||||
var indexContent strings.Builder
|
||||
if indexPage.Summary != "" && indexPage.Summary != "Index" {
|
||||
indexContent.WriteString(indexPage.Summary)
|
||||
indexContent.WriteString("\n")
|
||||
} else {
|
||||
indexContent.WriteString("# Wiki Index\n\n")
|
||||
}
|
||||
|
||||
// Group pages by type
|
||||
typeOrder := []string{types.WikiPageTypeSummary, types.WikiPageTypeEntity, types.WikiPageTypeConcept, types.WikiPageTypeSynthesis, types.WikiPageTypeComparison}
|
||||
grouped := make(map[string][]*types.WikiPage)
|
||||
for _, p := range allPages {
|
||||
if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog {
|
||||
continue
|
||||
}
|
||||
if p.Status == types.WikiPageStatusArchived {
|
||||
continue
|
||||
}
|
||||
if p.PageType != currentType {
|
||||
currentType = p.PageType
|
||||
label := currentType
|
||||
if len(label) > 0 {
|
||||
label = strings.ToUpper(label[:1]) + label[1:]
|
||||
}
|
||||
fmt.Fprintf(&indexContent, "\n## %s\n\n", label)
|
||||
}
|
||||
fmt.Fprintf(&indexContent, "- [[%s]] — %s\n", p.Slug, p.Summary)
|
||||
grouped[p.PageType] = append(grouped[p.PageType], p)
|
||||
}
|
||||
|
||||
indexPage, _ := s.wikiService.GetIndex(ctx, kbID)
|
||||
if indexPage != nil {
|
||||
indexPage.Content = indexContent.String()
|
||||
if _, err := s.wikiService.UpdatePage(ctx, indexPage); err != nil {
|
||||
logger.Warnf(ctx, "wiki cleanup: failed to rebuild index: %v", err)
|
||||
for _, pt := range typeOrder {
|
||||
pages := grouped[pt]
|
||||
if len(pages) == 0 {
|
||||
continue
|
||||
}
|
||||
label := pt
|
||||
if len(label) > 0 {
|
||||
label = strings.ToUpper(label[:1]) + label[1:]
|
||||
}
|
||||
fmt.Fprintf(&indexContent, "\n## %s\n\n", label)
|
||||
for _, p := range pages {
|
||||
fmt.Fprintf(&indexContent, "- [[%s]] — %s\n", p.Slug, p.Summary)
|
||||
}
|
||||
}
|
||||
|
||||
indexPage.Content = indexContent.String()
|
||||
if _, err := s.wikiService.UpdatePage(ctx, indexPage); err != nil {
|
||||
logger.Warnf(ctx, "wiki cleanup: failed to rebuild index: %v", err)
|
||||
}
|
||||
|
||||
// Append log entry
|
||||
|
||||
@@ -63,8 +63,9 @@ type WikiRetractPayload struct {
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||||
KnowledgeID string `json:"knowledge_id"`
|
||||
DocTitle string `json:"doc_title"`
|
||||
DocSummary string `json:"doc_summary,omitempty"` // one-line summary of the deleted document
|
||||
Language string `json:"language,omitempty"`
|
||||
PageSlugs []string `json:"page_slugs"` // pages to retract content from
|
||||
PageSlugs []string `json:"page_slugs"`
|
||||
}
|
||||
|
||||
// wikiIngestService handles the LLM-powered wiki generation pipeline
|
||||
@@ -205,51 +206,14 @@ func (s *wikiIngestService) ProcessWikiRetract(ctx context.Context, t *asynq.Tas
|
||||
|
||||
lang := types.LanguageNameFromContext(ctx)
|
||||
|
||||
for _, slug := range payload.PageSlugs {
|
||||
page, err := s.wikiService.GetPageBySlug(ctx, payload.KnowledgeBaseID, slug)
|
||||
if err != nil || page == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// Build remaining sources list
|
||||
var remainingSources string
|
||||
for _, ref := range page.SourceRefs {
|
||||
pipeIdx := strings.Index(ref, "|")
|
||||
if pipeIdx > 0 {
|
||||
remainingSources += "- " + ref[pipeIdx+1:] + "\n"
|
||||
} else {
|
||||
remainingSources += "- " + ref + "\n"
|
||||
}
|
||||
}
|
||||
if remainingSources == "" {
|
||||
remainingSources = "(no other sources)"
|
||||
}
|
||||
|
||||
// Call LLM to retract content
|
||||
updatedContent, err := s.generateWithTemplate(ctx, chatModel, agent.WikiPageRetractPrompt, map[string]string{
|
||||
"ExistingContent": page.Content,
|
||||
"DeletedDocTitle": payload.DocTitle,
|
||||
"RemainingSources": remainingSources,
|
||||
"Language": lang,
|
||||
})
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "wiki retract: LLM call failed for page %s: %v", slug, err)
|
||||
continue
|
||||
}
|
||||
|
||||
page.Content = updatedContent
|
||||
if _, err := s.wikiService.UpdatePage(ctx, page); err != nil {
|
||||
logger.Warnf(ctx, "wiki retract: failed to update page %s: %v", slug, err)
|
||||
} else {
|
||||
logger.Infof(ctx, "wiki retract: updated page %s after removing content from '%s'", slug, payload.DocTitle)
|
||||
}
|
||||
}
|
||||
s.retractPagesContent(ctx, chatModel, payload.KnowledgeBaseID, payload.DocTitle, payload.DocSummary, payload.PageSlugs, lang)
|
||||
|
||||
// Rebuild index page to reflect changes
|
||||
retractChangeDesc := fmt.Sprintf("Removed document '%s': %s", payload.DocTitle, payload.DocSummary)
|
||||
if err := s.rebuildIndexPage(ctx, chatModel, WikiIngestPayload{
|
||||
TenantID: payload.TenantID,
|
||||
KnowledgeBaseID: payload.KnowledgeBaseID,
|
||||
}, lang); err != nil {
|
||||
}, retractChangeDesc, lang); err != nil {
|
||||
logger.Warnf(ctx, "wiki retract: rebuild index failed: %v", err)
|
||||
}
|
||||
|
||||
@@ -266,6 +230,81 @@ func (s *wikiIngestService) ProcessWikiRetract(ctx context.Context, t *asynq.Tas
|
||||
return nil
|
||||
}
|
||||
|
||||
// retractPagesContent uses LLM to remove a deleted/stale document's contributions
|
||||
// from the given wiki pages. docContent is the deleted document's content (or summary)
|
||||
// so the LLM can accurately identify which parts to remove. It does NOT rebuild the
|
||||
// index or append log entries — callers are responsible for those post-processing steps.
|
||||
func (s *wikiIngestService) retractPagesContent(
|
||||
ctx context.Context,
|
||||
chatModel chat.Chat,
|
||||
kbID, docTitle, docContent string,
|
||||
pageSlugs []string,
|
||||
lang string,
|
||||
) {
|
||||
allPages, _ := s.wikiService.ListAllPages(ctx, kbID)
|
||||
slugTitleMap := make(map[string]string)
|
||||
for _, p := range allPages {
|
||||
if p.PageType != types.WikiPageTypeIndex && p.PageType != types.WikiPageTypeLog && p.Status != types.WikiPageStatusArchived {
|
||||
slugTitleMap[p.Slug] = p.Title
|
||||
}
|
||||
}
|
||||
|
||||
for _, slug := range pageSlugs {
|
||||
page, err := s.wikiService.GetPageBySlug(ctx, kbID, slug)
|
||||
if err != nil || page == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
var remainingSources string
|
||||
for _, ref := range page.SourceRefs {
|
||||
pipeIdx := strings.Index(ref, "|")
|
||||
if pipeIdx > 0 {
|
||||
remainingSources += "- " + ref[pipeIdx+1:] + "\n"
|
||||
} else {
|
||||
remainingSources += "- " + ref + "\n"
|
||||
}
|
||||
}
|
||||
if remainingSources == "" {
|
||||
remainingSources = "(no other sources)"
|
||||
}
|
||||
|
||||
var relatedSlugs strings.Builder
|
||||
for _, outSlug := range page.OutLinks {
|
||||
if title, ok := slugTitleMap[outSlug]; ok {
|
||||
fmt.Fprintf(&relatedSlugs, "- %s (%s)\n", outSlug, title)
|
||||
}
|
||||
}
|
||||
|
||||
updatedContent, err := s.generateWithTemplate(ctx, chatModel, agent.WikiPageRetractPrompt, map[string]string{
|
||||
"ExistingContent": page.Content,
|
||||
"DeletedDocTitle": docTitle,
|
||||
"DeletedDocContent": docContent,
|
||||
"RemainingSources": remainingSources,
|
||||
"AvailableSlugs": relatedSlugs.String(),
|
||||
"Language": lang,
|
||||
})
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "wiki retract: LLM call failed for page %s: %v", slug, err)
|
||||
continue
|
||||
}
|
||||
|
||||
retractedSummary, retractedBody := splitSummaryLine(updatedContent)
|
||||
if retractedBody != "" {
|
||||
page.Content = retractedBody
|
||||
} else {
|
||||
page.Content = updatedContent
|
||||
}
|
||||
if retractedSummary != "" {
|
||||
page.Summary = retractedSummary
|
||||
}
|
||||
if _, err := s.wikiService.UpdatePage(ctx, page); err != nil {
|
||||
logger.Warnf(ctx, "wiki retract: failed to update page %s: %v", slug, err)
|
||||
} else {
|
||||
logger.Infof(ctx, "wiki retract: updated page %s after removing content from '%s'", slug, docTitle)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ProcessWikiIngest processes a batch wiki ingest task.
|
||||
//
|
||||
// Concurrency model (Redis mode):
|
||||
@@ -356,19 +395,30 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
|
||||
// Process each document
|
||||
var allPagesAffected []string
|
||||
var docResults []*docIngestResult
|
||||
for _, knowledgeID := range knowledgeIDs {
|
||||
pages, err := s.processOneDocument(ctx, chatModel, payload, knowledgeID, lang)
|
||||
result, err := s.processOneDocument(ctx, chatModel, payload, knowledgeID, lang)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: failed to process knowledge %s: %v", knowledgeID, err)
|
||||
continue
|
||||
}
|
||||
allPagesAffected = append(allPagesAffected, pages...)
|
||||
if result != nil {
|
||||
allPagesAffected = append(allPagesAffected, result.Pages...)
|
||||
docResults = append(docResults, result)
|
||||
}
|
||||
}
|
||||
|
||||
// Batch post-processing (once for the whole batch, not per-doc)
|
||||
|
||||
// Build change description from processed documents
|
||||
var changeDesc strings.Builder
|
||||
fmt.Fprintf(&changeDesc, "Added %d documents:\n", len(docResults))
|
||||
for _, r := range docResults {
|
||||
fmt.Fprintf(&changeDesc, "- %s: %s\n", r.DocTitle, r.Summary)
|
||||
}
|
||||
|
||||
// Rebuild index page
|
||||
if err := s.rebuildIndexPage(ctx, chatModel, payload, lang); err != nil {
|
||||
if err := s.rebuildIndexPage(ctx, chatModel, payload, changeDesc.String(), lang); err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: rebuild index failed: %v", err)
|
||||
}
|
||||
|
||||
@@ -453,15 +503,22 @@ func (s *wikiIngestService) drainPendingList(ctx context.Context, kbID string) [
|
||||
return unique
|
||||
}
|
||||
|
||||
// docIngestResult captures per-document info for batch post-processing.
|
||||
type docIngestResult struct {
|
||||
KnowledgeID string
|
||||
DocTitle string
|
||||
Summary string // one-line summary of the document (from summary page)
|
||||
Pages []string // affected page slugs
|
||||
}
|
||||
|
||||
// processOneDocument handles wiki ingest for a single document.
|
||||
// Returns the list of affected page slugs.
|
||||
func (s *wikiIngestService) processOneDocument(
|
||||
ctx context.Context,
|
||||
chatModel chat.Chat,
|
||||
payload WikiIngestPayload,
|
||||
knowledgeID string,
|
||||
lang string,
|
||||
) ([]string, error) {
|
||||
) (*docIngestResult, error) {
|
||||
// Get document chunks and reconstruct content
|
||||
chunks, err := s.chunkRepo.ListChunksByKnowledgeID(ctx, payload.TenantID, knowledgeID)
|
||||
if err != nil {
|
||||
@@ -489,6 +546,7 @@ func (s *wikiIngestService) processOneDocument(
|
||||
}
|
||||
|
||||
var pagesAffected []string
|
||||
var docSummaryLine string
|
||||
sourceRef := fmt.Sprintf("%s|%s", knowledgeID, docTitle)
|
||||
|
||||
// Snapshot existing page slugs for stale detection
|
||||
@@ -503,7 +561,7 @@ func (s *wikiIngestService) processOneDocument(
|
||||
}
|
||||
|
||||
// Step 1: Extract entities and concepts
|
||||
extractedPages, extractedSlugs, err := s.extractEntitiesAndConcepts(ctx, chatModel, content, docTitle, lang, docPayload, sourceRef, oldPageSlugs)
|
||||
extractedPages, err := s.extractEntitiesAndConcepts(ctx, chatModel, content, docTitle, lang, docPayload, sourceRef, oldPageSlugs)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: knowledge extraction failed for %s: %v", knowledgeID, err)
|
||||
} else {
|
||||
@@ -513,7 +571,7 @@ func (s *wikiIngestService) processOneDocument(
|
||||
// Step 2: Generate summary page
|
||||
summarySlug := fmt.Sprintf("summary/%s", slugify(docTitle))
|
||||
var slugListing string
|
||||
for _, slug := range extractedSlugs {
|
||||
for _, slug := range extractedPages {
|
||||
slugListing += fmt.Sprintf("- [[%s]]\n", slug)
|
||||
}
|
||||
summaryContent, err := s.generateWithTemplate(ctx, chatModel, agent.WikiSummaryPrompt, map[string]string{
|
||||
@@ -527,6 +585,14 @@ func (s *wikiIngestService) processOneDocument(
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "wiki ingest: generate summary failed for %s: %v", knowledgeID, err)
|
||||
} else {
|
||||
sumLine, sumBody := splitSummaryLine(summaryContent)
|
||||
if sumBody == "" {
|
||||
sumBody = summaryContent
|
||||
}
|
||||
if sumLine == "" {
|
||||
sumLine = docTitle
|
||||
}
|
||||
docSummaryLine = sumLine
|
||||
_, err := s.wikiService.CreatePage(ctx, &types.WikiPage{
|
||||
ID: uuid.New().String(),
|
||||
TenantID: payload.TenantID,
|
||||
@@ -535,8 +601,8 @@ func (s *wikiIngestService) processOneDocument(
|
||||
Title: docTitle + " - Summary",
|
||||
PageType: types.WikiPageTypeSummary,
|
||||
Status: types.WikiPageStatusDraft,
|
||||
Content: summaryContent,
|
||||
Summary: truncateString(summaryContent, 200),
|
||||
Content: sumBody,
|
||||
Summary: sumLine,
|
||||
SourceRefs: types.StringArray{sourceRef},
|
||||
})
|
||||
if err != nil {
|
||||
@@ -547,34 +613,36 @@ func (s *wikiIngestService) processOneDocument(
|
||||
}
|
||||
|
||||
// Retract stale pages (pages this doc previously contributed to but no longer does)
|
||||
s.retractStalePages(ctx, docPayload, oldPageSlugs, pagesAffected, docTitle, lang)
|
||||
s.retractStalePages(ctx, chatModel, docPayload, oldPageSlugs, pagesAffected, docTitle, content, lang)
|
||||
|
||||
logger.Infof(ctx, "wiki ingest: processed knowledge %s, %d pages affected", knowledgeID, len(pagesAffected))
|
||||
return pagesAffected, nil
|
||||
return &docIngestResult{
|
||||
KnowledgeID: knowledgeID,
|
||||
DocTitle: docTitle,
|
||||
Summary: docSummaryLine,
|
||||
Pages: pagesAffected,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// cleanDeadLinks removes [[wiki-links]] that point to archived or deleted pages.
|
||||
// Scans all published pages, checks each out_link, and removes references to
|
||||
// pages that no longer exist or are archived. No LLM call — pure text cleanup.
|
||||
func (s *wikiIngestService) cleanDeadLinks(ctx context.Context, kbID string) {
|
||||
resp, err := s.wikiService.ListPages(ctx, &types.WikiPageListRequest{
|
||||
KnowledgeBaseID: kbID,
|
||||
PageSize: 500,
|
||||
})
|
||||
if err != nil || resp == nil {
|
||||
allPages, err := s.wikiService.ListAllPages(ctx, kbID)
|
||||
if err != nil || len(allPages) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// Build set of live (non-archived, non-system) slugs
|
||||
liveSlugs := make(map[string]bool)
|
||||
for _, p := range resp.Pages {
|
||||
for _, p := range allPages {
|
||||
if p.Status != types.WikiPageStatusArchived {
|
||||
liveSlugs[p.Slug] = true
|
||||
}
|
||||
}
|
||||
|
||||
var cleaned int
|
||||
for _, p := range resp.Pages {
|
||||
for _, p := range allPages {
|
||||
if p.Status == types.WikiPageStatusArchived {
|
||||
continue
|
||||
}
|
||||
@@ -622,11 +690,8 @@ func (s *wikiIngestService) cleanDeadLinks(ctx context.Context, kbID string) {
|
||||
// Only processes entity/concept/synthesis/comparison pages (not index/log/summary).
|
||||
func (s *wikiIngestService) injectCrossLinks(ctx context.Context, kbID string, affectedSlugs []string) {
|
||||
// Build a title→slug lookup from ALL pages in this KB
|
||||
resp, err := s.wikiService.ListPages(ctx, &types.WikiPageListRequest{
|
||||
KnowledgeBaseID: kbID,
|
||||
PageSize: 500,
|
||||
})
|
||||
if err != nil || resp == nil || len(resp.Pages) < 2 {
|
||||
allPages, err := s.wikiService.ListAllPages(ctx, kbID)
|
||||
if err != nil || len(allPages) < 2 {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -635,7 +700,7 @@ func (s *wikiIngestService) injectCrossLinks(ctx context.Context, kbID string, a
|
||||
title string
|
||||
}
|
||||
var allRefs []pageRef
|
||||
for _, p := range resp.Pages {
|
||||
for _, p := range allPages {
|
||||
if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog {
|
||||
continue
|
||||
}
|
||||
@@ -664,7 +729,7 @@ func (s *wikiIngestService) injectCrossLinks(ctx context.Context, kbID string, a
|
||||
}
|
||||
|
||||
var updated int
|
||||
for _, p := range resp.Pages {
|
||||
for _, p := range allPages {
|
||||
if !affectedSet[p.Slug] {
|
||||
continue
|
||||
}
|
||||
@@ -745,17 +810,14 @@ func replaceFirstOutsideLinks(s, old, newStr string) string {
|
||||
// getExistingPageSlugsForKnowledge returns all page slugs that currently reference
|
||||
// a given knowledge ID in their source_refs. Used to snapshot state before re-ingest.
|
||||
func (s *wikiIngestService) getExistingPageSlugsForKnowledge(ctx context.Context, kbID, knowledgeID string) map[string]bool {
|
||||
resp, err := s.wikiService.ListPages(ctx, &types.WikiPageListRequest{
|
||||
KnowledgeBaseID: kbID,
|
||||
PageSize: 500,
|
||||
})
|
||||
if err != nil || resp == nil {
|
||||
allPages, err := s.wikiService.ListAllPages(ctx, kbID)
|
||||
if err != nil || len(allPages) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
slugs := make(map[string]bool)
|
||||
prefix := knowledgeID + "|"
|
||||
for _, p := range resp.Pages {
|
||||
for _, p := range allPages {
|
||||
if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog {
|
||||
continue
|
||||
}
|
||||
@@ -771,14 +833,15 @@ func (s *wikiIngestService) getExistingPageSlugsForKnowledge(ctx context.Context
|
||||
|
||||
// retractStalePages handles pages that were previously linked to this document
|
||||
// but are no longer produced by the updated extraction.
|
||||
// - Single-source stale pages → archived
|
||||
// - Multi-source stale pages → enqueue LLM retract to clean content
|
||||
// - Single-source stale pages → deleted
|
||||
// - Multi-source stale pages → LLM retract to clean content synchronously
|
||||
func (s *wikiIngestService) retractStalePages(
|
||||
ctx context.Context,
|
||||
chatModel chat.Chat,
|
||||
payload WikiIngestPayload,
|
||||
oldSlugs map[string]bool,
|
||||
newSlugs []string,
|
||||
docTitle, lang string,
|
||||
docTitle, docContent, lang string,
|
||||
) {
|
||||
if len(oldSlugs) == 0 {
|
||||
return
|
||||
@@ -838,16 +901,8 @@ func (s *wikiIngestService) retractStalePages(
|
||||
}
|
||||
}
|
||||
|
||||
// Enqueue LLM retraction for multi-source stale pages
|
||||
if len(retractSlugs) > 0 {
|
||||
EnqueueWikiRetract(ctx, s.task, WikiRetractPayload{
|
||||
TenantID: payload.TenantID,
|
||||
KnowledgeBaseID: payload.KnowledgeBaseID,
|
||||
KnowledgeID: payload.KnowledgeID,
|
||||
DocTitle: docTitle,
|
||||
Language: payload.Language,
|
||||
PageSlugs: retractSlugs,
|
||||
})
|
||||
s.retractPagesContent(ctx, chatModel, payload.KnowledgeBaseID, docTitle, docContent, retractSlugs, lang)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -866,7 +921,7 @@ type combinedExtraction struct {
|
||||
}
|
||||
|
||||
// extractEntitiesAndConcepts performs a single LLM call to extract both entities and concepts,
|
||||
// then upserts pages for each. Returns the list of affected page slugs and all extracted slugs (for linking).
|
||||
// then upserts pages for each. Returns the list of successfully upserted page slugs.
|
||||
// oldPageSlugs contains slugs from the previous version of this document — passed to LLM for slug stability.
|
||||
func (s *wikiIngestService) extractEntitiesAndConcepts(
|
||||
ctx context.Context,
|
||||
@@ -875,7 +930,7 @@ func (s *wikiIngestService) extractEntitiesAndConcepts(
|
||||
payload WikiIngestPayload,
|
||||
sourceRef string,
|
||||
oldPageSlugs map[string]bool,
|
||||
) ([]string, []string, error) {
|
||||
) ([]string, error) {
|
||||
// Build previous slugs listing for the prompt
|
||||
var prevSlugsText string
|
||||
if len(oldPageSlugs) > 0 {
|
||||
@@ -896,7 +951,7 @@ func (s *wikiIngestService) extractEntitiesAndConcepts(
|
||||
"PreviousSlugs": prevSlugsText,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("combined extraction failed: %w", err)
|
||||
return nil, fmt.Errorf("combined extraction failed: %w", err)
|
||||
}
|
||||
|
||||
// Clean JSON - strip markdown code blocks if present
|
||||
@@ -909,20 +964,7 @@ func (s *wikiIngestService) extractEntitiesAndConcepts(
|
||||
var result combinedExtraction
|
||||
if err := json.Unmarshal([]byte(extractionJSON), &result); err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: failed to parse combined extraction JSON: %v\nRaw: %s", err, truncateString(extractionJSON, 500))
|
||||
return nil, nil, fmt.Errorf("parse combined extraction JSON: %w", err)
|
||||
}
|
||||
|
||||
// Collect all extracted slugs (before dedup remapping) for summary linking
|
||||
var allSlugs []string
|
||||
for _, item := range result.Entities {
|
||||
if item.Slug != "" {
|
||||
allSlugs = append(allSlugs, item.Slug)
|
||||
}
|
||||
}
|
||||
for _, item := range result.Concepts {
|
||||
if item.Slug != "" {
|
||||
allSlugs = append(allSlugs, item.Slug)
|
||||
}
|
||||
return nil, fmt.Errorf("parse combined extraction JSON: %w", err)
|
||||
}
|
||||
|
||||
var affected []string
|
||||
@@ -933,19 +975,6 @@ func (s *wikiIngestService) extractEntitiesAndConcepts(
|
||||
// Deduplicate concepts against existing wiki pages (LLM-based)
|
||||
result.Concepts = s.deduplicateItems(ctx, chatModel, result.Concepts, types.WikiPageTypeConcept, payload.KnowledgeBaseID)
|
||||
|
||||
// Update allSlugs after dedup (use final slugs)
|
||||
allSlugs = allSlugs[:0]
|
||||
for _, item := range result.Entities {
|
||||
if item.Slug != "" {
|
||||
allSlugs = append(allSlugs, item.Slug)
|
||||
}
|
||||
}
|
||||
for _, item := range result.Concepts {
|
||||
if item.Slug != "" {
|
||||
allSlugs = append(allSlugs, item.Slug)
|
||||
}
|
||||
}
|
||||
|
||||
// Upsert entity pages
|
||||
entitySlugs, err := s.upsertExtractedPages(ctx, chatModel, result.Entities, types.WikiPageTypeEntity, docTitle, lang, payload, sourceRef)
|
||||
if err != nil {
|
||||
@@ -962,7 +991,7 @@ func (s *wikiIngestService) extractEntitiesAndConcepts(
|
||||
affected = append(affected, conceptSlugs...)
|
||||
}
|
||||
|
||||
return affected, allSlugs, nil
|
||||
return affected, nil
|
||||
}
|
||||
|
||||
// upsertExtractedPages creates or updates wiki pages from pre-extracted items.
|
||||
@@ -996,8 +1025,16 @@ func (s *wikiIngestService) upsertExtractedPages(
|
||||
continue
|
||||
}
|
||||
|
||||
existing.Content = updatedContent
|
||||
existing.Summary = item.Description
|
||||
// LLM returns "SUMMARY: ..." on first line — use it as the authoritative summary
|
||||
updatedSummary, updatedBody := splitSummaryLine(updatedContent)
|
||||
if updatedBody != "" {
|
||||
existing.Content = updatedBody
|
||||
} else {
|
||||
existing.Content = updatedContent
|
||||
}
|
||||
if updatedSummary != "" {
|
||||
existing.Summary = updatedSummary
|
||||
}
|
||||
existing.SourceRefs = appendUnique(existing.SourceRefs, sourceRef)
|
||||
|
||||
if _, err := s.wikiService.UpdatePage(ctx, existing); err != nil {
|
||||
@@ -1032,47 +1069,143 @@ func (s *wikiIngestService) upsertExtractedPages(
|
||||
return affected, nil
|
||||
}
|
||||
|
||||
// rebuildIndexPage regenerates the index page from all existing pages
|
||||
func (s *wikiIngestService) rebuildIndexPage(ctx context.Context, chatModel chat.Chat, payload WikiIngestPayload, lang string) error {
|
||||
// List all pages
|
||||
resp, err := s.wikiService.ListPages(ctx, &types.WikiPageListRequest{
|
||||
KnowledgeBaseID: payload.KnowledgeBaseID,
|
||||
PageSize: 500,
|
||||
SortBy: "page_type",
|
||||
SortOrder: "asc",
|
||||
})
|
||||
// rebuildIndexPage regenerates the index page.
|
||||
//
|
||||
// Strategy: Index = LLM-generated intro (stored in Summary field) + code-generated directory.
|
||||
// - Intro: stored in indexPage.Summary. First time: generated from document summaries.
|
||||
// Subsequent: incrementally updated with changeDescription.
|
||||
// - Directory: pure code, rebuilt every time. O(N) string concat, no LLM.
|
||||
func (s *wikiIngestService) rebuildIndexPage(ctx context.Context, chatModel chat.Chat, payload WikiIngestPayload, changeDesc, lang string) error {
|
||||
indexPage, _ := s.wikiService.GetIndex(ctx, payload.KnowledgeBaseID)
|
||||
if indexPage == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// List all live pages
|
||||
allPages, err := s.wikiService.ListAllPages(ctx, payload.KnowledgeBaseID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Build page listing
|
||||
var listing strings.Builder
|
||||
for _, p := range resp.Pages {
|
||||
typeOrder := []string{
|
||||
types.WikiPageTypeSummary, types.WikiPageTypeEntity, types.WikiPageTypeConcept,
|
||||
types.WikiPageTypeSynthesis, types.WikiPageTypeComparison,
|
||||
}
|
||||
typeLabels := map[string]string{
|
||||
types.WikiPageTypeSummary: "Summary", types.WikiPageTypeEntity: "Entity",
|
||||
types.WikiPageTypeConcept: "Concept", types.WikiPageTypeSynthesis: "Synthesis",
|
||||
types.WikiPageTypeComparison: "Comparison",
|
||||
}
|
||||
|
||||
grouped := make(map[string][]*types.WikiPage)
|
||||
totalPages := 0
|
||||
for _, p := range allPages {
|
||||
if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog {
|
||||
continue
|
||||
}
|
||||
listing.WriteString(fmt.Sprintf("- [%s] [[%s]] | %s — %s\n", p.PageType, p.Slug, p.Title, p.Summary))
|
||||
if p.Status == types.WikiPageStatusArchived {
|
||||
continue
|
||||
}
|
||||
grouped[p.PageType] = append(grouped[p.PageType], p)
|
||||
totalPages++
|
||||
}
|
||||
|
||||
indexContent, err := s.generateWithTemplate(ctx, chatModel, agent.WikiIndexRebuildPrompt, map[string]string{
|
||||
"PageListing": listing.String(),
|
||||
"Language": lang,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("generate index: %w", err)
|
||||
// Build document summaries listing (only summary-type pages — they represent documents)
|
||||
var docSummaries strings.Builder
|
||||
for _, p := range grouped[types.WikiPageTypeSummary] {
|
||||
fmt.Fprintf(&docSummaries, "- %s: %s\n", p.Title, p.Summary)
|
||||
}
|
||||
if docSummaries.Len() == 0 {
|
||||
docSummaries.WriteString("(no documents yet)")
|
||||
}
|
||||
|
||||
// Get or create index page
|
||||
indexPage, _ := s.wikiService.GetIndex(ctx, payload.KnowledgeBaseID)
|
||||
if indexPage != nil {
|
||||
indexPage.Content = indexContent
|
||||
indexPage.Summary = "Wiki index - table of contents"
|
||||
if _, err := s.wikiService.UpdatePage(ctx, indexPage); err != nil {
|
||||
return fmt.Errorf("update index page: %w", err)
|
||||
// Generate or update intro
|
||||
existingIntro := indexPage.Summary
|
||||
var intro string
|
||||
|
||||
if existingIntro == "" || existingIntro == "Wiki index - table of contents" {
|
||||
// First time — generate intro from scratch
|
||||
generatedIntro, genErr := s.generateWithTemplate(ctx, chatModel, agent.WikiIndexIntroPrompt, map[string]string{
|
||||
"DocumentSummaries": docSummaries.String(),
|
||||
"Language": lang,
|
||||
})
|
||||
if genErr != nil {
|
||||
intro = "# Wiki Index\n\nThis wiki contains knowledge extracted from uploaded documents.\n"
|
||||
} else {
|
||||
intro = strings.TrimSpace(generatedIntro)
|
||||
}
|
||||
} else if changeDesc != "" {
|
||||
// Incremental update — tell LLM what changed
|
||||
updatedIntro, genErr := s.generateWithTemplate(ctx, chatModel, agent.WikiIndexIntroUpdatePrompt, map[string]string{
|
||||
"ExistingIntro": existingIntro,
|
||||
"ChangeDescription": changeDesc,
|
||||
"DocumentSummaries": docSummaries.String(),
|
||||
"Language": lang,
|
||||
})
|
||||
if genErr != nil {
|
||||
intro = existingIntro // keep existing on error
|
||||
} else {
|
||||
intro = strings.TrimSpace(updatedIntro)
|
||||
}
|
||||
} else {
|
||||
intro = existingIntro // no change description, keep as-is
|
||||
}
|
||||
|
||||
// Build directory (pure code, no LLM)
|
||||
var dir strings.Builder
|
||||
for _, pt := range typeOrder {
|
||||
pages := grouped[pt]
|
||||
if len(pages) == 0 {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(&dir, "\n## %s (%d)\n\n", typeLabels[pt], len(pages))
|
||||
for _, p := range pages {
|
||||
summary := p.Summary
|
||||
fmt.Fprintf(&dir, "[[%s]] — %s\n", p.Slug, summary)
|
||||
}
|
||||
}
|
||||
for pt, pages := range grouped {
|
||||
inOrder := false
|
||||
for _, o := range typeOrder {
|
||||
if o == pt {
|
||||
inOrder = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if inOrder || len(pages) == 0 {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(&dir, "\n## %s (%d)\n\n", pt, len(pages))
|
||||
for _, p := range pages {
|
||||
fmt.Fprintf(&dir, "[[%s]] — %s\n", p.Slug, p.Summary)
|
||||
}
|
||||
}
|
||||
if totalPages == 0 {
|
||||
dir.WriteString("\n*No wiki pages yet. Upload documents to get started.*\n")
|
||||
}
|
||||
|
||||
return nil
|
||||
indexPage.Content = intro + "\n" + dir.String()
|
||||
indexPage.Summary = intro // persist intro for next incremental update
|
||||
_, err = s.wikiService.UpdatePage(ctx, indexPage)
|
||||
return err
|
||||
}
|
||||
|
||||
// splitSummaryLine extracts the "SUMMARY: ..." line from LLM output.
|
||||
// Returns (summary, content). If no SUMMARY line found, summary is empty.
|
||||
func splitSummaryLine(raw string) (summary string, content string) {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if strings.HasPrefix(raw, "SUMMARY:") || strings.HasPrefix(raw, "SUMMARY:") {
|
||||
idx := strings.IndexByte(raw, '\n')
|
||||
if idx < 0 {
|
||||
// Only one line
|
||||
return strings.TrimSpace(strings.TrimPrefix(strings.TrimPrefix(raw, "SUMMARY:"), "SUMMARY:")), ""
|
||||
}
|
||||
summaryLine := raw[:idx]
|
||||
summaryLine = strings.TrimPrefix(summaryLine, "SUMMARY:")
|
||||
summaryLine = strings.TrimPrefix(summaryLine, "SUMMARY:")
|
||||
return strings.TrimSpace(summaryLine), strings.TrimSpace(raw[idx+1:])
|
||||
}
|
||||
return "", raw
|
||||
}
|
||||
|
||||
// appendLogEntry appends an entry to the log page, including any synthesis suggestions
|
||||
@@ -1136,18 +1269,25 @@ func (s *wikiIngestService) deduplicateItems(
|
||||
}
|
||||
|
||||
// Get existing pages of the same type
|
||||
resp, err := s.wikiService.ListPages(ctx, &types.WikiPageListRequest{
|
||||
KnowledgeBaseID: kbID,
|
||||
PageType: pageType,
|
||||
PageSize: 2000,
|
||||
})
|
||||
if err != nil || resp == nil || len(resp.Pages) == 0 {
|
||||
return items // No existing pages → nothing to deduplicate against
|
||||
allPages, err := s.wikiService.ListAllPages(ctx, kbID)
|
||||
if err != nil || len(allPages) == 0 {
|
||||
return items
|
||||
}
|
||||
|
||||
// Filter to the target type
|
||||
var typedPages []*types.WikiPage
|
||||
for _, p := range allPages {
|
||||
if p.PageType == pageType {
|
||||
typedPages = append(typedPages, p)
|
||||
}
|
||||
}
|
||||
if len(typedPages) == 0 {
|
||||
return items // No existing pages of this type → nothing to deduplicate against
|
||||
}
|
||||
|
||||
// Build existing pages listing
|
||||
var existingBuf strings.Builder
|
||||
for _, p := range resp.Pages {
|
||||
for _, p := range typedPages {
|
||||
fmt.Fprintf(&existingBuf, "- slug: %s | title: %s\n", p.Slug, p.Title)
|
||||
}
|
||||
|
||||
|
||||
@@ -20,9 +20,9 @@ var wikiLinkRegex = regexp.MustCompile(`\[\[([^\]]+)\]\]`)
|
||||
|
||||
// wikiPageService implements the WikiPageService interface
|
||||
type wikiPageService struct {
|
||||
repo interfaces.WikiPageRepository
|
||||
chunkRepo interfaces.ChunkRepository
|
||||
kbService interfaces.KnowledgeBaseService
|
||||
repo interfaces.WikiPageRepository
|
||||
chunkRepo interfaces.ChunkRepository
|
||||
kbService interfaces.KnowledgeBaseService
|
||||
}
|
||||
|
||||
// NewWikiPageService creates a new wiki page service
|
||||
@@ -70,11 +70,6 @@ func (s *wikiPageService) CreatePage(ctx context.Context, page *types.WikiPage)
|
||||
// Update inbound links on target pages
|
||||
s.updateInLinks(ctx, page.KnowledgeBaseID, page.Slug, page.OutLinks)
|
||||
|
||||
// Sync to chunks for retrieval pipeline integration
|
||||
if err := s.syncToChunk(ctx, page); err != nil {
|
||||
logger.Warnf(ctx, "wiki: failed to sync page %s to chunks: %v", page.Slug, err)
|
||||
}
|
||||
|
||||
return page, nil
|
||||
}
|
||||
|
||||
@@ -108,11 +103,6 @@ func (s *wikiPageService) UpdatePage(ctx context.Context, page *types.WikiPage)
|
||||
s.removeInLinks(ctx, existing.KnowledgeBaseID, existing.Slug, oldOutLinks)
|
||||
s.updateInLinks(ctx, existing.KnowledgeBaseID, existing.Slug, existing.OutLinks)
|
||||
|
||||
// Re-sync to chunks
|
||||
if err := s.syncToChunk(ctx, existing); err != nil {
|
||||
logger.Warnf(ctx, "wiki: failed to sync page %s to chunks: %v", existing.Slug, err)
|
||||
}
|
||||
|
||||
return existing, nil
|
||||
}
|
||||
|
||||
@@ -341,6 +331,11 @@ func (s *wikiPageService) RebuildLinks(ctx context.Context, kbID string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListAllPages retrieves all wiki pages without pagination.
|
||||
func (s *wikiPageService) ListAllPages(ctx context.Context, kbID string) ([]*types.WikiPage, error) {
|
||||
return s.repo.ListAll(ctx, kbID)
|
||||
}
|
||||
|
||||
// SearchPages performs full-text search over wiki pages
|
||||
func (s *wikiPageService) SearchPages(ctx context.Context, kbID string, query string, limit int) ([]*types.WikiPage, error) {
|
||||
return s.repo.Search(ctx, kbID, query, limit)
|
||||
@@ -413,33 +408,6 @@ func (s *wikiPageService) removeInLinks(ctx context.Context, kbID string, source
|
||||
}
|
||||
}
|
||||
|
||||
// syncToChunk syncs a wiki page to the chunk table for retrieval pipeline integration.
|
||||
// Each wiki page maps to a single chunk with ChunkType = "wiki_page".
|
||||
// The chunk's KnowledgeID is set to the wiki page ID for tracking.
|
||||
func (s *wikiPageService) syncToChunk(ctx context.Context, page *types.WikiPage) error {
|
||||
// Build content for the chunk: title + summary + content
|
||||
chunkContent := fmt.Sprintf("# %s\n\n%s\n\n%s", page.Title, page.Summary, page.Content)
|
||||
|
||||
chunk := &types.Chunk{
|
||||
ID: "wp-" + page.ID, // prefix to avoid collision with regular chunks
|
||||
TenantID: page.TenantID,
|
||||
KnowledgeID: page.ID, // link back to wiki page
|
||||
KnowledgeBaseID: page.KnowledgeBaseID,
|
||||
Content: chunkContent,
|
||||
ChunkType: types.ChunkTypeWikiPage,
|
||||
IsEnabled: page.Status == types.WikiPageStatusPublished,
|
||||
Status: int(types.ChunkStatusStored),
|
||||
}
|
||||
|
||||
// Try to create; if it already exists, update
|
||||
err := s.chunkRepo.CreateChunks(ctx, []*types.Chunk{chunk})
|
||||
if err != nil {
|
||||
// Fallback: update existing chunk
|
||||
return s.chunkRepo.UpdateChunk(ctx, chunk)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// deleteChunkForPage removes the synced chunk for a wiki page
|
||||
func (s *wikiPageService) deleteChunkForPage(ctx context.Context, page *types.WikiPage) {
|
||||
chunkID := "wp-" + page.ID
|
||||
|
||||
@@ -52,6 +52,10 @@ type WikiPageService interface {
|
||||
// RebuildLinks re-parses all pages and rebuilds bidirectional link references.
|
||||
RebuildLinks(ctx context.Context, kbID string) error
|
||||
|
||||
// ListAllPages retrieves all wiki pages in a knowledge base without pagination.
|
||||
// Used for index rebuild, graph generation, cross-link injection, etc.
|
||||
ListAllPages(ctx context.Context, kbID string) ([]*types.WikiPage, error)
|
||||
|
||||
// SearchPages performs full-text search over wiki pages.
|
||||
SearchPages(ctx context.Context, kbID string, query string, limit int) ([]*types.WikiPage, error)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user