diff --git a/internal/agent/prompts_wiki.go b/internal/agent/prompts_wiki.go
index 6bd23d90..79c264d6 100644
--- a/internal/agent/prompts_wiki.go
+++ b/internal/agent/prompts_wiki.go
@@ -112,70 +112,49 @@ Output ONLY valid JSON. Example:
]
}`
-// WikiPageUpdatePrompt incrementally updates an existing wiki page with new information.
-const WikiPageUpdatePrompt = `You are a wiki editor tasked with updating an existing wiki page with new information from a recently ingested document.
+// WikiPageModifyPrompt updates an existing wiki page with new additions and removes stale/deleted information in a single pass.
+const WikiPageModifyPrompt = `You are a wiki editor tasked with updating an existing wiki page. You must process a set of NEW information to add, AND/OR a set of deleted documents whose exclusive contributions must be REMOVED.
{{.ExistingContent}}
+{{if .HasAdditions}}
-{{.NewDocTitle}}
-
{{.NewContent}}
-
+{{end}}
-
-1. The FIRST line of your output MUST be: SUMMARY: {one sentence, 15-40 words, describing what this page is about after the update — for wiki index listing}
-2. Merge the new information into the existing page content.
-3. Preserve all existing information that is still valid.
-4. If the new information contradicts existing content, prefer the newer information and silently replace the old claim.
-5. Add new facts, details, and context from the new document.
-6. Preserve any existing [[slug|name]] wiki-link references in the content. Do NOT invent new wiki-link slugs.
-7. Maintain the existing page structure and formatting style.
-8. Add a source reference to the new document at the bottom.
-9. **Image rule**: If the new document contains tags with elements, you SHOULD include the relevant images in your updated page using the Markdown syntax: . Place the images where they are contextually relevant to the text.
-10. Write in {{.Language}}.
-
-
-Output the SUMMARY line first, then the updated Markdown content. Do not include any other preamble.`
-
-// WikiPageRetractPrompt removes information contributed by a deleted document from an existing wiki page.
-// It includes the actual content of remaining source documents (from their summary pages) so the LLM
-// can accurately determine which facts are still supported and which should be removed.
-const WikiPageRetractPrompt = `You are a wiki editor. A source document has been DELETED from the knowledge base. You must update the existing wiki page to remove any information that came exclusively from this deleted document.
-
-
-{{.ExistingContent}}
-
-
-
-{{.DeletedDocTitle}}
-
-{{.DeletedDocContent}}
-
-
+{{if .HasRetractions}}
+
+{{.DeletedContent}}
+
{{.RemainingSourcesContent}}
+{{end}}
{{.AvailableSlugs}}
-1. The FIRST line of your output MUST be: SUMMARY: {one sentence, 15-40 words, describing what this page is about after retraction — for wiki index listing}
-2. Carefully review the existing page content. Compare it against BOTH the deleted document and the remaining source documents above.
-3. Remove any facts, claims, or details that were ONLY sourced from the deleted document and are NOT present in any remaining source document.
-4. If a fact appears in both the deleted document and a remaining source, KEEP it.
-5. If you are unsure whether a fact came from the deleted document, keep it as-is — do NOT add any review notes or annotations.
-6. Update or remove the "Source: {{.DeletedDocTitle}}" reference line if present.
-7. Keep [[slug|name]] wiki-link references ONLY if the slug appears in the list above. Remove any [[slug|name]] whose slug is NOT in that list.
-8. Maintain the existing page structure, formatting style, and language.
-9. If after removing the deleted document's contributions the page becomes nearly empty, output just: "SUMMARY: (empty page)\n# [Title]\n\n*This page's primary source document was removed.*"
-10. Write in {{.Language}}.
+1. The FIRST line of your output MUST be: SUMMARY: {one sentence, 15-40 words, describing what this page is about after the update — for wiki index listing}
+{{if .HasRetractions}}
+2. REMOVE facts/claims that were ONLY sourced from the and are NOT present in any or .
+{{end}}
+{{if .HasAdditions}}
+3. ADD and MERGE the facts, details, and context from the into the page. If it contradicts old content, prefer the newer information.
+{{end}}
+4. Preserve existing information that is still valid.
+5. Keep [[slug|name]] wiki-link references ONLY if the slug appears in the list above. Remove any [[slug|name]] whose slug is NOT in that list. Do NOT invent new wiki-link slugs.
+6. Maintain the existing page structure and formatting style.
+7. **Image rule**: Include relevant images using Markdown syntax:  from new information if applicable.
+{{if .HasRetractions}}
+8. If after removing deleted content the page becomes nearly empty and there is no new information to add, output just: "SUMMARY: (empty page)\n# [Title]\n\n*This page's primary source document was removed.*"
+{{end}}
+9. Write in {{.Language}}.
Output the SUMMARY line first, then the updated Markdown content. Do not include any other preamble.`
diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go
index 95af880a..a42231ad 100644
--- a/internal/application/service/knowledge.go
+++ b/internal/application/service/knowledge.go
@@ -1327,8 +1327,10 @@ func (s *knowledgeService) cleanupWikiOnKnowledgeDelete(ctx context.Context, kbI
len(deletedSlugs), knowledgeID, deletedSlugs)
}
- // Enqueue async LLM retraction for multi-source pages
- if len(retractSlugs) > 0 {
+ allAffectedSlugs := append(retractSlugs, deletedSlugs...)
+
+ // Enqueue async LLM retraction and index rebuild
+ if len(allAffectedSlugs) > 0 {
lang, _ := types.LanguageFromContext(ctx)
tenantID, _ := types.TenantIDFromContext(ctx)
EnqueueWikiRetract(ctx, s.task, s.redisClient, WikiRetractPayload{
@@ -1338,16 +1340,9 @@ func (s *knowledgeService) cleanupWikiOnKnowledgeDelete(ctx context.Context, kbI
DocTitle: docTitle,
DocSummary: docSummary,
Language: lang,
- PageSlugs: retractSlugs,
+ PageSlugs: allAffectedSlugs,
})
- logger.Infof(ctx, "wiki cleanup: enqueued retract task for %d pages: %v", len(retractSlugs), retractSlugs)
- }
-
- // If pages were deleted but no retract task was enqueued,
- // we still need to update the index and log synchronously.
- // (When retractSlugs > 0, ProcessWikiRetract handles index/log.)
- if len(deletedSlugs) > 0 && len(retractSlugs) == 0 {
- s.rebuildWikiIndexSimple(ctx, kbID, knowledgeID, docTitle, deletedSlugs)
+ logger.Infof(ctx, "wiki cleanup: enqueued retract task for %d pages: %v", len(allAffectedSlugs), allAffectedSlugs)
}
}
@@ -1365,81 +1360,6 @@ func removeSourceRef(refs types.StringArray, knowledgeID string) types.StringArr
return result
}
-// rebuildWikiIndexSimple regenerates the index page from current page listing
-// and appends a deletion log entry. Does NOT use LLM — generates a simple
-// structured listing while preserving the LLM-generated intro from Summary field.
-func (s *knowledgeService) rebuildWikiIndexSimple(ctx context.Context, kbID, knowledgeID, docTitle string, deletedSlugs []string) {
- // Rebuild index
- allPages, err := s.wikiService.ListAllPages(ctx, kbID)
- if err != nil {
- logger.Warnf(ctx, "wiki cleanup: failed to list pages: %v", err)
- return
- }
-
- indexPage, _ := s.wikiService.GetIndex(ctx, kbID)
- if indexPage == nil {
- return
- }
-
- // Preserve LLM-generated intro from Summary field
- var indexContent strings.Builder
- if indexPage.Summary != "" && indexPage.Summary != "Index" {
- indexContent.WriteString(indexPage.Summary)
- indexContent.WriteString("\n")
- } else {
- indexContent.WriteString("# Wiki Index\n\n")
- }
-
- // Group pages by type
- typeOrder := []string{types.WikiPageTypeSummary, types.WikiPageTypeEntity, types.WikiPageTypeConcept, types.WikiPageTypeSynthesis, types.WikiPageTypeComparison}
- grouped := make(map[string][]*types.WikiPage)
- for _, p := range allPages {
- if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog {
- continue
- }
- if p.Status == types.WikiPageStatusArchived {
- continue
- }
- grouped[p.PageType] = append(grouped[p.PageType], p)
- }
-
- for _, pt := range typeOrder {
- pages := grouped[pt]
- if len(pages) == 0 {
- continue
- }
- label := pt
- if len(label) > 0 {
- label = strings.ToUpper(label[:1]) + label[1:]
- }
- fmt.Fprintf(&indexContent, "\n## %s\n\n", label)
- for _, p := range pages {
- fmt.Fprintf(&indexContent, "- [[%s]] — %s\n", p.Slug, p.Summary)
- }
- }
-
- indexPage.Content = indexContent.String()
- if _, err := s.wikiService.UpdatePage(ctx, indexPage); err != nil {
- logger.Warnf(ctx, "wiki cleanup: failed to rebuild index: %v", err)
- }
-
- // Append log entry
- logPage, _ := s.wikiService.GetLog(ctx, kbID)
- if logPage != nil {
- entry := fmt.Sprintf("\n## [%s] delete | %s\n- **Source**: knowledge/%s\n- **Pages deleted**: %d (%s)\n",
- time.Now().UTC().Format("2006-01-02 15:04:05"),
- docTitle,
- knowledgeID,
- len(deletedSlugs),
- strings.Join(deletedSlugs, ", "),
- )
- logPage.Content = logPage.Content + entry
- if _, err := s.wikiService.UpdatePage(ctx, logPage); err != nil {
- logger.Warnf(ctx, "wiki cleanup: failed to update log: %v", err)
- }
- }
-}
-
// DeleteKnowledgeList deletes a knowledge entry and all related resources
func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string) error {
if len(ids) == 0 {
diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go
index 907519f0..95132d73 100644
--- a/internal/application/service/wiki_ingest.go
+++ b/internal/application/service/wiki_ingest.go
@@ -14,7 +14,6 @@ import (
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
- "github.com/google/uuid"
"github.com/hibiken/asynq"
"github.com/redis/go-redis/v9"
)
@@ -223,421 +222,6 @@ func (s *wikiIngestService) Handle(ctx context.Context, t *asynq.Task) error {
return s.ProcessWikiIngest(ctx, t)
}
-// retractPagesContent uses LLM to remove a deleted/stale document's contributions
-// from the given wiki pages. docContent is the deleted document's content (or summary)
-// so the LLM can accurately identify which parts to remove. It does NOT rebuild the
-// index or append log entries — callers are responsible for those post-processing steps.
-func (s *wikiIngestService) retractPagesContent(
- ctx context.Context,
- chatModel chat.Chat,
- kbID, docTitle, docContent string,
- pageSlugs []string,
- lang string,
-) {
- allPages, _ := s.wikiService.ListAllPages(ctx, kbID)
- slugTitleMap := make(map[string]string)
- summaryContentByKnowledgeID := make(map[string]string)
- for _, p := range allPages {
- if p.PageType != types.WikiPageTypeIndex && p.PageType != types.WikiPageTypeLog && p.Status != types.WikiPageStatusArchived {
- slugTitleMap[p.Slug] = p.Title
- }
- if p.PageType == types.WikiPageTypeSummary && p.Content != "" {
- for _, ref := range p.SourceRefs {
- kid := ref
- if pipeIdx := strings.Index(ref, "|"); pipeIdx > 0 {
- kid = ref[:pipeIdx]
- }
- summaryContentByKnowledgeID[kid] = p.Content
- }
- }
- }
-
- for _, slug := range pageSlugs {
- page, err := s.wikiService.GetPageBySlug(ctx, kbID, slug)
- if err != nil || page == nil {
- continue
- }
-
- var remainingSourcesContent strings.Builder
- for _, ref := range page.SourceRefs {
- pipeIdx := strings.Index(ref, "|")
- var refKnowledgeID, refTitle string
- if pipeIdx > 0 {
- refKnowledgeID = ref[:pipeIdx]
- refTitle = ref[pipeIdx+1:]
- } else {
- refKnowledgeID = ref
- refTitle = ref
- }
- if content, ok := summaryContentByKnowledgeID[refKnowledgeID]; ok {
- fmt.Fprintf(&remainingSourcesContent, "\n%s\n\n\n", refTitle, content)
- } else {
- fmt.Fprintf(&remainingSourcesContent, "\n(summary not available)\n\n\n", refTitle)
- }
- }
- if remainingSourcesContent.Len() == 0 {
- remainingSourcesContent.WriteString("(no remaining sources)")
- }
-
- var relatedSlugs strings.Builder
- for _, outSlug := range page.OutLinks {
- if title, ok := slugTitleMap[outSlug]; ok {
- fmt.Fprintf(&relatedSlugs, "- %s (%s)\n", outSlug, title)
- }
- }
-
- updatedContent, err := s.generateWithTemplate(ctx, chatModel, agent.WikiPageRetractPrompt, map[string]string{
- "ExistingContent": page.Content,
- "DeletedDocTitle": docTitle,
- "DeletedDocContent": docContent,
- "RemainingSourcesContent": remainingSourcesContent.String(),
- "AvailableSlugs": relatedSlugs.String(),
- "Language": lang,
- })
- if err != nil {
- logger.Warnf(ctx, "wiki retract: LLM call failed for page %s: %v", slug, err)
- continue
- }
-
- retractedSummary, retractedBody := splitSummaryLine(updatedContent)
- if retractedBody != "" {
- page.Content = retractedBody
- } else {
- page.Content = updatedContent
- }
- if retractedSummary != "" {
- page.Summary = retractedSummary
- }
- if _, err := s.wikiService.UpdatePage(ctx, page); err != nil {
- logger.Warnf(ctx, "wiki retract: failed to update page %s: %v", slug, err)
- } else {
- logger.Infof(ctx, "wiki retract: updated page %s after removing content from '%s'", slug, docTitle)
- }
- }
-}
-
-// ProcessWikiIngest processes a batch wiki ingest task.
-//
-// Concurrency model (Redis mode):
-// 1. Try to set "wiki:active:{kbID}" flag via SetNX. If already set, another batch
-// is running → return nil (no-op). Documents are safe in the pending list.
-// 2. Atomically drain the pending list → process all documents sequentially.
-// 3. After processing, clear the active flag.
-// 4. Check if more documents arrived during processing. If so, enqueue a follow-up
-// task (no delay — docs are already waiting). This is safe because we just
-// cleared the active flag.
-//
-// This ensures: one batch per KB at a time, no locks, no blocking, no timeouts.
-func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task) error {
- taskStartedAt := time.Now()
- retryCount, _ := asynq.GetRetryCount(ctx)
- maxRetry, _ := asynq.GetMaxRetry(ctx)
-
- var payload WikiIngestPayload
- exitStatus := "success"
- mode := "redis"
- lockAcquired := false
- pendingOpsCount := 0
- ingestOps := 0
- retractOps := 0
- ingestSucceeded := 0
- ingestFailed := 0
- retractHandled := 0
- indexRebuildAttempted := false
- indexRebuildSucceeded := false
- followUpScheduled := false
- ingestPagesCount := 0
- retractPagesCount := 0
- totalPagesAffected := 0
- docPreview := make([]string, 0, 6)
- defer func() {
- logger.Infof(
- ctx,
- "wiki ingest stats: kb=%s tenant=%d retry=%d/%d status=%s elapsed=%s mode=%s lock_acquired=%v pending_ops=%d ops(ingest=%d,retract=%d) ingest(success=%d,failed=%d) retract_handled=%d pages(ingest=%d,retract=%d,total=%d) index(rebuild_attempted=%v,rebuild_succeeded=%v) followup=%v preview=%s",
- payload.KnowledgeBaseID,
- payload.TenantID,
- retryCount,
- maxRetry,
- exitStatus,
- time.Since(taskStartedAt).Round(time.Millisecond),
- mode,
- lockAcquired,
- pendingOpsCount,
- ingestOps,
- retractOps,
- ingestSucceeded,
- ingestFailed,
- retractHandled,
- ingestPagesCount,
- retractPagesCount,
- totalPagesAffected,
- indexRebuildAttempted,
- indexRebuildSucceeded,
- followUpScheduled,
- previewStringSlice(docPreview, 6),
- )
- }()
-
- if err := json.Unmarshal(t.Payload(), &payload); err != nil {
- exitStatus = "invalid_payload"
- return fmt.Errorf("wiki ingest: unmarshal payload: %w", err)
- }
-
- // Inject context
- ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
- if payload.Language != "" {
- ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language)
- }
-
- // Try to acquire the "active batch" flag (non-blocking)
- if s.redisClient != nil {
- activeKey := wikiActiveKeyPrefix + payload.KnowledgeBaseID
- // Use a 5-minute initial TTL
- acquired, err := s.redisClient.SetNX(ctx, activeKey, "1", 5*time.Minute).Result()
- if err != nil {
- logger.Warnf(ctx, "wiki ingest: redis SetNX failed: %v", err)
- // Proceed anyway — better to risk brief overlap than drop documents
- } else if !acquired {
- exitStatus = "active_lock_conflict"
- // Another batch is actively processing this KB.
- // By returning an error, we leverage Asynq's built-in exponential backoff to retry this task later.
- // This ensures that if the active task crashes (leaking the lock until its TTL expires),
- // this task will eventually retry and process the remaining items in the queue, preventing a stalled queue.
- logger.Infof(ctx, "wiki ingest: another batch active for KB %s, deferring to asynq retry", payload.KnowledgeBaseID)
- return fmt.Errorf("concurrent wiki task active, please retry")
- }
- lockAcquired = acquired
-
- // Create a context to cancel the keep-alive goroutine when we're done
- lockCtx, cancelLock := context.WithCancel(context.Background())
-
- // We own the flag — make sure to release it when done
- defer func() {
- cancelLock()
- // Use context.Background() to ensure release even if ctx is cancelled
- s.redisClient.Del(context.Background(), activeKey)
- }()
-
- // Keep-alive goroutine to extend lock TTL while the task is running
- go func() {
- ticker := time.NewTicker(2 * time.Minute)
- defer ticker.Stop()
- for {
- select {
- case <-lockCtx.Done():
- return
- case <-ticker.C:
- s.redisClient.Expire(context.Background(), activeKey, 5*time.Minute)
- }
- }
- }()
- } else {
- mode = "lite"
- }
-
- // Get KB and validate
- kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
- if err != nil {
- exitStatus = "get_kb_failed"
- return fmt.Errorf("wiki ingest: get KB: %w", err)
- }
- if !kb.IsWikiEnabled() {
- exitStatus = "kb_not_wiki_enabled"
- return fmt.Errorf("wiki ingest: KB %s is not wiki type", kb.ID)
- }
- if kb.WikiConfig == nil || !kb.WikiConfig.AutoIngest {
- exitStatus = "auto_ingest_disabled"
- logger.Infof(ctx, "wiki ingest: auto_ingest disabled for KB %s, skipping", kb.ID)
- return nil
- }
-
- // Get synthesis model
- synthesisModelID := kb.WikiConfig.SynthesisModelID
- if synthesisModelID == "" {
- synthesisModelID = kb.SummaryModelID
- }
- if synthesisModelID == "" {
- exitStatus = "missing_synthesis_model"
- return fmt.Errorf("wiki ingest: no synthesis model configured for KB %s", kb.ID)
- }
- chatModel, err := s.modelService.GetChatModel(ctx, synthesisModelID)
- if err != nil {
- exitStatus = "get_chat_model_failed"
- return fmt.Errorf("wiki ingest: get chat model: %w", err)
- }
-
- lang := types.LanguageNameFromContext(ctx)
-
- // Peek Redis pending list to get all operations queued for this KB without removing them
- pendingOps, peekedCount := s.peekPendingList(ctx, payload.KnowledgeBaseID)
- pendingOpsCount = len(pendingOps)
- if len(pendingOps) == 0 {
- if s.redisClient != nil {
- // Redis mode: list was already drained — nothing to do
- exitStatus = "no_pending_ops"
- logger.Infof(ctx, "wiki ingest: no pending operations for KB %s", payload.KnowledgeBaseID)
- return nil
- }
- // Lite mode (no Redis): use LiteOps from payload
- if len(payload.LiteOps) > 0 {
- pendingOps = payload.LiteOps
- peekedCount = len(pendingOps)
- pendingOpsCount = len(pendingOps)
- } else {
- exitStatus = "no_lite_ops"
- return nil
- }
- }
-
- logger.Infof(ctx, "wiki ingest: batch processing %d ops for KB %s",
- len(pendingOps), payload.KnowledgeBaseID)
-
- // Process each operation
- var ingestPagesAffected []string
- var retractPagesAffected []string
- var docResults []*docIngestResult
- var retractChangeDesc strings.Builder
-
- for _, op := range pendingOps {
- if op.Op == WikiOpRetract {
- retractOps++
- logger.Infof(ctx, "wiki ingest: retracting document '%s' (%s)", op.DocTitle, op.KnowledgeID)
- s.retractPagesContent(ctx, chatModel, payload.KnowledgeBaseID, op.DocTitle, op.DocSummary, op.PageSlugs, op.Language)
- retractPagesAffected = append(retractPagesAffected, op.PageSlugs...)
- retractPagesCount += len(op.PageSlugs)
- retractHandled++
- docPreview = append(docPreview,
- fmt.Sprintf("retract[%s]: %s", previewText(op.KnowledgeID, 24), previewText(op.DocTitle, 48)))
- fmt.Fprintf(&retractChangeDesc, "Removed document '%s': %s\n", op.DocTitle, op.DocSummary)
- s.appendLogEntry(ctx, WikiIngestPayload{
- TenantID: payload.TenantID,
- KnowledgeBaseID: payload.KnowledgeBaseID,
- }, "retract", op.KnowledgeID, op.DocTitle, op.PageSlugs, "")
- } else {
- ingestOps++
- // Default to ingest
- logger.Infof(ctx, "wiki ingest: processing document '%s' (%s)", op.DocTitle, op.KnowledgeID)
- result, err := s.processOneDocument(ctx, chatModel, payload, op.KnowledgeID, op.Language)
- if err != nil {
- ingestFailed++
- logger.Warnf(ctx, "wiki ingest: failed to process knowledge %s: %v", op.KnowledgeID, err)
- continue
- }
- if result != nil {
- ingestSucceeded++
- ingestPagesAffected = append(ingestPagesAffected, result.Pages...)
- ingestPagesCount += len(result.Pages)
- docResults = append(docResults, result)
- docPreview = append(docPreview,
- fmt.Sprintf(
- "ingest[%s]: title=%s summary=%s pages=%s",
- previewText(result.KnowledgeID, 24),
- previewText(result.DocTitle, 40),
- previewText(result.Summary, 64),
- previewStringSlice(result.Pages, 4),
- ))
- }
- }
- }
-
- allPagesAffected := append(ingestPagesAffected, retractPagesAffected...)
- totalPagesAffected = len(allPagesAffected)
-
- // Batch post-processing (once for the whole batch, not per-doc)
-
- // Build change description from processed documents
- var changeDesc strings.Builder
- if len(docResults) > 0 {
- fmt.Fprintf(&changeDesc, "Added %d documents:\n", len(docResults))
- for _, r := range docResults {
- fmt.Fprintf(&changeDesc, "- %s: %s\n", r.DocTitle, r.Summary)
- }
- }
- if retractChangeDesc.Len() > 0 {
- changeDesc.WriteString(retractChangeDesc.String())
- }
-
- // Rebuild index page
- if changeDesc.Len() > 0 {
- indexRebuildAttempted = true
- logger.Infof(ctx, "wiki ingest: rebuilding index page")
- if err := s.rebuildIndexPage(ctx, chatModel, payload, changeDesc.String(), lang); err != nil {
- logger.Warnf(ctx, "wiki ingest: rebuild index failed: %v", err)
- docPreview = append(docPreview,
- fmt.Sprintf("index_change=%s", previewText(changeDesc.String(), 160)))
- } else {
- indexRebuildSucceeded = true
- docPreview = append(docPreview,
- fmt.Sprintf("index_change=%s", previewText(changeDesc.String(), 160)))
- }
- }
-
- // Append log entry for ingests
- if len(docResults) > 0 {
- s.appendLogEntry(ctx, payload, "ingest", "",
- fmt.Sprintf("%d documents", len(docResults)),
- ingestPagesAffected, "")
- }
-
- // Clean dead links (needed after retracts)
- if len(retractPagesAffected) > 0 {
- logger.Infof(ctx, "wiki ingest: cleaning dead links")
- s.cleanDeadLinks(ctx, payload.KnowledgeBaseID)
- }
-
- if len(allPagesAffected) > 0 {
- // Cross-link injection
- logger.Infof(ctx, "wiki ingest: injecting cross links")
- s.injectCrossLinks(ctx, payload.KnowledgeBaseID, allPagesAffected)
-
- // Publish all draft pages
- logger.Infof(ctx, "wiki ingest: publishing draft pages")
- s.publishDraftPages(ctx, payload.KnowledgeBaseID, allPagesAffected)
- }
-
- // Trim the pending list now that processing is complete
- s.trimPendingList(ctx, payload.KnowledgeBaseID, peekedCount)
-
- logger.Infof(ctx, "wiki ingest: batch completed for KB %s, %d ops, %d pages affected",
- payload.KnowledgeBaseID, len(pendingOps), len(allPagesAffected))
-
- // After clearing active flag (via defer above), check for follow-up work.
- // Note: this runs BEFORE defer, but defer runs LIFO so active flag is still set here.
- // We need to clear it first, then check. Use a closure:
- followUpScheduled = s.scheduleFollowUp(ctx, payload)
-
- return nil
-}
-
-// scheduleFollowUp checks if documents arrived in the pending list during batch processing.
-// Called right before the active flag is released (via defer). Enqueues a new task with
-// minimal delay so the next batch picks up new docs promptly.
-func (s *wikiIngestService) scheduleFollowUp(ctx context.Context, payload WikiIngestPayload) bool {
- if s.redisClient == nil {
- return false
- }
- pendingKey := wikiPendingKeyPrefix + payload.KnowledgeBaseID
- count, err := s.redisClient.LLen(ctx, pendingKey).Result()
- if err != nil || count == 0 {
- return false
- }
-
- logger.Infof(ctx, "wiki ingest: %d more documents pending for KB %s, scheduling follow-up", count, payload.KnowledgeBaseID)
-
- payloadBytes, _ := json.Marshal(payload)
- t := asynq.NewTask(types.TypeWikiIngest, payloadBytes,
- asynq.Queue("low"),
- asynq.MaxRetry(10), // Increased from 3 to 10 to outlast the active lock TTL
- asynq.Timeout(60*time.Minute),
- asynq.ProcessIn(5*time.Second), // short delay — active flag will be released by then
- )
- if _, err := s.task.Enqueue(t); err != nil {
- logger.Warnf(ctx, "wiki ingest: follow-up enqueue failed: %v", err)
- return false
- }
- return true
-}
-
// peekPendingList gets up to wikiMaxDocsPerBatch entries from the Redis pending list
// WITHOUT removing them. It returns the unique ops and the actual number of items peeked.
func (s *wikiIngestService) peekPendingList(ctx context.Context, kbID string) ([]WikiPendingOp, int) {
@@ -708,161 +292,24 @@ type docIngestResult struct {
Pages []string // affected page slugs
}
-// processOneDocument handles wiki ingest for a single document.
-func (s *wikiIngestService) processOneDocument(
- ctx context.Context,
- chatModel chat.Chat,
- payload WikiIngestPayload,
- knowledgeID string,
- lang string,
-) (*docIngestResult, error) {
- docStartedAt := time.Now()
- // Get document chunks and reconstruct content
- chunks, err := s.chunkRepo.ListChunksByKnowledgeID(ctx, payload.TenantID, knowledgeID)
- if err != nil {
- return nil, fmt.Errorf("get chunks: %w", err)
- }
- if len(chunks) == 0 {
- logger.Infof(ctx, "wiki ingest: document %s has no chunks, skip", knowledgeID)
- return nil, nil
- }
+// WikiBatchContext holds data needed during the Reduce phase
+type WikiBatchContext struct {
+ SlugTitleMap map[string]string
+ SummaryContentByKnowledgeID map[string]string
+}
- content := reconstructContent(chunks)
- rawRuneCount := len([]rune(content))
- if len([]rune(content)) > maxContentForWiki {
- content = string([]rune(content)[:maxContentForWiki])
- }
- logger.Infof(ctx,
- "wiki ingest: doc %s chunks=%d content_len(raw=%d,truncated=%d) content_preview=%q",
- knowledgeID, len(chunks), rawRuneCount, len([]rune(content)), previewText(content, 120))
-
- // Get document title
- docTitle := knowledgeID
- if kn, err := s.knowledgeSvc.GetKnowledgeByIDOnly(ctx, knowledgeID); err == nil && kn != nil && kn.Title != "" {
- docTitle = kn.Title
- } else {
- for _, ch := range chunks {
- if ch.Content != "" {
- lines := strings.SplitN(ch.Content, "\n", 2)
- if len(lines) > 0 && len(lines[0]) > 0 && len(lines[0]) < 200 {
- docTitle = strings.TrimPrefix(strings.TrimSpace(lines[0]), "# ")
- break
- }
- }
- }
- }
-
- var pagesAffected []string
- var docSummaryLine string
- sourceRef := fmt.Sprintf("%s|%s", knowledgeID, docTitle)
-
- // Snapshot existing page slugs for stale detection
- oldPageSlugs := s.getExistingPageSlugsForKnowledge(ctx, payload.KnowledgeBaseID, knowledgeID)
-
- // Build a per-doc payload for functions that still need KnowledgeID
- // (docPayload removed as WikiIngestPayload no longer holds KnowledgeID)
-
- // Step 1: Extract entities and concepts
- logger.Infof(ctx, "wiki ingest: extracting entities and concepts for %s", knowledgeID)
- extractedPages, slugItems, err := s.extractEntitiesAndConcepts(ctx, chatModel, content, docTitle, lang, payload, sourceRef, oldPageSlugs)
- if err != nil {
- logger.Warnf(ctx, "wiki ingest: knowledge extraction failed for %s: %v", knowledgeID, err)
- } else {
- pagesAffected = append(pagesAffected, extractedPages...)
- }
-
- // Step 2: Generate summary page
- logger.Infof(ctx, "wiki ingest: generating summary page for %s", knowledgeID)
- summarySlug := fmt.Sprintf("summary/%s", slugify(docTitle))
- var slugListing string
- for _, slug := range extractedPages {
- if item, ok := slugItems[slug]; ok {
- aliases := ""
- if len(item.Aliases) > 0 {
- aliases = fmt.Sprintf(" (Aliases: %s)", strings.Join(item.Aliases, ", "))
- }
- slugListing += fmt.Sprintf("- [[%s]] = %s%s\n", slug, item.Name, aliases)
- } else {
- slugListing += fmt.Sprintf("- [[%s]]\n", slug)
- }
- }
- summaryContent, err := s.generateWithTemplate(ctx, chatModel, agent.WikiSummaryPrompt, map[string]string{
- "Title": docTitle,
- "FileName": docTitle,
- "FileType": "document",
- "Content": content,
- "Language": lang,
- "ExtractedSlugs": slugListing,
- })
- if err != nil {
- logger.Errorf(ctx, "wiki ingest: generate summary failed for %s: %v", knowledgeID, err)
- } else {
- sumLine, sumBody := splitSummaryLine(summaryContent)
- if sumBody == "" {
- sumBody = summaryContent
- }
- if sumLine == "" {
- sumLine = docTitle
- }
- docSummaryLine = sumLine
- logger.Infof(ctx, "wiki ingest: summary preview for %s => line=%q body=%q",
- knowledgeID, previewText(sumLine, 100), previewText(sumBody, 140))
-
- existingSummary, err := s.wikiService.GetPageBySlug(ctx, payload.KnowledgeBaseID, summarySlug)
- if err == nil && existingSummary != nil {
- // Update existing summary page (idempotent for retries)
- existingSummary.Title = docTitle + " - Summary"
- existingSummary.Content = sumBody
- existingSummary.Summary = sumLine
- existingSummary.Status = types.WikiPageStatusDraft
- existingSummary.SourceRefs = appendUnique(existingSummary.SourceRefs, sourceRef)
-
- if _, err := s.wikiService.UpdatePage(ctx, existingSummary); err != nil {
- logger.Warnf(ctx, "wiki ingest: update summary page failed for %s: %v", knowledgeID, err)
- } else {
- pagesAffected = append(pagesAffected, summarySlug)
- }
- } else {
- // Create new summary page
- _, err = s.wikiService.CreatePage(ctx, &types.WikiPage{
- ID: uuid.New().String(),
- TenantID: payload.TenantID,
- KnowledgeBaseID: payload.KnowledgeBaseID,
- Slug: summarySlug,
- Title: docTitle + " - Summary",
- PageType: types.WikiPageTypeSummary,
- Status: types.WikiPageStatusDraft,
- Content: sumBody,
- Summary: sumLine,
- SourceRefs: types.StringArray{sourceRef},
- })
- if err != nil {
- logger.Warnf(ctx, "wiki ingest: create summary page failed for %s: %v", knowledgeID, err)
- } else {
- pagesAffected = append(pagesAffected, summarySlug)
- }
- }
- }
-
- // Retract stale pages (pages this doc previously contributed to but no longer does)
- logger.Infof(ctx, "wiki ingest: retracting stale pages for %s", knowledgeID)
- s.retractStalePages(ctx, chatModel, payload, knowledgeID, oldPageSlugs, pagesAffected, docTitle, content, lang)
-
- logger.Infof(ctx,
- "wiki ingest: processed knowledge %s title=%q affected_pages=%d page_preview=%s extracted_pages=%s elapsed=%s",
- knowledgeID,
- previewText(docTitle, 80),
- len(pagesAffected),
- previewStringSlice(pagesAffected, 6),
- previewStringSlice(extractedPages, 6),
- time.Since(docStartedAt).Round(time.Millisecond),
- )
- return &docIngestResult{
- KnowledgeID: knowledgeID,
- DocTitle: docTitle,
- Summary: docSummaryLine,
- Pages: pagesAffected,
- }, nil
+// SlugUpdate represents a single update operation for a specific slug
+type SlugUpdate struct {
+ Slug string
+ Type string // "entity", "concept", "summary", "retract", "retractStale"
+ Item extractedItem // For entity/concept
+ DocTitle string
+ KnowledgeID string
+ SourceRef string
+ Language string
+ SummaryBody string // For summary
+ SummaryLine string // For summary
+ RetractDocContent string // For retract / retractStale
}
func previewText(s string, maxRunes int) string {
@@ -1112,77 +559,16 @@ func (s *wikiIngestService) getExistingPageSlugsForKnowledge(ctx context.Context
// but are no longer produced by the updated extraction.
// - Single-source stale pages → deleted
// - Multi-source stale pages → LLM retract to clean content synchronously
-func (s *wikiIngestService) retractStalePages(
- ctx context.Context,
- chatModel chat.Chat,
- payload WikiIngestPayload,
- knowledgeID string,
- oldSlugs map[string]bool,
- newSlugs []string,
- docTitle, docContent, lang string,
-) {
- if len(oldSlugs) == 0 {
- return
- }
- // Build set of newly affected slugs (including summary)
- newSet := make(map[string]bool, len(newSlugs))
- for _, s := range newSlugs {
- newSet[s] = true
- }
+// Build set of newly affected slugs (including summary)
- // Stale = was in old set but not in new set
- var staleSlugs []string
- for slug := range oldSlugs {
- if !newSet[slug] {
- staleSlugs = append(staleSlugs, slug)
- }
- }
- if len(staleSlugs) == 0 {
- return
- }
+// Stale = was in old set but not in new set
- logger.Infof(ctx, "wiki ingest: %d stale pages detected after document update: %v", len(staleSlugs), staleSlugs)
+// Remove this doc's source ref
- var retractSlugs []string
- sourceRef := fmt.Sprintf("%s|%s", knowledgeID, docTitle)
- prefix := knowledgeID + "|"
+// No other sources → delete the page
- for _, slug := range staleSlugs {
- page, err := s.wikiService.GetPageBySlug(ctx, payload.KnowledgeBaseID, slug)
- if err != nil || page == nil {
- continue
- }
-
- // Remove this doc's source ref
- var remaining types.StringArray
- for _, ref := range page.SourceRefs {
- if ref == knowledgeID || ref == sourceRef || strings.HasPrefix(ref, prefix) {
- continue
- }
- remaining = append(remaining, ref)
- }
-
- if len(remaining) == 0 {
- // No other sources → delete the page
- if err := s.wikiService.DeletePage(ctx, payload.KnowledgeBaseID, slug); err != nil {
- logger.Warnf(ctx, "wiki ingest: failed to delete stale page %s: %v", slug, err)
- }
- } else {
- // Multi-source → remove ref, queue retract
- page.SourceRefs = remaining
- if err := s.wikiService.UpdatePageMeta(ctx, page); err != nil {
- logger.Warnf(ctx, "wiki ingest: failed to update stale page %s: %v", slug, err)
- } else {
- retractSlugs = append(retractSlugs, slug)
- }
- }
- }
-
- if len(retractSlugs) > 0 {
- s.retractPagesContent(ctx, chatModel, payload.KnowledgeBaseID, docTitle, docContent, retractSlugs, lang)
- }
-}
+// Multi-source → remove ref, queue retract
// extractedItem represents a single extracted entity or concept
type extractedItem struct {
@@ -1199,189 +585,6 @@ type combinedExtraction struct {
Concepts []extractedItem `json:"concepts"`
}
-// extractEntitiesAndConcepts performs a single LLM call to extract both entities and concepts,
-// then upserts pages for each. Returns the list of successfully upserted page slugs and
-// a slug→display name map for building wiki-link references.
-// oldPageSlugs contains slugs from the previous version of this document — passed to LLM for slug stability.
-func (s *wikiIngestService) extractEntitiesAndConcepts(
- ctx context.Context,
- chatModel chat.Chat,
- content, docTitle, lang string,
- payload WikiIngestPayload,
- sourceRef string,
- oldPageSlugs map[string]bool,
-) ([]string, map[string]extractedItem, error) {
- // Build previous slugs listing for the prompt
- var prevSlugsText string
- if len(oldPageSlugs) > 0 {
- var sb strings.Builder
- for slug := range oldPageSlugs {
- fmt.Fprintf(&sb, "- %s\n", slug)
- }
- prevSlugsText = sb.String()
- } else {
- prevSlugsText = "(none — this is a new document)"
- }
-
- // Single LLM call for both entities and concepts
- extractionJSON, err := s.generateWithTemplate(ctx, chatModel, agent.WikiKnowledgeExtractPrompt, map[string]string{
- "Title": docTitle,
- "Content": content,
- "Language": lang,
- "PreviousSlugs": prevSlugsText,
- })
- if err != nil {
- return nil, nil, fmt.Errorf("combined extraction failed: %w", err)
- }
-
- // Clean JSON - strip markdown code blocks if present
- extractionJSON = strings.TrimSpace(extractionJSON)
- extractionJSON = strings.TrimPrefix(extractionJSON, "```json")
- extractionJSON = strings.TrimPrefix(extractionJSON, "```")
- extractionJSON = strings.TrimSuffix(extractionJSON, "```")
- extractionJSON = strings.TrimSpace(extractionJSON)
-
- extractionJSON = sanitizeJSONString(extractionJSON)
-
- var result combinedExtraction
- if err := json.Unmarshal([]byte(extractionJSON), &result); err != nil {
- logger.Warnf(ctx, "wiki ingest: failed to parse combined extraction JSON: %v\nRaw: %s", err, extractionJSON)
- return nil, nil, fmt.Errorf("parse combined extraction JSON: %w", err)
- }
-
- var affected []string
-
- // Deduplicate entities against existing wiki pages (LLM-based)
- logger.Infof(ctx, "wiki ingest: deduplicating %d entities", len(result.Entities))
- result.Entities = s.deduplicateItems(ctx, chatModel, result.Entities, types.WikiPageTypeEntity, payload.KnowledgeBaseID)
-
- // Deduplicate concepts against existing wiki pages (LLM-based)
- logger.Infof(ctx, "wiki ingest: deduplicating %d concepts", len(result.Concepts))
- result.Concepts = s.deduplicateItems(ctx, chatModel, result.Concepts, types.WikiPageTypeConcept, payload.KnowledgeBaseID)
-
- // Build slug→item map for wiki-link generation in summary pages
- slugItems := make(map[string]extractedItem)
- for _, item := range result.Entities {
- if item.Slug != "" && item.Name != "" {
- slugItems[item.Slug] = item
- }
- }
- for _, item := range result.Concepts {
- if item.Slug != "" && item.Name != "" {
- slugItems[item.Slug] = item
- }
- }
-
- // Upsert entity pages
- logger.Infof(ctx, "wiki ingest: upserting %d entity pages", len(result.Entities))
- entitySlugs, err := s.upsertExtractedPages(ctx, chatModel, result.Entities, types.WikiPageTypeEntity, docTitle, lang, payload, sourceRef)
- if err != nil {
- logger.Warnf(ctx, "wiki ingest: entity upsert failed: %v", err)
- } else {
- affected = append(affected, entitySlugs...)
- }
-
- // Upsert concept pages
- logger.Infof(ctx, "wiki ingest: upserting %d concept pages", len(result.Concepts))
- conceptSlugs, err := s.upsertExtractedPages(ctx, chatModel, result.Concepts, types.WikiPageTypeConcept, docTitle, lang, payload, sourceRef)
- if err != nil {
- logger.Warnf(ctx, "wiki ingest: concept upsert failed: %v", err)
- } else {
- affected = append(affected, conceptSlugs...)
- }
-
- return affected, slugItems, nil
-}
-
-// upsertExtractedPages creates or updates wiki pages from pre-extracted items.
-func (s *wikiIngestService) upsertExtractedPages(
- ctx context.Context,
- chatModel chat.Chat,
- items []extractedItem,
- pageType string,
- docTitle, lang string,
- payload WikiIngestPayload,
- sourceRef string,
-) ([]string, error) {
- var affected []string
- for _, item := range items {
- if item.Slug == "" || item.Name == "" {
- continue
- }
-
- // Check if page already exists
- existing, err := s.wikiService.GetPageBySlug(ctx, payload.KnowledgeBaseID, item.Slug)
- if err == nil && existing != nil {
- // Page exists → incremental update
- updatedContent, err := s.generateWithTemplate(ctx, chatModel, agent.WikiPageUpdatePrompt, map[string]string{
- "ExistingContent": existing.Content,
- "NewDocTitle": docTitle,
- "NewContent": fmt.Sprintf("**%s**: %s\n\n%s", item.Name, item.Description, item.Details),
- "Language": lang,
- })
- if err != nil {
- logger.Warnf(ctx, "wiki ingest: update page %s failed: %v", item.Slug, err)
- continue
- }
-
- // LLM returns "SUMMARY: ..." on first line — use it as the authoritative summary
- updatedSummary, updatedBody := splitSummaryLine(updatedContent)
- if updatedBody != "" {
- existing.Content = updatedBody
- } else {
- existing.Content = updatedContent
- }
- if updatedSummary != "" {
- existing.Summary = updatedSummary
- }
- if len(item.Aliases) > 0 {
- // Merge new aliases with existing ones, deduplicating
- aliasMap := make(map[string]bool)
- for _, alias := range existing.Aliases {
- aliasMap[alias] = true
- }
- for _, newAlias := range item.Aliases {
- if !aliasMap[newAlias] {
- existing.Aliases = append(existing.Aliases, newAlias)
- aliasMap[newAlias] = true
- }
- }
- }
- existing.SourceRefs = appendUnique(existing.SourceRefs, sourceRef)
-
- if _, err := s.wikiService.UpdatePage(ctx, existing); err != nil {
- logger.Warnf(ctx, "wiki ingest: save updated page %s failed: %v", item.Slug, err)
- continue
- }
- affected = append(affected, item.Slug)
- } else {
- // New page
- pageContent := fmt.Sprintf("# %s\n\n%s\n\n%s\n\n---\n*Source: %s*\n",
- item.Name, item.Description, item.Details, docTitle)
-
- if _, err := s.wikiService.CreatePage(ctx, &types.WikiPage{
- ID: uuid.New().String(),
- TenantID: payload.TenantID,
- KnowledgeBaseID: payload.KnowledgeBaseID,
- Slug: item.Slug,
- Title: item.Name,
- Aliases: item.Aliases,
- PageType: pageType,
- Status: types.WikiPageStatusDraft,
- Content: pageContent,
- Summary: item.Description,
- SourceRefs: types.StringArray{sourceRef},
- }); err != nil {
- logger.Warnf(ctx, "wiki ingest: create page %s failed: %v", item.Slug, err)
- continue
- }
- affected = append(affected, item.Slug)
- }
- }
-
- return affected, nil
-}
-
// rebuildIndexPage regenerates the index page.
//
// Strategy: Index = LLM-generated intro (stored in Summary field) + code-generated directory.
@@ -1426,7 +629,7 @@ func (s *wikiIngestService) rebuildIndexPage(ctx context.Context, chatModel chat
// Build document summaries listing (only summary-type pages — they represent documents)
var docSummaries strings.Builder
for _, p := range grouped[types.WikiPageTypeSummary] {
- fmt.Fprintf(&docSummaries, "- %s: %s\n", p.Title, p.Summary)
+ fmt.Fprintf(&docSummaries, "\n%s\n%s\n\n\n", p.Title, p.Summary)
}
if docSummaries.Len() == 0 {
docSummaries.WriteString("(no documents yet)")
diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go
new file mode 100644
index 00000000..ccdae79c
--- /dev/null
+++ b/internal/application/service/wiki_ingest_batch.go
@@ -0,0 +1,803 @@
+package service
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/agent"
+ "github.com/Tencent/WeKnora/internal/logger"
+ "github.com/Tencent/WeKnora/internal/models/chat"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/google/uuid"
+ "github.com/hibiken/asynq"
+ "golang.org/x/sync/errgroup"
+)
+
+func (s *wikiIngestService) scheduleFollowUp(ctx context.Context, payload WikiIngestPayload) bool {
+ if s.redisClient == nil {
+ return false
+ }
+ pendingKey := wikiPendingKeyPrefix + payload.KnowledgeBaseID
+ count, err := s.redisClient.LLen(ctx, pendingKey).Result()
+ if err != nil || count == 0 {
+ return false
+ }
+
+ logger.Infof(ctx, "wiki ingest: %d more documents pending for KB %s, scheduling follow-up", count, payload.KnowledgeBaseID)
+
+ payloadBytes, _ := json.Marshal(payload)
+ t := asynq.NewTask(types.TypeWikiIngest, payloadBytes,
+ asynq.Queue("low"),
+ asynq.MaxRetry(10), // Increased from 3 to 10 to outlast the active lock TTL
+ asynq.Timeout(60*time.Minute),
+ asynq.ProcessIn(5*time.Second), // short delay — active flag will be released by then
+ )
+ if _, err := s.task.Enqueue(t); err != nil {
+ logger.Warnf(ctx, "wiki ingest: follow-up enqueue failed: %v", err)
+ return false
+ }
+ return true
+}
+
+func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task) error {
+ taskStartedAt := time.Now()
+ retryCount, _ := asynq.GetRetryCount(ctx)
+ maxRetry, _ := asynq.GetMaxRetry(ctx)
+
+ var payload WikiIngestPayload
+ exitStatus := "success"
+ mode := "redis"
+ lockAcquired := false
+ pendingOpsCount := 0
+ ingestOps := 0
+ retractOps := 0
+ ingestSucceeded := 0
+ ingestFailed := 0
+ retractHandled := 0
+ indexRebuildAttempted := false
+ indexRebuildSucceeded := false
+ followUpScheduled := false
+ totalPagesAffected := 0
+ docPreview := make([]string, 0, 6)
+
+ defer func() {
+ logger.Infof(
+ ctx,
+ "wiki ingest stats: kb=%s tenant=%d retry=%d/%d status=%s elapsed=%s mode=%s lock_acquired=%v pending_ops=%d ops(ingest=%d,retract=%d) ingest(success=%d,failed=%d) retract_handled=%d pages(total=%d) index(rebuild_attempted=%v,rebuild_succeeded=%v) followup=%v preview=%s",
+ payload.KnowledgeBaseID,
+ payload.TenantID,
+ retryCount,
+ maxRetry,
+ exitStatus,
+ time.Since(taskStartedAt).Round(time.Millisecond),
+ mode,
+ lockAcquired,
+ pendingOpsCount,
+ ingestOps,
+ retractOps,
+ ingestSucceeded,
+ ingestFailed,
+ retractHandled,
+ totalPagesAffected,
+ indexRebuildAttempted,
+ indexRebuildSucceeded,
+ followUpScheduled,
+ previewStringSlice(docPreview, 6),
+ )
+ }()
+
+ if err := json.Unmarshal(t.Payload(), &payload); err != nil {
+ exitStatus = "invalid_payload"
+ return fmt.Errorf("wiki ingest: unmarshal payload: %w", err)
+ }
+
+ // Inject context
+ ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
+ if payload.Language != "" {
+ ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language)
+ }
+
+ // Try to acquire the "active batch" flag (non-blocking)
+ if s.redisClient != nil {
+ activeKey := wikiActiveKeyPrefix + payload.KnowledgeBaseID
+ acquired, err := s.redisClient.SetNX(ctx, activeKey, "1", 5*time.Minute).Result()
+ if err != nil {
+ logger.Warnf(ctx, "wiki ingest: redis SetNX failed: %v", err)
+ } else if !acquired {
+ exitStatus = "active_lock_conflict"
+ logger.Infof(ctx, "wiki ingest: another batch active for KB %s, deferring to asynq retry", payload.KnowledgeBaseID)
+ return fmt.Errorf("concurrent wiki task active, please retry")
+ }
+ lockAcquired = acquired
+
+ lockCtx, cancelLock := context.WithCancel(context.Background())
+ defer func() {
+ cancelLock()
+ s.redisClient.Del(context.Background(), activeKey)
+ }()
+
+ go func() {
+ ticker := time.NewTicker(2 * time.Minute)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-lockCtx.Done():
+ return
+ case <-ticker.C:
+ s.redisClient.Expire(context.Background(), activeKey, 5*time.Minute)
+ }
+ }
+ }()
+ } else {
+ mode = "lite"
+ }
+
+ kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
+ if err != nil {
+ exitStatus = "get_kb_failed"
+ return fmt.Errorf("wiki ingest: get KB: %w", err)
+ }
+ if !kb.IsWikiEnabled() {
+ exitStatus = "kb_not_wiki_enabled"
+ return fmt.Errorf("wiki ingest: KB %s is not wiki type", kb.ID)
+ }
+ if kb.WikiConfig == nil || !kb.WikiConfig.AutoIngest {
+ exitStatus = "auto_ingest_disabled"
+ logger.Infof(ctx, "wiki ingest: auto_ingest disabled for KB %s, skipping", kb.ID)
+ return nil
+ }
+
+ synthesisModelID := kb.WikiConfig.SynthesisModelID
+ if synthesisModelID == "" {
+ synthesisModelID = kb.SummaryModelID
+ }
+ if synthesisModelID == "" {
+ exitStatus = "missing_synthesis_model"
+ return fmt.Errorf("wiki ingest: no synthesis model configured for KB %s", kb.ID)
+ }
+ chatModel, err := s.modelService.GetChatModel(ctx, synthesisModelID)
+ if err != nil {
+ exitStatus = "get_chat_model_failed"
+ return fmt.Errorf("wiki ingest: get chat model: %w", err)
+ }
+
+ lang := types.LanguageNameFromContext(ctx)
+
+ pendingOps, peekedCount := s.peekPendingList(ctx, payload.KnowledgeBaseID)
+ pendingOpsCount = len(pendingOps)
+ if len(pendingOps) == 0 {
+ if s.redisClient != nil {
+ exitStatus = "no_pending_ops"
+ logger.Infof(ctx, "wiki ingest: no pending operations for KB %s", payload.KnowledgeBaseID)
+ return nil
+ }
+ if len(payload.LiteOps) > 0 {
+ pendingOps = payload.LiteOps
+ peekedCount = len(pendingOps)
+ pendingOpsCount = len(pendingOps)
+ } else {
+ exitStatus = "no_lite_ops"
+ return nil
+ }
+ }
+
+ logger.Infof(ctx, "wiki ingest: batch processing %d ops for KB %s", len(pendingOps), payload.KnowledgeBaseID)
+
+ // Fetch all existing pages to pass to the Map-Reduce phases
+ allPages, _ := s.wikiService.ListAllPages(ctx, payload.KnowledgeBaseID)
+ batchCtx := &WikiBatchContext{
+ SlugTitleMap: make(map[string]string),
+ SummaryContentByKnowledgeID: make(map[string]string),
+ }
+ for _, p := range allPages {
+ if p.PageType != types.WikiPageTypeIndex && p.PageType != types.WikiPageTypeLog && p.Status != types.WikiPageStatusArchived {
+ batchCtx.SlugTitleMap[p.Slug] = p.Title
+ }
+ if p.PageType == types.WikiPageTypeSummary && p.Content != "" {
+ for _, ref := range p.SourceRefs {
+ kid := ref
+ if pipeIdx := strings.Index(ref, "|"); pipeIdx > 0 {
+ kid = ref[:pipeIdx]
+ }
+ batchCtx.SummaryContentByKnowledgeID[kid] = p.Content
+ }
+ }
+ }
+
+ // 1. MAP PHASE (Parallel extraction and generation of updates)
+ var mapMu sync.Mutex
+ slugUpdates := make(map[string][]SlugUpdate)
+ var docResults []*docIngestResult
+ var retractChangeDesc strings.Builder
+
+ eg, mapCtx := errgroup.WithContext(ctx)
+ eg.SetLimit(10) // Map phase limit
+
+ for _, op := range pendingOps {
+ op := op
+ eg.Go(func() error {
+ if op.Op == WikiOpRetract {
+ mapMu.Lock()
+ retractOps++
+ retractHandled++
+ docPreview = append(docPreview, fmt.Sprintf("retract[%s]: %s", previewText(op.KnowledgeID, 24), previewText(op.DocTitle, 48)))
+ fmt.Fprintf(&retractChangeDesc, "\n%s\n%s\n\n\n", op.DocTitle, op.DocSummary)
+
+ for _, slug := range op.PageSlugs {
+ slugUpdates[slug] = append(slugUpdates[slug], SlugUpdate{
+ Slug: slug,
+ Type: "retract",
+ RetractDocContent: op.DocSummary,
+ DocTitle: op.DocTitle,
+ KnowledgeID: op.KnowledgeID,
+ Language: op.Language,
+ })
+ }
+ mapMu.Unlock()
+ return nil
+ }
+
+ // Ingest
+ mapMu.Lock()
+ ingestOps++
+ mapMu.Unlock()
+
+ logger.Infof(mapCtx, "wiki ingest: processing document '%s' (%s)", op.DocTitle, op.KnowledgeID)
+ result, updates, err := s.mapOneDocument(mapCtx, chatModel, payload, op, batchCtx)
+ if err != nil {
+ mapMu.Lock()
+ ingestFailed++
+ mapMu.Unlock()
+ logger.Warnf(mapCtx, "wiki ingest: failed to map knowledge %s: %v", op.KnowledgeID, err)
+ return nil // Don't fail the whole batch
+ }
+
+ if result != nil {
+ mapMu.Lock()
+ ingestSucceeded++
+ docResults = append(docResults, result)
+ docPreview = append(docPreview, fmt.Sprintf("ingest[%s]: title=%s summary=%s", previewText(result.KnowledgeID, 24), previewText(result.DocTitle, 40), previewText(result.Summary, 64)))
+ for _, u := range updates {
+ slugUpdates[u.Slug] = append(slugUpdates[u.Slug], u)
+ }
+ mapMu.Unlock()
+ }
+ return nil
+ })
+ }
+ _ = eg.Wait()
+
+ // 2. REDUCE PHASE (Parallel upserting grouped by Slug)
+ egReduce, reduceCtx := errgroup.WithContext(ctx)
+ egReduce.SetLimit(10) // Reduce phase limit (LLM + DB concurrent connections)
+
+ var reduceMu sync.Mutex
+ var allPagesAffected []string
+ var ingestPagesAffected []string
+ var retractPagesAffected []string
+
+ for slug, updates := range slugUpdates {
+ slug := slug
+ updates := updates
+ egReduce.Go(func() error {
+ changed, affectedType, err := s.reduceSlugUpdates(reduceCtx, chatModel, payload.KnowledgeBaseID, slug, updates, payload.TenantID, batchCtx)
+ if err != nil {
+ logger.Warnf(reduceCtx, "wiki ingest: reduce failed for slug %s: %v", slug, err)
+ }
+ if changed {
+ reduceMu.Lock()
+ allPagesAffected = append(allPagesAffected, slug)
+ if affectedType == "ingest" {
+ ingestPagesAffected = append(ingestPagesAffected, slug)
+ } else if affectedType == "retract" {
+ retractPagesAffected = append(retractPagesAffected, slug)
+ }
+ reduceMu.Unlock()
+ }
+ return nil
+ })
+ }
+ _ = egReduce.Wait()
+
+ totalPagesAffected = len(allPagesAffected)
+
+ // Append log entry for retracts (since retracts aren't in docResults)
+ for _, op := range pendingOps {
+ if op.Op == WikiOpRetract {
+ s.appendLogEntry(ctx, WikiIngestPayload{
+ TenantID: payload.TenantID,
+ KnowledgeBaseID: payload.KnowledgeBaseID,
+ }, "retract", op.KnowledgeID, op.DocTitle, op.PageSlugs, "")
+ }
+ }
+
+ // Build change description for the Index Intro LLM prompt
+ var changeDesc strings.Builder
+ if len(docResults) > 0 {
+ for _, r := range docResults {
+ fmt.Fprintf(&changeDesc, "\n%s\n%s\n\n\n", r.DocTitle, r.Summary)
+ }
+ }
+ if retractChangeDesc.Len() > 0 {
+ changeDesc.WriteString(retractChangeDesc.String())
+ }
+
+ // Rebuild index page
+ if changeDesc.Len() > 0 {
+ indexRebuildAttempted = true
+ logger.Infof(ctx, "wiki ingest: rebuilding index page")
+ if err := s.rebuildIndexPage(ctx, chatModel, payload, changeDesc.String(), lang); err != nil {
+ logger.Warnf(ctx, "wiki ingest: rebuild index failed: %v", err)
+ docPreview = append(docPreview, fmt.Sprintf("index_change=%s", previewText(changeDesc.String(), 160)))
+ } else {
+ indexRebuildSucceeded = true
+ docPreview = append(docPreview, fmt.Sprintf("index_change=%s", previewText(changeDesc.String(), 160)))
+ }
+ }
+
+ // Append log entry for ingests
+ if len(docResults) > 0 {
+ s.appendLogEntry(ctx, payload, "ingest", "", fmt.Sprintf("%d documents", len(docResults)), ingestPagesAffected, "")
+ }
+
+ if len(retractPagesAffected) > 0 {
+ logger.Infof(ctx, "wiki ingest: cleaning dead links")
+ s.cleanDeadLinks(ctx, payload.KnowledgeBaseID)
+ }
+
+ if len(allPagesAffected) > 0 {
+ logger.Infof(ctx, "wiki ingest: injecting cross links")
+ s.injectCrossLinks(ctx, payload.KnowledgeBaseID, allPagesAffected)
+
+ logger.Infof(ctx, "wiki ingest: publishing draft pages")
+ s.publishDraftPages(ctx, payload.KnowledgeBaseID, allPagesAffected)
+ }
+
+ s.trimPendingList(ctx, payload.KnowledgeBaseID, peekedCount)
+
+ logger.Infof(ctx, "wiki ingest: batch completed for KB %s, %d ops, %d pages affected", payload.KnowledgeBaseID, len(pendingOps), len(allPagesAffected))
+
+ followUpScheduled = s.scheduleFollowUp(ctx, payload)
+ return nil
+}
+
+func (s *wikiIngestService) mapOneDocument(
+ ctx context.Context,
+ chatModel chat.Chat,
+ payload WikiIngestPayload,
+ op WikiPendingOp,
+ batchCtx *WikiBatchContext,
+) (*docIngestResult, []SlugUpdate, error) {
+ docStartedAt := time.Now()
+ knowledgeID := op.KnowledgeID
+ lang := op.Language
+
+ chunks, err := s.chunkRepo.ListChunksByKnowledgeID(ctx, payload.TenantID, knowledgeID)
+ if err != nil {
+ return nil, nil, fmt.Errorf("get chunks: %w", err)
+ }
+ if len(chunks) == 0 {
+ logger.Infof(ctx, "wiki ingest: document %s has no chunks, skip", knowledgeID)
+ return nil, nil, nil
+ }
+
+ content := reconstructContent(chunks)
+ rawRuneCount := len([]rune(content))
+ if len([]rune(content)) > maxContentForWiki {
+ content = string([]rune(content)[:maxContentForWiki])
+ }
+ logger.Infof(ctx, "wiki ingest: doc %s chunks=%d content_len(raw=%d,truncated=%d)", knowledgeID, len(chunks), rawRuneCount, len([]rune(content)))
+
+ docTitle := knowledgeID
+ if kn, err := s.knowledgeSvc.GetKnowledgeByIDOnly(ctx, knowledgeID); err == nil && kn != nil && kn.Title != "" {
+ docTitle = kn.Title
+ } else {
+ for _, ch := range chunks {
+ if ch.Content != "" {
+ lines := strings.SplitN(ch.Content, "\n", 2)
+ if len(lines) > 0 && len(lines[0]) > 0 && len(lines[0]) < 200 {
+ docTitle = strings.TrimPrefix(strings.TrimSpace(lines[0]), "# ")
+ break
+ }
+ }
+ }
+ }
+
+ sourceRef := fmt.Sprintf("%s|%s", knowledgeID, docTitle)
+ oldPageSlugs := s.getExistingPageSlugsForKnowledge(ctx, payload.KnowledgeBaseID, knowledgeID)
+
+ logger.Infof(ctx, "wiki ingest: extracting entities and concepts for %s", knowledgeID)
+ extractedEntities, extractedConcepts, slugItems, err := s.extractEntitiesAndConceptsNoUpsert(ctx, chatModel, content, docTitle, lang, payload, oldPageSlugs)
+ if err != nil {
+ logger.Warnf(ctx, "wiki ingest: knowledge extraction failed for %s: %v", knowledgeID, err)
+ return nil, nil, err
+ }
+
+ var extractedPages []string
+ for slug := range slugItems {
+ extractedPages = append(extractedPages, slug)
+ }
+
+ // Summary
+ summarySlug := fmt.Sprintf("summary/%s", slugify(docTitle))
+ var slugListing string
+ for _, slug := range extractedPages {
+ if item, ok := slugItems[slug]; ok {
+ aliases := ""
+ if len(item.Aliases) > 0 {
+ aliases = fmt.Sprintf(" (Aliases: %s)", strings.Join(item.Aliases, ", "))
+ }
+ slugListing += fmt.Sprintf("- [[%s]] = %s%s\n", slug, item.Name, aliases)
+ } else {
+ slugListing += fmt.Sprintf("- [[%s]]\n", slug)
+ }
+ }
+
+ var docSummaryLine string
+ summaryContent, err := s.generateWithTemplate(ctx, chatModel, agent.WikiSummaryPrompt, map[string]string{
+ "Title": docTitle,
+ "FileName": docTitle,
+ "FileType": "document",
+ "Content": content,
+ "Language": lang,
+ "ExtractedSlugs": slugListing,
+ })
+
+ var updates []SlugUpdate
+
+ if err != nil {
+ logger.Errorf(ctx, "wiki ingest: generate summary failed for %s: %v", knowledgeID, err)
+ } else {
+ sumLine, sumBody := splitSummaryLine(summaryContent)
+ if sumBody == "" {
+ sumBody = summaryContent
+ }
+ if sumLine == "" {
+ sumLine = docTitle
+ }
+ docSummaryLine = sumLine
+ updates = append(updates, SlugUpdate{
+ Slug: summarySlug,
+ Type: types.WikiPageTypeSummary,
+ DocTitle: docTitle,
+ KnowledgeID: knowledgeID,
+ SourceRef: sourceRef,
+ Language: lang,
+ SummaryLine: sumLine,
+ SummaryBody: sumBody,
+ })
+ extractedPages = append(extractedPages, summarySlug)
+ }
+
+ // Entities
+ for _, item := range extractedEntities {
+ if item.Slug != "" {
+ updates = append(updates, SlugUpdate{
+ Slug: item.Slug,
+ Type: types.WikiPageTypeEntity,
+ Item: item,
+ DocTitle: docTitle,
+ KnowledgeID: knowledgeID,
+ SourceRef: sourceRef,
+ Language: lang,
+ })
+ }
+ }
+
+ // Concepts
+ for _, item := range extractedConcepts {
+ if item.Slug != "" {
+ updates = append(updates, SlugUpdate{
+ Slug: item.Slug,
+ Type: types.WikiPageTypeConcept,
+ Item: item,
+ DocTitle: docTitle,
+ KnowledgeID: knowledgeID,
+ SourceRef: sourceRef,
+ Language: lang,
+ })
+ }
+ }
+
+ // Stale Pages
+ for oldSlug := range oldPageSlugs {
+ found := false
+ for _, newSlug := range extractedPages {
+ if oldSlug == newSlug {
+ found = true
+ break
+ }
+ }
+ if !found {
+ updates = append(updates, SlugUpdate{
+ Slug: oldSlug,
+ Type: "retractStale",
+ RetractDocContent: content,
+ DocTitle: docTitle,
+ KnowledgeID: knowledgeID,
+ Language: lang,
+ })
+ }
+ }
+
+ logger.Infof(ctx, "wiki ingest: mapped knowledge %s title=%q generated_updates=%d elapsed=%s",
+ knowledgeID, previewText(docTitle, 80), len(updates), time.Since(docStartedAt).Round(time.Millisecond))
+
+ return &docIngestResult{
+ KnowledgeID: knowledgeID,
+ DocTitle: docTitle,
+ Summary: docSummaryLine,
+ }, updates, nil
+}
+
+func (s *wikiIngestService) extractEntitiesAndConceptsNoUpsert(
+ ctx context.Context,
+ chatModel chat.Chat,
+ content, docTitle, lang string,
+ payload WikiIngestPayload,
+ oldPageSlugs map[string]bool,
+) ([]extractedItem, []extractedItem, map[string]extractedItem, error) {
+ var prevSlugsText string
+ if len(oldPageSlugs) > 0 {
+ var sb strings.Builder
+ for slug := range oldPageSlugs {
+ fmt.Fprintf(&sb, "- %s\n", slug)
+ }
+ prevSlugsText = sb.String()
+ } else {
+ prevSlugsText = "(none — this is a new document)"
+ }
+
+ extractionJSON, err := s.generateWithTemplate(ctx, chatModel, agent.WikiKnowledgeExtractPrompt, map[string]string{
+ "Title": docTitle,
+ "Content": content,
+ "Language": lang,
+ "PreviousSlugs": prevSlugsText,
+ })
+ if err != nil {
+ return nil, nil, nil, fmt.Errorf("combined extraction failed: %w", err)
+ }
+
+ extractionJSON = strings.TrimSpace(extractionJSON)
+ extractionJSON = strings.TrimPrefix(extractionJSON, "```json")
+ extractionJSON = strings.TrimPrefix(extractionJSON, "```")
+ extractionJSON = strings.TrimSuffix(extractionJSON, "```")
+ extractionJSON = strings.TrimSpace(extractionJSON)
+ extractionJSON = sanitizeJSONString(extractionJSON)
+
+ var result combinedExtraction
+ if err := json.Unmarshal([]byte(extractionJSON), &result); err != nil {
+ logger.Warnf(ctx, "wiki ingest: failed to parse combined extraction JSON: %v\nRaw: %s", err, extractionJSON)
+ return nil, nil, nil, fmt.Errorf("parse combined extraction JSON: %w", err)
+ }
+
+ result.Entities = s.deduplicateItems(ctx, chatModel, result.Entities, types.WikiPageTypeEntity, payload.KnowledgeBaseID)
+ result.Concepts = s.deduplicateItems(ctx, chatModel, result.Concepts, types.WikiPageTypeConcept, payload.KnowledgeBaseID)
+
+ slugItems := make(map[string]extractedItem)
+ for _, item := range result.Entities {
+ if item.Slug != "" && item.Name != "" {
+ slugItems[item.Slug] = item
+ }
+ }
+ for _, item := range result.Concepts {
+ if item.Slug != "" && item.Name != "" {
+ slugItems[item.Slug] = item
+ }
+ }
+
+ return result.Entities, result.Concepts, slugItems, nil
+}
+
+func (s *wikiIngestService) reduceSlugUpdates(
+ ctx context.Context,
+ chatModel chat.Chat,
+ kbID string,
+ slug string,
+ updates []SlugUpdate,
+ tenantID uint64,
+ batchCtx *WikiBatchContext,
+) (bool, string, error) {
+ page, err := s.wikiService.GetPageBySlug(ctx, kbID, slug)
+ exists := (err == nil && page != nil)
+
+ if !exists {
+ hasAdditions := false
+ for _, u := range updates {
+ if u.Type == types.WikiPageTypeEntity || u.Type == types.WikiPageTypeConcept || u.Type == "summary" {
+ hasAdditions = true
+ break
+ }
+ }
+ if !hasAdditions {
+ return false, "", nil
+ }
+
+ page = &types.WikiPage{
+ ID: uuid.New().String(),
+ TenantID: tenantID,
+ KnowledgeBaseID: kbID,
+ Slug: slug,
+ Status: types.WikiPageStatusDraft,
+ SourceRefs: types.StringArray{},
+ Aliases: types.StringArray{},
+ }
+ }
+
+ changed := false
+ affectedType := "ingest"
+
+ var summaryUpdate *SlugUpdate
+ var retracts []SlugUpdate
+ var additions []SlugUpdate
+
+ for i, u := range updates {
+ if u.Type == "summary" {
+ summaryUpdate = &updates[i]
+ } else if u.Type == "retract" || u.Type == "retractStale" {
+ retracts = append(retracts, u)
+ affectedType = "retract"
+ } else if u.Type == types.WikiPageTypeEntity || u.Type == types.WikiPageTypeConcept {
+ additions = append(additions, u)
+ affectedType = "ingest" // Additions override retracts type
+ }
+ }
+
+ if summaryUpdate != nil {
+ page.Title = summaryUpdate.DocTitle + " - Summary"
+ page.Content = summaryUpdate.SummaryBody
+ page.Summary = summaryUpdate.SummaryLine
+ page.PageType = types.WikiPageTypeSummary
+ page.SourceRefs = appendUnique(page.SourceRefs, summaryUpdate.SourceRef)
+ changed = true
+
+ if exists {
+ _, err = s.wikiService.UpdatePage(ctx, page)
+ } else {
+ _, err = s.wikiService.CreatePage(ctx, page)
+ }
+ return changed, affectedType, err
+ }
+
+ var remainingSourcesContent strings.Builder
+ var deletedContent strings.Builder
+ var relatedSlugs strings.Builder
+ var newContentBuilder strings.Builder
+ var docTitles []string
+ var language string
+
+ if len(retracts) > 0 {
+ language = retracts[0].Language
+
+ for _, r := range retracts {
+ fmt.Fprintf(&deletedContent, "\n%s\n\n%s\n\n\n\n", r.DocTitle, r.RetractDocContent)
+ }
+
+ retractKIDs := make(map[string]bool)
+ for _, r := range retracts {
+ retractKIDs[r.KnowledgeID] = true
+ }
+
+ for _, ref := range page.SourceRefs {
+ pipeIdx := strings.Index(ref, "|")
+ var refKnowledgeID, refTitle string
+ if pipeIdx > 0 {
+ refKnowledgeID = ref[:pipeIdx]
+ refTitle = ref[pipeIdx+1:]
+ } else {
+ refKnowledgeID = ref
+ refTitle = ref
+ }
+
+ if retractKIDs[refKnowledgeID] {
+ continue
+ }
+
+ if content, ok := batchCtx.SummaryContentByKnowledgeID[refKnowledgeID]; ok {
+ fmt.Fprintf(&remainingSourcesContent, "\n%s\n\n%s\n\n\n\n", refTitle, content)
+ } else {
+ fmt.Fprintf(&remainingSourcesContent, "\n%s\n\n(summary not available)\n\n\n\n", refTitle)
+ }
+ }
+ if remainingSourcesContent.Len() == 0 {
+ remainingSourcesContent.WriteString("(no remaining sources)")
+ }
+
+ newRefs := types.StringArray{}
+ for _, ref := range page.SourceRefs {
+ pipeIdx := strings.Index(ref, "|")
+ refKnowledgeID := ref
+ if pipeIdx > 0 {
+ refKnowledgeID = ref[:pipeIdx]
+ }
+ if !retractKIDs[refKnowledgeID] {
+ newRefs = append(newRefs, ref)
+ }
+ }
+ page.SourceRefs = newRefs
+ }
+
+ if len(additions) > 0 {
+ language = additions[0].Language
+ for _, add := range additions {
+ fmt.Fprintf(&newContentBuilder, "\n%s\n\n**%s**: %s\n\n%s\n\n\n\n",
+ add.DocTitle, add.Item.Name, add.Item.Description, add.Item.Details)
+ docTitles = appendUnique(docTitles, add.DocTitle)
+
+ for _, alias := range add.Item.Aliases {
+ page.Aliases = appendUnique(page.Aliases, alias)
+ }
+ page.SourceRefs = appendUnique(page.SourceRefs, add.SourceRef)
+
+ if page.Title == "" {
+ page.Title = add.Item.Name
+ }
+ if page.PageType == "" {
+ page.PageType = add.Type
+ }
+ }
+ }
+
+ if len(additions) > 0 || len(retracts) > 0 {
+ for _, outSlug := range page.OutLinks {
+ if title, ok := batchCtx.SlugTitleMap[outSlug]; ok {
+ fmt.Fprintf(&relatedSlugs, "- %s (%s)\n", outSlug, title)
+ }
+ }
+
+ existingContent := page.Content
+ if !exists || existingContent == "" {
+ existingContent = "(New page)"
+ }
+
+ hasAdditionsStr := ""
+ if len(additions) > 0 {
+ hasAdditionsStr = "1"
+ }
+ hasRetractionsStr := ""
+ if len(retracts) > 0 {
+ hasRetractionsStr = "1"
+ }
+
+ updatedContent, err := s.generateWithTemplate(ctx, chatModel, agent.WikiPageModifyPrompt, map[string]string{
+ "HasAdditions": hasAdditionsStr,
+ "HasRetractions": hasRetractionsStr,
+ "ExistingContent": existingContent,
+ "NewContent": newContentBuilder.String(),
+ "DeletedContent": deletedContent.String(),
+ "RemainingSourcesContent": remainingSourcesContent.String(),
+ "AvailableSlugs": relatedSlugs.String(),
+ "Language": language,
+ })
+
+ if err == nil && updatedContent != "" {
+ updatedSummary, updatedBody := splitSummaryLine(updatedContent)
+ if updatedBody != "" {
+ page.Content = updatedBody
+ } else {
+ page.Content = updatedContent
+ }
+ if updatedSummary != "" {
+ page.Summary = updatedSummary
+ }
+ changed = true
+ } else if err != nil {
+ logger.Warnf(ctx, "wiki ingest: update/retract failed for slug %s: %v", slug, err)
+ }
+ }
+
+ if changed {
+ if exists {
+ _, err = s.wikiService.UpdatePage(ctx, page)
+ } else {
+ _, err = s.wikiService.CreatePage(ctx, page)
+ }
+ return true, affectedType, err
+ }
+
+ return false, "", nil
+}
diff --git a/internal/middleware/logger.go b/internal/middleware/logger.go
index 147b13f9..d2c73852 100644
--- a/internal/middleware/logger.go
+++ b/internal/middleware/logger.go
@@ -143,7 +143,9 @@ func Logger() gin.HandlerFunc {
start := time.Now()
path := c.Request.URL.Path
raw := c.Request.URL.RawQuery
- if strings.HasPrefix(path, "/assets/") {
+
+ isWikiStats := strings.HasPrefix(path, "/api/v1/knowledgebase/") && strings.HasSuffix(path, "/wiki/stats")
+ if strings.HasPrefix(path, "/assets/") || isWikiStats {
c.Next()
return
}