package service import ( "context" "encoding/json" "fmt" "strings" "sync" "time" "unicode/utf8" "github.com/Tencent/WeKnora/internal/agent" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/models/chat" "github.com/Tencent/WeKnora/internal/tracing/langfuse" "github.com/Tencent/WeKnora/internal/types" "github.com/google/uuid" "github.com/hibiken/asynq" "golang.org/x/sync/errgroup" ) // scheduleFollowUp enqueues another asynq trigger task if there are // still pending ops in task_pending_ops for this KB. Returns true when // a follow-up was scheduled. // // Post-Phase-3 this only backstops the case where a batch drained its // claimed window but more rows remain and no other trigger is pending // (e.g. steady trickle of uploads). Standard mode already fans a KB's // backlog across concurrent claiming batches, so the short delay is // normally just a light debounce rather than a lock-release wait. // // `delay` is the ProcessIn before the follow-up fires. Callers pass // wikiFollowUpDelay for the normal case and wikiRateLimitBackoff when the // batch tripped an upstream rate limit — the released failed rows are // eligible immediately, but nothing claims them until a trigger fires, so // stretching the follow-up interval is what actually paces retries down // during a 429 storm. func (s *wikiIngestService) scheduleFollowUp(ctx context.Context, payload WikiIngestPayload, delay time.Duration) bool { if s.pendingRepo == nil { return false } count, err := s.pendingRepo.PendingCount(ctx, wikiTaskType, wikiTaskScope, payload.KnowledgeBaseID) if err != nil || count == 0 { return false } logger.Infof(ctx, "wiki ingest: %d more documents pending for KB %s, scheduling follow-up in %s", count, payload.KnowledgeBaseID, delay) langfuse.InjectTracing(ctx, &payload) payloadBytes, _ := json.Marshal(payload) t := asynq.NewTask(types.TypeWikiIngest, payloadBytes, asynq.Queue(types.QueueWiki), asynq.MaxRetry(wikiIngestMaxRetry), asynq.Timeout(60*time.Minute), asynq.ProcessIn(delay), // debounce (or rate-limit backoff) before draining the remainder ) if _, err := s.task.Enqueue(t); err != nil { logger.Warnf(ctx, "wiki ingest: follow-up enqueue failed: %v", err) return false } return true } // newWikiBatchContext builds the per-run lazy fetchers used by both the ingest // batch and the debounced finalize task. These replace the legacy pre-batch // ListAllPages dump: instead of pulling ~100MB of rows up front (and walking // them several more times), callers pay only for the slugs / knowledge ids they // actually reach for. Cache hits keep repeat lookups within a single run free. // The cache is per-call (goroutine-safe via fetchMu), so each task gets a fresh, // isolated view. func (s *wikiIngestService) newWikiBatchContext( kbID string, wikiConfig *types.WikiConfig, ) *WikiBatchContext { var ( fetchMu sync.Mutex slugTitleCache = make(map[string]string) // slug -> title; "" = known-missing summaryKIDCache = make(map[string]string) // kid -> content; "" = known-missing ) resolveSlugs := func(ctx context.Context, slugs []string) map[string]string { // Filter to the slugs we don't already have cached. fetchMu.Lock() need := slugs[:0:0] for _, slug := range slugs { if _, ok := slugTitleCache[slug]; ok { continue } need = append(need, slug) } fetchMu.Unlock() if len(need) > 0 { pages, err := s.wikiService.ListBySlugs(ctx, kbID, need) if err != nil { logger.Warnf(ctx, "wiki ingest: ListBySlugs(%d slugs) failed: %v", len(need), err) } fetchMu.Lock() for _, slug := range need { if p, ok := pages[slug]; ok && p != nil { if p.Status == types.WikiPageStatusArchived || p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog { // Treat archived / system pages as missing from the // title-resolution map: cleanDeadLinks shouldn't link // to them, and the log-feed slug-title fallback // should degrade to slug-only display. slugTitleCache[slug] = "" continue } slugTitleCache[slug] = p.Title } else { slugTitleCache[slug] = "" } } fetchMu.Unlock() } out := make(map[string]string, len(slugs)) fetchMu.Lock() for _, slug := range slugs { if title := slugTitleCache[slug]; title != "" { out[slug] = title } } fetchMu.Unlock() return out } resolveSummaries := func(ctx context.Context, kids []string) map[string]string { fetchMu.Lock() need := kids[:0:0] for _, kid := range kids { if _, ok := summaryKIDCache[kid]; ok { continue } need = append(need, kid) } fetchMu.Unlock() if len(need) > 0 { contents, err := s.wikiService.ListSummariesByKnowledgeIDs(ctx, kbID, need) if err != nil { logger.Warnf(ctx, "wiki ingest: ListSummariesByKnowledgeIDs(%d kids) failed: %v", len(need), err) } fetchMu.Lock() for _, kid := range need { if c, ok := contents[kid]; ok && c != "" { summaryKIDCache[kid] = c } else { summaryKIDCache[kid] = "" } } fetchMu.Unlock() } out := make(map[string]string, len(kids)) fetchMu.Lock() for _, kid := range kids { if content := summaryKIDCache[kid]; content != "" { out[kid] = content } } fetchMu.Unlock() return out } granularity := types.WikiExtractionStandard contentInstructions := "" extractionInstructions := "" if wikiConfig != nil { granularity = wikiConfig.ExtractionGranularity.Normalize() contentInstructions = wikiConfig.ContentInstructions extractionInstructions = wikiConfig.ExtractionInstructions } return &WikiBatchContext{ SlugTitle: func(ctx context.Context, slug string) string { m := resolveSlugs(ctx, []string{slug}) return m[slug] }, SlugTitleMany: resolveSlugs, SummaryContentByKnowledgeID: func(ctx context.Context, kid string) string { m := resolveSummaries(ctx, []string{kid}) return m[kid] }, ExtractionGranularity: granularity, ContentInstructions: contentInstructions, ExtractionInstructions: extractionInstructions, } } func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task) error { taskStartedAt := time.Now() retryCount, _ := asynq.GetRetryCount(ctx) maxRetry, _ := asynq.GetMaxRetry(ctx) var payload WikiIngestPayload exitStatus := "success" mode := "redis" pendingOpsCount := 0 ingestOps := 0 retractOps := 0 ingestSucceeded := 0 ingestFailed := 0 retractHandled := 0 followUpScheduled := false totalPagesAffected := 0 docPreview := make([]string, 0, 6) // Tunables resolved from KB.WikiConfig once we've loaded the KB. // Captured up here so the deferred stats log can observe them // regardless of which exit path we took. (Index-intro rebuild moved to // the debounced wiki:finalize task in Phase 1, so it's logged there, not // here; the exclusive per-KB lock was removed in Phase 3.) loggedBatchSize := 0 loggedMapPar := 0 loggedReducePar := 0 loggedMaxInflight := 0 defer func() { logger.Infof( ctx, "wiki ingest stats: kb=%s tenant=%d retry=%d/%d status=%s elapsed=%s mode=%s pending_ops=%d ops(ingest=%d,retract=%d) ingest(success=%d,failed=%d) retract_handled=%d pages(total=%d) followup=%v tunables(batch=%d,map_par=%d,reduce_par=%d,max_inflight=%d) preview=%s", payload.KnowledgeBaseID, payload.TenantID, retryCount, maxRetry, exitStatus, time.Since(taskStartedAt).Round(time.Millisecond), mode, pendingOpsCount, ingestOps, retractOps, ingestSucceeded, ingestFailed, retractHandled, totalPagesAffected, followUpScheduled, loggedBatchSize, loggedMapPar, loggedReducePar, loggedMaxInflight, previewStringSlice(docPreview, 6), ) }() if err := json.Unmarshal(t.Payload(), &payload); err != nil { exitStatus = "invalid_payload" return fmt.Errorf("wiki ingest: unmarshal payload: %w", err) } // Inject context ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID) if payload.Language != "" { ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language) } // Concurrency model (Phase 3): // // - Standard (Redis) mode: NO exclusive per-KB lock. Multiple batches // for the same KB may run at once, each claiming a DISJOINT set of // rows via claimPendingList (below). This lets one KB's backlog fan // out across the whole wiki worker pool instead of draining one // 5-doc batch at a time. Same-slug reduce safety is provided by // withSlugLock, not by a batch-wide lock. // // - Lite mode (no Redis, single process): keep the in-process // liteLocks guard so exactly one batch per KB runs at a time. Lite // targets small/local scale where serial-per-KB is simplest and the // claiming machinery (which needs FOR UPDATE SKIP LOCKED) buys // nothing. if s.redisClient == nil { mode = "lite" if _, loaded := s.liteLocks.LoadOrStore(payload.KnowledgeBaseID, struct{}{}); loaded { exitStatus = "active_lock_conflict" logger.Infof(ctx, "wiki ingest: another batch active for KB %s (lite lock), deferring to asynq retry", payload.KnowledgeBaseID) return ErrWikiIngestConcurrent } defer s.liteLocks.Delete(payload.KnowledgeBaseID) } kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID) if err != nil { exitStatus = "get_kb_failed" return fmt.Errorf("wiki ingest: get KB: %w", err) } if !kb.IsWikiEnabled() { exitStatus = "kb_not_wiki_enabled" return fmt.Errorf("wiki ingest: KB %s is not wiki type", kb.ID) } var synthesisModelID string if kb.WikiConfig != nil { synthesisModelID = kb.WikiConfig.SynthesisModelID } if synthesisModelID == "" { synthesisModelID = kb.SummaryModelID } if synthesisModelID == "" { exitStatus = "missing_synthesis_model" return fmt.Errorf("wiki ingest: no synthesis model configured for KB %s", kb.ID) } chatModel, err := s.modelService.GetChatModel(ctx, synthesisModelID) if err != nil { exitStatus = "get_chat_model_failed" return fmt.Errorf("wiki ingest: get chat model: %w", err) } // Resolve per-KB tunables once. WikiConfig.IngestBatchSize / // IngestMapParallel / IngestReduceParallel let operators on // 4w-document KBs raise the throughput knob (more docs per batch + // more concurrent LLM calls) without a code deploy. Zero falls back // to the historical defaults so existing KBs see no behaviour // change until they opt in. batchSize := kb.WikiConfig.IngestBatchSizeOrDefault(wikiMaxDocsPerBatch) mapParallel := kb.WikiConfig.IngestMapParallelOrDefault(10) reduceParallel := kb.WikiConfig.IngestReduceParallelOrDefault(10) loggedBatchSize = batchSize loggedMapPar = mapParallel loggedReducePar = reduceParallel lang := types.LanguageNameFromContext(ctx) // Per-KB in-flight cap (Phase 4, standard mode): keep one KB's bulk // import from monopolizing the shared wiki pool. If the KB is already at // its cap we schedule a coalesced retry and bail without claiming, so the // rows stay available for whichever running batch frees a slot first. maxInflight := kb.WikiConfig.IngestMaxInflightOrDefault(wikiInflightDefault) loggedMaxInflight = maxInflight releaseSlot, gotSlot := s.reserveInflightSlot(ctx, payload.KnowledgeBaseID, maxInflight) if !gotSlot { exitStatus = "inflight_cap" logger.Infof(ctx, "wiki ingest: KB %s at in-flight cap (%d), rescheduling", payload.KnowledgeBaseID, maxInflight) s.scheduleCappedRetry(ctx, payload) return nil } defer releaseSlot() // Standard mode claims rows (marks claimed_at, disjoint across concurrent // batches); Lite mode peeks under its in-process lock. var pendingOps []WikiPendingOp var peekedIDs []int64 var loadErr error if s.redisClient != nil { pendingOps, peekedIDs, loadErr = s.claimPendingList(ctx, payload.KnowledgeBaseID, batchSize) } else { pendingOps, peekedIDs, loadErr = s.peekPendingList(ctx, payload.KnowledgeBaseID, batchSize) } if loadErr != nil { // Transient failure loading the pending list. Return an error so // asynq retries this trigger with backoff — acking here would strand // the queue until an unrelated upload happened to re-trigger it. exitStatus = "load_pending_failed" logger.Warnf(ctx, "wiki ingest: failed to load pending list for KB %s: %v", payload.KnowledgeBaseID, loadErr) return fmt.Errorf("wiki ingest: load pending list: %w", loadErr) } pendingOpsCount = len(pendingOps) if len(pendingOps) == 0 { exitStatus = "no_pending_ops" logger.Infof(ctx, "wiki ingest: no pending operations for KB %s", payload.KnowledgeBaseID) // We claimed nothing, but rows may still exist held by a FRESH claim // (a concurrent batch that is still running, or one that crashed // mid-flight and left claimed_at stamped). This no-op return does not // chain a follow-up, so without a safety net a crashed batch's rows // would sit unclaimable until wikiClaimStaleAfter AND never get // re-triggered afterwards — stranding the KB indefinitely. Schedule a // coalesced recheck timed past the stale threshold so those rows are // reclaimed automatically once eligible. followUpScheduled = s.scheduleStaleClaimRecheck(ctx, payload) return nil } logger.Infof(ctx, "wiki ingest: batch processing %d ops for KB %s", len(pendingOps), payload.KnowledgeBaseID) // Crash/abort safety net (standard/claim mode only). If this batch exits // abnormally — panic, ctx timeout, or an early error return — BEFORE it // settles its claimed rows (trim + requeueFailedOps), release the claims // so the next trigger can re-claim within seconds instead of waiting out // wikiClaimStaleAfter (~90m). On the normal path claimsSettled flips true // once the rows reach their terminal state, making this a no-op. Uses a // background context because ctx may already be cancelled on the timeout // path. Lite mode peeks without claiming, so there is nothing to release. claimsSettled := false if s.redisClient != nil && len(peekedIDs) > 0 { defer func() { if claimsSettled { return } if err := s.pendingRepo.ReleaseByIDs(context.Background(), peekedIDs); err != nil { logger.Warnf(ctx, "wiki ingest: failed to release %d claims on abnormal exit for KB %s: %v", len(peekedIDs), payload.KnowledgeBaseID, err) return } logger.Warnf(ctx, "wiki ingest: released %d claimed rows on abnormal exit for KB %s (re-claimable immediately)", len(peekedIDs), payload.KnowledgeBaseID) }() } // Resolve extraction granularity once per batch. Historical rows with // empty/unknown values fall back to Standard via Normalize(). Failures // to load the KB (unlikely since we're already acting on it) also // degrade gracefully to Standard. var wikiConfig *types.WikiConfig if kb, kbErr := s.kbService.GetKnowledgeBaseByID(ctx, payload.KnowledgeBaseID); kbErr == nil && kb != nil && kb.WikiConfig != nil { wikiConfig = kb.WikiConfig } batchCtx := s.newWikiBatchContext(payload.KnowledgeBaseID, wikiConfig) // 1. MAP PHASE (Parallel extraction and generation of updates) var mapMu sync.Mutex var failedOps []WikiPendingOp slugUpdates := make(map[string][]SlugUpdate) var docResults []*docIngestResult // rateLimited flips true when any map/reduce LLM failure looks like an // upstream 429/quota trip. It steers the follow-up scheduler onto the // longer wikiRateLimitBackoff so retries don't keep hammering an already // saturated rpm budget. Guarded by mapMu (map phase) / reduceMu (reduce // phase) — both are held when written. rateLimited := false eg, mapCtx := errgroup.WithContext(ctx) eg.SetLimit(mapParallel) // Map phase limit (configurable via WikiConfig) for _, op := range pendingOps { op := op eg.Go(func() error { if op.Op == WikiOpRetract { // Resolve the authoritative page set at run-time. The caller // (knowledgeService.cleanupWikiOnKnowledgeDelete) captures // PageSlugs from a DB snapshot taken *before* this task fires, // but there is a window where: // - cleanup ran before ingest → snapshot is empty, but a // concurrent ingest may have already created pages by now // - a previous ingest batch created new pages after cleanup // captured its snapshot // Re-querying ListPagesBySourceRef here unions the caller's // slugs with whatever currently references the knowledge, so // no page is left un-retracted. It also lets us support // callers that deliberately enqueue retract with empty // PageSlugs as "figure it out yourself" — see // cleanupWikiOnKnowledgeDelete's comment (3). slugSet := make(map[string]struct{}, len(op.PageSlugs)) for _, slug := range op.PageSlugs { if slug == "" { continue } slugSet[slug] = struct{}{} } if op.KnowledgeID != "" { livePages, err := s.wikiService.ListPagesBySourceRef(mapCtx, payload.KnowledgeBaseID, op.KnowledgeID) if err != nil { logger.Warnf(mapCtx, "wiki ingest: retract lookup failed for %s: %v", op.KnowledgeID, err) } else { for _, p := range livePages { if p == nil || p.Slug == "" { continue } // Index/log pages never carry real source_refs; // if they somehow surface here, skip — the // reduce stage would be a no-op anyway. if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog { continue } slugSet[p.Slug] = struct{}{} } } } mapMu.Lock() retractOps++ retractHandled++ docPreview = append(docPreview, fmt.Sprintf("retract[%s]: %s (%d slugs)", previewText(op.KnowledgeID, 24), previewText(op.DocTitle, 48), len(slugSet))) for slug := range slugSet { slugUpdates[slug] = append(slugUpdates[slug], SlugUpdate{ Slug: slug, Type: "retract", RetractDocContent: op.DocSummary, DocTitle: op.DocTitle, KnowledgeID: op.KnowledgeID, Language: types.LanguageLocaleName(op.Language), }) } mapMu.Unlock() return nil } // Ingest mapMu.Lock() ingestOps++ mapMu.Unlock() logger.Infof(mapCtx, "wiki ingest: processing document '%s' (%s)", op.DocTitle, op.KnowledgeID) result, updates, err := s.mapOneDocument(mapCtx, chatModel, payload, op, batchCtx) if err != nil { mapMu.Lock() ingestFailed++ failedOps = append(failedOps, op) if isLikelyRateLimitError(err) { rateLimited = true } mapMu.Unlock() logger.Warnf(mapCtx, "wiki ingest: failed to map knowledge %s: %v", op.KnowledgeID, err) return nil // Don't fail the whole batch } if result != nil { mapMu.Lock() ingestSucceeded++ docResults = append(docResults, result) docPreview = append(docPreview, fmt.Sprintf("ingest[%s]: title=%s summary=%s", previewText(result.KnowledgeID, 24), previewText(result.DocTitle, 40), previewText(result.Summary, 64))) for _, u := range updates { slugUpdates[u.Slug] = append(slugUpdates[u.Slug], u) } mapMu.Unlock() // No fail-count reset needed: a successful op is added // to peekedIDs and gets DELETEd from task_pending_ops at // trim time, so there is no stale fail_count column to // scrub. Compare with the legacy Redis path, which kept // a separate wiki:failcount:<...> key alive for 24h // regardless of whether the original op had drained. // // The finalizing slot is drained later (after reduce + // publish) in the docResults loop, so "completed" only // arrives once wiki is fully written. } else { // err == nil && result == nil: mapOneDocument skipped this // doc at a terminal, non-retryable state (knowledge // deleted / no chunks / insufficient text). It produces no // docResult and is not a failedOp, so neither the success // nor the dead-letter drain path will fire. Release the // finalizing slot here so the row doesn't hang in // "finalizing" until the housekeeping sweep marks it // failed. The matching +1 was seeded by // KnowledgePostProcess.SetFinalizing. s.finalizeWikiSubtask(mapCtx, op.KnowledgeID) } return nil }) } _ = eg.Wait() // Plan the directory once for the whole batch BEFORE reduce. Reduce writes // pages in parallel, so it can't converge on shared folders on its own; this // single pass assigns every new entity/concept slug a coherent category_path // that reuses existing folders. Reduce then only applies the plan to pages // that don't already have a category (user-curated pages are never churned). batchCtx.PlannedFolderID = s.resolvePlannedFolders(ctx, kb, s.planBatchTaxonomy(ctx, chatModel, kb, slugUpdates, lang)) // 2. REDUCE PHASE (Parallel upserting grouped by Slug) egReduce, reduceCtx := errgroup.WithContext(ctx) egReduce.SetLimit(reduceParallel) // Reduce phase limit (LLM + DB concurrent connections, configurable) var reduceMu sync.Mutex var allPagesAffected []string var ingestPagesAffected []string var retractPagesAffected []string // failedAdditionSlugs collects entity/concept slugs whose page // generation LLM call failed (so the page was never written). The // post-reduce cleanup step uses this set to (a) strip dead [[slug]] // references from the same batch's summary pages, and (b) prune the // slugs out of the wiki log feed so users don't see clickable entries // pointing at missing pages. failedAdditionSlugs := make(map[string]struct{}) // unappliedSlugKIDs collects the knowledge_ids that contributed to a // slug whose update never landed — either because we could NOT acquire // the per-slug lock within wikiSlugLockWait, or because reduce returned // an error. In both cases the page keeps its prior content, so the // owning document(s) must be re-queued rather than trimmed — otherwise // the row is deleted and the contribution is silently lost forever // (finalize only rebuilds the index / cross-links, it does not re-run // reduce). requeueFailedOps' fail_count budget bounds the retries and // dead-letters a document whose slug fails/stays hot permanently. unappliedSlugKIDs := make(map[string]struct{}) // collectUnapplied records every knowledge_id backing a slug we failed // to apply. Caller holds no lock; it takes reduceMu itself. collectUnapplied := func(updates []SlugUpdate) { reduceMu.Lock() for _, u := range updates { if u.KnowledgeID != "" { unappliedSlugKIDs[u.KnowledgeID] = struct{}{} } } reduceMu.Unlock() } // Build the kid → wikiSpan lookup before kicking off reduce. Each // per-slug reduce attaches a postprocess.wiki.page[slug] subspan // under the FIRST contributing doc's wiki span — see comment in // reduceSlugUpdates for the multi-contributor attribution rule. kidToWikiSpan := make(map[string]*Span, len(docResults)) for _, r := range docResults { if r != nil && r.WikiSpan != nil { kidToWikiSpan[r.KnowledgeID] = r.WikiSpan } } for slug, updates := range slugUpdates { slug := slug updates := updates egReduce.Go(func() error { var ( changed bool affectedType string additionFailed bool reduceErr error ) // Serialize same-slug read-modify-write across concurrent batches // (standard mode). runs fn directly in Lite mode. acquired, lockErr := s.withSlugLock(reduceCtx, payload.KnowledgeBaseID, slug, func() error { changed, affectedType, additionFailed, reduceErr = s.reduceSlugUpdates( reduceCtx, chatModel, payload.KnowledgeBaseID, slug, updates, payload.TenantID, batchCtx, kidToWikiSpan) return reduceErr }) if lockErr != nil { // ctx cancelled (batch timeout / shutdown) — stop quietly. return nil } if !acquired { // Contended slug we couldn't get in time. The page keeps its // prior content, so the documents that fed this slug are NOT // done: record their knowledge_ids so the trim phase re-queues // them (via the failed-op retry budget) for a later, // hopefully-uncontended batch instead of deleting their rows. logger.Warnf(reduceCtx, "wiki ingest: slug %s busy > %s, deferring update", slug, wikiSlugLockWait) collectUnapplied(updates) return nil } if reduceErr != nil { // The page's read-modify-write failed, so this slug's update // never landed. Re-queue the contributing documents (same // fail_count budget as a map failure) so a later batch retries // instead of silently dropping the row at trim time. logger.Warnf(reduceCtx, "wiki ingest: reduce failed for slug %s: %v", slug, reduceErr) collectUnapplied(updates) if isLikelyRateLimitError(reduceErr) { reduceMu.Lock() rateLimited = true reduceMu.Unlock() } } if changed { reduceMu.Lock() allPagesAffected = append(allPagesAffected, slug) if affectedType == "ingest" { ingestPagesAffected = append(ingestPagesAffected, slug) } else if affectedType == "retract" { retractPagesAffected = append(retractPagesAffected, slug) } reduceMu.Unlock() } if additionFailed { reduceMu.Lock() failedAdditionSlugs[slug] = struct{}{} reduceMu.Unlock() } return nil }) } _ = egReduce.Wait() // Sanitize the doc summary pages produced by this batch BEFORE we // build log entries / rebuild the index. The summary LLM (run during // map) was free to inject [[entity/foo|name]] links to every slug it // saw extracted, but reduce may have failed to materialize some of // those slugs into actual pages. Rewrite those dead links to plain // text so the summary doesn't contain unresolvable references. if len(failedAdditionSlugs) > 0 && len(docResults) > 0 { s.sanitizeDeadSummaryLinks(ctx, payload.KnowledgeBaseID, docResults, failedAdditionSlugs, batchCtx) } totalPagesAffected = len(allPagesAffected) // Collect log entries for this batch and flush them in a single INSERT. // Historically each op triggered its own `GetLog + UpdatePage` round // trip, which rewrote the entire log page TEXT column and caused O(n^2) // write amplification as the log grew. AppendBatch writes one row per // event into wiki_log_entries instead. // // slugsToRefs resolves each retract slug against the batch-start // snapshot (batchCtx.SlugTitleMap) so the log feed carries titles for // pages that existed when the batch began. Pages created or renamed // during this batch fall through the map lookup and log as slug-only // refs, which the frontend renders as the slug itself — a sensible // fallback given retracts only touch pre-existing pages. slugsToRefs := func(slugs []string) []types.WikiLogPageRef { if len(slugs) == 0 { return nil } titles := batchCtx.SlugTitleMany(ctx, slugs) out := make([]types.WikiLogPageRef, 0, len(slugs)) for _, slug := range slugs { out = append(out, types.WikiLogPageRef{Slug: slug, Title: titles[slug]}) } return out } logEntries := make([]*types.WikiLogEntry, 0, len(pendingOps)+len(docResults)) for _, op := range pendingOps { if op.Op == WikiOpRetract { logEntries = append(logEntries, s.buildLogEntry(payload.TenantID, payload.KnowledgeBaseID, "retract", op.KnowledgeID, op.DocTitle, op.DocSummary, slugsToRefs(op.PageSlugs))) } } for _, r := range docResults { // Drop any slugs whose page generation failed in reduce so the // log feed never offers a clickable entry that 404s. The summary // page itself (slug = summary/) is always created // unconditionally upstream, so it survives the filter. pages := r.Pages if len(failedAdditionSlugs) > 0 { pages = pages[:0:0] for _, ref := range r.Pages { if _, bad := failedAdditionSlugs[ref.Slug]; bad { continue } pages = append(pages, ref) } } logEntries = append(logEntries, s.buildLogEntry(payload.TenantID, payload.KnowledgeBaseID, "ingest", r.KnowledgeID, r.DocTitle, r.Summary, pages)) } if len(logEntries) > 0 && s.logEntrySvc != nil { if err := s.logEntrySvc.AppendBatch(ctx, logEntries); err != nil { logger.Warnf(ctx, "wiki ingest: failed to append %d log entries: %v", len(logEntries), err) } } // Publish freshly-generated pages immediately (NOT deferred to finalize): // users should see a document's wiki pages as soon as their content is // written, not after the debounce window. This is a cheap status flip. if len(allPagesAffected) > 0 { logger.Infof(ctx, "wiki ingest: publishing draft pages") s.publishDraftPages(ctx, payload.KnowledgeBaseID, allPagesAffected) } // Defer KB-global convergence (index-intro rebuild + dead-link cleanup + // cross-link injection) to a debounced per-KB wiki:finalize task instead of // running it at the tail of every 5-doc batch. We record what changed into // the finalize lane and schedule a coalesced trigger; a burst of N documents // then rebuilds the index ONCE. See ProcessWikiFinalize. // // freshTitleBySlug carries the (slug → title) pairs this batch successfully // wrote (minus reduce-phase failures) so finalize's cross-link pass can // linkify mentions of the new pages. freshTitleBySlug := make(map[string]string, len(docResults)*4) for _, dr := range docResults { if dr == nil { continue } for _, p := range dr.Pages { if p.Slug == "" || p.Title == "" { continue } if _, bad := failedAdditionSlugs[p.Slug]; bad { continue } freshTitleBySlug[p.Slug] = p.Title } } if len(allPagesAffected) > 0 || len(docResults) > 0 { changes := make([]wikiFinalizeChange, 0, len(docResults)+len(pendingOps)) for _, r := range docResults { changes = append(changes, wikiFinalizeChange{ Action: wikiFinalizeAdded, DocTitle: r.DocTitle, DocSummary: r.Summary, }) } for _, op := range pendingOps { if op.Op == WikiOpRetract { changes = append(changes, wikiFinalizeChange{ Action: wikiFinalizeRemoved, DocTitle: op.DocTitle, DocSummary: op.DocSummary, }) } } s.enqueueFinalize(ctx, payload, allPagesAffected, freshTitleBySlug, changes) } // Close postprocess.wiki spans for every successfully-mapped doc. // Span duration now spans map + reduce + index rebuild + cleanup + // cross-link injection + publish, matching the wall-clock window // the user thinks of as "wiki processing for this knowledge". // Per-doc page write outcomes are summarised in the output so the // trace viewer can show how many of the doc's extracted pages // actually landed (vs. dropped because reduce-phase generation // failed). failedAdditionSlugCount := len(failedAdditionSlugs) for _, r := range docResults { if r == nil { continue } // A successfully-mapped doc is terminal for its wiki op, so // release the knowledge's slot in pending_subtasks_count (the row // promotes to completed once the counter hits zero). Done before // the WikiSpan nil-check below so a doc that had no attempt to // attach a span to still drains its counter slot. The matching +1 // is seeded by KnowledgePostProcess.SetFinalizing. // // EXCEPT docs with an unapplied slug (contended lock or reduce // error): those are re-queued below, so keep their finalizing slot // held — the retry (or the dead-letter drain in requeueFailedOps) // releases it once the op reaches a real terminal state. if _, unapplied := unappliedSlugKIDs[r.KnowledgeID]; !unapplied { s.finalizeWikiSubtask(ctx, r.KnowledgeID) } if r.WikiSpan == nil { continue } writtenPages := make([]map[string]string, 0, len(r.Pages)) droppedPages := make([]map[string]string, 0) for _, p := range r.Pages { entry := map[string]string{ "slug": p.Slug, "title": previewText(p.Title, 80), } if _, bad := failedAdditionSlugs[p.Slug]; bad { droppedPages = append(droppedPages, entry) continue } writtenPages = append(writtenPages, entry) } output := types.JSONMap{ "pages_written": len(writtenPages), "pages_dropped": len(droppedPages), "pages_total": len(r.Pages), "failed_slug_writes": failedAdditionSlugCount, "pages_written_preview": writtenPages, } if len(droppedPages) > 0 { output["pages_dropped_preview"] = droppedPages } for k, v := range r.MapStats { output[k] = v } s.tracker().EndSpan(ctx, r.WikiSpan, output) } // Failed-map docs already had FailSpan called inside // mapOneDocument (the failedOps path returns before reaching // docResults). Nothing extra to do here for them. // Fold documents with an unapplied slug (contended lock or reduce error) // into failedOps so they are neither trimmed nor promoted to completed: // requeueFailedOps then runs them through the same fail_count budget // (retry now, dead-letter once the slug stays permanently hot/broken) as // a map-phase failure. A doc already counted as a map failure is skipped // to avoid a double fail_count bump. if len(unappliedSlugKIDs) > 0 { failedKIDs := make(map[string]struct{}, len(failedOps)) for _, op := range failedOps { failedKIDs[op.KnowledgeID] = struct{}{} } for _, op := range pendingOps { if _, unapplied := unappliedSlugKIDs[op.KnowledgeID]; !unapplied { continue } if _, already := failedKIDs[op.KnowledgeID]; already { continue } failedOps = append(failedOps, op) failedKIDs[op.KnowledgeID] = struct{}{} } } // Build the trim set: rows that should be removed from // task_pending_ops. We start from the full peekedIDs (every row we // pulled, even ones de-duplicated by knowledge_id) and subtract // any failed op's dbID — those need to stay in place so the // requeueFailedOps path can decide between retry and dead-letter. failedIDSet := make(map[int64]struct{}, len(failedOps)) for _, op := range failedOps { if op.dbID != 0 { failedIDSet[op.dbID] = struct{}{} } } trimIDs := make([]int64, 0, len(peekedIDs)) for _, id := range peekedIDs { if _, fail := failedIDSet[id]; fail { continue } trimIDs = append(trimIDs, id) } s.trimPendingList(ctx, trimIDs) // Process failed ops: increment fail_count and dead-letter once // the cap is hit. Must come AFTER trim so successful siblings are // already gone from the queue — otherwise a follow-up batch could // re-pick them up. if len(failedOps) > 0 { s.requeueFailedOps(ctx, payload, failedOps) } // All claimed rows have now reached a terminal state (deleted on success, // released for retry, or dead-lettered), so disarm the abnormal-exit // release net. claimsSettled = true logger.Infof(ctx, "wiki ingest: batch completed for KB %s, %d ops, %d pages affected", payload.KnowledgeBaseID, len(pendingOps), len(allPagesAffected)) // Pace the follow-up: on a rate-limit trip, back off so the per-minute // window can reset instead of retrying the failed docs immediately. followUpDelay := wikiFollowUpDelay if rateLimited { followUpDelay = wikiRateLimitBackoff logger.Warnf(ctx, "wiki ingest: KB %s hit upstream rate limiting, backing off follow-up to %s", payload.KnowledgeBaseID, followUpDelay) } followUpScheduled = s.scheduleFollowUp(ctx, payload, followUpDelay) return nil } // ProcessWikiFinalize runs the debounced, per-KB KB-global convergence pass: // index-intro rebuild, dead-link cleanup, and cross-link injection. It drains // the finalize lane of task_pending_ops (written by ProcessWikiIngest via // enqueueFinalize) so a burst of documents rebuilds the index ONCE instead of // once per 5-doc batch. func (s *wikiIngestService) ProcessWikiFinalize(ctx context.Context, t *asynq.Task) error { startedAt := time.Now() var payload WikiIngestPayload if err := json.Unmarshal(t.Payload(), &payload); err != nil { return fmt.Errorf("wiki finalize: unmarshal payload: %w", err) } ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID) if payload.Language != "" { ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language) } if s.pendingRepo == nil { return nil } // Per-KB finalize lock, separate from the ingest active lock so finalize // and ingest batches never block each other. Coalescing via asynq.TaskID // already keeps at most one finalize pending per KB; this guards the // reschedule-overlap window against a concurrent index-page write. if s.redisClient != nil { key := "wiki:finalize:active:" + payload.KnowledgeBaseID acquired, err := s.redisClient.SetNX(ctx, key, "1", wikiFinalizeLockTTL).Result() if err != nil { // Fail CLOSED: proceeding unlocked would let two finalize runs // drain the same PeekBatch rows and double-rebuild the index page // (GetPage→modify→UpdatePage lost update). Return an error so // asynq retries the (coalesced) finalize once Redis recovers, // instead of racing a concurrent finalize. logger.Warnf(ctx, "wiki finalize: SetNX failed for KB %s: %v (retrying)", payload.KnowledgeBaseID, err) return fmt.Errorf("wiki finalize: acquire lock: %w", err) } else if !acquired { // Another finalize is running; it will drain the lane and // reschedule if rows remain. Safe to no-op. return nil } lockCtx, cancelLock := context.WithCancel(context.Background()) defer func() { cancelLock() s.redisClient.Del(context.Background(), key) }() go func() { ticker := time.NewTicker(wikiFinalizeLockRenew) defer ticker.Stop() for { select { case <-lockCtx.Done(): return case <-ticker.C: s.redisClient.Expire(context.Background(), key, wikiFinalizeLockTTL) } } }() } else { if _, loaded := s.liteFinalizeLocks.LoadOrStore(payload.KnowledgeBaseID, struct{}{}); loaded { return nil } defer s.liteFinalizeLocks.Delete(payload.KnowledgeBaseID) } rows, err := s.pendingRepo.PeekBatch(ctx, wikiFinalizeTaskType, wikiTaskScope, payload.KnowledgeBaseID, wikiFinalizeMaxRows) if err != nil { return fmt.Errorf("wiki finalize: peek: %w", err) } if len(rows) == 0 { return nil } kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID) if err != nil { return fmt.Errorf("wiki finalize: get KB: %w", err) } // Aggregate the drained rows into: affected slugs (dedup), fresh cross-link // refs, and the index-intro change description. We collect ids up front so // we can drain the lane even on the KB-disabled short-circuit below. ids := make([]int64, 0, len(rows)) affectedSet := make(map[string]struct{}, len(rows)) var affectedSlugs []string var freshRefs []linkRef var changeDesc strings.Builder for _, r := range rows { ids = append(ids, r.ID) if len(r.Payload) == 0 { continue } var row wikiFinalizeRow if err := json.Unmarshal(r.Payload, &row); err != nil { logger.Warnf(ctx, "wiki finalize: unmarshal row id=%d failed: %v", r.ID, err) continue } if row.Change != nil { if row.Change.Action == wikiFinalizeRemoved { fmt.Fprintf(&changeDesc, "\n%s\n%s\n\n\n", row.Change.DocTitle, row.Change.DocSummary) } else { fmt.Fprintf(&changeDesc, "\n%s\n%s\n\n\n", row.Change.DocTitle, row.Change.DocSummary) } continue } if row.Slug != "" { if _, ok := affectedSet[row.Slug]; !ok { affectedSet[row.Slug] = struct{}{} affectedSlugs = append(affectedSlugs, row.Slug) } if row.Title != "" { freshRefs = append(freshRefs, linkRef{slug: row.Slug, matchText: row.Title}) } } } // KB flipped away from wiki (deleted / type change) — drain the lane so the // rows don't accumulate, then stop. if !kb.IsWikiEnabled() { s.trimPendingList(ctx, ids) return nil } synthesisModelID := "" if kb.WikiConfig != nil { synthesisModelID = kb.WikiConfig.SynthesisModelID } if synthesisModelID == "" { synthesisModelID = kb.SummaryModelID } if synthesisModelID == "" { // No model to rebuild the index with; still run the pure-text passes, // then drain. Missing model is a config gap, not a transient error. logger.Warnf(ctx, "wiki finalize: no synthesis model for KB %s, skipping index rebuild", payload.KnowledgeBaseID) } batchCtx := s.newWikiBatchContext(payload.KnowledgeBaseID, kb.WikiConfig) lang := types.LanguageNameFromContext(ctx) indexRebuilt := false if changeDesc.Len() > 0 && synthesisModelID != "" { chatModel, mErr := s.modelService.GetChatModel(ctx, synthesisModelID) if mErr != nil { logger.Warnf(ctx, "wiki finalize: get chat model failed: %v", mErr) } else if err := s.rebuildIndexPage(ctx, chatModel, payload, changeDesc.String(), lang, batchCtx.ContentInstructions); err != nil { logger.Warnf(ctx, "wiki finalize: rebuild index failed: %v", err) } else { indexRebuilt = true } } if len(affectedSlugs) > 0 { s.cleanDeadLinks(ctx, payload.KnowledgeBaseID, affectedSlugs, batchCtx) s.injectCrossLinks(ctx, payload.KnowledgeBaseID, affectedSlugs, freshRefs, batchCtx) } // Drain the processed rows. Best-effort convergence mirrors the legacy // in-batch behaviour: a failed index rebuild is logged (not retried), // so we delete regardless to avoid re-doing the whole pass forever. s.trimPendingList(ctx, ids) // If more finalize rows landed while we were working, reschedule so they // get their own convergence pass. rescheduled := false if n, cErr := s.pendingRepo.PendingCount(ctx, wikiFinalizeTaskType, wikiTaskScope, payload.KnowledgeBaseID); cErr == nil && n > 0 { s.scheduleFinalize(ctx, payload) rescheduled = true } logger.Infof(ctx, "wiki finalize: kb=%s rows=%d affected_slugs=%d index_rebuilt=%v rescheduled=%v elapsed=%s", payload.KnowledgeBaseID, len(rows), len(affectedSlugs), indexRebuilt, rescheduled, time.Since(startedAt).Round(time.Millisecond), ) return nil } func (s *wikiIngestService) mapOneDocument( ctx context.Context, chatModel chat.Chat, payload WikiIngestPayload, op WikiPendingOp, batchCtx *WikiBatchContext, ) (*docIngestResult, []SlugUpdate, error) { docStartedAt := time.Now() knowledgeID := op.KnowledgeID lang := types.LanguageLocaleName(op.Language) // Open a postprocess.wiki subspan under the parent attempt's // postprocess stage so the actual per-doc work (LLM extraction + // summary + classification) shows up in the trace tree. Returns // nil when the parent attempt is gone (no panic on missing // lookups — span tracker is best-effort). wikiSpan := s.beginWikiSubspan(ctx, knowledgeID, types.JSONMap{ "language": lang, "knowledge_base_id": payload.KnowledgeBaseID, }) // Guard against the ingest/delete race: if the user deleted the doc while // this task was queued (wikiIngestDelay = 30s) or while an earlier stage // was in flight, we must NOT proceed to LLM extraction — doing so would // create wiki pages whose source_refs point at a ghost knowledge ID, // permanently unreachable via wiki_read_source_doc. if s.isKnowledgeGone(ctx, payload.KnowledgeBaseID, knowledgeID) { logger.Infof(ctx, "wiki ingest: knowledge %s has been deleted, skip map", knowledgeID) s.tracker().SkipSpan(ctx, wikiSpan, "knowledge_deleted") return nil, nil, nil } chunks, err := s.chunkRepo.ListChunksByKnowledgeID(ctx, payload.TenantID, knowledgeID) if err != nil { s.tracker().FailSpan(ctx, wikiSpan, "LIST_CHUNKS_FAILED", err.Error(), err) return nil, nil, fmt.Errorf("get chunks: %w", err) } if len(chunks) == 0 { logger.Infof(ctx, "wiki ingest: document %s has no chunks, skip", knowledgeID) s.tracker().SkipSpan(ctx, wikiSpan, "no_chunks") return nil, nil, nil } content := reconstructEnrichedContent(ctx, s.chunkRepo, payload.TenantID, chunks) rawRuneCount := len([]rune(content)) if len([]rune(content)) > maxContentForWiki { content = string([]rune(content)[:maxContentForWiki]) } logger.Infof(ctx, "wiki ingest: doc %s chunks=%d content_len(raw=%d,truncated=%d)", knowledgeID, len(chunks), rawRuneCount, len([]rune(content))) // Refuse to run LLM-based extraction when the document carries no real // text — e.g. a scanned PDF whose pages were converted to images but where // VLM OCR produced nothing usable. Without this guard the LLM would have // only image markup left and would happily fabricate entities/concepts. if !hasSufficientTextContent(content) { logger.Warnf(ctx, "wiki ingest: doc %s has insufficient text content after stripping image markup (raw_len=%d), skipping LLM extraction", knowledgeID, rawRuneCount, ) s.tracker().SkipSpan(ctx, wikiSpan, "insufficient_text_content") return nil, nil, nil } docTitle := knowledgeID if kn, err := s.knowledgeSvc.GetKnowledgeByIDOnly(ctx, knowledgeID); err == nil && kn != nil && kn.Title != "" { docTitle = kn.Title } else { for _, ch := range chunks { if ch.Content != "" { lines := strings.SplitN(ch.Content, "\n", 2) if len(lines) > 0 && len(lines[0]) > 0 && len(lines[0]) < 200 { docTitle = strings.TrimPrefix(strings.TrimSpace(lines[0]), "# ") break } } } } // Citation source reference. We deliberately use only the knowledge ID // (not docTitle, which is typically the upload filename) so the filename // does not leak into citation strings that downstream LLM prompts may // surface during wiki page editing. sourceRef := knowledgeID oldPageSlugs := s.getExistingPageSlugsForKnowledge(ctx, payload.KnowledgeBaseID, knowledgeID) // Pass 0: lightweight candidate slug extraction (skeleton only). // On failure we fall back to the legacy single-shot extractor so the doc // still gets ingested, just without chunk-level citations. var ( extractedEntities []extractedItem extractedConcepts []extractedItem slugItems map[string]extractedItem pass0Failed bool ) logger.Infof(ctx, "wiki ingest: pass 0 — extracting candidate slugs for %s", knowledgeID) extractSpan := s.tracker().BeginSubSpan(ctx, wikiSpan, "postprocess.wiki.extract", types.SpanKindSubSpan, types.JSONMap{ "content_chars": utf8.RuneCountInString(content), "old_pages": len(oldPageSlugs), }) extractedEntities, extractedConcepts, slugItems, err = s.extractCandidateSlugs(ctx, chatModel, payload.KnowledgeBaseID, content, lang, oldPageSlugs, batchCtx) if err != nil { logger.Warnf(ctx, "wiki ingest: pass 0 failed for %s (%v) — falling back to legacy extractor", knowledgeID, err) pass0Failed = true extractedEntities, extractedConcepts, slugItems, err = s.extractEntitiesAndConceptsNoUpsert(ctx, chatModel, payload.KnowledgeBaseID, content, lang, oldPageSlugs, batchCtx) if err != nil { logger.Warnf(ctx, "wiki ingest: legacy fallback also failed for %s: %v", knowledgeID, err) s.tracker().FailSpan(ctx, extractSpan, "EXTRACT_FAILED", err.Error(), err) s.tracker().FailSpan(ctx, wikiSpan, "EXTRACT_FAILED", err.Error(), err) return nil, nil, err } } s.tracker().EndSpan(ctx, extractSpan, types.JSONMap{ "entities": len(extractedEntities), "concepts": len(extractedConcepts), "pass0_fallback": pass0Failed, "entities_preview": previewExtractedItems(extractedEntities, 8), "concepts_preview": previewExtractedItems(extractedConcepts, 8), }) // Build slug listing for Summary's wiki-link input. var summaryExtractedPages []string for slug := range slugItems { summaryExtractedPages = append(summaryExtractedPages, slug) } // Wiki summary slug is derived from the knowledge ID rather than the // docTitle (which is typically the upload filename). Filename-based slugs // like "summary/mx5280-pdf" expose the filename in cross-link contexts // that downstream LLM prompts read; a UUID-based slug is uglier but // hallucination-safe. summarySlug := fmt.Sprintf("summary/%s", slugify(knowledgeID)) var slugListing string for _, slug := range summaryExtractedPages { if item, ok := slugItems[slug]; ok { aliases := "" if len(item.Aliases) > 0 { aliases = fmt.Sprintf(" (Aliases: %s)", strings.Join(item.Aliases, ", ")) } slugListing += fmt.Sprintf("- [[%s]] = %s%s\n", slug, item.Name, aliases) } else { slugListing += fmt.Sprintf("- [[%s]]\n", slug) } } // Summary and chunk classification are independent given Pass 0 output — // run them in parallel. Summary handles wiki-link injection; classification // attaches concrete chunk IDs to each candidate slug. var ( summaryContent string summaryErr error citations map[string][]string newSlugs []newSlugFromCitation batchCount int ) // Both calls run in parallel goroutines under the same wikiSpan // parent — their subspans will visually overlap in the trace view, // which correctly reflects their wall-clock concurrency. summarySpan := s.tracker().BeginSubSpan(ctx, wikiSpan, "postprocess.wiki.summary", types.SpanKindSubSpan, types.JSONMap{ "content_chars": utf8.RuneCountInString(content), "extracted_slugs": len(summaryExtractedPages), }) var classifySpan *Span if !pass0Failed { classifySpan = s.tracker().BeginSubSpan(ctx, wikiSpan, "postprocess.wiki.classify", types.SpanKindSubSpan, types.JSONMap{ "chunks": len(chunks), "candidates": len(extractedEntities) + len(extractedConcepts), }) } var wg sync.WaitGroup wg.Add(2) go func() { defer wg.Done() summaryContent, summaryErr = s.generateWithTemplate(ctx, chatModel, agent.WikiSummaryPrompt, map[string]string{ "Content": content, "Language": lang, "ExtractedSlugs": slugListing, "CustomInstructions": batchCtx.ContentInstructions, "InstructionScope": "wiki_content", }) if summaryErr != nil { s.tracker().FailSpan(ctx, summarySpan, "SUMMARY_FAILED", summaryErr.Error(), summaryErr) } else { sumLine, sumBody := splitSummaryLine(summaryContent) s.tracker().EndSpan(ctx, summarySpan, types.JSONMap{ "chars": utf8.RuneCountInString(summaryContent), "summary_line": previewText(sumLine, 160), "body_preview": previewText(sumBody, 320), }) } }() go func() { defer wg.Done() // Skip citation pass when Pass 0 has fallen back to the legacy path — // the legacy output already contains paraphrased Details, so chunk // citations would be redundant and we'd spend LLM calls for nothing. if pass0Failed { citations = map[string][]string{} return } candidatesXML := renderCandidateSlugsXML(extractedEntities, extractedConcepts) citations, newSlugs, batchCount = s.classifyChunkCitations(ctx, chatModel, candidatesXML, chunks, lang, batchCtx) s.tracker().EndSpan(ctx, classifySpan, types.JSONMap{ "cited_slugs": len(citations), "new_slugs": len(newSlugs), "batches": batchCount, "top_cited": topCitedSlugs(citations, 8), "new_slugs_sample": previewNewSlugs(newSlugs, 8), }) }() wg.Wait() // Merge citations back into the item structs (non-failing; items without // citations simply keep their Description+Details fallback). var uncited int extractedEntities, extractedConcepts, uncited = mergeCitationsIntoItems(extractedEntities, extractedConcepts, citations, newSlugs) // Rebuild slugItems so stale entries (for slugs that did not survive the // merge) and brand-new slugs discovered by the citation pass are both // reflected in summaryExtractedPages tracking. slugItems = make(map[string]extractedItem, len(extractedEntities)+len(extractedConcepts)) for _, item := range extractedEntities { if item.Slug != "" && item.Name != "" { slugItems[item.Slug] = item } } for _, item := range extractedConcepts { if item.Slug != "" && item.Name != "" { slugItems[item.Slug] = item } } // extractedPages records every wiki page this document materialized // (entities, concepts, plus the summary page appended below). The // slug is used for link/retract bookkeeping; the title is captured // for the log feed so the user sees "提供本学位在线验证报告查询…" // rather than "entity/xue-xin-wang". extractedPages := make([]types.WikiLogPageRef, 0, len(slugItems)+1) for slug, item := range slugItems { title := item.Name if title == "" { title = slug } extractedPages = append(extractedPages, types.WikiLogPageRef{Slug: slug, Title: title}) } // Count total distinct chunks cited across all slugs for logging. citedChunkSet := make(map[string]bool) for _, ids := range citations { for _, id := range ids { citedChunkSet[id] = true } } var updates []SlugUpdate // docSummaryLine is the one-sentence headline used for terse log/audit // previews and for blocks in retract prompts. // docSummary is the full summary body attached to each entity/concept // update so the editor model gets rich framing in . var docSummaryLine string var docSummary string if summaryErr != nil { // Summary is the headline artifact of an ingested document — a // document with no summary page is half-ingested and leaves the // entity/concept updates hanging without a root to link back to // from the index. Historically we just logged and moved on, // which meant a single transient 504 permanently dropped the // summary page for that document. // // Returning an error here sends the op to failedOps (see the // map-phase loop in ProcessWikiIngest), which requeueFailedOps // appends back onto the pending list so the next batch retries. // The internal retries in generateWithTemplate already exhaust // the LLM's own transient-error budget before we give up here. logger.Errorf(ctx, "wiki ingest: generate summary failed for %s, will requeue: %v", knowledgeID, summaryErr) s.tracker().FailSpan(ctx, wikiSpan, "SUMMARY_FAILED", summaryErr.Error(), summaryErr) return nil, nil, fmt.Errorf("generate summary: %w", summaryErr) } sumLine, sumBody := splitSummaryLine(summaryContent) if sumBody == "" { sumBody = summaryContent } if sumLine == "" { sumLine = docTitle } docSummaryLine = sumLine docSummary = sumBody if strings.TrimSpace(docSummary) == "" { docSummary = sumLine } updates = append(updates, SlugUpdate{ Slug: summarySlug, Type: types.WikiPageTypeSummary, DocTitle: docTitle, KnowledgeID: knowledgeID, SourceRef: sourceRef, Language: lang, SummaryLine: sumLine, SummaryBody: sumBody, }) extractedPages = append(extractedPages, types.WikiLogPageRef{Slug: summarySlug, Title: docTitle}) // Entities for _, item := range extractedEntities { if item.Slug != "" { updates = append(updates, SlugUpdate{ Slug: item.Slug, Type: types.WikiPageTypeEntity, Item: item, DocTitle: docTitle, KnowledgeID: knowledgeID, SourceRef: sourceRef, Language: lang, SourceChunks: item.SourceChunks, DocSummary: docSummary, }) } } // Concepts for _, item := range extractedConcepts { if item.Slug != "" { updates = append(updates, SlugUpdate{ Slug: item.Slug, Type: types.WikiPageTypeConcept, Item: item, DocTitle: docTitle, KnowledgeID: knowledgeID, SourceRef: sourceRef, Language: lang, SourceChunks: item.SourceChunks, DocSummary: docSummary, }) } } // Reconcile old page set against new extraction. // // Three cases: // // (a) oldSlug ∉ new → "retractStale": the doc no longer mentions this // page's subject, so strip its ref (and possibly delete the page // if this was the only source). Passes the NEW content as the // retract context — if the LLM finds matching facts it trims // them, otherwise the retract is a near no-op, which is fine. // // (b) oldSlug ∈ new AND slug is an entity/concept page → reparse // swap: emit BOTH a "retract" (carrying the doc's PRIOR summary // body as the old-version signal) AND the normal addition. The // reduce stage sees HasAdditions=1 + HasRetractions=1 and the // WikiPageModifyPrompt correctly tells the editor model to // remove the old K section and add the new K section in one // pass — giving us replace-not-append semantics that "append // new K on top of old K" would otherwise violate. // // (c) oldSlug ∈ new AND slug is a summary page (summary/...) → // nothing to do here. reduceSlugUpdates' summary branch // unconditionally overwrites the whole page from the new // SummaryBody, so emitting an extra retract would just be // dead weight that the summary branch discards anyway. // // priorContribution is the doc's LAST summary body, fetched lazily // at this point (rather than pre-loaded into the batch context). // Empty on first-ever ingest — in that case oldPageSlugs is also // empty, so we never consult it. priorContribution := batchCtx.SummaryContentByKnowledgeID(ctx, knowledgeID) newSlugSet := make(map[string]bool, len(extractedPages)) for _, ns := range extractedPages { newSlugSet[ns.Slug] = true } var reparseOverlap, staleCount int for oldSlug := range oldPageSlugs { if newSlugSet[oldSlug] { // Skip summary slugs — they're overwritten wholesale by the // summary update, retract would be ignored downstream. if strings.HasPrefix(oldSlug, "summary/") { continue } reparseOverlap++ updates = append(updates, SlugUpdate{ Slug: oldSlug, Type: "retract", RetractDocContent: priorContribution, DocTitle: docTitle, KnowledgeID: knowledgeID, Language: lang, }) continue } staleCount++ updates = append(updates, SlugUpdate{ Slug: oldSlug, Type: "retractStale", RetractDocContent: content, DocTitle: docTitle, KnowledgeID: knowledgeID, Language: lang, }) } logger.Infof(ctx, "wiki ingest: mapped knowledge %s title=%q candidates=%d chunks=%d batches=%d cited_chunks=%d uncited_slugs=%d new_slugs=%d updates=%d reparse_slugs=%d stale_slugs=%d pass0_fallback=%v elapsed=%s", knowledgeID, previewText(docTitle, 80), len(slugItems), len(chunks), batchCount, len(citedChunkSet), uncited, len(newSlugs), len(updates), reparseOverlap, staleCount, pass0Failed, time.Since(docStartedAt).Round(time.Millisecond), ) // Map-phase metrics get attached to the postprocess.wiki span's // output, but we do NOT EndSpan here — the batch driver keeps the // span open through reduce + index rebuild + cross-link injection // + page publish, then closes it once this doc's pages have all // been written. That way the span's duration reflects the full // "wiki processing for this knowledge" time the user sees in the // trace viewer, not just the LLM extraction slice. mapStats := types.JSONMap{ "doc_title": previewText(docTitle, 120), "chunks": len(chunks), "candidate_slugs": len(slugItems), "cited_chunks": len(citedChunkSet), "uncited_slugs": uncited, "new_slugs": len(newSlugs), "updates": len(updates), "reparse_slugs": reparseOverlap, "stale_slugs": staleCount, "extracted_pages": len(extractedPages), "summary_chars": utf8.RuneCountInString(docSummary), "pass0_fallback": pass0Failed, "classify_batches": batchCount, "summary_preview": previewText(docSummaryLine, 160), } return &docIngestResult{ KnowledgeID: knowledgeID, DocTitle: docTitle, Summary: docSummaryLine, Pages: extractedPages, MapStats: mapStats, WikiSpan: wikiSpan, }, updates, nil } func (s *wikiIngestService) extractEntitiesAndConceptsNoUpsert( ctx context.Context, chatModel chat.Chat, kbID string, content, lang string, oldPageSlugs map[string]bool, batchCtx *WikiBatchContext, ) ([]extractedItem, []extractedItem, map[string]extractedItem, error) { // Only entity/* and concept/* slugs are relevant for LLM slug-continuity — // summary slugs are code-generated from the knowledge ID and never appear // in the extraction output, so including them just wastes tokens and risks // confusing the model. var prevSlugsText string if len(oldPageSlugs) > 0 { var sb strings.Builder for slug := range oldPageSlugs { if !strings.HasPrefix(slug, "entity/") && !strings.HasPrefix(slug, "concept/") { continue } fmt.Fprintf(&sb, "- %s\n", slug) } prevSlugsText = sb.String() } if prevSlugsText == "" { prevSlugsText = "(none — this is a new document)" } extractionJSON, err := s.generateWithTemplate(ctx, chatModel, agent.WikiKnowledgeExtractPrompt, map[string]string{ "Content": content, "Language": lang, "PreviousSlugs": prevSlugsText, "CustomInstructions": batchCtx.ExtractionInstructions, "InstructionScope": "wiki_extraction", }) if err != nil { return nil, nil, nil, fmt.Errorf("combined extraction failed: %w", err) } extractionJSON = cleanLLMJSON(extractionJSON) var result combinedExtraction if err := json.Unmarshal([]byte(extractionJSON), &result); err != nil { logger.Warnf(ctx, "wiki ingest: failed to parse combined extraction JSON: %v\nRaw: %s", err, extractionJSON) return nil, nil, nil, fmt.Errorf("parse combined extraction JSON: %w", err) } // Dedup pre-filter is dispatched against the wiki page repo via // pg_trgm (see deduplicateExtractedBatch). Until the trgm path // lands the dedup pre-filter degrades to "no dedup" which is the // safe default — the LLM merge call simply doesn't get a candidate // list and the items pass through unchanged. result.Entities, result.Concepts = s.deduplicateExtractedBatch( ctx, chatModel, kbID, result.Entities, result.Concepts, ) slugItems := make(map[string]extractedItem) for _, item := range result.Entities { if item.Slug != "" && item.Name != "" { slugItems[item.Slug] = item } } for _, item := range result.Concepts { if item.Slug != "" && item.Name != "" { slugItems[item.Slug] = item } } return result.Entities, result.Concepts, slugItems, nil } // reduceSlugUpdates returns: // - changed: whether the wiki page was created or updated // - affectedType: "ingest" or "retract" — drives downstream bookkeeping // - additionFailed: true iff the slug had entity/concept additions queued // AND the WikiPageModifyPrompt LLM call failed, so no page exists/was // refreshed for it. Callers use this to sanitize dead [[slug]] links // elsewhere (e.g. in the doc's summary page) and to drop the slug from // the wiki log feed so users don't see a clickable entry that 404s. // - err: transport / repo error from the persisted upsert. func (s *wikiIngestService) reduceSlugUpdates( ctx context.Context, chatModel chat.Chat, kbID string, slug string, updates []SlugUpdate, tenantID uint64, batchCtx *WikiBatchContext, kidToWikiSpan map[string]*Span, ) (changed bool, affectedType string, additionFailed bool, err error) { // Final safety net for the ingest/delete race: between Map (which already // checks isKnowledgeGone) and Reduce there is a long LLM call where the // source document may be deleted. Drop any addition/summary updates whose // knowledge no longer exists so we don't resurrect a ghost source_ref. // Retract updates are kept — they actively remove refs, which is what we // want when the doc is gone. updates = s.filterLiveUpdates(ctx, kbID, updates) if len(updates) == 0 { return false, "", false, nil } // Per-slug page span attribution: a single slug can receive // contributions from multiple docs in the same batch (entity / // concept pages aggregate across sources). We attach the // postprocess.wiki.page[slug] subspan under whichever // contributing doc's wikiSpan is encountered first in the updates // list — span tree topology only allows one parent. Every // contributing knowledge id is recorded in the span's `contributors` // output so users can still see the full attribution. Pages whose // only contributors had no wikiSpan (e.g. their parse attempt // already closed and was archived) simply get a nil pageSpan, // which the tracker helpers no-op on. var ( pageSpan *Span contributors []string ) { seen := make(map[string]bool, len(updates)) for _, u := range updates { kid := u.KnowledgeID if kid == "" || seen[kid] { continue } seen[kid] = true contributors = append(contributors, kid) if pageSpan == nil { if sp, ok := kidToWikiSpan[kid]; ok && sp != nil { pageSpan = s.tracker().BeginSubSpan(ctx, sp, fmt.Sprintf("postprocess.wiki.page[%s]", slug), types.SpanKindSubSpan, types.JSONMap{ "slug": slug, "updates": len(updates), "contributors": contributors, }) } } } } var page *types.WikiPage // Deferred output captures `&page` so it observes the post-merge // state (title, page type, content snippet) at function return — // that's what's actually useful in the trace viewer, not the // stale pre-reduce shell that exists when the defer is registered. defer func() { if pageSpan == nil { return } if err != nil { s.tracker().FailSpan(ctx, pageSpan, "REDUCE_FAILED", err.Error(), err) return } if !changed { s.tracker().SkipSpan(ctx, pageSpan, "no_change") return } out := types.JSONMap{ "affected_type": affectedType, "addition_failed": additionFailed, "contributors": contributors, } if page != nil { out["page_title"] = previewText(page.Title, 160) out["page_type"] = string(page.PageType) out["page_summary"] = previewText(page.Summary, 200) out["content_preview"] = previewText(page.Content, 320) out["source_refs"] = len(page.SourceRefs) out["chunk_refs"] = len(page.ChunkRefs) out["aliases"] = []string(page.Aliases) } s.tracker().EndSpan(ctx, pageSpan, out) }() page, err = s.wikiService.GetPageBySlug(ctx, kbID, slug) exists := (err == nil && page != nil) if !exists { hasAdditions := false for _, u := range updates { if u.Type == types.WikiPageTypeEntity || u.Type == types.WikiPageTypeConcept || u.Type == "summary" { hasAdditions = true break } } if !hasAdditions { return false, "", false, nil } page = &types.WikiPage{ ID: uuid.New().String(), TenantID: tenantID, KnowledgeBaseID: kbID, Slug: slug, Status: types.WikiPageStatusDraft, SourceRefs: types.StringArray{}, Aliases: types.StringArray{}, } // Reset err: GetPageBySlug returned "not found" which we just // handled by synthesizing the page. Don't leak that error to // the named return — subsequent assignments would mask it // anyway, but be explicit. err = nil } affectedType = "ingest" var summaryUpdate *SlugUpdate var retracts []SlugUpdate var additions []SlugUpdate for i, u := range updates { if u.Type == "summary" { summaryUpdate = &updates[i] } else if u.Type == "retract" || u.Type == "retractStale" { retracts = append(retracts, u) affectedType = "retract" } else if u.Type == types.WikiPageTypeEntity || u.Type == types.WikiPageTypeConcept { additions = append(additions, u) affectedType = "ingest" // Additions override retracts type } } if summaryUpdate != nil { page.Title = summaryUpdate.DocTitle + " - Summary" page.Content = summaryUpdate.SummaryBody page.Summary = summaryUpdate.SummaryLine page.PageType = types.WikiPageTypeSummary page.SourceRefs = appendUnique(page.SourceRefs, summaryUpdate.SourceRef) // Summary pages don't carry chunk-level citations (they are document- // level synopses generated from the whole content). Clear any stale // chunk refs that may remain if this slug was once an entity page // and got converted to a summary page. page.ChunkRefs = types.StringArray{} changed = true if exists { _, err = s.wikiService.UpdatePage(ctx, page) } else { _, err = s.wikiService.CreatePage(ctx, page) } return changed, affectedType, false, err } var remainingSourcesContent strings.Builder var deletedContent strings.Builder var relatedSlugs strings.Builder var newContentBuilder strings.Builder var docTitles []string var language string if len(retracts) > 0 { language = retracts[0].Language for _, r := range retracts { fmt.Fprintf(&deletedContent, "\n%s\n\n%s\n\n\n\n", r.DocTitle, r.RetractDocContent) } retractKIDs := make(map[string]bool) for _, r := range retracts { retractKIDs[r.KnowledgeID] = true } for _, ref := range page.SourceRefs { pipeIdx := strings.Index(ref, "|") var refKnowledgeID, refTitle string if pipeIdx > 0 { refKnowledgeID = ref[:pipeIdx] refTitle = ref[pipeIdx+1:] } else { refKnowledgeID = ref refTitle = ref } if retractKIDs[refKnowledgeID] { continue } if content := batchCtx.SummaryContentByKnowledgeID(ctx, refKnowledgeID); content != "" { fmt.Fprintf(&remainingSourcesContent, "\n%s\n\n%s\n\n\n\n", refTitle, content) } else { fmt.Fprintf(&remainingSourcesContent, "\n%s\n\n(summary not available)\n\n\n\n", refTitle) } } if remainingSourcesContent.Len() == 0 { remainingSourcesContent.WriteString("(no remaining sources)") } newRefs := types.StringArray{} for _, ref := range page.SourceRefs { pipeIdx := strings.Index(ref, "|") refKnowledgeID := ref if pipeIdx > 0 { refKnowledgeID = ref[:pipeIdx] } if !retractKIDs[refKnowledgeID] { newRefs = append(newRefs, ref) } } page.SourceRefs = newRefs } if len(additions) > 0 { language = additions[0].Language // Resolve SourceChunks → chunk contents in a single batched query per // knowledge ID, so the block can quote the chunks // verbatim instead of relying on the short Details paraphrase. chunkContentByID := s.resolveCitedChunks(ctx, tenantID, additions) for _, add := range additions { cited := collectCitedChunkContent(add.SourceChunks, chunkContentByID) // Frame the chunks with the document-level summary body so the // editor model knows BOTH what the document is about AND what // kind of document it is (resume vs announcement vs product // page vs schedule). The one-sentence headline alone was too // terse to keep the editor grounded on longer or multi-topic // source documents, and calibrating tone (self-reported vs // third-party authoritative) benefits from the richer context. sourceCtx := strings.TrimSpace(add.DocSummary) sourceCtxBlock := "" if sourceCtx != "" { sourceCtxBlock = fmt.Sprintf("\n%s\n\n", sourceCtx) } if cited != "" { fmt.Fprintf(&newContentBuilder, "\n%s\n%s\n**%s**: %s\n\n%s\n\n\n\n", add.DocTitle, sourceCtxBlock, add.Item.Name, add.Item.Description, cited) } else { // Fallback: no citations available (legacy path, citation pass // failed, or bad chunk IDs were filtered out) — stick with // the short Details summary so the page still gets real text. fmt.Fprintf(&newContentBuilder, "\n%s\n%s\n**%s**: %s\n\n%s\n\n\n\n", add.DocTitle, sourceCtxBlock, add.Item.Name, add.Item.Description, add.Item.Details) } docTitles = appendUnique(docTitles, add.DocTitle) for _, alias := range add.Item.Aliases { page.Aliases = appendUnique(page.Aliases, alias) } page.SourceRefs = appendUnique(page.SourceRefs, add.SourceRef) if page.Title == "" { page.Title = add.Item.Name } if page.PageType == "" { page.PageType = add.Type } } } if len(additions) > 0 || len(retracts) > 0 { titles := batchCtx.SlugTitleMany(ctx, []string(page.OutLinks)) for _, outSlug := range page.OutLinks { if title := titles[outSlug]; title != "" { fmt.Fprintf(&relatedSlugs, "- %s (%s)\n", outSlug, title) } } // Older generated pages may still contain short chunk aliases such as // [c003]. They are internal ingest metadata; keep the editor context // clean so a subsequent update cannot copy them into rewritten prose. existingContent := stripWikiInlineChunkCitations(page.Content) if !exists || existingContent == "" { existingContent = "(New page)" } hasAdditionsStr := "" if len(additions) > 0 { hasAdditionsStr = "1" } hasRetractionsStr := "" if len(retracts) > 0 { hasRetractionsStr = "1" } // Fall back gracefully if title/type are still unset (shouldn't happen // for well-formed updates — both get populated from `additions` above, // and retract-only paths require an existing page — but stay defensive // so we never feed the LLM an empty identity block). pageTitle := page.Title if pageTitle == "" { pageTitle = slug } pageType := string(page.PageType) if pageType == "" { pageType = "wiki page" } pageAliases := strings.Join(page.Aliases, ", ") var updatedContent string updatedContent, err = s.generateWithTemplate(ctx, chatModel, agent.WikiPageModifyPrompt, map[string]string{ "HasAdditions": hasAdditionsStr, "HasRetractions": hasRetractionsStr, "PageSlug": slug, "PageTitle": pageTitle, "PageType": pageType, "PageAliases": pageAliases, "ExistingContent": existingContent, "NewContent": newContentBuilder.String(), "DeletedContent": deletedContent.String(), "RemainingSourcesContent": remainingSourcesContent.String(), "AvailableSlugs": relatedSlugs.String(), "Language": language, "CustomInstructions": batchCtx.ContentInstructions, "InstructionScope": "wiki_content", }) if err == nil && updatedContent != "" { updatedSummary, updatedBody := splitSummaryLine(updatedContent) if updatedBody != "" { page.Content = updatedBody } else { page.Content = updatedContent } if updatedSummary != "" { page.Summary = updatedSummary } changed = true } else if err != nil { logger.Warnf(ctx, "wiki ingest: update/retract failed for slug %s: %v", slug, err) // Flag addition failures so the batch can sanitize stale // [[slug]] references in the doc's summary page and prune // the slug from log entries — otherwise the wiki feed shows // a clickable entry whose target page doesn't exist. // Retract-only failures don't poison anything (they leave // the existing page unchanged), so don't flag those. if len(additions) > 0 { additionFailed = true } // Don't propagate the LLM error to the named return: it has // already been logged, and the eg.Go caller would otherwise // log it a second time as "reduce failed for slug". err = nil } } // Apply the batch taxonomy plan, but only to pages that aren't already // filed — so brand-new pages get a coherent folder while previously-filed // or user-moved pages keep their placement (manual edits are authoritative). // The page's category_path cache is derived from folder_id downstream by // CreatePage/UpdatePage, so assigning the folder id is sufficient here. if page.FolderID == "" && batchCtx != nil { if fid := batchCtx.PlannedFolderID[slug]; fid != "" { page.FolderID = fid } } if changed { // Refresh chunk refs in-place on the page so they persist alongside // the rest of the row. Retract-only updates (no additions) preserve // the existing refs; addition rounds append the newly-cited chunks // on top of what was already there, deduplicated. page.ChunkRefs = mergeChunkRefs(page.ChunkRefs, additions) if exists { _, err = s.wikiService.UpdatePage(ctx, page) } else { _, err = s.wikiService.CreatePage(ctx, page) } return true, affectedType, additionFailed, err } return false, "", additionFailed, nil } // mergeChunkRefs unions the chunk IDs currently on the page with the ones // cited by this batch's additions, preserving insertion order and dropping // duplicates. Empty strings are filtered out so a malformed source_chunks // array can't leave junk in the column. // // A retract round with no additions leaves the current refs untouched — // retract-only paths don't carry chunk IDs (only knowledge IDs), and we // can't surgically filter without that info. The next time the slug is // re-materialized via additions the fresh chunks will overlay on top. func mergeChunkRefs(current types.StringArray, additions []SlugUpdate) types.StringArray { seen := make(map[string]bool, len(current)) out := make(types.StringArray, 0, len(current)) for _, id := range current { if id == "" || seen[id] { continue } seen[id] = true out = append(out, id) } for _, add := range additions { for _, chunkID := range add.SourceChunks { if chunkID == "" || seen[chunkID] { continue } seen[chunkID] = true out = append(out, chunkID) } } return out }