diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index 2ea2affe..0473d275 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -1297,7 +1297,7 @@ func (s *knowledgeService) cleanupWikiOnKnowledgeDelete(ctx context.Context, kno // wiki_ingest task that wakes up between here and the retract enqueue // below sees "knowledge gone" and bails out. s.markKnowledgeDeletedForWiki(ctx, kbID, knowledgeID) - s.scrubWikiPendingIngest(ctx, kbID, knowledgeID) + s.scrubWikiPendingIngest(ctx, kbID, knowledgeID, "cleanup") // Pull title/summary from the knowledge itself — do NOT read them from // existing wiki pages. In the race window wiki pages may not exist yet, @@ -1398,14 +1398,19 @@ func (s *knowledgeService) markKnowledgeDeletedForWiki(ctx context.Context, kbID } // scrubWikiPendingIngest removes queued WikiOpIngest entries for a knowledge -// from the debounced pending list, so the next batch doesn't re-ingest a -// document we're about to soft-delete. Retract entries stay put — the wiki -// still needs them to unlink referencing pages. +// from the debounced pending list. Used by both the delete path (we're about +// to soft-delete the doc, no point ingesting it) and the reparse path (the +// old chunks are about to vanish, so any pending ingest would either race +// with the cleanup or no-op on an empty chunk set — and the post-process +// task will enqueue a fresh ingest once new chunks land anyway). +// +// Retract entries stay put — delete still needs them to unlink referencing +// pages, and reparse never enqueues retracts for the doc being reparsed. // // We use LREM against JSON-encoded entries plus a best-effort raw-UUID // fallback for backward compatibility with the legacy format documented in // peekPendingList. -func (s *knowledgeService) scrubWikiPendingIngest(ctx context.Context, kbID, knowledgeID string) { +func (s *knowledgeService) scrubWikiPendingIngest(ctx context.Context, kbID, knowledgeID, reason string) { if s.redisClient == nil || kbID == "" || knowledgeID == "" { return } @@ -1417,7 +1422,7 @@ func (s *knowledgeService) scrubWikiPendingIngest(ctx context.Context, kbID, kno // is safe. items, err := s.redisClient.LRange(ctx, pendingKey, 0, -1).Result() if err != nil { - logger.Warnf(ctx, "wiki cleanup: failed to read pending list %s: %v", pendingKey, err) + logger.Warnf(ctx, "wiki %s: failed to read pending list %s: %v", reason, pendingKey, err) return } removed := 0 @@ -1444,10 +1449,38 @@ func (s *knowledgeService) scrubWikiPendingIngest(ctx context.Context, kbID, kno } } if removed > 0 { - logger.Infof(ctx, "wiki cleanup: scrubbed %d pending ingest ops for knowledge %s", removed, knowledgeID) + logger.Infof(ctx, "wiki %s: scrubbed %d pending ingest ops for knowledge %s", reason, removed, knowledgeID) } } +// prepareWikiForReparse is the reparse counterpart to +// cleanupWikiOnKnowledgeDelete. It aligns reparse with the same "pending +// queue hygiene" the delete path already enforces, without taking any +// destructive action against existing pages. +// +// Why no retract / tombstone here: reparse is not a "K is gone" event, it's +// a "K's contribution is about to be swapped for a new version" event. The +// actual swap happens asynchronously inside mapOneDocument (see its +// oldPageSlugs handling) — that's where we have both the old page set and +// the freshly extracted candidate slugs, which is exactly the information +// the WikiPageModifyPrompt needs to do a correct replace-not-append. +// +// So the only thing worth doing synchronously at reparse time is keeping +// the Redis pending list clean so the re-ingest enqueued by +// KnowledgePostProcess doesn't race with a stale ingest op that would +// fire mid-flight against zero chunks. +func (s *knowledgeService) prepareWikiForReparse(ctx context.Context, knowledge *types.Knowledge) { + if knowledge == nil { + return + } + kbID := knowledge.KnowledgeBaseID + knowledgeID := knowledge.ID + if kbID == "" || knowledgeID == "" { + return + } + s.scrubWikiPendingIngest(ctx, kbID, knowledgeID, "reparse") +} + // removeSourceRef removes entries from source_refs that match a knowledge ID. // Handles both old format ("knowledgeID") and new format ("knowledgeID|title"). func removeSourceRef(refs types.StringArray, knowledgeID string) types.StringArray { @@ -3076,6 +3109,16 @@ func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID str return nil, err } + // Keep wiki's pending queue consistent across both manual and non-manual + // paths. The destructive work (swapping old wiki contributions for new) + // happens asynchronously inside mapOneDocument — see its oldPageSlugs + // handling — once post-process re-enqueues wiki ingest. All we need to + // do here is stop any stale pending ingest op from firing against the + // pre-reparse chunk set. + if kb != nil && kb.IsWikiEnabled() { + s.prepareWikiForReparse(ctx, existing) + } + // For manual knowledge, use async manual processing (cleanup + re-indexing in worker) if existing.IsManual() { meta, metaErr := existing.ManualMetadata() diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go index 34b875db..c9c3bb19 100644 --- a/internal/application/service/wiki_ingest_batch.go +++ b/internal/application/service/wiki_ingest_batch.go @@ -653,32 +653,76 @@ func (s *wikiIngestService) mapOneDocument( } } - // Stale Pages + // Reconcile old page set against new extraction. + // + // Three cases: + // + // (a) oldSlug ∉ new → "retractStale": the doc no longer mentions this + // page's subject, so strip its ref (and possibly delete the page + // if this was the only source). Passes the NEW content as the + // retract context — if the LLM finds matching facts it trims + // them, otherwise the retract is a near no-op, which is fine. + // + // (b) oldSlug ∈ new AND slug is an entity/concept page → reparse + // swap: emit BOTH a "retract" (carrying the doc's PRIOR summary + // body as the old-version signal) AND the normal addition. The + // reduce stage sees HasAdditions=1 + HasRetractions=1 and the + // WikiPageModifyPrompt correctly tells the editor model to + // remove the old K section and add the new K section in one + // pass — giving us replace-not-append semantics that "append + // new K on top of old K" would otherwise violate. + // + // (c) oldSlug ∈ new AND slug is a summary page (summary/...) → + // nothing to do here. reduceSlugUpdates' summary branch + // unconditionally overwrites the whole page from the new + // SummaryBody, so emitting an extra retract would just be + // dead weight that the summary branch discards anyway. + // + // priorContribution is the doc's LAST summary body snapshotted at the + // start of this batch (from allPages scan). Empty on first-ever ingest + // — in that case oldPageSlugs is also empty, so we never consult it. + priorContribution := batchCtx.SummaryContentByKnowledgeID[knowledgeID] + + newSlugSet := make(map[string]bool, len(extractedPages)) + for _, ns := range extractedPages { + newSlugSet[ns] = true + } + + var reparseOverlap, staleCount int for oldSlug := range oldPageSlugs { - found := false - for _, newSlug := range extractedPages { - if oldSlug == newSlug { - found = true - break + if newSlugSet[oldSlug] { + // Skip summary slugs — they're overwritten wholesale by the + // summary update, retract would be ignored downstream. + if strings.HasPrefix(oldSlug, "summary/") { + continue } - } - if !found { + reparseOverlap++ updates = append(updates, SlugUpdate{ Slug: oldSlug, - Type: "retractStale", - RetractDocContent: content, + Type: "retract", + RetractDocContent: priorContribution, DocTitle: docTitle, KnowledgeID: knowledgeID, Language: lang, }) + continue } + staleCount++ + updates = append(updates, SlugUpdate{ + Slug: oldSlug, + Type: "retractStale", + RetractDocContent: content, + DocTitle: docTitle, + KnowledgeID: knowledgeID, + Language: lang, + }) } logger.Infof(ctx, - "wiki ingest: mapped knowledge %s title=%q candidates=%d chunks=%d batches=%d cited_chunks=%d uncited_slugs=%d new_slugs=%d updates=%d pass0_fallback=%v elapsed=%s", + "wiki ingest: mapped knowledge %s title=%q candidates=%d chunks=%d batches=%d cited_chunks=%d uncited_slugs=%d new_slugs=%d updates=%d reparse_slugs=%d stale_slugs=%d pass0_fallback=%v elapsed=%s", knowledgeID, previewText(docTitle, 80), len(slugItems), len(chunks), batchCount, len(citedChunkSet), uncited, len(newSlugs), - len(updates), pass0Failed, + len(updates), reparseOverlap, staleCount, pass0Failed, time.Since(docStartedAt).Round(time.Millisecond), )