feat: Enhance wiki cleanup and reparse logic for improved consistency

- Updated the scrubWikiPendingIngest method to accept a reason parameter, improving logging clarity during wiki cleanup and reparse operations.
- Introduced prepareWikiForReparse to maintain the integrity of the pending ingest queue during reparse events, preventing stale operations from executing.
- Enhanced the mapping logic in the wiki ingest process to differentiate between stale and reparse updates, ensuring accurate handling of knowledge contributions.
- Improved logging to provide better insights into the number of reparse and stale slugs processed during wiki ingestion.
This commit is contained in:
wizardchen
2026-04-23 10:47:53 +08:00
parent 0d429613e2
commit 1d5b86ddf5
2 changed files with 106 additions and 19 deletions
+50 -7
View File
@@ -1297,7 +1297,7 @@ func (s *knowledgeService) cleanupWikiOnKnowledgeDelete(ctx context.Context, kno
// wiki_ingest task that wakes up between here and the retract enqueue
// below sees "knowledge gone" and bails out.
s.markKnowledgeDeletedForWiki(ctx, kbID, knowledgeID)
s.scrubWikiPendingIngest(ctx, kbID, knowledgeID)
s.scrubWikiPendingIngest(ctx, kbID, knowledgeID, "cleanup")
// Pull title/summary from the knowledge itself — do NOT read them from
// existing wiki pages. In the race window wiki pages may not exist yet,
@@ -1398,14 +1398,19 @@ func (s *knowledgeService) markKnowledgeDeletedForWiki(ctx context.Context, kbID
}
// scrubWikiPendingIngest removes queued WikiOpIngest entries for a knowledge
// from the debounced pending list, so the next batch doesn't re-ingest a
// document we're about to soft-delete. Retract entries stay put — the wiki
// still needs them to unlink referencing pages.
// from the debounced pending list. Used by both the delete path (we're about
// to soft-delete the doc, no point ingesting it) and the reparse path (the
// old chunks are about to vanish, so any pending ingest would either race
// with the cleanup or no-op on an empty chunk set — and the post-process
// task will enqueue a fresh ingest once new chunks land anyway).
//
// Retract entries stay put — delete still needs them to unlink referencing
// pages, and reparse never enqueues retracts for the doc being reparsed.
//
// We use LREM against JSON-encoded entries plus a best-effort raw-UUID
// fallback for backward compatibility with the legacy format documented in
// peekPendingList.
func (s *knowledgeService) scrubWikiPendingIngest(ctx context.Context, kbID, knowledgeID string) {
func (s *knowledgeService) scrubWikiPendingIngest(ctx context.Context, kbID, knowledgeID, reason string) {
if s.redisClient == nil || kbID == "" || knowledgeID == "" {
return
}
@@ -1417,7 +1422,7 @@ func (s *knowledgeService) scrubWikiPendingIngest(ctx context.Context, kbID, kno
// is safe.
items, err := s.redisClient.LRange(ctx, pendingKey, 0, -1).Result()
if err != nil {
logger.Warnf(ctx, "wiki cleanup: failed to read pending list %s: %v", pendingKey, err)
logger.Warnf(ctx, "wiki %s: failed to read pending list %s: %v", reason, pendingKey, err)
return
}
removed := 0
@@ -1444,10 +1449,38 @@ func (s *knowledgeService) scrubWikiPendingIngest(ctx context.Context, kbID, kno
}
}
if removed > 0 {
logger.Infof(ctx, "wiki cleanup: scrubbed %d pending ingest ops for knowledge %s", removed, knowledgeID)
logger.Infof(ctx, "wiki %s: scrubbed %d pending ingest ops for knowledge %s", reason, removed, knowledgeID)
}
}
// prepareWikiForReparse is the reparse counterpart to
// cleanupWikiOnKnowledgeDelete. It aligns reparse with the same "pending
// queue hygiene" the delete path already enforces, without taking any
// destructive action against existing pages.
//
// Why no retract / tombstone here: reparse is not a "K is gone" event, it's
// a "K's contribution is about to be swapped for a new version" event. The
// actual swap happens asynchronously inside mapOneDocument (see its
// oldPageSlugs handling) — that's where we have both the old page set and
// the freshly extracted candidate slugs, which is exactly the information
// the WikiPageModifyPrompt needs to do a correct replace-not-append.
//
// So the only thing worth doing synchronously at reparse time is keeping
// the Redis pending list clean so the re-ingest enqueued by
// KnowledgePostProcess doesn't race with a stale ingest op that would
// fire mid-flight against zero chunks.
func (s *knowledgeService) prepareWikiForReparse(ctx context.Context, knowledge *types.Knowledge) {
if knowledge == nil {
return
}
kbID := knowledge.KnowledgeBaseID
knowledgeID := knowledge.ID
if kbID == "" || knowledgeID == "" {
return
}
s.scrubWikiPendingIngest(ctx, kbID, knowledgeID, "reparse")
}
// removeSourceRef removes entries from source_refs that match a knowledge ID.
// Handles both old format ("knowledgeID") and new format ("knowledgeID|title").
func removeSourceRef(refs types.StringArray, knowledgeID string) types.StringArray {
@@ -3076,6 +3109,16 @@ func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID str
return nil, err
}
// Keep wiki's pending queue consistent across both manual and non-manual
// paths. The destructive work (swapping old wiki contributions for new)
// happens asynchronously inside mapOneDocument — see its oldPageSlugs
// handling — once post-process re-enqueues wiki ingest. All we need to
// do here is stop any stale pending ingest op from firing against the
// pre-reparse chunk set.
if kb != nil && kb.IsWikiEnabled() {
s.prepareWikiForReparse(ctx, existing)
}
// For manual knowledge, use async manual processing (cleanup + re-indexing in worker)
if existing.IsManual() {
meta, metaErr := existing.ManualMetadata()
@@ -653,32 +653,76 @@ func (s *wikiIngestService) mapOneDocument(
}
}
// Stale Pages
// Reconcile old page set against new extraction.
//
// Three cases:
//
// (a) oldSlug ∉ new → "retractStale": the doc no longer mentions this
// page's subject, so strip its ref (and possibly delete the page
// if this was the only source). Passes the NEW content as the
// retract context — if the LLM finds matching facts it trims
// them, otherwise the retract is a near no-op, which is fine.
//
// (b) oldSlug ∈ new AND slug is an entity/concept page → reparse
// swap: emit BOTH a "retract" (carrying the doc's PRIOR summary
// body as the old-version signal) AND the normal addition. The
// reduce stage sees HasAdditions=1 + HasRetractions=1 and the
// WikiPageModifyPrompt correctly tells the editor model to
// remove the old K section and add the new K section in one
// pass — giving us replace-not-append semantics that "append
// new K on top of old K" would otherwise violate.
//
// (c) oldSlug ∈ new AND slug is a summary page (summary/...) →
// nothing to do here. reduceSlugUpdates' summary branch
// unconditionally overwrites the whole page from the new
// SummaryBody, so emitting an extra retract would just be
// dead weight that the summary branch discards anyway.
//
// priorContribution is the doc's LAST summary body snapshotted at the
// start of this batch (from allPages scan). Empty on first-ever ingest
// — in that case oldPageSlugs is also empty, so we never consult it.
priorContribution := batchCtx.SummaryContentByKnowledgeID[knowledgeID]
newSlugSet := make(map[string]bool, len(extractedPages))
for _, ns := range extractedPages {
newSlugSet[ns] = true
}
var reparseOverlap, staleCount int
for oldSlug := range oldPageSlugs {
found := false
for _, newSlug := range extractedPages {
if oldSlug == newSlug {
found = true
break
if newSlugSet[oldSlug] {
// Skip summary slugs — they're overwritten wholesale by the
// summary update, retract would be ignored downstream.
if strings.HasPrefix(oldSlug, "summary/") {
continue
}
}
if !found {
reparseOverlap++
updates = append(updates, SlugUpdate{
Slug: oldSlug,
Type: "retractStale",
RetractDocContent: content,
Type: "retract",
RetractDocContent: priorContribution,
DocTitle: docTitle,
KnowledgeID: knowledgeID,
Language: lang,
})
continue
}
staleCount++
updates = append(updates, SlugUpdate{
Slug: oldSlug,
Type: "retractStale",
RetractDocContent: content,
DocTitle: docTitle,
KnowledgeID: knowledgeID,
Language: lang,
})
}
logger.Infof(ctx,
"wiki ingest: mapped knowledge %s title=%q candidates=%d chunks=%d batches=%d cited_chunks=%d uncited_slugs=%d new_slugs=%d updates=%d pass0_fallback=%v elapsed=%s",
"wiki ingest: mapped knowledge %s title=%q candidates=%d chunks=%d batches=%d cited_chunks=%d uncited_slugs=%d new_slugs=%d updates=%d reparse_slugs=%d stale_slugs=%d pass0_fallback=%v elapsed=%s",
knowledgeID, previewText(docTitle, 80),
len(slugItems), len(chunks), batchCount, len(citedChunkSet), uncited, len(newSlugs),
len(updates), pass0Failed,
len(updates), reparseOverlap, staleCount, pass0Failed,
time.Since(docStartedAt).Round(time.Millisecond),
)