feat: Improve wiki cleanup and ingest handling for deleted knowledge

- Enhanced the `cleanupWikiOnKnowledgeDelete` function to pass the full knowledge object, allowing for more accurate title and summary retrieval during wiki cleanup.
- Introduced a tombstone mechanism in Redis to mark recently deleted knowledge, enabling in-flight wiki ingest tasks to skip processing for deleted documents.
- Updated the `isKnowledgeGone` function to check for deleted knowledge using the tombstone, improving efficiency in handling concurrent delete and ingest operations.
- Added logic in the `wikiIngestService` to filter out updates for deleted knowledge, ensuring that no ghost references are created during the ingest process.
- Implemented lint checks for stale source references in the `WikiLintService`, allowing for automatic cleanup of pages referencing deleted knowledge.

These changes significantly enhance the robustness of the wiki system, ensuring consistency and accuracy in handling knowledge deletions and related wiki pages.
This commit is contained in:
wizardchen
2026-04-21 14:20:52 +08:00
parent 8461d5ce4d
commit fdccb68ee4
6 changed files with 387 additions and 56 deletions
+148 -49
View File
@@ -1239,10 +1239,12 @@ func (s *knowledgeService) DeleteKnowledge(ctx context.Context, id string) error
return nil
})
// Clean up wiki pages that reference this knowledge
// Clean up wiki pages that reference this knowledge. Pass the full
// knowledge object so cleanup can source title/summary from the row
// itself rather than reaching into possibly-not-yet-written wiki pages.
if kb != nil && kb.IsWikiEnabled() {
wg.Go(func() error {
s.cleanupWikiOnKnowledgeDelete(ctx, knowledge.KnowledgeBaseID, knowledge.ID)
s.cleanupWikiOnKnowledgeDelete(ctx, knowledge)
return nil
})
}
@@ -1255,39 +1257,69 @@ func (s *knowledgeService) DeleteKnowledge(ctx context.Context, id string) error
}
// cleanupWikiOnKnowledgeDelete handles wiki pages when a source document is deleted.
// - Pages with only this source ref → archived (kept for audit, hidden from normal views)
// - Pages with multiple source refs → source ref removed, page stays published
// - Index and log pages are rebuilt afterward
func (s *knowledgeService) cleanupWikiOnKnowledgeDelete(ctx context.Context, kbID, knowledgeID string) {
// Find all wiki pages referencing this knowledge ID
//
// There are three sources of truth we must keep consistent:
// - The knowledge row (being soft-deleted right now by the caller)
// - Wiki pages whose source_refs include this knowledge
// - Pending/in-flight wiki_ingest tasks that may create *new* pages pointing at it
//
// The function is deliberately best-effort and idempotent:
// - It writes a tombstone + scrubs pending ingest ops so new pages cannot be
// born with a stale source_ref (guards (a) queued ingest and (b) ingest
// tasks mid-LLM call — both consult the tombstone before writing).
// - It immediately reconciles any pages already present (delete-if-only-ref
// or strip-ref-if-multi).
// - It *unconditionally* enqueues a retract task. Crucially we DO NOT gate
// enqueue on "pages currently exist": in the ingest/delete race the
// knowledge may have pages that exist only after this function returns
// (the ingest task fires later and, absent the tombstone, would have
// created them). The retract handler re-queries ListPagesBySourceRef at
// run time, so even with an empty PageSlugs it will do the right thing —
// and at worst it's a cheap no-op.
func (s *knowledgeService) cleanupWikiOnKnowledgeDelete(ctx context.Context, knowledge *types.Knowledge) {
if knowledge == nil {
return
}
kbID := knowledge.KnowledgeBaseID
knowledgeID := knowledge.ID
if kbID == "" || knowledgeID == "" {
return
}
// (1) Tombstone + scrub pending ingest — must happen first so any
// wiki_ingest task that wakes up between here and the retract enqueue
// below sees "knowledge gone" and bails out.
s.markKnowledgeDeletedForWiki(ctx, kbID, knowledgeID)
s.scrubWikiPendingIngest(ctx, kbID, knowledgeID)
// Pull title/summary from the knowledge itself — do NOT read them from
// existing wiki pages. In the race window wiki pages may not exist yet,
// and even when they do their "summary" is the LLM-extracted one which
// we're about to invalidate anyway. The knowledge row still has the
// original Title/FileName/Description, which is what the retract prompt
// actually wants.
docTitle := knowledge.Title
if docTitle == "" {
docTitle = knowledge.FileName
}
if docTitle == "" {
docTitle = knowledgeID
}
docSummary := knowledge.Description
// (2) Immediate reconciliation for pages already present. If ingest
// hasn't run yet this simply finds nothing; that's fine — see (3).
pages, err := s.wikiRepo.ListBySourceRef(ctx, kbID, knowledgeID)
if err != nil {
logger.Warnf(ctx, "wiki cleanup: failed to list pages by source ref %s: %v", knowledgeID, err)
return
}
if len(pages) == 0 {
return
pages = nil
}
// Extract doc title from source refs for retract prompt
docTitle := knowledgeID
// Prefer the on-disk summary if the summary page already exists (it's
// richer than the raw user-provided description). Leave docSummary
// untouched otherwise so we still pass something meaningful downstream.
for _, page := range pages {
for _, ref := range page.SourceRefs {
prefix := knowledgeID + "|"
if strings.HasPrefix(ref, prefix) {
docTitle = ref[len(prefix):]
break
}
}
if docTitle != knowledgeID {
break
}
}
// Capture the summary page's description before deleting (for retract changeDesc)
var docSummary string
for _, page := range pages {
if page.PageType == types.WikiPageTypeSummary {
if page.PageType == types.WikiPageTypeSummary && page.Summary != "" {
docSummary = page.Summary
break
}
@@ -1296,23 +1328,19 @@ func (s *knowledgeService) cleanupWikiOnKnowledgeDelete(ctx context.Context, kbI
var deletedSlugs []string
var retractSlugs []string
for _, page := range pages {
// Skip index/log system pages
if page.PageType == types.WikiPageTypeIndex || page.PageType == types.WikiPageTypeLog {
continue
}
// Check if this is the only source ref (match by prefix "knowledgeID|" or exact "knowledgeID")
remaining := removeSourceRef(page.SourceRefs, knowledgeID)
if len(remaining) == 0 {
// No other sources → delete the page entirely
if err := s.wikiService.DeletePage(ctx, kbID, page.Slug); err != nil {
logger.Warnf(ctx, "wiki cleanup: failed to delete page %s: %v", page.Slug, err)
} else {
deletedSlugs = append(deletedSlugs, page.Slug)
}
} else {
// Other sources remain → remove source ref and queue LLM content retraction
page.SourceRefs = remaining
if err := s.wikiService.UpdatePageMeta(ctx, page); err != nil {
logger.Warnf(ctx, "wiki cleanup: failed to update source refs for page %s: %v", page.Slug, err)
@@ -1329,20 +1357,87 @@ func (s *knowledgeService) cleanupWikiOnKnowledgeDelete(ctx context.Context, kbI
allAffectedSlugs := append(retractSlugs, deletedSlugs...)
// Enqueue async LLM retraction and index rebuild
if len(allAffectedSlugs) > 0 {
lang, _ := types.LanguageFromContext(ctx)
tenantID, _ := types.TenantIDFromContext(ctx)
EnqueueWikiRetract(ctx, s.task, s.redisClient, WikiRetractPayload{
TenantID: tenantID,
KnowledgeBaseID: kbID,
KnowledgeID: knowledgeID,
DocTitle: docTitle,
DocSummary: docSummary,
Language: lang,
PageSlugs: allAffectedSlugs,
})
logger.Infof(ctx, "wiki cleanup: enqueued retract task for %d pages: %v", len(allAffectedSlugs), allAffectedSlugs)
// (3) Unconditionally enqueue the retract task. See function comment —
// an empty PageSlugs is not a bug, it's the signal "re-query at run
// time". The handler will ListPagesBySourceRef again, pick up any
// pages that materialised after we looked, and also rebuild index/log
// so the knowledge's disappearance is reflected in the UI.
lang, _ := types.LanguageFromContext(ctx)
tenantID, _ := types.TenantIDFromContext(ctx)
EnqueueWikiRetract(ctx, s.task, s.redisClient, WikiRetractPayload{
TenantID: tenantID,
KnowledgeBaseID: kbID,
KnowledgeID: knowledgeID,
DocTitle: docTitle,
DocSummary: docSummary,
Language: lang,
PageSlugs: allAffectedSlugs,
})
logger.Infof(ctx, "wiki cleanup: enqueued retract task for knowledge %s (%d known slugs: %v)",
knowledgeID, len(allAffectedSlugs), allAffectedSlugs)
}
// markKnowledgeDeletedForWiki writes a short-TTL tombstone so any wiki_ingest
// task still running or queued for this knowledge can short-circuit before
// resurrecting a page with a stale source_ref. No-op when Redis is absent.
func (s *knowledgeService) markKnowledgeDeletedForWiki(ctx context.Context, kbID, knowledgeID string) {
if s.redisClient == nil || kbID == "" || knowledgeID == "" {
return
}
key := WikiDeletedTombstoneKey(kbID, knowledgeID)
if err := s.redisClient.Set(ctx, key, "1", wikiDeletedTTL).Err(); err != nil {
logger.Warnf(ctx, "wiki cleanup: failed to write tombstone %s: %v", key, err)
}
}
// scrubWikiPendingIngest removes queued WikiOpIngest entries for a knowledge
// from the debounced pending list, so the next batch doesn't re-ingest a
// document we're about to soft-delete. Retract entries stay put — the wiki
// still needs them to unlink referencing pages.
//
// We use LREM against JSON-encoded entries plus a best-effort raw-UUID
// fallback for backward compatibility with the legacy format documented in
// peekPendingList.
func (s *knowledgeService) scrubWikiPendingIngest(ctx context.Context, kbID, knowledgeID string) {
if s.redisClient == nil || kbID == "" || knowledgeID == "" {
return
}
pendingKey := wikiPendingKeyPrefix + kbID
// Best-effort: inspect the list, remove matching ingest entries one by one.
// The list is bounded (wikiMaxDocsPerBatch at a time on the consumer
// side, practical uploads rarely exceed a few dozen), so a single LRange
// is safe.
items, err := s.redisClient.LRange(ctx, pendingKey, 0, -1).Result()
if err != nil {
logger.Warnf(ctx, "wiki cleanup: failed to read pending list %s: %v", pendingKey, err)
return
}
removed := 0
for _, item := range items {
// Legacy raw-UUID form
if item == knowledgeID {
if n, err := s.redisClient.LRem(ctx, pendingKey, 0, item).Result(); err == nil {
removed += int(n)
}
continue
}
if !strings.HasPrefix(item, "{") {
continue
}
var op WikiPendingOp
if err := json.Unmarshal([]byte(item), &op); err != nil {
continue
}
if op.KnowledgeID != knowledgeID || op.Op != WikiOpIngest {
continue
}
if n, err := s.redisClient.LRem(ctx, pendingKey, 0, item).Result(); err == nil {
removed += int(n)
}
}
if removed > 0 {
logger.Infof(ctx, "wiki cleanup: scrubbed %d pending ingest ops for knowledge %s", removed, knowledgeID)
}
}
@@ -1503,12 +1598,16 @@ func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string
return nil
})
// Clean up wiki pages that reference deleted knowledge
// Clean up wiki pages that reference deleted knowledge. cleanup needs
// the full knowledge object (Title / Description) so the retract prompt
// can describe the vanished document even when wiki pages haven't been
// ingested yet — which is common in the batch-delete-shortly-after-upload
// flow.
wg.Go(func() error {
for _, knowledge := range knowledgeList {
kb, _ := s.kbService.GetKnowledgeBaseByID(ctx, knowledge.KnowledgeBaseID)
if kb != nil && kb.IsWikiEnabled() {
s.cleanupWikiOnKnowledgeDelete(ctx, knowledge.KnowledgeBaseID, knowledge.ID)
s.cleanupWikiOnKnowledgeDelete(ctx, knowledge)
}
}
return nil