feat: Improve wiki cleanup and ingest handling for deleted knowledge
- Enhanced the `cleanupWikiOnKnowledgeDelete` function to pass the full knowledge object, allowing for more accurate title and summary retrieval during wiki cleanup. - Introduced a tombstone mechanism in Redis to mark recently deleted knowledge, enabling in-flight wiki ingest tasks to skip processing for deleted documents. - Updated the `isKnowledgeGone` function to check for deleted knowledge using the tombstone, improving efficiency in handling concurrent delete and ingest operations. - Added logic in the `wikiIngestService` to filter out updates for deleted knowledge, ensuring that no ghost references are created during the ingest process. - Implemented lint checks for stale source references in the `WikiLintService`, allowing for automatic cleanup of pages referencing deleted knowledge. These changes significantly enhance the robustness of the wiki system, ensuring consistency and accuracy in handling knowledge deletions and related wiki pages.
This commit is contained in:
@@ -1239,10 +1239,12 @@ func (s *knowledgeService) DeleteKnowledge(ctx context.Context, id string) error
|
||||
return nil
|
||||
})
|
||||
|
||||
// Clean up wiki pages that reference this knowledge
|
||||
// Clean up wiki pages that reference this knowledge. Pass the full
|
||||
// knowledge object so cleanup can source title/summary from the row
|
||||
// itself rather than reaching into possibly-not-yet-written wiki pages.
|
||||
if kb != nil && kb.IsWikiEnabled() {
|
||||
wg.Go(func() error {
|
||||
s.cleanupWikiOnKnowledgeDelete(ctx, knowledge.KnowledgeBaseID, knowledge.ID)
|
||||
s.cleanupWikiOnKnowledgeDelete(ctx, knowledge)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
@@ -1255,39 +1257,69 @@ func (s *knowledgeService) DeleteKnowledge(ctx context.Context, id string) error
|
||||
}
|
||||
|
||||
// cleanupWikiOnKnowledgeDelete handles wiki pages when a source document is deleted.
|
||||
// - Pages with only this source ref → archived (kept for audit, hidden from normal views)
|
||||
// - Pages with multiple source refs → source ref removed, page stays published
|
||||
// - Index and log pages are rebuilt afterward
|
||||
func (s *knowledgeService) cleanupWikiOnKnowledgeDelete(ctx context.Context, kbID, knowledgeID string) {
|
||||
// Find all wiki pages referencing this knowledge ID
|
||||
//
|
||||
// There are three sources of truth we must keep consistent:
|
||||
// - The knowledge row (being soft-deleted right now by the caller)
|
||||
// - Wiki pages whose source_refs include this knowledge
|
||||
// - Pending/in-flight wiki_ingest tasks that may create *new* pages pointing at it
|
||||
//
|
||||
// The function is deliberately best-effort and idempotent:
|
||||
// - It writes a tombstone + scrubs pending ingest ops so new pages cannot be
|
||||
// born with a stale source_ref (guards (a) queued ingest and (b) ingest
|
||||
// tasks mid-LLM call — both consult the tombstone before writing).
|
||||
// - It immediately reconciles any pages already present (delete-if-only-ref
|
||||
// or strip-ref-if-multi).
|
||||
// - It *unconditionally* enqueues a retract task. Crucially we DO NOT gate
|
||||
// enqueue on "pages currently exist": in the ingest/delete race the
|
||||
// knowledge may have pages that exist only after this function returns
|
||||
// (the ingest task fires later and, absent the tombstone, would have
|
||||
// created them). The retract handler re-queries ListPagesBySourceRef at
|
||||
// run time, so even with an empty PageSlugs it will do the right thing —
|
||||
// and at worst it's a cheap no-op.
|
||||
func (s *knowledgeService) cleanupWikiOnKnowledgeDelete(ctx context.Context, knowledge *types.Knowledge) {
|
||||
if knowledge == nil {
|
||||
return
|
||||
}
|
||||
kbID := knowledge.KnowledgeBaseID
|
||||
knowledgeID := knowledge.ID
|
||||
if kbID == "" || knowledgeID == "" {
|
||||
return
|
||||
}
|
||||
|
||||
// (1) Tombstone + scrub pending ingest — must happen first so any
|
||||
// wiki_ingest task that wakes up between here and the retract enqueue
|
||||
// below sees "knowledge gone" and bails out.
|
||||
s.markKnowledgeDeletedForWiki(ctx, kbID, knowledgeID)
|
||||
s.scrubWikiPendingIngest(ctx, kbID, knowledgeID)
|
||||
|
||||
// Pull title/summary from the knowledge itself — do NOT read them from
|
||||
// existing wiki pages. In the race window wiki pages may not exist yet,
|
||||
// and even when they do their "summary" is the LLM-extracted one which
|
||||
// we're about to invalidate anyway. The knowledge row still has the
|
||||
// original Title/FileName/Description, which is what the retract prompt
|
||||
// actually wants.
|
||||
docTitle := knowledge.Title
|
||||
if docTitle == "" {
|
||||
docTitle = knowledge.FileName
|
||||
}
|
||||
if docTitle == "" {
|
||||
docTitle = knowledgeID
|
||||
}
|
||||
docSummary := knowledge.Description
|
||||
|
||||
// (2) Immediate reconciliation for pages already present. If ingest
|
||||
// hasn't run yet this simply finds nothing; that's fine — see (3).
|
||||
pages, err := s.wikiRepo.ListBySourceRef(ctx, kbID, knowledgeID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "wiki cleanup: failed to list pages by source ref %s: %v", knowledgeID, err)
|
||||
return
|
||||
}
|
||||
if len(pages) == 0 {
|
||||
return
|
||||
pages = nil
|
||||
}
|
||||
|
||||
// Extract doc title from source refs for retract prompt
|
||||
docTitle := knowledgeID
|
||||
// Prefer the on-disk summary if the summary page already exists (it's
|
||||
// richer than the raw user-provided description). Leave docSummary
|
||||
// untouched otherwise so we still pass something meaningful downstream.
|
||||
for _, page := range pages {
|
||||
for _, ref := range page.SourceRefs {
|
||||
prefix := knowledgeID + "|"
|
||||
if strings.HasPrefix(ref, prefix) {
|
||||
docTitle = ref[len(prefix):]
|
||||
break
|
||||
}
|
||||
}
|
||||
if docTitle != knowledgeID {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Capture the summary page's description before deleting (for retract changeDesc)
|
||||
var docSummary string
|
||||
for _, page := range pages {
|
||||
if page.PageType == types.WikiPageTypeSummary {
|
||||
if page.PageType == types.WikiPageTypeSummary && page.Summary != "" {
|
||||
docSummary = page.Summary
|
||||
break
|
||||
}
|
||||
@@ -1296,23 +1328,19 @@ func (s *knowledgeService) cleanupWikiOnKnowledgeDelete(ctx context.Context, kbI
|
||||
var deletedSlugs []string
|
||||
var retractSlugs []string
|
||||
for _, page := range pages {
|
||||
// Skip index/log system pages
|
||||
if page.PageType == types.WikiPageTypeIndex || page.PageType == types.WikiPageTypeLog {
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if this is the only source ref (match by prefix "knowledgeID|" or exact "knowledgeID")
|
||||
remaining := removeSourceRef(page.SourceRefs, knowledgeID)
|
||||
|
||||
if len(remaining) == 0 {
|
||||
// No other sources → delete the page entirely
|
||||
if err := s.wikiService.DeletePage(ctx, kbID, page.Slug); err != nil {
|
||||
logger.Warnf(ctx, "wiki cleanup: failed to delete page %s: %v", page.Slug, err)
|
||||
} else {
|
||||
deletedSlugs = append(deletedSlugs, page.Slug)
|
||||
}
|
||||
} else {
|
||||
// Other sources remain → remove source ref and queue LLM content retraction
|
||||
page.SourceRefs = remaining
|
||||
if err := s.wikiService.UpdatePageMeta(ctx, page); err != nil {
|
||||
logger.Warnf(ctx, "wiki cleanup: failed to update source refs for page %s: %v", page.Slug, err)
|
||||
@@ -1329,20 +1357,87 @@ func (s *knowledgeService) cleanupWikiOnKnowledgeDelete(ctx context.Context, kbI
|
||||
|
||||
allAffectedSlugs := append(retractSlugs, deletedSlugs...)
|
||||
|
||||
// Enqueue async LLM retraction and index rebuild
|
||||
if len(allAffectedSlugs) > 0 {
|
||||
lang, _ := types.LanguageFromContext(ctx)
|
||||
tenantID, _ := types.TenantIDFromContext(ctx)
|
||||
EnqueueWikiRetract(ctx, s.task, s.redisClient, WikiRetractPayload{
|
||||
TenantID: tenantID,
|
||||
KnowledgeBaseID: kbID,
|
||||
KnowledgeID: knowledgeID,
|
||||
DocTitle: docTitle,
|
||||
DocSummary: docSummary,
|
||||
Language: lang,
|
||||
PageSlugs: allAffectedSlugs,
|
||||
})
|
||||
logger.Infof(ctx, "wiki cleanup: enqueued retract task for %d pages: %v", len(allAffectedSlugs), allAffectedSlugs)
|
||||
// (3) Unconditionally enqueue the retract task. See function comment —
|
||||
// an empty PageSlugs is not a bug, it's the signal "re-query at run
|
||||
// time". The handler will ListPagesBySourceRef again, pick up any
|
||||
// pages that materialised after we looked, and also rebuild index/log
|
||||
// so the knowledge's disappearance is reflected in the UI.
|
||||
lang, _ := types.LanguageFromContext(ctx)
|
||||
tenantID, _ := types.TenantIDFromContext(ctx)
|
||||
EnqueueWikiRetract(ctx, s.task, s.redisClient, WikiRetractPayload{
|
||||
TenantID: tenantID,
|
||||
KnowledgeBaseID: kbID,
|
||||
KnowledgeID: knowledgeID,
|
||||
DocTitle: docTitle,
|
||||
DocSummary: docSummary,
|
||||
Language: lang,
|
||||
PageSlugs: allAffectedSlugs,
|
||||
})
|
||||
logger.Infof(ctx, "wiki cleanup: enqueued retract task for knowledge %s (%d known slugs: %v)",
|
||||
knowledgeID, len(allAffectedSlugs), allAffectedSlugs)
|
||||
}
|
||||
|
||||
// markKnowledgeDeletedForWiki writes a short-TTL tombstone so any wiki_ingest
|
||||
// task still running or queued for this knowledge can short-circuit before
|
||||
// resurrecting a page with a stale source_ref. No-op when Redis is absent.
|
||||
func (s *knowledgeService) markKnowledgeDeletedForWiki(ctx context.Context, kbID, knowledgeID string) {
|
||||
if s.redisClient == nil || kbID == "" || knowledgeID == "" {
|
||||
return
|
||||
}
|
||||
key := WikiDeletedTombstoneKey(kbID, knowledgeID)
|
||||
if err := s.redisClient.Set(ctx, key, "1", wikiDeletedTTL).Err(); err != nil {
|
||||
logger.Warnf(ctx, "wiki cleanup: failed to write tombstone %s: %v", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
// scrubWikiPendingIngest removes queued WikiOpIngest entries for a knowledge
|
||||
// from the debounced pending list, so the next batch doesn't re-ingest a
|
||||
// document we're about to soft-delete. Retract entries stay put — the wiki
|
||||
// still needs them to unlink referencing pages.
|
||||
//
|
||||
// We use LREM against JSON-encoded entries plus a best-effort raw-UUID
|
||||
// fallback for backward compatibility with the legacy format documented in
|
||||
// peekPendingList.
|
||||
func (s *knowledgeService) scrubWikiPendingIngest(ctx context.Context, kbID, knowledgeID string) {
|
||||
if s.redisClient == nil || kbID == "" || knowledgeID == "" {
|
||||
return
|
||||
}
|
||||
pendingKey := wikiPendingKeyPrefix + kbID
|
||||
|
||||
// Best-effort: inspect the list, remove matching ingest entries one by one.
|
||||
// The list is bounded (wikiMaxDocsPerBatch at a time on the consumer
|
||||
// side, practical uploads rarely exceed a few dozen), so a single LRange
|
||||
// is safe.
|
||||
items, err := s.redisClient.LRange(ctx, pendingKey, 0, -1).Result()
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "wiki cleanup: failed to read pending list %s: %v", pendingKey, err)
|
||||
return
|
||||
}
|
||||
removed := 0
|
||||
for _, item := range items {
|
||||
// Legacy raw-UUID form
|
||||
if item == knowledgeID {
|
||||
if n, err := s.redisClient.LRem(ctx, pendingKey, 0, item).Result(); err == nil {
|
||||
removed += int(n)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !strings.HasPrefix(item, "{") {
|
||||
continue
|
||||
}
|
||||
var op WikiPendingOp
|
||||
if err := json.Unmarshal([]byte(item), &op); err != nil {
|
||||
continue
|
||||
}
|
||||
if op.KnowledgeID != knowledgeID || op.Op != WikiOpIngest {
|
||||
continue
|
||||
}
|
||||
if n, err := s.redisClient.LRem(ctx, pendingKey, 0, item).Result(); err == nil {
|
||||
removed += int(n)
|
||||
}
|
||||
}
|
||||
if removed > 0 {
|
||||
logger.Infof(ctx, "wiki cleanup: scrubbed %d pending ingest ops for knowledge %s", removed, knowledgeID)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1503,12 +1598,16 @@ func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string
|
||||
return nil
|
||||
})
|
||||
|
||||
// Clean up wiki pages that reference deleted knowledge
|
||||
// Clean up wiki pages that reference deleted knowledge. cleanup needs
|
||||
// the full knowledge object (Title / Description) so the retract prompt
|
||||
// can describe the vanished document even when wiki pages haven't been
|
||||
// ingested yet — which is common in the batch-delete-shortly-after-upload
|
||||
// flow.
|
||||
wg.Go(func() error {
|
||||
for _, knowledge := range knowledgeList {
|
||||
kb, _ := s.kbService.GetKnowledgeBaseByID(ctx, knowledge.KnowledgeBaseID)
|
||||
if kb != nil && kb.IsWikiEnabled() {
|
||||
s.cleanupWikiOnKnowledgeDelete(ctx, knowledge.KnowledgeBaseID, knowledge.ID)
|
||||
s.cleanupWikiOnKnowledgeDelete(ctx, knowledge)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
||||
Reference in New Issue
Block a user