From 4c3ae69b492936e6e3fd145d3c552f34c26b48c1 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Wed, 22 Jul 2026 17:57:55 +0800 Subject: [PATCH] feat(wiki): prune empty folders after document retract Automatically remove wiki folders that become empty when knowledge is retracted, using the durable finalize lane so pruning waits until ingest drains. Repository DeleteFolder now atomically checks emptiness to avoid races with concurrent page or child-folder writes. --- internal/application/repository/wiki_page.go | 33 ++++- .../application/repository/wiki_page_test.go | 8 ++ .../application/service/knowledge_delete.go | 5 + .../wiki_folder_prune_finalize_test.go | 122 ++++++++++++++++++ internal/application/service/wiki_ingest.go | 79 +++++++++++- .../application/service/wiki_ingest_batch.go | 79 +++++++++++- internal/application/service/wiki_page.go | 83 +++++++++++- .../application/service/wiki_page_test.go | 47 +++++++ internal/handler/wiki_page.go | 2 +- internal/types/interfaces/wiki_page.go | 4 + 10 files changed, 444 insertions(+), 18 deletions(-) create mode 100644 internal/application/service/wiki_folder_prune_finalize_test.go diff --git a/internal/application/repository/wiki_page.go b/internal/application/repository/wiki_page.go index 1db1f0ca..b59979f4 100644 --- a/internal/application/repository/wiki_page.go +++ b/internal/application/repository/wiki_page.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "strings" + "time" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" @@ -455,6 +456,10 @@ var ErrWikiFolderNotFound = errors.New("wiki folder not found") // already exists under the same parent. var ErrWikiFolderConflict = errors.New("wiki folder name conflict") +// ErrWikiFolderNotEmpty is returned when a folder still has a live page or +// child folder at the instant an atomic delete is attempted. +var ErrWikiFolderNotEmpty = errors.New("wiki folder is not empty") + func (r *wikiPageRepository) CreateFolder(ctx context.Context, folder *types.WikiFolder) error { return r.db.WithContext(ctx).Create(folder).Error } @@ -535,14 +540,34 @@ func (r *wikiPageRepository) UpdateFolder(ctx context.Context, folder *types.Wik } func (r *wikiPageRepository) DeleteFolder(ctx context.Context, kbID string, id string) error { - result := r.db.WithContext(ctx). - Where("knowledge_base_id = ? AND id = ?", kbID, id). - Delete(&types.WikiFolder{}) + // Keep the emptiness test in the same SQL statement as the soft delete. + // A page move or child-folder create can race the service's earlier checks; + // a check-then-delete sequence would otherwise leave a dangling folder_id. + result := r.db.WithContext(ctx).Exec(` +UPDATE wiki_folders +SET deleted_at = ? +WHERE knowledge_base_id = ? AND id = ? AND deleted_at IS NULL + AND NOT EXISTS ( + SELECT 1 FROM wiki_pages + WHERE knowledge_base_id = ? AND folder_id = ? AND deleted_at IS NULL + ) + AND NOT EXISTS ( + SELECT 1 FROM wiki_folders AS child + WHERE child.knowledge_base_id = ? AND child.parent_id = ? AND child.deleted_at IS NULL + )`, time.Now(), kbID, id, kbID, id, kbID, id) if result.Error != nil { return result.Error } if result.RowsAffected == 0 { - return ErrWikiFolderNotFound + var count int64 + if err := r.db.WithContext(ctx).Model(&types.WikiFolder{}). + Where("knowledge_base_id = ? AND id = ?", kbID, id).Count(&count).Error; err != nil { + return err + } + if count == 0 { + return ErrWikiFolderNotFound + } + return ErrWikiFolderNotEmpty } return nil } diff --git a/internal/application/repository/wiki_page_test.go b/internal/application/repository/wiki_page_test.go index cd184ed3..71be1359 100644 --- a/internal/application/repository/wiki_page_test.go +++ b/internal/application/repository/wiki_page_test.go @@ -205,6 +205,14 @@ func TestFolderTree_CRUDAndChildListing(t *testing.T) { pages, err := repo.ListPagesByFolderIDs(ctx, "kb-f", []string{"f-ai", "f-llm"}) require.NoError(t, err) assert.Len(t, pages, 3) + + // Repository deletion re-checks emptiness atomically, so a concurrent page + // move / child create cannot slip between the service check and soft delete. + err = repo.DeleteFolder(ctx, "kb-f", "f-ai") + assert.ErrorIs(t, err, ErrWikiFolderNotEmpty) + require.NoError(t, repo.DeleteFolder(ctx, "kb-f", "f-people")) + _, err = repo.GetFolderByID(ctx, "kb-f", "f-people") + assert.ErrorIs(t, err, ErrWikiFolderNotFound) } // TestListByTypeLight_ProjectsNarrowColumnsAndExcludesArchived verifies diff --git a/internal/application/service/knowledge_delete.go b/internal/application/service/knowledge_delete.go index 4616117f..d7518ebc 100644 --- a/internal/application/service/knowledge_delete.go +++ b/internal/application/service/knowledge_delete.go @@ -272,10 +272,14 @@ func (s *knowledgeService) cleanupWikiOnKnowledgeDelete(ctx context.Context, kno var deletedSlugs []string var retractSlugs []string + var affectedFolderIDs []string for _, page := range pages { if page.PageType == types.WikiPageTypeIndex || page.PageType == types.WikiPageTypeLog { continue } + if page.FolderID != "" { + affectedFolderIDs = append(affectedFolderIDs, page.FolderID) + } remaining := removeSourceRef(page.SourceRefs, knowledgeID) @@ -318,6 +322,7 @@ func (s *knowledgeService) cleanupWikiOnKnowledgeDelete(ctx context.Context, kno DocSummary: docSummary, Language: lang, PageSlugs: allAffectedSlugs, + FolderIDs: uniqueWikiFolderIDs(affectedFolderIDs), }) logger.Infof(ctx, "wiki cleanup: enqueued retract task for knowledge %s (%d known slugs: %v)", knowledgeID, len(allAffectedSlugs), allAffectedSlugs) diff --git a/internal/application/service/wiki_folder_prune_finalize_test.go b/internal/application/service/wiki_folder_prune_finalize_test.go new file mode 100644 index 00000000..f5dacbd5 --- /dev/null +++ b/internal/application/service/wiki_folder_prune_finalize_test.go @@ -0,0 +1,122 @@ +package service + +import ( + "context" + "encoding/json" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/hibiken/asynq" + "github.com/stretchr/testify/require" +) + +type folderPruneWikiServiceStub struct { + interfaces.WikiPageService + calls int + folderIDs []string +} + +func (s *folderPruneWikiServiceStub) PruneEmptyFolderChains( + _ context.Context, _ string, folderIDs []string, +) ([]string, error) { + s.calls++ + s.folderIDs = append([]string(nil), folderIDs...) + return folderIDs, nil +} + +type folderPruneKBServiceStub struct { + interfaces.KnowledgeBaseService +} + +func (s *folderPruneKBServiceStub) GetKnowledgeBaseByIDOnly( + _ context.Context, id string, +) (*types.KnowledgeBase, error) { + return &types.KnowledgeBase{ + ID: id, + IndexingStrategy: types.IndexingStrategy{ + WikiEnabled: true, + }, + }, nil +} + +type folderPrunePendingRepoStub struct { + interfaces.TaskPendingOpsRepository + rows []*types.TaskPendingOp + ingestPending int64 + deletedRowIDs []int64 + finalizeCounts int +} + +func (s *folderPrunePendingRepoStub) PeekBatch( + _ context.Context, _, _, _ string, _ int, +) ([]*types.TaskPendingOp, error) { + return s.rows, nil +} + +func (s *folderPrunePendingRepoStub) PendingCount( + _ context.Context, taskType, _, _ string, +) (int64, error) { + if taskType == wikiTaskType { + return s.ingestPending, nil + } + s.finalizeCounts++ + return int64(len(s.rows)), nil +} + +func (s *folderPrunePendingRepoStub) DeleteByIDs(_ context.Context, ids []int64) error { + s.deletedRowIDs = append(s.deletedRowIDs, ids...) + return nil +} + +type folderPruneTaskStub struct { + interfaces.TaskEnqueuer + enqueuedTypes []string +} + +func (s *folderPruneTaskStub) Enqueue(task *asynq.Task, _ ...asynq.Option) (*asynq.TaskInfo, error) { + s.enqueuedTypes = append(s.enqueuedTypes, task.Type()) + return &asynq.TaskInfo{}, nil +} + +func makeFolderPruneFinalizeTask(t *testing.T) (*asynq.Task, *types.TaskPendingOp) { + t.Helper() + rowPayload, err := json.Marshal(wikiFinalizeRow{FolderIDs: []string{"folder-a"}}) + require.NoError(t, err) + taskPayload, err := json.Marshal(WikiIngestPayload{TenantID: 1, KnowledgeBaseID: "kb-1"}) + require.NoError(t, err) + return asynq.NewTask(types.TypeWikiFinalize, taskPayload), &types.TaskPendingOp{ + ID: 11, Op: wikiFinalizeOpFolderPrune, Payload: rowPayload, + } +} + +func TestProcessWikiFinalizeDefersFolderPruneWhileIngestIsPending(t *testing.T) { + task, row := makeFolderPruneFinalizeTask(t) + wikiSvc := &folderPruneWikiServiceStub{} + pendingRepo := &folderPrunePendingRepoStub{rows: []*types.TaskPendingOp{row}, ingestPending: 1} + taskQueue := &folderPruneTaskStub{} + svc := &wikiIngestService{ + wikiService: wikiSvc, kbService: &folderPruneKBServiceStub{}, + pendingRepo: pendingRepo, task: taskQueue, + } + + require.NoError(t, svc.ProcessWikiFinalize(context.Background(), task)) + require.Zero(t, wikiSvc.calls, "must not prune a folder reserved by an in-flight ingest") + require.Empty(t, pendingRepo.deletedRowIDs, "durable prune row must remain for retry") + require.Equal(t, []string{types.TypeWikiFinalize}, taskQueue.enqueuedTypes) +} + +func TestProcessWikiFinalizePrunesFolderAfterIngestDrains(t *testing.T) { + task, row := makeFolderPruneFinalizeTask(t) + wikiSvc := &folderPruneWikiServiceStub{} + pendingRepo := &folderPrunePendingRepoStub{rows: []*types.TaskPendingOp{row}} + svc := &wikiIngestService{ + wikiService: wikiSvc, kbService: &folderPruneKBServiceStub{}, + pendingRepo: pendingRepo, task: &folderPruneTaskStub{}, + } + + require.NoError(t, svc.ProcessWikiFinalize(context.Background(), task)) + require.Equal(t, 1, wikiSvc.calls) + require.Equal(t, []string{"folder-a"}, wikiSvc.folderIDs) + require.Equal(t, []int64{11}, pendingRepo.deletedRowIDs) +} diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go index b8723e19..4b207a3c 100644 --- a/internal/application/service/wiki_ingest.go +++ b/internal/application/service/wiki_ingest.go @@ -183,6 +183,13 @@ const ( // wikiFinalizeOpChange rows carry a doc-level add/remove change entry for // the index-intro change description. wikiFinalizeOpChange = "change" + // wikiFinalizeOpFolderPrune rows carry folders that may have become empty + // after a document retract. Keeping this in the durable finalize lane lets + // us wait until every ingest op for the KB has settled before deleting the + // directories; taxonomy planning creates folders before reduce writes the + // corresponding pages, so pruning any earlier can invalidate in-flight + // folder assignments. + wikiFinalizeOpFolderPrune = "folder_prune" wikiFinalizeAdded = "added" wikiFinalizeRemoved = "removed" @@ -201,6 +208,11 @@ const ( wikiFinalizeLockTTL = 60 * time.Second wikiFinalizeLockRenew = 20 * time.Second + // Folder cleanup is maintenance, not user-blocking work. When an ingest is + // still active, retry slowly so pruning never competes with the primary wiki + // pipeline for worker capacity. + wikiFolderPruneRetryDelay = 1 * time.Minute + // wikiIngestCleanupTimeout bounds detached tail cleanup after the asynq // task context has been cancelled or hit its timeout. wikiIngestCleanupTimeout = 10 * time.Second @@ -215,12 +227,13 @@ type wikiFinalizeChange struct { } // wikiFinalizeRow is the JSON payload of a task_pending_ops row in the -// finalize lane. Exactly one of {Slug, Change} is set, distinguished by the -// row's Op column (wikiFinalizeOpSlug / wikiFinalizeOpChange). +// finalize lane. Exactly one of {Slug, Change, FolderIDs} is set, +// distinguished by the row's Op column. type wikiFinalizeRow struct { - Slug string `json:"slug,omitempty"` - Title string `json:"title,omitempty"` - Change *wikiFinalizeChange `json:"change,omitempty"` + Slug string `json:"slug,omitempty"` + Title string `json:"title,omitempty"` + Change *wikiFinalizeChange `json:"change,omitempty"` + FolderIDs []string `json:"folder_ids,omitempty"` } // WikiDeletedTombstoneKey returns the Redis key used to mark a knowledge as @@ -253,6 +266,7 @@ type WikiRetractPayload struct { DocSummary string `json:"doc_summary,omitempty"` // one-line summary of the deleted document Language string `json:"language,omitempty"` PageSlugs []string `json:"page_slugs"` + FolderIDs []string `json:"folder_ids,omitempty"` } const ( @@ -281,6 +295,7 @@ type WikiPendingOp struct { DocTitle string `json:"doc_title,omitempty"` DocSummary string `json:"doc_summary,omitempty"` PageSlugs []string `json:"page_slugs,omitempty"` + FolderIDs []string `json:"folder_ids,omitempty"` // dbID is set by peekPendingList from task_pending_ops.id. Zero in // constructions made outside the queue (e.g. legacy tests). @@ -489,6 +504,7 @@ func EnqueueWikiRetract( DocTitle: payload.DocTitle, DocSummary: payload.DocSummary, PageSlugs: payload.PageSlugs, + FolderIDs: payload.FolderIDs, Language: payload.Language, } payloadBytes, err := json.Marshal(op) @@ -559,6 +575,7 @@ func (s *wikiIngestService) enqueueFinalize( affectedSlugs []string, freshTitleBySlug map[string]string, changes []wikiFinalizeChange, + folderIDs []string, ) { if s.pendingRepo == nil { return @@ -599,9 +616,42 @@ func (s *wikiIngestService) enqueueFinalize( logger.Warnf(ctx, "wiki finalize: enqueue change row failed: %v", err) } } + if len(folderIDs) > 0 { + row := wikiFinalizeRow{FolderIDs: uniqueWikiFolderIDs(folderIDs)} + if b, err := json.Marshal(row); err == nil { + if err := s.pendingRepo.Enqueue(ctx, &types.TaskPendingOp{ + TenantID: payload.TenantID, + TaskType: wikiFinalizeTaskType, + Scope: wikiTaskScope, + ScopeID: payload.KnowledgeBaseID, + Op: wikiFinalizeOpFolderPrune, + DedupKey: "", + Payload: b, + }); err != nil { + logger.Warnf(ctx, "wiki finalize: enqueue folder prune row failed: %v", err) + } + } + } s.scheduleFinalize(ctx, payload) } +func uniqueWikiFolderIDs(values []string) []string { + seen := make(map[string]struct{}, len(values)) + out := make([]string, 0, len(values)) + for _, value := range values { + value = strings.TrimSpace(value) + if value == "" { + continue + } + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + out = append(out, value) + } + return out +} + // scheduleFinalize enqueues a debounced, coalesced KB-global finalize trigger. // asynq.TaskID ("wiki-finalize-") makes concurrent schedules within the // debounce window collapse into one pending task; the conflict error is the @@ -626,6 +676,25 @@ func (s *wikiIngestService) scheduleFinalize(ctx context.Context, payload WikiIn } } +// scheduleFinalizeRetry is used when folder pruning is waiting for ingest +// rows to drain. It deliberately has no stable TaskID: the currently-running +// finalize task still owns that ID until it returns, so reusing it here would +// coalesce the only retry away. Duplicate retries are harmless because the +// durable prune row is deleted exactly once and an empty lane is a no-op. +func (s *wikiIngestService) scheduleFinalizeRetry(ctx context.Context, payload WikiIngestPayload) { + langfuse.InjectTracing(ctx, &payload) + b, _ := json.Marshal(payload) + t := asynq.NewTask(types.TypeWikiFinalize, b, + asynq.Queue(types.QueueWiki), + asynq.MaxRetry(wikiIngestMaxRetry), + asynq.Timeout(30*time.Minute), + asynq.ProcessIn(wikiFolderPruneRetryDelay), + ) + if _, err := s.task.Enqueue(t); err != nil { + logger.Warnf(ctx, "wiki finalize: schedule deferred folder prune failed: %v", err) + } +} + // peekPendingList loads up to `limit` ops from task_pending_ops for // this KB, ordered FIFO. Rows are NOT removed; callers must // DeleteByIDs once they have been consumed (or IncrFailCount + leave diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go index 3321eaa8..f533e5b6 100644 --- a/internal/application/service/wiki_ingest_batch.go +++ b/internal/application/service/wiki_ingest_batch.go @@ -413,6 +413,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task var failedOps []WikiPendingOp slugUpdates := make(map[string][]SlugUpdate) var docResults []*docIngestResult + var retractFolderIDs []string // rateLimited flips true when any map/reduce LLM failure looks like an // upstream 429/quota trip. It steers the follow-up scheduler onto the // longer wikiRateLimitBackoff so retries don't keep hammering an already @@ -442,12 +443,18 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task // PageSlugs as "figure it out yourself" — see // cleanupWikiOnKnowledgeDelete's comment (3). slugSet := make(map[string]struct{}, len(op.PageSlugs)) + folderSet := make(map[string]struct{}, len(op.FolderIDs)) for _, slug := range op.PageSlugs { if slug == "" { continue } slugSet[slug] = struct{}{} } + for _, folderID := range op.FolderIDs { + if folderID != "" { + folderSet[folderID] = struct{}{} + } + } if op.KnowledgeID != "" { livePages, err := s.wikiService.ListPagesBySourceRef(mapCtx, payload.KnowledgeBaseID, op.KnowledgeID) if err != nil { @@ -464,6 +471,9 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task continue } slugSet[p.Slug] = struct{}{} + if p.FolderID != "" { + folderSet[p.FolderID] = struct{}{} + } } } } @@ -483,6 +493,9 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task Language: types.LanguageLocaleName(op.Language), }) } + for folderID := range folderSet { + retractFolderIDs = append(retractFolderIDs, folderID) + } mapMu.Unlock() return nil } @@ -763,7 +776,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task freshTitleBySlug[p.Slug] = p.Title } } - if len(allPagesAffected) > 0 || len(docResults) > 0 { + if len(allPagesAffected) > 0 || len(docResults) > 0 || retractHandled > 0 || len(retractFolderIDs) > 0 { changes := make([]wikiFinalizeChange, 0, len(docResults)+len(pendingOps)) for _, r := range docResults { changes = append(changes, wikiFinalizeChange{ @@ -777,7 +790,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task }) } } - s.enqueueFinalize(tailCtx, payload, allPagesAffected, freshTitleBySlug, changes) + s.enqueueFinalize(tailCtx, payload, allPagesAffected, freshTitleBySlug, changes, retractFolderIDs) } // Close postprocess.wiki spans for every successfully-mapped doc. @@ -1001,12 +1014,17 @@ func (s *wikiIngestService) ProcessWikiFinalize(ctx context.Context, t *asynq.Ta // refs, and the index-intro change description. We collect ids up front so // we can drain the lane even on the KB-disabled short-circuit below. ids := make([]int64, 0, len(rows)) + pruneRowIDs := make([]int64, 0) affectedSet := make(map[string]struct{}, len(rows)) var affectedSlugs []string var freshRefs []linkRef + var folderPruneIDs []string var changeDesc strings.Builder for _, r := range rows { ids = append(ids, r.ID) + if r.Op == wikiFinalizeOpFolderPrune { + pruneRowIDs = append(pruneRowIDs, r.ID) + } if len(r.Payload) == 0 { continue } @@ -1015,6 +1033,10 @@ func (s *wikiIngestService) ProcessWikiFinalize(ctx context.Context, t *asynq.Ta logger.Warnf(ctx, "wiki finalize: unmarshal row id=%d failed: %v", r.ID, err) continue } + if r.Op == wikiFinalizeOpFolderPrune { + folderPruneIDs = append(folderPruneIDs, row.FolderIDs...) + continue + } if row.Change != nil { if row.Change.Action == wikiFinalizeRemoved { fmt.Fprintf(&changeDesc, "\n%s\n%s\n\n\n", row.Change.DocTitle, row.Change.DocSummary) @@ -1080,11 +1102,50 @@ func (s *wikiIngestService) ProcessWikiFinalize(ctx context.Context, t *asynq.Ta s.injectCrossLinks(ctx, payload.KnowledgeBaseID, affectedSlugs, freshRefs, batchCtx) } + // A retract may leave one or more generated folders empty. Do not prune + // while any ingest row for this KB is queued or claimed: taxonomy planning + // creates folders before reduce writes pages, so an apparently-empty folder + // can still be owned by an in-flight batch. The durable prune rows stay in + // the finalize lane and are retried after the ingest lane drains. + pruneDeferred := false + deletedFolders := 0 + if len(folderPruneIDs) > 0 { + pending, pErr := s.pendingRepo.PendingCount(ctx, wikiTaskType, wikiTaskScope, payload.KnowledgeBaseID) + if pErr != nil { + logger.Warnf(ctx, "wiki finalize: cannot verify ingest drain before folder prune: %v", pErr) + pruneDeferred = true + } else if pending > 0 { + pruneDeferred = true + } else { + deleted, pruneErr := s.wikiService.PruneEmptyFolderChains( + ctx, payload.KnowledgeBaseID, uniqueWikiFolderIDs(folderPruneIDs)) + if pruneErr != nil { + logger.Warnf(ctx, "wiki finalize: prune empty folders failed: %v", pruneErr) + pruneDeferred = true + } else { + deletedFolders = len(deleted) + } + } + } + // Drain the processed rows. Best-effort convergence mirrors the legacy // in-batch behaviour: a failed index rebuild is logged (not retried), // so we delete regardless to avoid re-doing the whole pass forever. + idsToTrim := ids + if pruneDeferred && len(pruneRowIDs) > 0 { + deferred := make(map[int64]struct{}, len(pruneRowIDs)) + for _, id := range pruneRowIDs { + deferred[id] = struct{}{} + } + idsToTrim = idsToTrim[:0:0] + for _, id := range ids { + if _, keep := deferred[id]; !keep { + idsToTrim = append(idsToTrim, id) + } + } + } drainCtx, drainCancel := wikiIngestCleanupContext(ctx) - err = s.trimPendingList(drainCtx, ids) + err = s.trimPendingList(drainCtx, idsToTrim) drainCancel() if err != nil { return fmt.Errorf("wiki finalize: trim pending rows: %w", err) @@ -1093,14 +1154,20 @@ func (s *wikiIngestService) ProcessWikiFinalize(ctx context.Context, t *asynq.Ta // If more finalize rows landed while we were working, reschedule so they // get their own convergence pass. rescheduled := false + if pruneDeferred { + s.scheduleFinalizeRetry(ctx, payload) + rescheduled = true + } if n, cErr := s.pendingRepo.PendingCount(ctx, wikiFinalizeTaskType, wikiTaskScope, payload.KnowledgeBaseID); cErr == nil && n > 0 { - s.scheduleFinalize(ctx, payload) + if !pruneDeferred { + s.scheduleFinalize(ctx, payload) + } rescheduled = true } logger.Infof(ctx, - "wiki finalize: kb=%s rows=%d affected_slugs=%d index_rebuilt=%v rescheduled=%v elapsed=%s", - payload.KnowledgeBaseID, len(rows), len(affectedSlugs), indexRebuilt, rescheduled, + "wiki finalize: kb=%s rows=%d affected_slugs=%d deleted_folders=%d folder_prune_deferred=%v index_rebuilt=%v rescheduled=%v elapsed=%s", + payload.KnowledgeBaseID, len(rows), len(affectedSlugs), deletedFolders, pruneDeferred, indexRebuilt, rescheduled, time.Since(startedAt).Round(time.Millisecond), ) return nil diff --git a/internal/application/service/wiki_page.go b/internal/application/service/wiki_page.go index 06b0e7f9..55889bbf 100644 --- a/internal/application/service/wiki_page.go +++ b/internal/application/service/wiki_page.go @@ -1460,18 +1460,97 @@ func (s *wikiPageService) DeleteFolder(ctx context.Context, kbID string, id stri return err } if len(children) > 0 { - return errors.New("folder is not empty: it still has sub-folders") + return repository.ErrWikiFolderNotEmpty } pages, err := s.repo.ListPagesByFolderIDs(ctx, kbID, []string{id}) if err != nil { return err } if len(pages) > 0 { - return errors.New("folder is not empty: it still contains pages") + return repository.ErrWikiFolderNotEmpty } return s.repo.DeleteFolder(ctx, kbID, id) } +// PruneEmptyFolderChains removes folders that became empty after retracting a +// document, followed by any ancestors made empty by those removals. It only +// considers the supplied folder chains, so intentionally-empty folders +// elsewhere in the wiki are preserved. Callers must wait for the KB's ingest +// queue to drain before invoking this method: taxonomy planning creates a +// folder before reduce writes the page that will reference it. +func (s *wikiPageService) PruneEmptyFolderChains( + ctx context.Context, kbID string, folderIDs []string, +) ([]string, error) { + if len(folderIDs) == 0 { + return nil, nil + } + all, err := s.repo.ListAllFolders(ctx, kbID) + if err != nil { + return nil, err + } + byID := make(map[string]*types.WikiFolder, len(all)) + for _, folder := range all { + if folder != nil { + byID[folder.ID] = folder + } + } + + candidates := make(map[string]*types.WikiFolder) + for _, id := range folderIDs { + seen := make(map[string]struct{}) + for id != types.WikiFolderRootID { + if _, cycle := seen[id]; cycle { + break + } + seen[id] = struct{}{} + folder := byID[id] + if folder == nil { + break + } + candidates[id] = folder + id = folder.ParentID + } + } + + ordered := make([]*types.WikiFolder, 0, len(candidates)) + for _, folder := range candidates { + ordered = append(ordered, folder) + } + sort.Slice(ordered, func(i, j int) bool { + if ordered[i].Depth == ordered[j].Depth { + return ordered[i].Path > ordered[j].Path + } + return ordered[i].Depth > ordered[j].Depth + }) + + deleted := make([]string, 0, len(ordered)) + for _, folder := range ordered { + children, err := s.repo.ListChildFolders(ctx, kbID, folder.ID) + if err != nil { + return deleted, err + } + if len(children) > 0 { + continue + } + pages, err := s.repo.ListPagesByFolderIDs(ctx, kbID, []string{folder.ID}) + if err != nil { + return deleted, err + } + if len(pages) > 0 { + continue + } + if err := s.repo.DeleteFolder(ctx, kbID, folder.ID); err != nil { + if errors.Is(err, repository.ErrWikiFolderNotFound) || + errors.Is(err, repository.ErrWikiFolderNotEmpty) { + continue + } + return deleted, err + } + deleted = append(deleted, folder.ID) + } + return deleted, nil +} + // InjectCrossLinks scans affected pages and injects [[wiki-links]] for mentions // of other wiki page titles in the content. Pure text replacement, no LLM call. // Shares the linkifyContent helper with the ingest pipeline so both paths honor diff --git a/internal/application/service/wiki_page_test.go b/internal/application/service/wiki_page_test.go index eb7d34c5..84c4b9ab 100644 --- a/internal/application/service/wiki_page_test.go +++ b/internal/application/service/wiki_page_test.go @@ -1,11 +1,58 @@ package service import ( + "context" + "fmt" "testing" + "time" + "github.com/Tencent/WeKnora/internal/application/repository" "github.com/Tencent/WeKnora/internal/types" + "github.com/stretchr/testify/require" + "gorm.io/driver/sqlite" + "gorm.io/gorm" ) +func TestPruneEmptyFolderChainsDeletesOnlyEmptyCandidateAncestors(t *testing.T) { + db, err := gorm.Open(sqlite.Open(fmt.Sprintf("file:%s?mode=memory&cache=shared", t.Name())), &gorm.Config{}) + require.NoError(t, err) + require.NoError(t, db.AutoMigrate(&types.WikiFolder{}, &types.WikiPage{})) + + ctx := context.Background() + repo := repository.NewWikiPageRepository(db) + svc := NewWikiPageService(repo, nil, nil, nil, nil) + now := time.Now() + createFolder := func(id, parentID, name, path string, depth int) { + require.NoError(t, repo.CreateFolder(ctx, &types.WikiFolder{ + ID: id, TenantID: 1, KnowledgeBaseID: "kb-prune", ParentID: parentID, + Name: name, Path: path, Depth: depth, CreatedAt: now, UpdatedAt: now, + })) + } + createFolder("topic", "", "Topic", "Topic", 1) + createFolder("empty-leaf", "topic", "Empty", "Topic/Empty", 2) + createFolder("occupied-leaf", "topic", "Occupied", "Topic/Occupied", 2) + createFolder("empty-chain", "", "Empty chain", "Empty chain", 1) + createFolder("empty-chain-leaf", "empty-chain", "Leaf", "Empty chain/Leaf", 2) + createFolder("unrelated-empty", "", "Keep me", "Keep me", 1) + + require.NoError(t, repo.Create(ctx, &types.WikiPage{ + ID: "page-1", TenantID: 1, KnowledgeBaseID: "kb-prune", Slug: "entity/occupied", + Title: "Occupied", PageType: types.WikiPageTypeEntity, Status: types.WikiPageStatusPublished, + FolderID: "occupied-leaf", Version: 1, CreatedAt: now, UpdatedAt: now, + })) + + deleted, err := svc.PruneEmptyFolderChains(ctx, "kb-prune", []string{"empty-leaf", "empty-chain-leaf"}) + require.NoError(t, err) + require.ElementsMatch(t, []string{"empty-leaf", "empty-chain-leaf", "empty-chain"}, deleted) + + _, err = repo.GetFolderByID(ctx, "kb-prune", "topic") + require.NoError(t, err, "ancestor with another occupied child must remain") + _, err = repo.GetFolderByID(ctx, "kb-prune", "occupied-leaf") + require.NoError(t, err) + _, err = repo.GetFolderByID(ctx, "kb-prune", "unrelated-empty") + require.NoError(t, err, "empty folders outside the affected chains must remain") +} + func TestStripWikiInlineChunkCitations(t *testing.T) { input := "[**橡皮障夹**](#)**钳**\n\n夹钳是用于夹持橡皮障夹的专用器械[c003]。手柄便于操作 [c003]。多个来源[c003, c1000]。" want := "[**橡皮障夹**](#)**钳**\n\n夹钳是用于夹持橡皮障夹的专用器械。手柄便于操作。多个来源。" diff --git a/internal/handler/wiki_page.go b/internal/handler/wiki_page.go index 2d7ea4cd..a954d9d3 100644 --- a/internal/handler/wiki_page.go +++ b/internal/handler/wiki_page.go @@ -314,7 +314,7 @@ func writeWikiFolderError(c *gin.Context, err error) { switch { case stderrors.Is(err, repository.ErrWikiFolderNotFound), stderrors.Is(err, repository.ErrWikiPageNotFound): c.JSON(http.StatusNotFound, gin.H{"error": err.Error()}) - case stderrors.Is(err, repository.ErrWikiFolderConflict): + case stderrors.Is(err, repository.ErrWikiFolderConflict), stderrors.Is(err, repository.ErrWikiFolderNotEmpty): c.JSON(http.StatusConflict, gin.H{"error": err.Error()}) default: c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) diff --git a/internal/types/interfaces/wiki_page.go b/internal/types/interfaces/wiki_page.go index 30c73dc6..9885009e 100644 --- a/internal/types/interfaces/wiki_page.go +++ b/internal/types/interfaces/wiki_page.go @@ -185,6 +185,10 @@ type WikiPageService interface { // DeleteFolder removes an empty folder. Fails if it still contains pages // or child folders (the UI must move or delete contents first). DeleteFolder(ctx context.Context, kbID string, id string) error + // PruneEmptyFolderChains deletes candidate folders that are still empty, + // walking upward through newly-empty ancestors. Candidates are supplied by + // document retract cleanup so unrelated user-created empty folders remain. + PruneEmptyFolderChains(ctx context.Context, kbID string, folderIDs []string) ([]string, error) // FindOrCreateFolderPath resolves a category path (e.g. ["AI","RAG"]) to a // folder id, creating any missing intermediate folders. Returns the leaf // folder id and the canonical (cleaned) path. An empty/blank path resolves