From 72272b2d62e3e112ae83997ccbe09065c275563e Mon Sep 17 00:00:00 2001 From: wizardchen Date: Fri, 29 May 2026 18:00:29 +0800 Subject: [PATCH] feat(knowledge): count wiki ingest in the finalizing subtask counter Wiki ingest runs asynchronously after the parse pipeline and was not counted in pending_subtasks_count, so a document flipped to "completed" while wiki generation was still minutes away (30s debounce + batch + retries). That hid the in-progress state and dropped the stop-parse affordance before wiki actually finished. Count wiki as a single enrichment subtask (when WikiEnabled and the doc has text chunks, matching the enqueue condition) so the row stays in "finalizing" until wiki is done. The batch worker drains the slot at the op's terminal state: once on successful map, and once when an op is dead-lettered after exhausting in-batch retries. Retract ops (deleted knowledge) are skipped. FinalizeSubtask guards both the decrement and the promote, so a wiki op enqueued before this accounting shipped is a harmless no-op on an already-completed row. A wiki op that never drains is bounded by the existing housekeeping finalizing sweep. --- .../service/knowledge_post_process.go | 18 ++++++++--- internal/application/service/wiki_ingest.go | 32 ++++++++++++++++++- .../application/service/wiki_ingest_batch.go | 12 ++++++- 3 files changed, 56 insertions(+), 6 deletions(-) diff --git a/internal/application/service/knowledge_post_process.go b/internal/application/service/knowledge_post_process.go index 8a04e95e..dd2656a6 100644 --- a/internal/application/service/knowledge_post_process.go +++ b/internal/application/service/knowledge_post_process.go @@ -143,12 +143,19 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta // on its terminal exit; the row promotes itself to "completed" when // the counter hits zero (see knowledgeRepository.FinalizeSubtask). // - // Wiki ingest is NOT counted here — it's a KB-scoped debounced - // batch with its own dedup queue; per-knowledge accounting is not - // meaningful for it. + // Wiki ingest IS counted (as a single subtask): although it's a + // KB-scoped debounced batch, each upload enqueues exactly one + // per-knowledge op, and the batch worker calls FinalizeSubtask once + // when that op reaches a terminal state (mapped successfully or + // dead-lettered after exhausting retries). Counting it keeps the row + // in "finalizing" — i.e. shown as in-progress and still cancellable — + // until wiki generation actually finishes instead of flipping to + // completed while wiki runs minutes later. A wiki op that never + // drains is bounded by the housekeeping finalizing sweep. willSpawnSummary := len(textChunks) > 0 willSpawnQuestion := willSpawnSummary && kb.NeedsEmbeddingModel() && kb.QuestionGenerationConfig != nil && kb.QuestionGenerationConfig.Enabled + willSpawnWiki := kb.IndexingStrategy.WikiEnabled && len(textChunks) > 0 graphChunkCount := 0 if kb.IsGraphEnabled() { graphChunkCount = len(textChunks) @@ -160,6 +167,9 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta if willSpawnQuestion { expectedSubtasks++ } + if willSpawnWiki { + expectedSubtasks++ + } expectedSubtasks += graphChunkCount switch { @@ -265,7 +275,7 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta // 6. Spawn Wiki Ingest Task if wiki indexing is enabled in IndexingStrategy enqueuedWiki := false - if kb.IndexingStrategy.WikiEnabled && len(textChunks) > 0 { + if willSpawnWiki { EnqueueWikiIngest(ctx, s.taskEnqueuer, s.pendingRepo, payload.TenantID, payload.KnowledgeBaseID, payload.KnowledgeID) logger.Infof(ctx, "[KnowledgePostProcess] Enqueued wiki ingest task for %s", payload.KnowledgeID) enqueuedWiki = true diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go index cc6d6d75..67f48ac6 100644 --- a/internal/application/service/wiki_ingest.go +++ b/internal/application/service/wiki_ingest.go @@ -194,6 +194,7 @@ type wikiIngestService struct { wikiService interfaces.WikiPageService kbService interfaces.KnowledgeBaseService knowledgeSvc interfaces.KnowledgeService + knowledgeRepo interfaces.KnowledgeRepository chunkRepo interfaces.ChunkRepository modelService interfaces.ModelService task interfaces.TaskEnqueuer @@ -218,6 +219,7 @@ func NewWikiIngestService( wikiService interfaces.WikiPageService, kbService interfaces.KnowledgeBaseService, knowledgeSvc interfaces.KnowledgeService, + knowledgeRepo interfaces.KnowledgeRepository, chunkRepo interfaces.ChunkRepository, modelService interfaces.ModelService, task interfaces.TaskEnqueuer, @@ -231,6 +233,7 @@ func NewWikiIngestService( wikiService: wikiService, kbService: kbService, knowledgeSvc: knowledgeSvc, + knowledgeRepo: knowledgeRepo, chunkRepo: chunkRepo, modelService: modelService, task: task, @@ -506,6 +509,26 @@ func (s *wikiIngestService) trimPendingList(ctx context.Context, ids []int64) { } } +// finalizeWikiSubtask releases this knowledge's slot in the finalizing +// counter once its wiki op reaches a terminal state (mapped successfully +// or dead-lettered). The matching +1 is seeded by +// KnowledgePostProcess.SetFinalizing when willSpawnWiki is true. Callers +// must only invoke this for ingest ops — retract ops are for deleted +// knowledge that has no counter to drain. +// +// Safe to call on a row that is already completed or whose counter is +// already zero: FinalizeSubtask guards both the decrement (count > 0) and +// the promote (parse_status = finalizing AND count = 0), so an op enqueued +// before this accounting shipped is a harmless no-op. +func (s *wikiIngestService) finalizeWikiSubtask(ctx context.Context, knowledgeID string) { + if s.knowledgeRepo == nil || knowledgeID == "" { + return + } + if _, _, err := s.knowledgeRepo.FinalizeSubtask(ctx, knowledgeID); err != nil { + logger.Warnf(ctx, "wiki ingest: FinalizeSubtask failed for %s: %v", knowledgeID, err) + } +} + // requeueFailedOps records in-batch failures. // // For each failed op: @@ -544,7 +567,14 @@ func (s *wikiIngestService) requeueFailedOps(ctx context.Context, payload WikiIn continue } - // Exhausted in-batch retries — archive and remove. + // Exhausted in-batch retries — archive and remove. This is the + // terminal failure point for the op, so release its slot in the + // knowledge's finalizing counter (ingest ops only; retracts are + // for deleted knowledge that has no counter to drain). The + // matching +1 was seeded by KnowledgePostProcess.SetFinalizing. + if op.Op == WikiOpIngest { + s.finalizeWikiSubtask(ctx, op.KnowledgeID) + } logger.Warnf(ctx, "wiki ingest: dropping op %s (%s) after %d failures (limit %d)", op.KnowledgeID, op.DocTitle, count, wikiMaxFailRetries) if s.deadLetterRepo != nil { payloadBytes, _ := json.Marshal(op) diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go index b5d8b401..8be3dc85 100644 --- a/internal/application/service/wiki_ingest_batch.go +++ b/internal/application/service/wiki_ingest_batch.go @@ -667,7 +667,17 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task // failed). failedAdditionSlugCount := len(failedAdditionSlugs) for _, r := range docResults { - if r == nil || r.WikiSpan == nil { + if r == nil { + continue + } + // A successfully-mapped doc is terminal for its wiki op, so + // release the knowledge's slot in pending_subtasks_count (the row + // promotes to completed once the counter hits zero). Done before + // the WikiSpan nil-check below so a doc that had no attempt to + // attach a span to still drains its counter slot. The matching +1 + // is seeded by KnowledgePostProcess.SetFinalizing. + s.finalizeWikiSubtask(ctx, r.KnowledgeID) + if r.WikiSpan == nil { continue } writtenPages := make([]map[string]string, 0, len(r.Pages))