diff --git a/internal/application/service/knowledge_post_process.go b/internal/application/service/knowledge_post_process.go index 8a04e95e..dd2656a6 100644 --- a/internal/application/service/knowledge_post_process.go +++ b/internal/application/service/knowledge_post_process.go @@ -143,12 +143,19 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta // on its terminal exit; the row promotes itself to "completed" when // the counter hits zero (see knowledgeRepository.FinalizeSubtask). // - // Wiki ingest is NOT counted here — it's a KB-scoped debounced - // batch with its own dedup queue; per-knowledge accounting is not - // meaningful for it. + // Wiki ingest IS counted (as a single subtask): although it's a + // KB-scoped debounced batch, each upload enqueues exactly one + // per-knowledge op, and the batch worker calls FinalizeSubtask once + // when that op reaches a terminal state (mapped successfully or + // dead-lettered after exhausting retries). Counting it keeps the row + // in "finalizing" — i.e. shown as in-progress and still cancellable — + // until wiki generation actually finishes instead of flipping to + // completed while wiki runs minutes later. A wiki op that never + // drains is bounded by the housekeeping finalizing sweep. willSpawnSummary := len(textChunks) > 0 willSpawnQuestion := willSpawnSummary && kb.NeedsEmbeddingModel() && kb.QuestionGenerationConfig != nil && kb.QuestionGenerationConfig.Enabled + willSpawnWiki := kb.IndexingStrategy.WikiEnabled && len(textChunks) > 0 graphChunkCount := 0 if kb.IsGraphEnabled() { graphChunkCount = len(textChunks) @@ -160,6 +167,9 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta if willSpawnQuestion { expectedSubtasks++ } + if willSpawnWiki { + expectedSubtasks++ + } expectedSubtasks += graphChunkCount switch { @@ -265,7 +275,7 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta // 6. Spawn Wiki Ingest Task if wiki indexing is enabled in IndexingStrategy enqueuedWiki := false - if kb.IndexingStrategy.WikiEnabled && len(textChunks) > 0 { + if willSpawnWiki { EnqueueWikiIngest(ctx, s.taskEnqueuer, s.pendingRepo, payload.TenantID, payload.KnowledgeBaseID, payload.KnowledgeID) logger.Infof(ctx, "[KnowledgePostProcess] Enqueued wiki ingest task for %s", payload.KnowledgeID) enqueuedWiki = true diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go index cc6d6d75..67f48ac6 100644 --- a/internal/application/service/wiki_ingest.go +++ b/internal/application/service/wiki_ingest.go @@ -194,6 +194,7 @@ type wikiIngestService struct { wikiService interfaces.WikiPageService kbService interfaces.KnowledgeBaseService knowledgeSvc interfaces.KnowledgeService + knowledgeRepo interfaces.KnowledgeRepository chunkRepo interfaces.ChunkRepository modelService interfaces.ModelService task interfaces.TaskEnqueuer @@ -218,6 +219,7 @@ func NewWikiIngestService( wikiService interfaces.WikiPageService, kbService interfaces.KnowledgeBaseService, knowledgeSvc interfaces.KnowledgeService, + knowledgeRepo interfaces.KnowledgeRepository, chunkRepo interfaces.ChunkRepository, modelService interfaces.ModelService, task interfaces.TaskEnqueuer, @@ -231,6 +233,7 @@ func NewWikiIngestService( wikiService: wikiService, kbService: kbService, knowledgeSvc: knowledgeSvc, + knowledgeRepo: knowledgeRepo, chunkRepo: chunkRepo, modelService: modelService, task: task, @@ -506,6 +509,26 @@ func (s *wikiIngestService) trimPendingList(ctx context.Context, ids []int64) { } } +// finalizeWikiSubtask releases this knowledge's slot in the finalizing +// counter once its wiki op reaches a terminal state (mapped successfully +// or dead-lettered). The matching +1 is seeded by +// KnowledgePostProcess.SetFinalizing when willSpawnWiki is true. Callers +// must only invoke this for ingest ops — retract ops are for deleted +// knowledge that has no counter to drain. +// +// Safe to call on a row that is already completed or whose counter is +// already zero: FinalizeSubtask guards both the decrement (count > 0) and +// the promote (parse_status = finalizing AND count = 0), so an op enqueued +// before this accounting shipped is a harmless no-op. +func (s *wikiIngestService) finalizeWikiSubtask(ctx context.Context, knowledgeID string) { + if s.knowledgeRepo == nil || knowledgeID == "" { + return + } + if _, _, err := s.knowledgeRepo.FinalizeSubtask(ctx, knowledgeID); err != nil { + logger.Warnf(ctx, "wiki ingest: FinalizeSubtask failed for %s: %v", knowledgeID, err) + } +} + // requeueFailedOps records in-batch failures. // // For each failed op: @@ -544,7 +567,14 @@ func (s *wikiIngestService) requeueFailedOps(ctx context.Context, payload WikiIn continue } - // Exhausted in-batch retries — archive and remove. + // Exhausted in-batch retries — archive and remove. This is the + // terminal failure point for the op, so release its slot in the + // knowledge's finalizing counter (ingest ops only; retracts are + // for deleted knowledge that has no counter to drain). The + // matching +1 was seeded by KnowledgePostProcess.SetFinalizing. + if op.Op == WikiOpIngest { + s.finalizeWikiSubtask(ctx, op.KnowledgeID) + } logger.Warnf(ctx, "wiki ingest: dropping op %s (%s) after %d failures (limit %d)", op.KnowledgeID, op.DocTitle, count, wikiMaxFailRetries) if s.deadLetterRepo != nil { payloadBytes, _ := json.Marshal(op) diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go index b5d8b401..8be3dc85 100644 --- a/internal/application/service/wiki_ingest_batch.go +++ b/internal/application/service/wiki_ingest_batch.go @@ -667,7 +667,17 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task // failed). failedAdditionSlugCount := len(failedAdditionSlugs) for _, r := range docResults { - if r == nil || r.WikiSpan == nil { + if r == nil { + continue + } + // A successfully-mapped doc is terminal for its wiki op, so + // release the knowledge's slot in pending_subtasks_count (the row + // promotes to completed once the counter hits zero). Done before + // the WikiSpan nil-check below so a doc that had no attempt to + // attach a span to still drains its counter slot. The matching +1 + // is seeded by KnowledgePostProcess.SetFinalizing. + s.finalizeWikiSubtask(ctx, r.KnowledgeID) + if r.WikiSpan == nil { continue } writtenPages := make([]map[string]string, 0, len(r.Pages))