diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go index e69346b2..b8723e19 100644 --- a/internal/application/service/wiki_ingest.go +++ b/internal/application/service/wiki_ingest.go @@ -200,6 +200,10 @@ const ( // top of the asynq.TaskID coalescing). wikiFinalizeLockTTL = 60 * time.Second wikiFinalizeLockRenew = 20 * time.Second + + // wikiIngestCleanupTimeout bounds detached tail cleanup after the asynq + // task context has been cancelled or hit its timeout. + wikiIngestCleanupTimeout = 10 * time.Second ) // wikiFinalizeChange is a doc-level add/remove entry for the index-intro @@ -537,6 +541,13 @@ func (s *wikiIngestService) Handle(ctx context.Context, t *asynq.Task) error { } } +func wikiIngestCleanupContext(ctx context.Context) (context.Context, context.CancelFunc) { + if ctx == nil { + ctx = context.Background() + } + return context.WithTimeout(context.WithoutCancel(ctx), wikiIngestCleanupTimeout) +} + // enqueueFinalize persists this batch's KB-global convergence work into the // finalize lane of task_pending_ops and schedules a debounced trigger. One // "slug" row per affected page (carrying its fresh title when this batch wrote @@ -920,13 +931,15 @@ func (s *wikiIngestService) decodePendingRows(ctx context.Context, rows []*types // trimPendingList deletes consumed rows from task_pending_ops. Empty // input is a no-op so callers can invoke unconditionally at the end // of a batch. -func (s *wikiIngestService) trimPendingList(ctx context.Context, ids []int64) { +func (s *wikiIngestService) trimPendingList(ctx context.Context, ids []int64) error { if s.pendingRepo == nil || len(ids) == 0 { - return + return nil } if err := s.pendingRepo.DeleteByIDs(ctx, ids); err != nil { logger.Warnf(ctx, "wiki ingest: failed to trim %d pending rows: %v", len(ids), err) + return err } + return nil } // finalizeWikiSubtask releases this knowledge's slot in the finalizing @@ -960,13 +973,13 @@ func (s *wikiIngestService) finalizeWikiSubtask(ctx context.Context, knowledgeID // (rows are ordered by id ASC and we never moved/touched it). // - If the count exceeds the retry cap: archive the op into // task_dead_letters and DeleteByIDs to remove it from the queue. -// Both writes are best-effort — a DB failure here is logged and -// swallowed so a single transient blip doesn't recursively spawn -// more failures. -func (s *wikiIngestService) requeueFailedOps(ctx context.Context, payload WikiIngestPayload, ops []WikiPendingOp) { +// Settlement failures are returned so the caller does not mark claims +// settled while rows are still claimed or undeleted. +func (s *wikiIngestService) requeueFailedOps(ctx context.Context, payload WikiIngestPayload, ops []WikiPendingOp) error { if s.pendingRepo == nil || len(ops) == 0 { - return + return nil } + var settleErrs []error for _, op := range ops { if op.dbID == 0 { // Op was never persisted (synthetic / test) — nothing to @@ -976,6 +989,7 @@ func (s *wikiIngestService) requeueFailedOps(ctx context.Context, payload WikiIn count, err := s.pendingRepo.IncrFailCount(ctx, op.dbID) if err != nil { logger.Warnf(ctx, "wiki ingest: failed to increment fail count for %s (id=%d): %v", op.KnowledgeID, op.dbID, err) + settleErrs = append(settleErrs, fmt.Errorf("increment fail count id=%d: %w", op.dbID, err)) // Without a fresh count we can't tell whether to drop. Be // conservative: leave the row in place; the next PeekBatch // will see it again and we'll try once more. @@ -989,6 +1003,7 @@ func (s *wikiIngestService) requeueFailedOps(ctx context.Context, payload WikiIn // budget still counts down. if err := s.pendingRepo.ReleaseByIDs(ctx, []int64{op.dbID}); err != nil { logger.Warnf(ctx, "wiki ingest: failed to release claim for retry id=%d: %v", op.dbID, err) + settleErrs = append(settleErrs, fmt.Errorf("release retry claim id=%d: %w", op.dbID, err)) } logger.Infof(ctx, "wiki ingest: re-queued failed op %s (%s) for retry (attempt %d/%d)", op.KnowledgeID, op.DocTitle, count, wikiMaxFailRetries) continue @@ -1016,12 +1031,15 @@ func (s *wikiIngestService) requeueFailedOps(ctx context.Context, payload WikiIn FailCount: count, }); dlErr != nil { logger.Warnf(ctx, "wiki ingest: failed to archive op %s to dead letters: %v", op.KnowledgeID, dlErr) + settleErrs = append(settleErrs, fmt.Errorf("archive dead letter id=%d: %w", op.dbID, dlErr)) } } if err := s.pendingRepo.DeleteByIDs(ctx, []int64{op.dbID}); err != nil { logger.Warnf(ctx, "wiki ingest: failed to drop dead-lettered row id=%d: %v", op.dbID, err) + settleErrs = append(settleErrs, fmt.Errorf("drop dead-lettered row id=%d: %w", op.dbID, err)) } } + return errors.Join(settleErrs...) } // docIngestResult captures per-document info for batch post-processing. diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go index da45d070..3321eaa8 100644 --- a/internal/application/service/wiki_ingest_batch.go +++ b/internal/application/service/wiki_ingest_batch.go @@ -3,6 +3,7 @@ package service import ( "context" "encoding/json" + "errors" "fmt" "strings" "sync" @@ -377,15 +378,18 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task // so the next trigger can re-claim within seconds instead of waiting out // wikiClaimStaleAfter (~90m). On the normal path claimsSettled flips true // once the rows reach their terminal state, making this a no-op. Uses a - // background context because ctx may already be cancelled on the timeout - // path. Lite mode peeks without claiming, so there is nothing to release. + // bounded detached cleanup context because ctx may already be cancelled on + // the timeout path. Lite mode peeks without claiming, so there is nothing + // to release. claimsSettled := false if s.redisClient != nil && len(peekedIDs) > 0 { defer func() { if claimsSettled { return } - if err := s.pendingRepo.ReleaseByIDs(context.Background(), peekedIDs); err != nil { + releaseCtx, releaseCancel := wikiIngestCleanupContext(ctx) + defer releaseCancel() + if err := s.pendingRepo.ReleaseByIDs(releaseCtx, peekedIDs); err != nil { logger.Warnf(ctx, "wiki ingest: failed to release %d claims on abnormal exit for KB %s: %v", len(peekedIDs), payload.KnowledgeBaseID, err) return } @@ -613,6 +617,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task return reduceErr }) if lockErr != nil { + collectUnapplied(updates) // ctx cancelled (batch timeout / shutdown) — stop quietly. return nil } @@ -659,6 +664,9 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task } _ = egReduce.Wait() + tailCtx, tailCancel := wikiIngestCleanupContext(ctx) + defer tailCancel() + // Sanitize the doc summary pages produced by this batch BEFORE we // build log entries / rebuild the index. The summary LLM (run during // map) was free to inject [[entity/foo|name]] links to every slug it @@ -666,7 +674,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task // those slugs into actual pages. Rewrite those dead links to plain // text so the summary doesn't contain unresolvable references. if len(failedAdditionSlugs) > 0 && len(docResults) > 0 { - s.sanitizeDeadSummaryLinks(ctx, payload.KnowledgeBaseID, docResults, failedAdditionSlugs, batchCtx) + s.sanitizeDeadSummaryLinks(tailCtx, payload.KnowledgeBaseID, docResults, failedAdditionSlugs, batchCtx) } totalPagesAffected = len(allPagesAffected) @@ -687,7 +695,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task if len(slugs) == 0 { return nil } - titles := batchCtx.SlugTitleMany(ctx, slugs) + titles := batchCtx.SlugTitleMany(tailCtx, slugs) out := make([]types.WikiLogPageRef, 0, len(slugs)) for _, slug := range slugs { out = append(out, types.WikiLogPageRef{Slug: slug, Title: titles[slug]}) @@ -718,7 +726,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task logEntries = append(logEntries, s.buildLogEntry(payload.TenantID, payload.KnowledgeBaseID, "ingest", r.KnowledgeID, r.DocTitle, r.Summary, pages)) } if len(logEntries) > 0 && s.logEntrySvc != nil { - if err := s.logEntrySvc.AppendBatch(ctx, logEntries); err != nil { + if err := s.logEntrySvc.AppendBatch(tailCtx, logEntries); err != nil { logger.Warnf(ctx, "wiki ingest: failed to append %d log entries: %v", len(logEntries), err) } } @@ -728,7 +736,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task // written, not after the debounce window. This is a cheap status flip. if len(allPagesAffected) > 0 { logger.Infof(ctx, "wiki ingest: publishing draft pages") - s.publishDraftPages(ctx, payload.KnowledgeBaseID, allPagesAffected) + s.publishDraftPages(tailCtx, payload.KnowledgeBaseID, allPagesAffected) } // Defer KB-global convergence (index-intro rebuild + dead-link cleanup + @@ -769,7 +777,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task }) } } - s.enqueueFinalize(ctx, payload, allPagesAffected, freshTitleBySlug, changes) + s.enqueueFinalize(tailCtx, payload, allPagesAffected, freshTitleBySlug, changes) } // Close postprocess.wiki spans for every successfully-mapped doc. @@ -781,6 +789,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task // actually landed (vs. dropped because reduce-phase generation // failed). failedAdditionSlugCount := len(failedAdditionSlugs) + spanCtx, spanCancel := wikiIngestCleanupContext(ctx) for _, r := range docResults { if r == nil { continue @@ -828,8 +837,9 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task for k, v := range r.MapStats { output[k] = v } - s.tracker().EndSpan(ctx, r.WikiSpan, output) + s.tracker().EndSpan(spanCtx, r.WikiSpan, output) } + spanCancel() // Failed-map docs already had FailSpan called inside // mapOneDocument (the failedOps path returns before reaching // docResults). Nothing extra to do here for them. @@ -875,14 +885,21 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task } trimIDs = append(trimIDs, id) } - s.trimPendingList(ctx, trimIDs) + settleCtx, settleCancel := wikiIngestCleanupContext(ctx) + trimErr := s.trimPendingList(settleCtx, trimIDs) // Process failed ops: increment fail_count and dead-letter once // the cap is hit. Must come AFTER trim so successful siblings are // already gone from the queue — otherwise a follow-up batch could // re-pick them up. + var requeueErr error if len(failedOps) > 0 { - s.requeueFailedOps(ctx, payload, failedOps) + requeueErr = s.requeueFailedOps(settleCtx, payload, failedOps) + } + settleCancel() + if err := errors.Join(trimErr, requeueErr); err != nil { + exitStatus = "settle_failed" + return fmt.Errorf("wiki ingest: settle claimed rows: %w", err) } // All claimed rows have now reached a terminal state (deleted on success, // released for retry, or dead-lettered), so disarm the abnormal-exit @@ -898,7 +915,9 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task followUpDelay = wikiRateLimitBackoff logger.Warnf(ctx, "wiki ingest: KB %s hit upstream rate limiting, backing off follow-up to %s", payload.KnowledgeBaseID, followUpDelay) } - followUpScheduled = s.scheduleFollowUp(ctx, payload, followUpDelay) + followCtx, followCancel := wikiIngestCleanupContext(ctx) + followUpScheduled = s.scheduleFollowUp(followCtx, payload, followUpDelay) + followCancel() return nil } @@ -1018,7 +1037,12 @@ func (s *wikiIngestService) ProcessWikiFinalize(ctx context.Context, t *asynq.Ta // KB flipped away from wiki (deleted / type change) — drain the lane so the // rows don't accumulate, then stop. if !kb.IsWikiEnabled() { - s.trimPendingList(ctx, ids) + drainCtx, drainCancel := wikiIngestCleanupContext(ctx) + err := s.trimPendingList(drainCtx, ids) + drainCancel() + if err != nil { + return fmt.Errorf("wiki finalize: trim disabled lane: %w", err) + } return nil } @@ -1059,7 +1083,12 @@ func (s *wikiIngestService) ProcessWikiFinalize(ctx context.Context, t *asynq.Ta // Drain the processed rows. Best-effort convergence mirrors the legacy // in-batch behaviour: a failed index rebuild is logged (not retried), // so we delete regardless to avoid re-doing the whole pass forever. - s.trimPendingList(ctx, ids) + drainCtx, drainCancel := wikiIngestCleanupContext(ctx) + err = s.trimPendingList(drainCtx, ids) + drainCancel() + if err != nil { + return fmt.Errorf("wiki finalize: trim pending rows: %w", err) + } // If more finalize rows landed while we were working, reschedule so they // get their own convergence pass. diff --git a/internal/application/service/wiki_ingest_test.go b/internal/application/service/wiki_ingest_test.go index 16d4e35a..dc45284a 100644 --- a/internal/application/service/wiki_ingest_test.go +++ b/internal/application/service/wiki_ingest_test.go @@ -2,8 +2,10 @@ package service import ( "context" + "errors" "strings" "testing" + "time" "github.com/Tencent/WeKnora/internal/models/chat" "github.com/Tencent/WeKnora/internal/types" @@ -355,3 +357,133 @@ func (m *templateCaptureChatModel) ChatStream( func (m *templateCaptureChatModel) GetModelName() string { return "capture" } func (m *templateCaptureChatModel) GetModelID() string { return "capture" } + +func TestWikiIngestCleanupContextDetachedFromCancelledParent(t *testing.T) { + parent := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(42)) + ctx, cancel := context.WithCancel(parent) + cancel() + + cleanupCtx, cleanupCancel := wikiIngestCleanupContext(ctx) + defer cleanupCancel() + + repo := &wikiPendingRepoForCleanupTest{} + svc := &wikiIngestService{pendingRepo: repo} + if err := svc.trimPendingList(cleanupCtx, []int64{7}); err != nil { + t.Fatalf("trimPendingList() error = %v", err) + } + if repo.deleteCtxErr != nil { + t.Fatalf("cleanup context should be detached from parent cancellation, got %v", repo.deleteCtxErr) + } + if got := cleanupCtx.Value(types.TenantIDContextKey); got != uint64(42) { + t.Fatalf("cleanup context lost tenant value: %v", got) + } +} + +func TestTrimPendingListReturnsDeleteError(t *testing.T) { + wantErr := errors.New("delete failed") + repo := &wikiPendingRepoForCleanupTest{deleteErr: wantErr} + svc := &wikiIngestService{pendingRepo: repo} + + err := svc.trimPendingList(context.Background(), []int64{1, 2, 3}) + if !errors.Is(err, wantErr) { + t.Fatalf("trimPendingList() error = %v, want %v", err, wantErr) + } +} + +func TestRequeueFailedOpsReturnsReleaseError(t *testing.T) { + wantErr := errors.New("release failed") + repo := &wikiPendingRepoForCleanupTest{incrCount: 1, releaseErr: wantErr} + svc := &wikiIngestService{pendingRepo: repo} + + err := svc.requeueFailedOps(context.Background(), WikiIngestPayload{}, []WikiPendingOp{{ + Op: WikiOpIngest, + KnowledgeID: "k1", + DocTitle: "doc", + dbID: 99, + }}) + if !errors.Is(err, wantErr) { + t.Fatalf("requeueFailedOps() error = %v, want %v", err, wantErr) + } +} + +type wikiPendingRepoForCleanupTest struct { + deleteErr error + releaseErr error + incrErr error + incrCount int + deleteCtxErr error + releaseCtxErr error + incrCtxErr error +} + +func (r *wikiPendingRepoForCleanupTest) Enqueue(context.Context, *types.TaskPendingOp) error { + return nil +} + +func (r *wikiPendingRepoForCleanupTest) PeekBatch( + context.Context, + string, + string, + string, + int, +) ([]*types.TaskPendingOp, error) { + return nil, nil +} + +func (r *wikiPendingRepoForCleanupTest) ClaimBatch( + context.Context, + string, + string, + string, + int, + time.Time, +) ([]*types.TaskPendingOp, error) { + return nil, nil +} + +func (r *wikiPendingRepoForCleanupTest) ReleaseByIDs(ctx context.Context, _ []int64) error { + r.releaseCtxErr = ctx.Err() + if r.releaseErr != nil { + return r.releaseErr + } + return nil +} + +func (r *wikiPendingRepoForCleanupTest) DeleteByIDs(ctx context.Context, _ []int64) error { + r.deleteCtxErr = ctx.Err() + if r.deleteErr != nil { + return r.deleteErr + } + return nil +} + +func (r *wikiPendingRepoForCleanupTest) IncrFailCount(ctx context.Context, _ int64) (int, error) { + r.incrCtxErr = ctx.Err() + if r.incrErr != nil { + return 0, r.incrErr + } + if r.incrCount == 0 { + r.incrCount = 1 + } + return r.incrCount, nil +} + +func (r *wikiPendingRepoForCleanupTest) PendingCount( + context.Context, + string, + string, + string, +) (int64, error) { + return 0, nil +} + +func (r *wikiPendingRepoForCleanupTest) DeleteByDedupKey( + context.Context, + string, + string, + string, + string, + string, +) error { + return nil +}