fix(wiki): detach ingest cleanup from task cancellation
Use bounded detached contexts for wiki ingest tail cleanup so pending row settlement, span closure, finalize enqueueing, and follow-up scheduling can finish even after the asynq task context is cancelled. Return settlement errors from pending row trimming and failed-op requeueing, and only mark claims settled after rows are deleted, released, or dead-lettered successfully. Requeue documents whose reduce updates were not applied after context cancellation to avoid silently dropping wiki work. Add unit coverage for detached cleanup contexts and settlement error handling.
This commit is contained in:
@@ -200,6 +200,10 @@ const (
|
||||
// top of the asynq.TaskID coalescing).
|
||||
wikiFinalizeLockTTL = 60 * time.Second
|
||||
wikiFinalizeLockRenew = 20 * time.Second
|
||||
|
||||
// wikiIngestCleanupTimeout bounds detached tail cleanup after the asynq
|
||||
// task context has been cancelled or hit its timeout.
|
||||
wikiIngestCleanupTimeout = 10 * time.Second
|
||||
)
|
||||
|
||||
// wikiFinalizeChange is a doc-level add/remove entry for the index-intro
|
||||
@@ -537,6 +541,13 @@ func (s *wikiIngestService) Handle(ctx context.Context, t *asynq.Task) error {
|
||||
}
|
||||
}
|
||||
|
||||
func wikiIngestCleanupContext(ctx context.Context) (context.Context, context.CancelFunc) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
return context.WithTimeout(context.WithoutCancel(ctx), wikiIngestCleanupTimeout)
|
||||
}
|
||||
|
||||
// enqueueFinalize persists this batch's KB-global convergence work into the
|
||||
// finalize lane of task_pending_ops and schedules a debounced trigger. One
|
||||
// "slug" row per affected page (carrying its fresh title when this batch wrote
|
||||
@@ -920,13 +931,15 @@ func (s *wikiIngestService) decodePendingRows(ctx context.Context, rows []*types
|
||||
// trimPendingList deletes consumed rows from task_pending_ops. Empty
|
||||
// input is a no-op so callers can invoke unconditionally at the end
|
||||
// of a batch.
|
||||
func (s *wikiIngestService) trimPendingList(ctx context.Context, ids []int64) {
|
||||
func (s *wikiIngestService) trimPendingList(ctx context.Context, ids []int64) error {
|
||||
if s.pendingRepo == nil || len(ids) == 0 {
|
||||
return
|
||||
return nil
|
||||
}
|
||||
if err := s.pendingRepo.DeleteByIDs(ctx, ids); err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: failed to trim %d pending rows: %v", len(ids), err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// finalizeWikiSubtask releases this knowledge's slot in the finalizing
|
||||
@@ -960,13 +973,13 @@ func (s *wikiIngestService) finalizeWikiSubtask(ctx context.Context, knowledgeID
|
||||
// (rows are ordered by id ASC and we never moved/touched it).
|
||||
// - If the count exceeds the retry cap: archive the op into
|
||||
// task_dead_letters and DeleteByIDs to remove it from the queue.
|
||||
// Both writes are best-effort — a DB failure here is logged and
|
||||
// swallowed so a single transient blip doesn't recursively spawn
|
||||
// more failures.
|
||||
func (s *wikiIngestService) requeueFailedOps(ctx context.Context, payload WikiIngestPayload, ops []WikiPendingOp) {
|
||||
// Settlement failures are returned so the caller does not mark claims
|
||||
// settled while rows are still claimed or undeleted.
|
||||
func (s *wikiIngestService) requeueFailedOps(ctx context.Context, payload WikiIngestPayload, ops []WikiPendingOp) error {
|
||||
if s.pendingRepo == nil || len(ops) == 0 {
|
||||
return
|
||||
return nil
|
||||
}
|
||||
var settleErrs []error
|
||||
for _, op := range ops {
|
||||
if op.dbID == 0 {
|
||||
// Op was never persisted (synthetic / test) — nothing to
|
||||
@@ -976,6 +989,7 @@ func (s *wikiIngestService) requeueFailedOps(ctx context.Context, payload WikiIn
|
||||
count, err := s.pendingRepo.IncrFailCount(ctx, op.dbID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: failed to increment fail count for %s (id=%d): %v", op.KnowledgeID, op.dbID, err)
|
||||
settleErrs = append(settleErrs, fmt.Errorf("increment fail count id=%d: %w", op.dbID, err))
|
||||
// Without a fresh count we can't tell whether to drop. Be
|
||||
// conservative: leave the row in place; the next PeekBatch
|
||||
// will see it again and we'll try once more.
|
||||
@@ -989,6 +1003,7 @@ func (s *wikiIngestService) requeueFailedOps(ctx context.Context, payload WikiIn
|
||||
// budget still counts down.
|
||||
if err := s.pendingRepo.ReleaseByIDs(ctx, []int64{op.dbID}); err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: failed to release claim for retry id=%d: %v", op.dbID, err)
|
||||
settleErrs = append(settleErrs, fmt.Errorf("release retry claim id=%d: %w", op.dbID, err))
|
||||
}
|
||||
logger.Infof(ctx, "wiki ingest: re-queued failed op %s (%s) for retry (attempt %d/%d)", op.KnowledgeID, op.DocTitle, count, wikiMaxFailRetries)
|
||||
continue
|
||||
@@ -1016,12 +1031,15 @@ func (s *wikiIngestService) requeueFailedOps(ctx context.Context, payload WikiIn
|
||||
FailCount: count,
|
||||
}); dlErr != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: failed to archive op %s to dead letters: %v", op.KnowledgeID, dlErr)
|
||||
settleErrs = append(settleErrs, fmt.Errorf("archive dead letter id=%d: %w", op.dbID, dlErr))
|
||||
}
|
||||
}
|
||||
if err := s.pendingRepo.DeleteByIDs(ctx, []int64{op.dbID}); err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: failed to drop dead-lettered row id=%d: %v", op.dbID, err)
|
||||
settleErrs = append(settleErrs, fmt.Errorf("drop dead-lettered row id=%d: %w", op.dbID, err))
|
||||
}
|
||||
}
|
||||
return errors.Join(settleErrs...)
|
||||
}
|
||||
|
||||
// docIngestResult captures per-document info for batch post-processing.
|
||||
|
||||
@@ -3,6 +3,7 @@ package service
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -377,15 +378,18 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
// so the next trigger can re-claim within seconds instead of waiting out
|
||||
// wikiClaimStaleAfter (~90m). On the normal path claimsSettled flips true
|
||||
// once the rows reach their terminal state, making this a no-op. Uses a
|
||||
// background context because ctx may already be cancelled on the timeout
|
||||
// path. Lite mode peeks without claiming, so there is nothing to release.
|
||||
// bounded detached cleanup context because ctx may already be cancelled on
|
||||
// the timeout path. Lite mode peeks without claiming, so there is nothing
|
||||
// to release.
|
||||
claimsSettled := false
|
||||
if s.redisClient != nil && len(peekedIDs) > 0 {
|
||||
defer func() {
|
||||
if claimsSettled {
|
||||
return
|
||||
}
|
||||
if err := s.pendingRepo.ReleaseByIDs(context.Background(), peekedIDs); err != nil {
|
||||
releaseCtx, releaseCancel := wikiIngestCleanupContext(ctx)
|
||||
defer releaseCancel()
|
||||
if err := s.pendingRepo.ReleaseByIDs(releaseCtx, peekedIDs); err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: failed to release %d claims on abnormal exit for KB %s: %v", len(peekedIDs), payload.KnowledgeBaseID, err)
|
||||
return
|
||||
}
|
||||
@@ -613,6 +617,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
return reduceErr
|
||||
})
|
||||
if lockErr != nil {
|
||||
collectUnapplied(updates)
|
||||
// ctx cancelled (batch timeout / shutdown) — stop quietly.
|
||||
return nil
|
||||
}
|
||||
@@ -659,6 +664,9 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
}
|
||||
_ = egReduce.Wait()
|
||||
|
||||
tailCtx, tailCancel := wikiIngestCleanupContext(ctx)
|
||||
defer tailCancel()
|
||||
|
||||
// Sanitize the doc summary pages produced by this batch BEFORE we
|
||||
// build log entries / rebuild the index. The summary LLM (run during
|
||||
// map) was free to inject [[entity/foo|name]] links to every slug it
|
||||
@@ -666,7 +674,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
// those slugs into actual pages. Rewrite those dead links to plain
|
||||
// text so the summary doesn't contain unresolvable references.
|
||||
if len(failedAdditionSlugs) > 0 && len(docResults) > 0 {
|
||||
s.sanitizeDeadSummaryLinks(ctx, payload.KnowledgeBaseID, docResults, failedAdditionSlugs, batchCtx)
|
||||
s.sanitizeDeadSummaryLinks(tailCtx, payload.KnowledgeBaseID, docResults, failedAdditionSlugs, batchCtx)
|
||||
}
|
||||
|
||||
totalPagesAffected = len(allPagesAffected)
|
||||
@@ -687,7 +695,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
if len(slugs) == 0 {
|
||||
return nil
|
||||
}
|
||||
titles := batchCtx.SlugTitleMany(ctx, slugs)
|
||||
titles := batchCtx.SlugTitleMany(tailCtx, slugs)
|
||||
out := make([]types.WikiLogPageRef, 0, len(slugs))
|
||||
for _, slug := range slugs {
|
||||
out = append(out, types.WikiLogPageRef{Slug: slug, Title: titles[slug]})
|
||||
@@ -718,7 +726,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
logEntries = append(logEntries, s.buildLogEntry(payload.TenantID, payload.KnowledgeBaseID, "ingest", r.KnowledgeID, r.DocTitle, r.Summary, pages))
|
||||
}
|
||||
if len(logEntries) > 0 && s.logEntrySvc != nil {
|
||||
if err := s.logEntrySvc.AppendBatch(ctx, logEntries); err != nil {
|
||||
if err := s.logEntrySvc.AppendBatch(tailCtx, logEntries); err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: failed to append %d log entries: %v", len(logEntries), err)
|
||||
}
|
||||
}
|
||||
@@ -728,7 +736,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
// written, not after the debounce window. This is a cheap status flip.
|
||||
if len(allPagesAffected) > 0 {
|
||||
logger.Infof(ctx, "wiki ingest: publishing draft pages")
|
||||
s.publishDraftPages(ctx, payload.KnowledgeBaseID, allPagesAffected)
|
||||
s.publishDraftPages(tailCtx, payload.KnowledgeBaseID, allPagesAffected)
|
||||
}
|
||||
|
||||
// Defer KB-global convergence (index-intro rebuild + dead-link cleanup +
|
||||
@@ -769,7 +777,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
})
|
||||
}
|
||||
}
|
||||
s.enqueueFinalize(ctx, payload, allPagesAffected, freshTitleBySlug, changes)
|
||||
s.enqueueFinalize(tailCtx, payload, allPagesAffected, freshTitleBySlug, changes)
|
||||
}
|
||||
|
||||
// Close postprocess.wiki spans for every successfully-mapped doc.
|
||||
@@ -781,6 +789,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
// actually landed (vs. dropped because reduce-phase generation
|
||||
// failed).
|
||||
failedAdditionSlugCount := len(failedAdditionSlugs)
|
||||
spanCtx, spanCancel := wikiIngestCleanupContext(ctx)
|
||||
for _, r := range docResults {
|
||||
if r == nil {
|
||||
continue
|
||||
@@ -828,8 +837,9 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
for k, v := range r.MapStats {
|
||||
output[k] = v
|
||||
}
|
||||
s.tracker().EndSpan(ctx, r.WikiSpan, output)
|
||||
s.tracker().EndSpan(spanCtx, r.WikiSpan, output)
|
||||
}
|
||||
spanCancel()
|
||||
// Failed-map docs already had FailSpan called inside
|
||||
// mapOneDocument (the failedOps path returns before reaching
|
||||
// docResults). Nothing extra to do here for them.
|
||||
@@ -875,14 +885,21 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
}
|
||||
trimIDs = append(trimIDs, id)
|
||||
}
|
||||
s.trimPendingList(ctx, trimIDs)
|
||||
settleCtx, settleCancel := wikiIngestCleanupContext(ctx)
|
||||
trimErr := s.trimPendingList(settleCtx, trimIDs)
|
||||
|
||||
// Process failed ops: increment fail_count and dead-letter once
|
||||
// the cap is hit. Must come AFTER trim so successful siblings are
|
||||
// already gone from the queue — otherwise a follow-up batch could
|
||||
// re-pick them up.
|
||||
var requeueErr error
|
||||
if len(failedOps) > 0 {
|
||||
s.requeueFailedOps(ctx, payload, failedOps)
|
||||
requeueErr = s.requeueFailedOps(settleCtx, payload, failedOps)
|
||||
}
|
||||
settleCancel()
|
||||
if err := errors.Join(trimErr, requeueErr); err != nil {
|
||||
exitStatus = "settle_failed"
|
||||
return fmt.Errorf("wiki ingest: settle claimed rows: %w", err)
|
||||
}
|
||||
// All claimed rows have now reached a terminal state (deleted on success,
|
||||
// released for retry, or dead-lettered), so disarm the abnormal-exit
|
||||
@@ -898,7 +915,9 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
followUpDelay = wikiRateLimitBackoff
|
||||
logger.Warnf(ctx, "wiki ingest: KB %s hit upstream rate limiting, backing off follow-up to %s", payload.KnowledgeBaseID, followUpDelay)
|
||||
}
|
||||
followUpScheduled = s.scheduleFollowUp(ctx, payload, followUpDelay)
|
||||
followCtx, followCancel := wikiIngestCleanupContext(ctx)
|
||||
followUpScheduled = s.scheduleFollowUp(followCtx, payload, followUpDelay)
|
||||
followCancel()
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1018,7 +1037,12 @@ func (s *wikiIngestService) ProcessWikiFinalize(ctx context.Context, t *asynq.Ta
|
||||
// KB flipped away from wiki (deleted / type change) — drain the lane so the
|
||||
// rows don't accumulate, then stop.
|
||||
if !kb.IsWikiEnabled() {
|
||||
s.trimPendingList(ctx, ids)
|
||||
drainCtx, drainCancel := wikiIngestCleanupContext(ctx)
|
||||
err := s.trimPendingList(drainCtx, ids)
|
||||
drainCancel()
|
||||
if err != nil {
|
||||
return fmt.Errorf("wiki finalize: trim disabled lane: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1059,7 +1083,12 @@ func (s *wikiIngestService) ProcessWikiFinalize(ctx context.Context, t *asynq.Ta
|
||||
// Drain the processed rows. Best-effort convergence mirrors the legacy
|
||||
// in-batch behaviour: a failed index rebuild is logged (not retried),
|
||||
// so we delete regardless to avoid re-doing the whole pass forever.
|
||||
s.trimPendingList(ctx, ids)
|
||||
drainCtx, drainCancel := wikiIngestCleanupContext(ctx)
|
||||
err = s.trimPendingList(drainCtx, ids)
|
||||
drainCancel()
|
||||
if err != nil {
|
||||
return fmt.Errorf("wiki finalize: trim pending rows: %w", err)
|
||||
}
|
||||
|
||||
// If more finalize rows landed while we were working, reschedule so they
|
||||
// get their own convergence pass.
|
||||
|
||||
@@ -2,8 +2,10 @@ package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/chat"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
@@ -355,3 +357,133 @@ func (m *templateCaptureChatModel) ChatStream(
|
||||
|
||||
func (m *templateCaptureChatModel) GetModelName() string { return "capture" }
|
||||
func (m *templateCaptureChatModel) GetModelID() string { return "capture" }
|
||||
|
||||
func TestWikiIngestCleanupContextDetachedFromCancelledParent(t *testing.T) {
|
||||
parent := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(42))
|
||||
ctx, cancel := context.WithCancel(parent)
|
||||
cancel()
|
||||
|
||||
cleanupCtx, cleanupCancel := wikiIngestCleanupContext(ctx)
|
||||
defer cleanupCancel()
|
||||
|
||||
repo := &wikiPendingRepoForCleanupTest{}
|
||||
svc := &wikiIngestService{pendingRepo: repo}
|
||||
if err := svc.trimPendingList(cleanupCtx, []int64{7}); err != nil {
|
||||
t.Fatalf("trimPendingList() error = %v", err)
|
||||
}
|
||||
if repo.deleteCtxErr != nil {
|
||||
t.Fatalf("cleanup context should be detached from parent cancellation, got %v", repo.deleteCtxErr)
|
||||
}
|
||||
if got := cleanupCtx.Value(types.TenantIDContextKey); got != uint64(42) {
|
||||
t.Fatalf("cleanup context lost tenant value: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrimPendingListReturnsDeleteError(t *testing.T) {
|
||||
wantErr := errors.New("delete failed")
|
||||
repo := &wikiPendingRepoForCleanupTest{deleteErr: wantErr}
|
||||
svc := &wikiIngestService{pendingRepo: repo}
|
||||
|
||||
err := svc.trimPendingList(context.Background(), []int64{1, 2, 3})
|
||||
if !errors.Is(err, wantErr) {
|
||||
t.Fatalf("trimPendingList() error = %v, want %v", err, wantErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequeueFailedOpsReturnsReleaseError(t *testing.T) {
|
||||
wantErr := errors.New("release failed")
|
||||
repo := &wikiPendingRepoForCleanupTest{incrCount: 1, releaseErr: wantErr}
|
||||
svc := &wikiIngestService{pendingRepo: repo}
|
||||
|
||||
err := svc.requeueFailedOps(context.Background(), WikiIngestPayload{}, []WikiPendingOp{{
|
||||
Op: WikiOpIngest,
|
||||
KnowledgeID: "k1",
|
||||
DocTitle: "doc",
|
||||
dbID: 99,
|
||||
}})
|
||||
if !errors.Is(err, wantErr) {
|
||||
t.Fatalf("requeueFailedOps() error = %v, want %v", err, wantErr)
|
||||
}
|
||||
}
|
||||
|
||||
type wikiPendingRepoForCleanupTest struct {
|
||||
deleteErr error
|
||||
releaseErr error
|
||||
incrErr error
|
||||
incrCount int
|
||||
deleteCtxErr error
|
||||
releaseCtxErr error
|
||||
incrCtxErr error
|
||||
}
|
||||
|
||||
func (r *wikiPendingRepoForCleanupTest) Enqueue(context.Context, *types.TaskPendingOp) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *wikiPendingRepoForCleanupTest) PeekBatch(
|
||||
context.Context,
|
||||
string,
|
||||
string,
|
||||
string,
|
||||
int,
|
||||
) ([]*types.TaskPendingOp, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (r *wikiPendingRepoForCleanupTest) ClaimBatch(
|
||||
context.Context,
|
||||
string,
|
||||
string,
|
||||
string,
|
||||
int,
|
||||
time.Time,
|
||||
) ([]*types.TaskPendingOp, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (r *wikiPendingRepoForCleanupTest) ReleaseByIDs(ctx context.Context, _ []int64) error {
|
||||
r.releaseCtxErr = ctx.Err()
|
||||
if r.releaseErr != nil {
|
||||
return r.releaseErr
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *wikiPendingRepoForCleanupTest) DeleteByIDs(ctx context.Context, _ []int64) error {
|
||||
r.deleteCtxErr = ctx.Err()
|
||||
if r.deleteErr != nil {
|
||||
return r.deleteErr
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *wikiPendingRepoForCleanupTest) IncrFailCount(ctx context.Context, _ int64) (int, error) {
|
||||
r.incrCtxErr = ctx.Err()
|
||||
if r.incrErr != nil {
|
||||
return 0, r.incrErr
|
||||
}
|
||||
if r.incrCount == 0 {
|
||||
r.incrCount = 1
|
||||
}
|
||||
return r.incrCount, nil
|
||||
}
|
||||
|
||||
func (r *wikiPendingRepoForCleanupTest) PendingCount(
|
||||
context.Context,
|
||||
string,
|
||||
string,
|
||||
string,
|
||||
) (int64, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func (r *wikiPendingRepoForCleanupTest) DeleteByDedupKey(
|
||||
context.Context,
|
||||
string,
|
||||
string,
|
||||
string,
|
||||
string,
|
||||
string,
|
||||
) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user