package router import ( "context" "encoding/json" "errors" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" "github.com/hibiken/asynq" ) // NewAsynqInspector constructs an *asynq.Inspector pointed at the same // Redis used by the asynq client. Only registered in asynq mode. func NewAsynqInspector() *asynq.Inspector { return asynq.NewInspector(getAsynqRedisClientOpt()) } // asynqTaskInspector implements interfaces.TaskInspector backed by an // *asynq.Inspector. Scans the queues we actually use ("default", // "critical", "low") and matches tasks whose payload carries the given // knowledge_id. Best-effort: any scan/delete error is logged and // swallowed so the cancel API still returns success even when Redis is // flaky. type asynqTaskInspector struct { inspector *asynq.Inspector } // NewAsynqTaskInspector returns a TaskInspector wrapping the given // *asynq.Inspector. nil-safe: a nil inspector degrades to a no-op so // the cancel path remains usable when the inspector failed to init. func NewAsynqTaskInspector(inspector *asynq.Inspector) interfaces.TaskInspector { if inspector == nil { return noopTaskInspector{} } return &asynqTaskInspector{inspector: inspector} } // knowledgeIDProbe is the minimal payload shape we need to filter // tasks. All pipeline payload types embed a json:"knowledge_id" field, // so a single struct covers Document / ImageMultimodal / PostProcess / // Question / Summary / Extract / Manual. type knowledgeIDProbe struct { KnowledgeID string `json:"knowledge_id,omitempty"` } // queuesScanned is the fixed set of queue names this codebase enqueues // into. Kept tight on purpose — we never scan user-defined queues. // MUST include every queue any cancelable task type can land in; the // multimodal queue is required here so cancelling a knowledge also purges // its (potentially hundreds of) pending image:multimodal tasks. var queuesScanned = []string{ types.QueueDefault, types.QueueCritical, types.QueueLow, types.QueueMultimodal, types.QueueGraph, types.QueueQuestion, } // taskTypesForKnowledgeCancel lists every asynq task type that carries // a knowledge_id in its payload and should be cancelable. The set is // deliberately narrow: we don't touch FAQ import / KB-level tasks // because the cancel API is per-knowledge. var taskTypesForKnowledgeCancel = map[string]struct{}{ types.TypeDocumentProcess: {}, types.TypeManualProcess: {}, types.TypeImageMultimodal: {}, types.TypeKnowledgePostProcess: {}, types.TypeQuestionGeneration: {}, types.TypeSummaryGeneration: {}, types.TypeChunkExtract: {}, } // listPageSize caps each Redis LIST call. Asynq pages tasks, so we // loop until a short page comes back. 100 matches asynq's default. const listPageSize = 100 // CancelTasksForKnowledge removes queued tasks whose payload references // the given knowledge_id and signals active workers running such tasks // to stop. func (a *asynqTaskInspector) CancelTasksForKnowledge( ctx context.Context, knowledgeID string, ) (int, int, error) { if a == nil || a.inspector == nil || knowledgeID == "" { return 0, 0, nil } deleted := 0 cancelled := 0 for _, queue := range queuesScanned { // Pending / Scheduled / Retry can all be deleted by task ID. // Archived tasks are NOT touched: dead-letter rows are // already final and should remain visible to operators. deleted += a.deletePendingMatches(ctx, queue, knowledgeID) deleted += a.deleteScheduledMatches(ctx, queue, knowledgeID) deleted += a.deleteRetryMatches(ctx, queue, knowledgeID) cancelled += a.cancelActiveMatches(ctx, queue, knowledgeID) } logger.Infof(ctx, "[TaskInspector] knowledge=%s cancel summary: deleted_from_queue=%d active_cancel_signaled=%d", knowledgeID, deleted, cancelled, ) return deleted, cancelled, nil } // HasQueuedTasksForKnowledge reports whether any pending / scheduled / // retry / active task referencing knowledgeID still lives in the queue. // Read-only counterpart of CancelTasksForKnowledge — the housekeeping // sweep uses it to avoid flagging a backlogged-but-not-orphaned row as // failed. Short-circuits on the first match and never deletes anything. func (a *asynqTaskInspector) HasQueuedTasksForKnowledge( ctx context.Context, knowledgeID string, ) (bool, error) { if a == nil || a.inspector == nil || knowledgeID == "" { return false, nil } listers := []struct { state string list func(string, ...asynq.ListOption) ([]*asynq.TaskInfo, error) }{ {"pending", a.inspector.ListPendingTasks}, {"scheduled", a.inspector.ListScheduledTasks}, {"retry", a.inspector.ListRetryTasks}, {"active", a.inspector.ListActiveTasks}, } for _, queue := range queuesScanned { for _, l := range listers { if a.queueStateHasMatch(ctx, queue, knowledgeID, l.state, l.list) { return true, nil } } } return false, nil } // queueStateHasMatch pages through one (queue, state) list looking for a // task that references knowledgeID. Mirrors the delete* scanners but is // strictly read-only and returns early on the first hit. A backend error // is logged and treated as "no match" (false); the caller's fail-safe // then errs toward recovering the row rather than preserving it forever. func (a *asynqTaskInspector) queueStateHasMatch( ctx context.Context, queue, knowledgeID, state string, list func(string, ...asynq.ListOption) ([]*asynq.TaskInfo, error), ) bool { page := 1 for { tasks, err := list(queue, asynq.PageSize(listPageSize), asynq.Page(page)) if err != nil { if !errors.Is(err, asynq.ErrQueueNotFound) { logger.Warnf(ctx, "[TaskInspector] probe %s queue=%s page=%d: %v", state, queue, page, err) } return false } if len(tasks) == 0 { return false } for _, t := range tasks { if matchesKnowledge(t.Type, t.Payload, knowledgeID) { return true } } if len(tasks) < listPageSize { return false } page++ } } // matchesKnowledge returns true when the task type is one we cancel // AND its payload references the target knowledge ID. func matchesKnowledge(taskType string, payload []byte, knowledgeID string) bool { if _, ok := taskTypesForKnowledgeCancel[taskType]; !ok { return false } var probe knowledgeIDProbe if err := json.Unmarshal(payload, &probe); err != nil { return false } return probe.KnowledgeID == knowledgeID } func (a *asynqTaskInspector) deletePendingMatches(ctx context.Context, queue, knowledgeID string) int { deleted := 0 page := 1 for { tasks, err := a.inspector.ListPendingTasks(queue, asynq.PageSize(listPageSize), asynq.Page(page)) if err != nil { if !errors.Is(err, asynq.ErrQueueNotFound) { logger.Warnf(ctx, "[TaskInspector] list pending queue=%s page=%d: %v", queue, page, err) } return deleted } if len(tasks) == 0 { return deleted } for _, t := range tasks { if !matchesKnowledge(t.Type, t.Payload, knowledgeID) { continue } if err := a.inspector.DeleteTask(queue, t.ID); err != nil { logger.Warnf(ctx, "[TaskInspector] delete pending type=%s id=%s: %v", t.Type, t.ID, err) continue } deleted++ } if len(tasks) < listPageSize { return deleted } page++ } } func (a *asynqTaskInspector) deleteScheduledMatches(ctx context.Context, queue, knowledgeID string) int { deleted := 0 page := 1 for { tasks, err := a.inspector.ListScheduledTasks(queue, asynq.PageSize(listPageSize), asynq.Page(page)) if err != nil { if !errors.Is(err, asynq.ErrQueueNotFound) { logger.Warnf(ctx, "[TaskInspector] list scheduled queue=%s page=%d: %v", queue, page, err) } return deleted } if len(tasks) == 0 { return deleted } for _, t := range tasks { if !matchesKnowledge(t.Type, t.Payload, knowledgeID) { continue } if err := a.inspector.DeleteTask(queue, t.ID); err != nil { logger.Warnf(ctx, "[TaskInspector] delete scheduled type=%s id=%s: %v", t.Type, t.ID, err) continue } deleted++ } if len(tasks) < listPageSize { return deleted } page++ } } func (a *asynqTaskInspector) deleteRetryMatches(ctx context.Context, queue, knowledgeID string) int { deleted := 0 page := 1 for { tasks, err := a.inspector.ListRetryTasks(queue, asynq.PageSize(listPageSize), asynq.Page(page)) if err != nil { if !errors.Is(err, asynq.ErrQueueNotFound) { logger.Warnf(ctx, "[TaskInspector] list retry queue=%s page=%d: %v", queue, page, err) } return deleted } if len(tasks) == 0 { return deleted } for _, t := range tasks { if !matchesKnowledge(t.Type, t.Payload, knowledgeID) { continue } if err := a.inspector.DeleteTask(queue, t.ID); err != nil { logger.Warnf(ctx, "[TaskInspector] delete retry type=%s id=%s: %v", t.Type, t.ID, err) continue } deleted++ } if len(tasks) < listPageSize { return deleted } page++ } } // cancelActiveMatches signals active workers to abort via // Inspector.CancelProcessing. The worker's ctx becomes Done() so the // next blocking call (or our checkpoint reads) bails. The DB-level // abort flag (parse_status=cancelled) remains the durable signal — // this is a latency optimization, not the correctness mechanism. func (a *asynqTaskInspector) cancelActiveMatches(ctx context.Context, queue, knowledgeID string) int { cancelled := 0 page := 1 for { tasks, err := a.inspector.ListActiveTasks(queue, asynq.PageSize(listPageSize), asynq.Page(page)) if err != nil { if !errors.Is(err, asynq.ErrQueueNotFound) { logger.Warnf(ctx, "[TaskInspector] list active queue=%s page=%d: %v", queue, page, err) } return cancelled } if len(tasks) == 0 { return cancelled } for _, t := range tasks { if !matchesKnowledge(t.Type, t.Payload, knowledgeID) { continue } if err := a.inspector.CancelProcessing(t.ID); err != nil { logger.Warnf(ctx, "[TaskInspector] cancel active type=%s id=%s: %v", t.Type, t.ID, err) continue } cancelled++ } if len(tasks) < listPageSize { return cancelled } page++ } } // noopTaskInspector is the Lite-mode (no Redis) inspector. Inline // goroutines spawned by SyncTaskExecutor cannot be dequeued before // they start; the checkpoint-based abort in worker code is the only // stop signal in that mode. type noopTaskInspector struct{} // NewNoopTaskInspector returns a no-op TaskInspector for Lite mode. func NewNoopTaskInspector() interfaces.TaskInspector { return noopTaskInspector{} } func (noopTaskInspector) CancelTasksForKnowledge( ctx context.Context, knowledgeID string, ) (int, int, error) { return 0, 0, nil } // HasQueuedTasksForKnowledge always reports false in Lite mode: inline // executors never enqueue, so there is no backlog to protect against and // the housekeeping sweep's span/updated_at checks stay authoritative. func (noopTaskInspector) HasQueuedTasksForKnowledge( ctx context.Context, knowledgeID string, ) (bool, error) { return false, nil }