From c74b0caa35280330ac2e9f5e8c083c061f4b2ed9 Mon Sep 17 00:00:00 2001 From: lhy Date: Thu, 23 Jul 2026 13:05:47 +0800 Subject: [PATCH] fix(task-inspector): handle lazy queue not found errors --- internal/router/task_inspector.go | 8 ++-- internal/router/task_inspector_errors.go | 36 ++++++++++++++++++ internal/router/task_inspector_errors_test.go | 37 +++++++++++++++++++ .../router/task_inspector_runtime_test.go | 34 +++++++++++++++++ 4 files changed, 111 insertions(+), 4 deletions(-) create mode 100644 internal/router/task_inspector_errors.go create mode 100644 internal/router/task_inspector_errors_test.go diff --git a/internal/router/task_inspector.go b/internal/router/task_inspector.go index 34063738..6ed5af03 100644 --- a/internal/router/task_inspector.go +++ b/internal/router/task_inspector.go @@ -187,9 +187,9 @@ func (a *asynqTaskInspector) HasQueuedTasksForKnowledge( // into. Read-only: it calls Inspector.GetQueueInfo per queue and maps // the result onto types.QueueStat, attaching static pool/weight metadata // from the central queue registry. A queue that has never received a task yields -// ErrQueueNotFound from asynq; we still surface it as a zeroed row so the -// dashboard shows the complete lane set even before a queue receives its -// first task. +// either ErrQueueNotFound or an internal NOT_FOUND error from asynq; we still +// surface it as a zeroed row so the dashboard shows the complete lane set even +// before a queue receives its first task. func (a *asynqTaskInspector) QueueStats( ctx context.Context, ) ([]types.QueueStat, bool, error) { @@ -207,7 +207,7 @@ func (a *asynqTaskInspector) QueueStats( } info, err := a.inspector.GetQueueInfo(queue) if err != nil { - if !errors.Is(err, asynq.ErrQueueNotFound) { + if !isAsynqQueueNotFound(err) { logger.Warnf(ctx, "[TaskInspector] queue info queue=%s: %v", queue, err) } // Zeroed row: queue not created yet (or transient error). diff --git a/internal/router/task_inspector_errors.go b/internal/router/task_inspector_errors.go new file mode 100644 index 00000000..d18effcc --- /dev/null +++ b/internal/router/task_inspector_errors.go @@ -0,0 +1,36 @@ +package router + +import ( + "errors" + "strings" + + "github.com/hibiken/asynq" +) + +const ( + asynqQueueNotFoundPrefix = `NOT_FOUND: queue "` + asynqQueueNotFoundSuffix = `" does not exist` +) + +// isAsynqQueueNotFound handles both the public sentinel and the internal error +// leaked by Inspector.GetQueueInfo in asynq v0.26.0. +func isAsynqQueueNotFound(err error) bool { + if err == nil { + return false + } + if errors.Is(err, asynq.ErrQueueNotFound) { + return true + } + + message := err.Error() + start := strings.LastIndex(message, asynqQueueNotFoundPrefix) + if start < 0 || (start > 0 && !strings.HasSuffix(message[:start], ": ")) { + return false + } + queueAndSuffix := message[start+len(asynqQueueNotFoundPrefix):] + if !strings.HasSuffix(queueAndSuffix, asynqQueueNotFoundSuffix) { + return false + } + queue := strings.TrimSuffix(queueAndSuffix, asynqQueueNotFoundSuffix) + return queue != "" && !strings.Contains(queue, `"`) +} diff --git a/internal/router/task_inspector_errors_test.go b/internal/router/task_inspector_errors_test.go new file mode 100644 index 00000000..2f93356a --- /dev/null +++ b/internal/router/task_inspector_errors_test.go @@ -0,0 +1,37 @@ +package router + +import ( + "errors" + "fmt" + "testing" + + "github.com/hibiken/asynq" +) + +func TestIsAsynqQueueNotFound(t *testing.T) { + internalQueueNotFound := errors.New(`NOT_FOUND: queue "default" does not exist`) + tests := []struct { + name string + err error + want bool + }{ + {name: "nil", err: nil, want: false}, + {name: "public sentinel", err: asynq.ErrQueueNotFound, want: true}, + {name: "wrapped public sentinel", err: fmt.Errorf("inspect queue: %w", asynq.ErrQueueNotFound), want: true}, + {name: "internal queue error", err: internalQueueNotFound, want: true}, + {name: "wrapped internal queue error", err: fmt.Errorf("inspect queue: %w", internalQueueNotFound), want: true}, + {name: "task not found", err: errors.New(`NOT_FOUND: task "default" does not exist`), want: false}, + {name: "different code", err: errors.New(`UNKNOWN: queue "default" does not exist`), want: false}, + {name: "different reason", err: errors.New(`NOT_FOUND: queue "default" is unavailable`), want: false}, + {name: "empty queue name", err: errors.New(`NOT_FOUND: queue "" does not exist`), want: false}, + {name: "trailing context", err: errors.New(`NOT_FOUND: queue "default" does not exist: retry later`), want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isAsynqQueueNotFound(tt.err); got != tt.want { + t.Fatalf("isAsynqQueueNotFound(%v) = %v, want %v", tt.err, got, tt.want) + } + }) + } +} diff --git a/internal/router/task_inspector_runtime_test.go b/internal/router/task_inspector_runtime_test.go index 755a409c..45634aa3 100644 --- a/internal/router/task_inspector_runtime_test.go +++ b/internal/router/task_inspector_runtime_test.go @@ -1,17 +1,51 @@ package router import ( + "bytes" "context" "encoding/json" + "os" + "strings" "testing" "time" + "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" "github.com/alicebob/miniredis/v2" "github.com/hibiken/asynq" "github.com/redis/go-redis/v9" ) +func TestQueueStatsDoesNotWarnForQueuesThatDoNotExist(t *testing.T) { + server := miniredis.RunT(t) + client := redis.NewClient(&redis.Options{Addr: server.Addr()}) + t.Cleanup(func() { _ = client.Close() }) + inspector := &asynqTaskInspector{ + inspector: asynq.NewInspectorFromRedisClient(client), + redis: client, + } + + var logs bytes.Buffer + logger.SetOutput(&logs) + t.Cleanup(func() { logger.SetOutput(os.Stdout) }) + + stats, supported, err := inspector.QueueStats(context.Background()) + if err != nil || !supported { + t.Fatalf("QueueStats: supported=%v err=%v", supported, err) + } + if got, want := len(stats), len(types.QueueDefinitions()); got != want { + t.Fatalf("QueueStats returned %d rows, want %d", got, want) + } + for _, stat := range stats { + if stat.Size != 0 || stat.Pending != 0 || stat.Active != 0 { + t.Fatalf("missing queue %q should have zero depth: %+v", stat.Name, stat) + } + } + if strings.Contains(logs.String(), "[TaskInspector] queue info") { + t.Fatalf("missing queues should not emit warnings:\n%s", logs.String()) + } +} + func TestProjectRuntimeTaskRedactsPayloadAndBuildsSafeActions(t *testing.T) { payload, err := json.Marshal(map[string]any{ "tenant_id": 42,