fix(task-inspector): handle lazy queue not found errors

This commit is contained in:
lhy
2026-07-23 13:05:47 +08:00
committed by lyingbug
parent 0869f3e829
commit c74b0caa35
4 changed files with 111 additions and 4 deletions
+4 -4
View File
@@ -187,9 +187,9 @@ func (a *asynqTaskInspector) HasQueuedTasksForKnowledge(
// into. Read-only: it calls Inspector.GetQueueInfo per queue and maps
// the result onto types.QueueStat, attaching static pool/weight metadata
// from the central queue registry. A queue that has never received a task yields
// ErrQueueNotFound from asynq; we still surface it as a zeroed row so the
// dashboard shows the complete lane set even before a queue receives its
// first task.
// either ErrQueueNotFound or an internal NOT_FOUND error from asynq; we still
// surface it as a zeroed row so the dashboard shows the complete lane set even
// before a queue receives its first task.
func (a *asynqTaskInspector) QueueStats(
ctx context.Context,
) ([]types.QueueStat, bool, error) {
@@ -207,7 +207,7 @@ func (a *asynqTaskInspector) QueueStats(
}
info, err := a.inspector.GetQueueInfo(queue)
if err != nil {
if !errors.Is(err, asynq.ErrQueueNotFound) {
if !isAsynqQueueNotFound(err) {
logger.Warnf(ctx, "[TaskInspector] queue info queue=%s: %v", queue, err)
}
// Zeroed row: queue not created yet (or transient error).
+36
View File
@@ -0,0 +1,36 @@
package router
import (
"errors"
"strings"
"github.com/hibiken/asynq"
)
const (
asynqQueueNotFoundPrefix = `NOT_FOUND: queue "`
asynqQueueNotFoundSuffix = `" does not exist`
)
// isAsynqQueueNotFound handles both the public sentinel and the internal error
// leaked by Inspector.GetQueueInfo in asynq v0.26.0.
func isAsynqQueueNotFound(err error) bool {
if err == nil {
return false
}
if errors.Is(err, asynq.ErrQueueNotFound) {
return true
}
message := err.Error()
start := strings.LastIndex(message, asynqQueueNotFoundPrefix)
if start < 0 || (start > 0 && !strings.HasSuffix(message[:start], ": ")) {
return false
}
queueAndSuffix := message[start+len(asynqQueueNotFoundPrefix):]
if !strings.HasSuffix(queueAndSuffix, asynqQueueNotFoundSuffix) {
return false
}
queue := strings.TrimSuffix(queueAndSuffix, asynqQueueNotFoundSuffix)
return queue != "" && !strings.Contains(queue, `"`)
}
@@ -0,0 +1,37 @@
package router
import (
"errors"
"fmt"
"testing"
"github.com/hibiken/asynq"
)
func TestIsAsynqQueueNotFound(t *testing.T) {
internalQueueNotFound := errors.New(`NOT_FOUND: queue "default" does not exist`)
tests := []struct {
name string
err error
want bool
}{
{name: "nil", err: nil, want: false},
{name: "public sentinel", err: asynq.ErrQueueNotFound, want: true},
{name: "wrapped public sentinel", err: fmt.Errorf("inspect queue: %w", asynq.ErrQueueNotFound), want: true},
{name: "internal queue error", err: internalQueueNotFound, want: true},
{name: "wrapped internal queue error", err: fmt.Errorf("inspect queue: %w", internalQueueNotFound), want: true},
{name: "task not found", err: errors.New(`NOT_FOUND: task "default" does not exist`), want: false},
{name: "different code", err: errors.New(`UNKNOWN: queue "default" does not exist`), want: false},
{name: "different reason", err: errors.New(`NOT_FOUND: queue "default" is unavailable`), want: false},
{name: "empty queue name", err: errors.New(`NOT_FOUND: queue "" does not exist`), want: false},
{name: "trailing context", err: errors.New(`NOT_FOUND: queue "default" does not exist: retry later`), want: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := isAsynqQueueNotFound(tt.err); got != tt.want {
t.Fatalf("isAsynqQueueNotFound(%v) = %v, want %v", tt.err, got, tt.want)
}
})
}
}
@@ -1,17 +1,51 @@
package router
import (
"bytes"
"context"
"encoding/json"
"os"
"strings"
"testing"
"time"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/alicebob/miniredis/v2"
"github.com/hibiken/asynq"
"github.com/redis/go-redis/v9"
)
func TestQueueStatsDoesNotWarnForQueuesThatDoNotExist(t *testing.T) {
server := miniredis.RunT(t)
client := redis.NewClient(&redis.Options{Addr: server.Addr()})
t.Cleanup(func() { _ = client.Close() })
inspector := &asynqTaskInspector{
inspector: asynq.NewInspectorFromRedisClient(client),
redis: client,
}
var logs bytes.Buffer
logger.SetOutput(&logs)
t.Cleanup(func() { logger.SetOutput(os.Stdout) })
stats, supported, err := inspector.QueueStats(context.Background())
if err != nil || !supported {
t.Fatalf("QueueStats: supported=%v err=%v", supported, err)
}
if got, want := len(stats), len(types.QueueDefinitions()); got != want {
t.Fatalf("QueueStats returned %d rows, want %d", got, want)
}
for _, stat := range stats {
if stat.Size != 0 || stat.Pending != 0 || stat.Active != 0 {
t.Fatalf("missing queue %q should have zero depth: %+v", stat.Name, stat)
}
}
if strings.Contains(logs.String(), "[TaskInspector] queue info") {
t.Fatalf("missing queues should not emit warnings:\n%s", logs.String())
}
}
func TestProjectRuntimeTaskRedactsPayloadAndBuildsSafeActions(t *testing.T) {
payload, err := json.Marshal(map[string]any{
"tenant_id": 42,