fix(task-inspector): handle lazy queue not found errors
This commit is contained in:
@@ -187,9 +187,9 @@ func (a *asynqTaskInspector) HasQueuedTasksForKnowledge(
|
||||
// into. Read-only: it calls Inspector.GetQueueInfo per queue and maps
|
||||
// the result onto types.QueueStat, attaching static pool/weight metadata
|
||||
// from the central queue registry. A queue that has never received a task yields
|
||||
// ErrQueueNotFound from asynq; we still surface it as a zeroed row so the
|
||||
// dashboard shows the complete lane set even before a queue receives its
|
||||
// first task.
|
||||
// either ErrQueueNotFound or an internal NOT_FOUND error from asynq; we still
|
||||
// surface it as a zeroed row so the dashboard shows the complete lane set even
|
||||
// before a queue receives its first task.
|
||||
func (a *asynqTaskInspector) QueueStats(
|
||||
ctx context.Context,
|
||||
) ([]types.QueueStat, bool, error) {
|
||||
@@ -207,7 +207,7 @@ func (a *asynqTaskInspector) QueueStats(
|
||||
}
|
||||
info, err := a.inspector.GetQueueInfo(queue)
|
||||
if err != nil {
|
||||
if !errors.Is(err, asynq.ErrQueueNotFound) {
|
||||
if !isAsynqQueueNotFound(err) {
|
||||
logger.Warnf(ctx, "[TaskInspector] queue info queue=%s: %v", queue, err)
|
||||
}
|
||||
// Zeroed row: queue not created yet (or transient error).
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
|
||||
"github.com/hibiken/asynq"
|
||||
)
|
||||
|
||||
const (
|
||||
asynqQueueNotFoundPrefix = `NOT_FOUND: queue "`
|
||||
asynqQueueNotFoundSuffix = `" does not exist`
|
||||
)
|
||||
|
||||
// isAsynqQueueNotFound handles both the public sentinel and the internal error
|
||||
// leaked by Inspector.GetQueueInfo in asynq v0.26.0.
|
||||
func isAsynqQueueNotFound(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, asynq.ErrQueueNotFound) {
|
||||
return true
|
||||
}
|
||||
|
||||
message := err.Error()
|
||||
start := strings.LastIndex(message, asynqQueueNotFoundPrefix)
|
||||
if start < 0 || (start > 0 && !strings.HasSuffix(message[:start], ": ")) {
|
||||
return false
|
||||
}
|
||||
queueAndSuffix := message[start+len(asynqQueueNotFoundPrefix):]
|
||||
if !strings.HasSuffix(queueAndSuffix, asynqQueueNotFoundSuffix) {
|
||||
return false
|
||||
}
|
||||
queue := strings.TrimSuffix(queueAndSuffix, asynqQueueNotFoundSuffix)
|
||||
return queue != "" && !strings.Contains(queue, `"`)
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/hibiken/asynq"
|
||||
)
|
||||
|
||||
func TestIsAsynqQueueNotFound(t *testing.T) {
|
||||
internalQueueNotFound := errors.New(`NOT_FOUND: queue "default" does not exist`)
|
||||
tests := []struct {
|
||||
name string
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{name: "nil", err: nil, want: false},
|
||||
{name: "public sentinel", err: asynq.ErrQueueNotFound, want: true},
|
||||
{name: "wrapped public sentinel", err: fmt.Errorf("inspect queue: %w", asynq.ErrQueueNotFound), want: true},
|
||||
{name: "internal queue error", err: internalQueueNotFound, want: true},
|
||||
{name: "wrapped internal queue error", err: fmt.Errorf("inspect queue: %w", internalQueueNotFound), want: true},
|
||||
{name: "task not found", err: errors.New(`NOT_FOUND: task "default" does not exist`), want: false},
|
||||
{name: "different code", err: errors.New(`UNKNOWN: queue "default" does not exist`), want: false},
|
||||
{name: "different reason", err: errors.New(`NOT_FOUND: queue "default" is unavailable`), want: false},
|
||||
{name: "empty queue name", err: errors.New(`NOT_FOUND: queue "" does not exist`), want: false},
|
||||
{name: "trailing context", err: errors.New(`NOT_FOUND: queue "default" does not exist: retry later`), want: false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := isAsynqQueueNotFound(tt.err); got != tt.want {
|
||||
t.Fatalf("isAsynqQueueNotFound(%v) = %v, want %v", tt.err, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,17 +1,51 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
func TestQueueStatsDoesNotWarnForQueuesThatDoNotExist(t *testing.T) {
|
||||
server := miniredis.RunT(t)
|
||||
client := redis.NewClient(&redis.Options{Addr: server.Addr()})
|
||||
t.Cleanup(func() { _ = client.Close() })
|
||||
inspector := &asynqTaskInspector{
|
||||
inspector: asynq.NewInspectorFromRedisClient(client),
|
||||
redis: client,
|
||||
}
|
||||
|
||||
var logs bytes.Buffer
|
||||
logger.SetOutput(&logs)
|
||||
t.Cleanup(func() { logger.SetOutput(os.Stdout) })
|
||||
|
||||
stats, supported, err := inspector.QueueStats(context.Background())
|
||||
if err != nil || !supported {
|
||||
t.Fatalf("QueueStats: supported=%v err=%v", supported, err)
|
||||
}
|
||||
if got, want := len(stats), len(types.QueueDefinitions()); got != want {
|
||||
t.Fatalf("QueueStats returned %d rows, want %d", got, want)
|
||||
}
|
||||
for _, stat := range stats {
|
||||
if stat.Size != 0 || stat.Pending != 0 || stat.Active != 0 {
|
||||
t.Fatalf("missing queue %q should have zero depth: %+v", stat.Name, stat)
|
||||
}
|
||||
}
|
||||
if strings.Contains(logs.String(), "[TaskInspector] queue info") {
|
||||
t.Fatalf("missing queues should not emit warnings:\n%s", logs.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProjectRuntimeTaskRedactsPayloadAndBuildsSafeActions(t *testing.T) {
|
||||
payload, err := json.Marshal(map[string]any{
|
||||
"tenant_id": 42,
|
||||
|
||||
Reference in New Issue
Block a user