feat(task-queue): split upstream workers into isolated core/enrichment/maintenance pools

Route summary/sync/maintenance tasks to dedicated queues and servers so parsing, enrichment, and batch work no longer share one concurrency budget, while extending the runtime dashboard and settings validation to reflect the new topology.
This commit is contained in:
wizardchen
2026-07-10 23:19:41 +08:00
committed by lyingbug
parent fe47ac1f96
commit d496d0fc73
31 changed files with 847 additions and 290 deletions
+4 -3
View File
@@ -256,7 +256,7 @@ func (h *KnowledgeHandler) enqueueKnowledgeListDelete(
return "", fmt.Errorf("marshal payload: %w", err)
}
task := asynq.NewTask(types.TypeKnowledgeListDelete, payloadBytes,
asynq.Queue("low"), asynq.MaxRetry(3))
asynq.Queue(types.QueueMaintenance), asynq.MaxRetry(3), asynq.Timeout(2*time.Hour))
info, err := h.asynqClient.Enqueue(task)
if err != nil {
return "", fmt.Errorf("enqueue task: %w", err)
@@ -280,7 +280,7 @@ func (h *KnowledgeHandler) enqueueKnowledgeListReparse(
return "", fmt.Errorf("marshal payload: %w", err)
}
task := asynq.NewTask(types.TypeKnowledgeListReparse, payloadBytes,
asynq.Queue("low"), asynq.MaxRetry(3))
asynq.Queue(types.QueueMaintenance), asynq.MaxRetry(3), asynq.Timeout(time.Hour))
info, err := h.asynqClient.Enqueue(task)
if err != nil {
return "", fmt.Errorf("enqueue task: %w", err)
@@ -2186,7 +2186,8 @@ func (h *KnowledgeHandler) MoveKnowledge(c *gin.Context) {
// Enqueue move task
task := asynq.NewTask(types.TypeKnowledgeMove, payloadBytes,
asynq.TaskID(taskID), asynq.Queue("default"), asynq.MaxRetry(3))
asynq.TaskID(taskID), asynq.Queue(types.QueueMaintenance),
asynq.MaxRetry(3), asynq.Timeout(2*time.Hour))
info, err := h.asynqClient.Enqueue(task)
if err != nil {
logger.Errorf(ctx, "MoveKnowledge: failed to enqueue task: %v", err)
+2 -1
View File
@@ -1083,7 +1083,8 @@ func (h *KnowledgeBaseHandler) CopyKnowledgeBase(c *gin.Context) {
// Enqueue KB clone task to Asynq
task := asynq.NewTask(types.TypeKBClone, payloadBytes,
asynq.TaskID(taskID), asynq.Queue("default"), asynq.MaxRetry(3))
asynq.TaskID(taskID), asynq.Queue(types.QueueMaintenance),
asynq.MaxRetry(3), asynq.Timeout(2*time.Hour))
info, err := h.asynqClient.Enqueue(task)
if err != nil {
logger.Errorf(ctx, "Failed to enqueue KB clone task: %v", err)
+45 -33
View File
@@ -1384,26 +1384,24 @@ func (h *SystemHandler) ListSystemAdmins(c *gin.Context) {
// RuntimeQueuesResponse is the payload for the SystemAdmin runtime queue
// dashboard. `available` is false in Lite mode (no Redis/asynq) so the
// UI can render an "unavailable in this deployment" state instead of an
// empty table. ParseConcurrency / WikiConcurrency are the *configured*
// worker-pool sizes (resolved DB > ENV > default); they are static
// capacity, not the live active-worker count (see QueueStat.Active for
// what's actually running).
type RuntimeQueuesResponse struct {
Available bool `json:"available"`
ParseConcurrency int `json:"parse_concurrency"`
WikiConcurrency int `json:"wiki_concurrency"`
Queues []types.QueueStat `json:"queues"`
Timestamp int64 `json:"timestamp"`
// empty table. Pool concurrency is the configured per-process capacity
// (resolved DB > ENV > default), not live busy-worker count or cluster-wide
// capacity (see QueueStat.Active for currently running tasks).
type RuntimeWorkerPool struct {
Name string `json:"name"`
Concurrency int `json:"concurrency"`
QueueCount int `json:"queue_count"`
}
// defaultParseConcurrency / defaultWikiConcurrency mirror the unexported
// router constants (defaultAsynqConcurrency / defaultWikiAsynqConcurrency).
// Duplicated here rather than exported from router to avoid an import
// cycle (router already imports handler); kept in sync by convention.
const (
defaultParseConcurrency = 32
defaultWikiConcurrency = 16
)
type RuntimeQueuesResponse struct {
Available bool `json:"available"`
UpstreamConcurrency int `json:"upstream_concurrency"`
ParseConcurrency int `json:"parse_concurrency"` // compatibility alias for upstream_concurrency
WikiConcurrency int `json:"wiki_concurrency"` // compatibility field
Pools []RuntimeWorkerPool `json:"pools"`
Queues []types.QueueStat `json:"queues"`
Timestamp int64 `json:"timestamp"`
}
// GetRuntimeQueues godoc
// @Summary 获取解析任务队列运行时状态
@@ -1414,10 +1412,36 @@ const (
// @Router /system/admin/runtime/queues [get]
func (h *SystemHandler) GetRuntimeQueues(c *gin.Context) {
ctx := logger.CloneContext(c.Request.Context())
upstreamConcurrency := types.DefaultUpstreamWorkerConcurrency
wikiConcurrency := types.DefaultWikiWorkerConcurrency
if h.systemSettingSvc != nil {
if configured := h.systemSettingSvc.GetInt(
ctx, "asynq.concurrency", "WEKNORA_ASYNQ_CONCURRENCY", types.DefaultUpstreamWorkerConcurrency,
); configured > 0 {
upstreamConcurrency = int(configured)
}
if configured := h.systemSettingSvc.GetInt(
ctx, "asynq.wiki_concurrency", "WEKNORA_WIKI_ASYNQ_CONCURRENCY", types.DefaultWikiWorkerConcurrency,
); configured > 0 {
wikiConcurrency = int(configured)
}
}
allocation := types.AllocateWorkerPoolConcurrency(upstreamConcurrency)
queueCounts := make(map[string]int)
for _, definition := range types.QueueDefinitions() {
queueCounts[definition.Pool]++
}
resp := RuntimeQueuesResponse{
ParseConcurrency: defaultParseConcurrency,
WikiConcurrency: defaultWikiConcurrency,
Timestamp: time.Now().Unix(),
UpstreamConcurrency: allocation.Total,
ParseConcurrency: allocation.Total,
WikiConcurrency: wikiConcurrency,
Pools: []RuntimeWorkerPool{
{Name: types.WorkerPoolCore, Concurrency: allocation.Core, QueueCount: queueCounts[types.WorkerPoolCore]},
{Name: types.WorkerPoolEnrichment, Concurrency: allocation.Enrichment, QueueCount: queueCounts[types.WorkerPoolEnrichment]},
{Name: types.WorkerPoolMaintenance, Concurrency: allocation.Maintenance, QueueCount: queueCounts[types.WorkerPoolMaintenance]},
{Name: types.WorkerPoolWiki, Concurrency: wikiConcurrency, QueueCount: queueCounts[types.WorkerPoolWiki]},
},
Timestamp: time.Now().Unix(),
}
if h.taskInspector != nil {
@@ -1436,18 +1460,6 @@ func (h *SystemHandler) GetRuntimeQueues(c *gin.Context) {
resp.Queues = []types.QueueStat{}
}
// Resolve configured worker-pool sizes (DB > ENV > default). These
// are the same keys NewParseAsynqServer / NewWikiAsynqServer read at
// startup, so operators see the value that will apply on next restart.
if h.systemSettingSvc != nil {
resp.ParseConcurrency = int(h.systemSettingSvc.GetInt(
ctx, "asynq.concurrency", "WEKNORA_ASYNQ_CONCURRENCY", defaultParseConcurrency,
))
resp.WikiConcurrency = int(h.systemSettingSvc.GetInt(
ctx, "asynq.wiki_concurrency", "WEKNORA_WIKI_ASYNQ_CONCURRENCY", defaultWikiConcurrency,
))
}
c.JSON(http.StatusOK, resp)
}
+133
View File
@@ -0,0 +1,133 @@
package handler
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"github.com/Tencent/WeKnora/internal/types"
"github.com/gin-gonic/gin"
)
type runtimeTestSettings struct{}
func (runtimeTestSettings) GetInt(_ context.Context, key, _ string, def int64) int64 {
switch key {
case "asynq.concurrency":
return 32
case "asynq.wiki_concurrency":
return 16
default:
return def
}
}
func (runtimeTestSettings) GetString(_ context.Context, _, _, def string) string { return def }
func (runtimeTestSettings) GetBool(_ context.Context, _, _ string, def bool) bool { return def }
func (runtimeTestSettings) GetStringList(_ context.Context, _, _ string, def []string) []string {
return def
}
func (runtimeTestSettings) List(context.Context) ([]*types.SystemSetting, error) { return nil, nil }
func (runtimeTestSettings) Get(context.Context, string) (*types.SystemSetting, error) {
return nil, nil
}
func (runtimeTestSettings) Update(context.Context, string, any) (*types.SystemSetting, error) {
return nil, nil
}
func (runtimeTestSettings) Reset(context.Context, string) error { return nil }
func (runtimeTestSettings) SubscribeRedis(context.Context) error { return nil }
type runtimeInvalidSettings struct{ runtimeTestSettings }
func (runtimeInvalidSettings) GetInt(_ context.Context, _ string, _ string, _ int64) int64 {
return 0
}
type runtimeTestInspector struct{}
func (runtimeTestInspector) CancelTasksForKnowledge(context.Context, string) (int, int, error) {
return 0, 0, nil
}
func (runtimeTestInspector) HasQueuedTasksForKnowledge(context.Context, string) (bool, error) {
return false, nil
}
func (runtimeTestInspector) QueueStats(context.Context) ([]types.QueueStat, bool, error) {
return []types.QueueStat{}, true, nil
}
func TestGetRuntimeQueuesReportsIsolatedPoolCapacity(t *testing.T) {
gin.SetMode(gin.TestMode)
handler := &SystemHandler{
systemSettingSvc: runtimeTestSettings{},
taskInspector: runtimeTestInspector{},
}
recorder := httptest.NewRecorder()
ctx, _ := gin.CreateTestContext(recorder)
ctx.Request = httptest.NewRequest(http.MethodGet, "/api/v1/system/admin/runtime/queues", nil)
handler.GetRuntimeQueues(ctx)
if recorder.Code != http.StatusOK {
t.Fatalf("status = %d, want %d; body=%s", recorder.Code, http.StatusOK, recorder.Body.String())
}
var response RuntimeQueuesResponse
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
t.Fatalf("decode response: %v", err)
}
if !response.Available {
t.Fatal("queue inspection should be available")
}
if response.UpstreamConcurrency != 32 || response.ParseConcurrency != 32 {
t.Fatalf("upstream compatibility values are wrong: %+v", response)
}
want := map[string]struct {
concurrency int
queueCount int
}{
types.WorkerPoolCore: {16, 1},
types.WorkerPoolEnrichment: {12, 4},
types.WorkerPoolMaintenance: {4, 2},
types.WorkerPoolWiki: {16, 1},
}
if len(response.Pools) != len(want) {
t.Fatalf("pool count = %d, want %d", len(response.Pools), len(want))
}
for _, pool := range response.Pools {
expected, ok := want[pool.Name]
if !ok {
t.Fatalf("unexpected pool %q", pool.Name)
}
if pool.Concurrency != expected.concurrency || pool.QueueCount != expected.queueCount {
t.Fatalf("pool %q = %+v, want concurrency=%d queue_count=%d",
pool.Name, pool, expected.concurrency, expected.queueCount)
}
}
}
func TestGetRuntimeQueuesFallsBackFromInvalidHistoricalConcurrency(t *testing.T) {
gin.SetMode(gin.TestMode)
handler := &SystemHandler{
systemSettingSvc: runtimeInvalidSettings{},
taskInspector: runtimeTestInspector{},
}
recorder := httptest.NewRecorder()
ctx, _ := gin.CreateTestContext(recorder)
ctx.Request = httptest.NewRequest(http.MethodGet, "/api/v1/system/admin/runtime/queues", nil)
handler.GetRuntimeQueues(ctx)
var response RuntimeQueuesResponse
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
t.Fatalf("decode response: %v", err)
}
if response.UpstreamConcurrency != types.DefaultUpstreamWorkerConcurrency ||
response.WikiConcurrency != types.DefaultWikiWorkerConcurrency {
t.Fatalf("invalid stored values should use worker defaults: %+v", response)
}
for _, pool := range response.Pools {
if pool.Concurrency < 1 {
t.Fatalf("pool %q reported non-positive concurrency: %+v", pool.Name, pool)
}
}
}