feat(worker): replace aggregate asynq concurrency with per-pool governance

Split the monolithic upstream worker budget into explicit core, post-process,
enrichment, maintenance, and shared elastic pools, expose live cluster capacity
on the runtime dashboard, and retire the legacy asynq.concurrency setting.
This commit is contained in:
wizardchen
2026-07-13 14:55:59 +08:00
committed by lyingbug
parent 5eefa70e6f
commit 07ba01ec93
27 changed files with 645 additions and 209 deletions
+78 -23
View File
@@ -1385,13 +1385,16 @@ func (h *SystemHandler) ListSystemAdmins(c *gin.Context) {
// RuntimeQueuesResponse is the payload for the SystemAdmin runtime queue
// dashboard. `available` is false in Lite mode (no Redis/asynq) so the
// UI can render an "unavailable in this deployment" state instead of an
// empty table. Pool concurrency is the configured per-process capacity
// (resolved DB > ENV > default), not live busy-worker count or cluster-wide
// capacity (see QueueStat.Active for currently running tasks).
// empty table. Each pool reports configured per-process concurrency plus live
// cluster capacity/active workers aggregated from asynq server heartbeats.
type RuntimeWorkerPool struct {
Name string `json:"name"`
Concurrency int `json:"concurrency"`
QueueCount int `json:"queue_count"`
Name string `json:"name"`
Concurrency int `json:"concurrency"` // configured per instance
QueueCount int `json:"queue_count"`
Instances int `json:"instances"` // live asynq server heartbeats
ClusterCapacity int `json:"cluster_capacity"` // sum of live server concurrency
Active int `json:"active"` // live workers assigned to this pool
Utilization float64 `json:"utilization"` // Active / ClusterCapacity
}
type RuntimeQueuesResponse struct {
@@ -1406,6 +1409,56 @@ type RuntimeQueuesResponse struct {
Timestamp int64 `json:"timestamp"`
}
func aggregateRuntimeWorkerPools(pools []RuntimeWorkerPool, servers []types.WorkerServerStat) {
queueConfigs := map[string]map[string]int{
types.WorkerPoolCore: types.QueueWeightsForPool(types.WorkerPoolCore),
types.WorkerPoolPostProcess: types.QueueWeightsForPool(types.WorkerPoolPostProcess),
types.WorkerPoolEnrichment: types.QueueWeightsForPool(types.WorkerPoolEnrichment),
types.WorkerPoolMaintenance: types.QueueWeightsForPool(types.WorkerPoolMaintenance),
types.WorkerPoolShared: types.QueueWeightsForSharedPool(),
types.WorkerPoolWiki: types.QueueWeightsForPool(types.WorkerPoolWiki),
}
indexes := make(map[string]int, len(pools))
for i := range pools {
indexes[pools[i].Name] = i
}
for _, server := range servers {
if server.Status != "active" {
continue
}
for poolName, expectedQueues := range queueConfigs {
if !sameQueueWeights(server.Queues, expectedQueues) {
continue
}
idx, ok := indexes[poolName]
if !ok {
break
}
pools[idx].Instances++
pools[idx].ClusterCapacity += server.Concurrency
pools[idx].Active += server.Active
break
}
}
for i := range pools {
if pools[i].ClusterCapacity > 0 {
pools[i].Utilization = float64(pools[i].Active) / float64(pools[i].ClusterCapacity)
}
}
}
func sameQueueWeights(left, right map[string]int) bool {
if len(left) != len(right) {
return false
}
for name, weight := range left {
if right[name] != weight {
return false
}
}
return true
}
// GetRuntimeQueues godoc
// @Summary 获取解析任务队列运行时状态
// @Description 返回各 asynq 队列的实时深度(pending/active/scheduled/retry 等)与 worker 并发配置,仅系统管理员可见
@@ -1415,34 +1468,30 @@ type RuntimeQueuesResponse struct {
// @Router /system/admin/runtime/queues [get]
func (h *SystemHandler) GetRuntimeQueues(c *gin.Context) {
ctx := logger.CloneContext(c.Request.Context())
upstreamConcurrency := types.DefaultUpstreamWorkerConcurrency
wikiConcurrency := types.DefaultWikiWorkerConcurrency
var allocation types.WorkerPoolConcurrency
if h.systemSettingSvc != nil {
if configured := h.systemSettingSvc.GetInt(
ctx, "asynq.concurrency", "WEKNORA_ASYNQ_CONCURRENCY", types.DefaultUpstreamWorkerConcurrency,
); configured > 0 {
upstreamConcurrency = int(configured)
}
if configured := h.systemSettingSvc.GetInt(
ctx, "asynq.wiki_concurrency", "WEKNORA_WIKI_ASYNQ_CONCURRENCY", types.DefaultWikiWorkerConcurrency,
); configured > 0 {
wikiConcurrency = int(configured)
}
allocation = types.ResolveWorkerPoolConcurrency(func(key, env string, fallback int) int {
configured := h.systemSettingSvc.GetInt(ctx, key, env, int64(fallback))
return int(configured)
})
} else {
allocation = types.DefaultWorkerPoolConcurrency()
}
allocation := types.AllocateWorkerPoolConcurrency(upstreamConcurrency)
queueCounts := make(map[string]int)
for _, definition := range types.QueueDefinitions() {
queueCounts[definition.Pool]++
}
resp := RuntimeQueuesResponse{
UpstreamConcurrency: allocation.Total,
ParseConcurrency: allocation.Total,
WikiConcurrency: wikiConcurrency,
UpstreamConcurrency: allocation.UpstreamTotal(),
ParseConcurrency: allocation.UpstreamTotal(),
WikiConcurrency: allocation.Wiki,
Pools: []RuntimeWorkerPool{
{Name: types.WorkerPoolCore, Concurrency: allocation.Core, QueueCount: queueCounts[types.WorkerPoolCore]},
{Name: types.WorkerPoolPostProcess, Concurrency: allocation.PostProcess, QueueCount: queueCounts[types.WorkerPoolPostProcess]},
{Name: types.WorkerPoolEnrichment, Concurrency: allocation.Enrichment, QueueCount: queueCounts[types.WorkerPoolEnrichment]},
{Name: types.WorkerPoolMaintenance, Concurrency: allocation.Maintenance, QueueCount: queueCounts[types.WorkerPoolMaintenance]},
{Name: types.WorkerPoolWiki, Concurrency: wikiConcurrency, QueueCount: queueCounts[types.WorkerPoolWiki]},
{Name: types.WorkerPoolShared, Concurrency: allocation.Shared, QueueCount: len(types.QueueWeightsForSharedPool())},
{Name: types.WorkerPoolWiki, Concurrency: allocation.Wiki, QueueCount: queueCounts[types.WorkerPoolWiki]},
},
Timestamp: time.Now().Unix(),
}
@@ -1456,6 +1505,12 @@ func (h *SystemHandler) GetRuntimeQueues(c *gin.Context) {
if stats != nil {
resp.Queues = stats
}
serverStats, _, serverErr := h.taskInspector.WorkerServerStats(ctx)
if serverErr != nil {
logger.Errorf(ctx, "get worker server stats failed: %v", serverErr)
} else {
aggregateRuntimeWorkerPools(resp.Pools, serverStats)
}
}
// Always emit a non-nil array so the JSON serialises to `[]` rather
// than `null`, keeping the frontend iteration simple.
+29 -5
View File
@@ -15,10 +15,18 @@ type runtimeTestSettings struct{}
func (runtimeTestSettings) GetInt(_ context.Context, key, _ string, def int64) int64 {
switch key {
case "asynq.concurrency":
return 32
case "asynq.core_concurrency":
return 8
case "asynq.postprocess_concurrency":
return 2
case "asynq.enrichment_concurrency":
return 12
case "asynq.maintenance_concurrency":
return 4
case "asynq.shared_concurrency":
return 6
case "asynq.wiki_concurrency":
return 16
return 8
default:
return def
}
@@ -55,6 +63,17 @@ func (runtimeTestInspector) HasQueuedTasksForKnowledge(context.Context, string)
func (runtimeTestInspector) QueueStats(context.Context) ([]types.QueueStat, bool, error) {
return []types.QueueStat{}, true, nil
}
func (runtimeTestInspector) WorkerServerStats(context.Context) ([]types.WorkerServerStat, bool, error) {
return []types.WorkerServerStat{
{Concurrency: 8, Active: 4, Status: "active", Queues: types.QueueWeightsForPool(types.WorkerPoolCore)},
{Concurrency: 2, Active: 1, Status: "active", Queues: types.QueueWeightsForPool(types.WorkerPoolPostProcess)},
{Concurrency: 12, Active: 6, Status: "active", Queues: types.QueueWeightsForPool(types.WorkerPoolEnrichment)},
{Concurrency: 4, Active: 1, Status: "active", Queues: types.QueueWeightsForPool(types.WorkerPoolMaintenance)},
{Concurrency: 6, Active: 3, Status: "active", Queues: types.QueueWeightsForSharedPool()},
{Concurrency: 8, Active: 2, Status: "active", Queues: types.QueueWeightsForPool(types.WorkerPoolWiki)},
{Concurrency: 99, Active: 0, Status: "stopped", Queues: types.QueueWeightsForPool(types.WorkerPoolCore)},
}, true, nil
}
func TestGetRuntimeQueuesReportsIsolatedPoolCapacity(t *testing.T) {
gin.SetMode(gin.TestMode)
@@ -85,10 +104,12 @@ func TestGetRuntimeQueuesReportsIsolatedPoolCapacity(t *testing.T) {
concurrency int
queueCount int
}{
types.WorkerPoolCore: {16, 1},
types.WorkerPoolCore: {8, 1},
types.WorkerPoolPostProcess: {2, 1},
types.WorkerPoolEnrichment: {12, 4},
types.WorkerPoolMaintenance: {4, 2},
types.WorkerPoolWiki: {16, 1},
types.WorkerPoolShared: {6, 5},
types.WorkerPoolWiki: {8, 1},
}
if len(response.Pools) != len(want) {
t.Fatalf("pool count = %d, want %d", len(response.Pools), len(want))
@@ -102,6 +123,9 @@ func TestGetRuntimeQueuesReportsIsolatedPoolCapacity(t *testing.T) {
t.Fatalf("pool %q = %+v, want concurrency=%d queue_count=%d",
pool.Name, pool, expected.concurrency, expected.queueCount)
}
if pool.Instances != 1 || pool.ClusterCapacity != expected.concurrency {
t.Fatalf("pool %q live capacity = %+v", pool.Name, pool)
}
}
}