feat(worker): replace aggregate asynq concurrency with per-pool governance

Split the monolithic upstream worker budget into explicit core, post-process,
enrichment, maintenance, and shared elastic pools, expose live cluster capacity
on the runtime dashboard, and retire the legacy asynq.concurrency setting.
This commit is contained in:
wizardchen
2026-07-13 14:55:59 +08:00
committed by lyingbug
parent 5eefa70e6f
commit 07ba01ec93
27 changed files with 645 additions and 209 deletions
@@ -626,7 +626,7 @@ func (s *ImageMultimodalService) enqueueKnowledgePostProcessTask(ctx context.Con
}
task := asynq.NewTask(types.TypeKnowledgePostProcess, payloadBytes,
asynq.Queue(types.QueueDefault), asynq.MaxRetry(3), asynq.Timeout(30*time.Minute))
knowledgePostProcessTaskOptions()...)
if _, err := s.taskEnqueuer.Enqueue(task); err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to enqueue post process task for %s: %v", payload.KnowledgeID, err)
} else {
@@ -186,8 +186,9 @@ func (h *HousekeepingService) runSweep(ctx context.Context) {
if queueSkipped > 0 {
// Visibility into "stale span heartbeat but tasks still queued"
// — i.e. backpressure, not a stuck row. Persistent counts here
// mean the queue is the bottleneck (raise WEKNORA_ASYNQ_CONCURRENCY
// or document_process_timeout), not that housekeeping misfires.
// mean the queue is the bottleneck (raise the matching per-pool or
// shared asynq concurrency, or document_process_timeout), not that
// housekeeping misfires.
logger.Infof(ctx,
"[Housekeeping] %d candidate(s) skipped — tasks still queued (backpressure, not stuck)",
queueSkipped)
@@ -126,6 +126,12 @@ func (f fakeTaskInspector) QueueStats(
return nil, false, nil
}
func (f fakeTaskInspector) WorkerServerStats(
_ context.Context,
) ([]types.WorkerServerStat, bool, error) {
return nil, false, nil
}
func newHousekeepingSvcForTest(db *gorm.DB) *HousekeepingService {
return newHousekeepingSvcWithInspector(db, fakeTaskInspector{})
}
@@ -675,7 +675,7 @@ func (s *knowledgeService) processChunks(ctx context.Context,
payloadBytes, err := json.Marshal(postProcessPayload)
if err == nil {
task := asynq.NewTask(types.TypeKnowledgePostProcess, payloadBytes,
asynq.Queue(types.QueueDefault), asynq.MaxRetry(3), asynq.Timeout(30*time.Minute))
knowledgePostProcessTaskOptions()...)
if _, err := s.task.Enqueue(task); err != nil {
logger.Errorf(ctx, "Failed to enqueue knowledge post process task: %v", err)
} else {
@@ -1,6 +1,8 @@
package service
import (
"time"
"github.com/Tencent/WeKnora/internal/config"
"github.com/Tencent/WeKnora/internal/types"
"github.com/hibiken/asynq"
@@ -15,3 +17,11 @@ func documentProcessTaskOptions(cfg *config.Config, extra ...asynq.Option) []asy
opts = append(opts, extra...)
return opts
}
func knowledgePostProcessTaskOptions() []asynq.Option {
return []asynq.Option{
asynq.Queue(types.QueuePostProcess),
asynq.MaxRetry(3),
asynq.Timeout(30 * time.Minute),
}
}
@@ -79,3 +79,12 @@ func TestDocumentProcessTaskOptions_extraMaxRetry(t *testing.T) {
require.NotNil(t, maxRetry)
assert.Equal(t, 3, *maxRetry)
}
func TestKnowledgePostProcessTaskOptionsUseDedicatedQueue(t *testing.T) {
t.Parallel()
queue, timeout, maxRetry := parseDocumentProcessOpts(t, knowledgePostProcessTaskOptions())
assert.Equal(t, types.QueuePostProcess, queue)
assert.Equal(t, 30*time.Minute, timeout)
require.NotNil(t, maxRetry)
assert.Equal(t, 3, *maxRetry)
}
+47 -19
View File
@@ -183,24 +183,50 @@ var registry = map[string]settingSpec{
"用于兼容旧版本「创建租户即下发默认 API Key」的行为(属于破坏性变更的回退开关)。" +
"每次创建租户时实时读取,修改后立即生效。默认 false(不自动创建,需通过 API Key 管理显式创建)。",
},
// asynq.concurrency is the total per-process upstream worker budget.
// It is divided deterministically across independent core, enrichment,
// and maintenance servers. Read once at startup; changing it requires a
// process restart. Mirrors WEKNORA_ASYNQ_CONCURRENCY (default 32).
"asynq.concurrency": {
"asynq.core_concurrency": {
Type: "int",
EnvName: "WEKNORA_ASYNQ_CONCURRENCY",
Default: int64(types.DefaultUpstreamWorkerConcurrency),
EnvName: "WEKNORA_ASYNQ_CORE_CONCURRENCY",
Default: int64(types.DefaultCoreWorkerConcurrency),
Category: "worker",
RequiresRestart: true,
Description: "上游任务的每实例 worker 总并发预算(不含 Wiki)。系统会按核心解析 1/2、" +
"内容富化 3/8、维护与同步为剩余容量,拆分为相互隔离的 worker 池。" +
"最小值为 3;修改后需重启服务进程方可生效。",
Description: "文档解析、手工重解析等核心任务的每实例保底并发。可额外使用共享弹性池;修改后需重启。",
},
"asynq.postprocess_concurrency": {
Type: "int",
EnvName: "WEKNORA_ASYNQ_POSTPROCESS_CONCURRENCY",
Default: int64(types.DefaultPostProcessWorkerConcurrency),
Category: "worker",
RequiresRestart: true,
Description: "解析完成后的轻量编排与富化扇出专用并发,避免被长时间文档解析阻塞;修改后需重启。",
},
"asynq.enrichment_concurrency": {
Type: "int",
EnvName: "WEKNORA_ASYNQ_ENRICHMENT_CONCURRENCY",
Default: int64(types.DefaultEnrichmentWorkerConcurrency),
Category: "worker",
RequiresRestart: true,
Description: "摘要、图片、图谱和问题生成的每实例保底并发。可额外使用共享弹性池;修改后需重启。",
},
"asynq.maintenance_concurrency": {
Type: "int",
EnvName: "WEKNORA_ASYNQ_MAINTENANCE_CONCURRENCY",
Default: int64(types.DefaultMaintenanceWorkerConcurrency),
Category: "worker",
RequiresRestart: true,
Description: "数据源同步、批处理、移动和删除清理的每实例保底并发,与用户面流水线硬隔离;修改后需重启。",
},
"asynq.shared_concurrency": {
Type: "int",
EnvName: "WEKNORA_ASYNQ_SHARED_CONCURRENCY",
Default: int64(types.DefaultSharedWorkerConcurrency),
Category: "worker",
RequiresRestart: true,
Description: "核心解析与内容富化共用的每实例弹性并发。空闲容量由有积压的一侧借用;修改后需重启。",
},
// asynq.wiki_concurrency is the size of the DEDICATED wiki worker pool,
// separate from asynq.concurrency. Read once when the wiki asynq server
// separate from the upstream pools. Read once when the wiki asynq server
// starts — changing it in the UI requires a process restart. Mirrors
// WEKNORA_WIKI_ASYNQ_CONCURRENCY (default 16).
// WEKNORA_WIKI_ASYNQ_CONCURRENCY (default 8).
"asynq.wiki_concurrency": {
Type: "int",
EnvName: "WEKNORA_WIKI_ASYNQ_CONCURRENCY",
@@ -669,6 +695,10 @@ func (s *systemSettingService) List(ctx context.Context) ([]*types.SystemSetting
for _, row := range rows {
byKey[row.Key] = row
}
// asynq.concurrency was the old fixed-ratio aggregate. Keeping an old DB
// row visible would suggest it still controls runtime capacity, so retire it
// explicitly while preserving all genuinely unknown rows for diagnostics.
delete(byKey, "asynq.concurrency")
keys := make([]string, 0, len(registry))
for key := range registry {
@@ -1228,17 +1258,15 @@ func encodeForType(declared string, rawValue any) (types.JSON, error) {
// 400 body verbatim).
func validateRegistryEntry(key string, rawValue any) error {
switch key {
case "asynq.concurrency", "asynq.wiki_concurrency":
case "asynq.core_concurrency", "asynq.postprocess_concurrency",
"asynq.enrichment_concurrency", "asynq.maintenance_concurrency",
"asynq.shared_concurrency", "asynq.wiki_concurrency":
n, err := coerceToPositiveInt64(rawValue)
if err != nil {
return err
}
minimum := int64(1)
if key == "asynq.concurrency" {
minimum = 3
}
if n < minimum {
return fmt.Errorf("concurrency must be at least %d", minimum)
if n < 1 {
return errors.New("concurrency must be at least 1")
}
case "ssrf.whitelist":
// Coerce into the same shape encodeForType produced. We don't
@@ -9,9 +9,12 @@ func TestValidateWorkerConcurrencyMinimums(t *testing.T) {
value any
wantErr bool
}{
{name: "upstream below one worker per pool", key: "asynq.concurrency", value: 2, wantErr: true},
{name: "upstream minimum", key: "asynq.concurrency", value: 3},
{name: "upstream normal", key: "asynq.concurrency", value: 32},
{name: "core zero", key: "asynq.core_concurrency", value: 0, wantErr: true},
{name: "core minimum", key: "asynq.core_concurrency", value: 1},
{name: "postprocess minimum", key: "asynq.postprocess_concurrency", value: 1},
{name: "enrichment minimum", key: "asynq.enrichment_concurrency", value: 1},
{name: "maintenance minimum", key: "asynq.maintenance_concurrency", value: 1},
{name: "shared minimum", key: "asynq.shared_concurrency", value: 1},
{name: "wiki zero", key: "asynq.wiki_concurrency", value: 0, wantErr: true},
{name: "wiki minimum", key: "asynq.wiki_concurrency", value: 1},
}
+1 -1
View File
@@ -79,7 +79,7 @@ const (
// wikiInflightPrefix keys the per-KB slot set: wiki:inflight:{kbID}.
wikiInflightPrefix = "wiki:inflight:"
// wikiInflightDefault is the fallback cap when WikiConfig.IngestMaxInflight
// is unset. 4 leaves headroom in a default 16-worker wiki pool for other
// is unset. 4 leaves half of the default 8-worker wiki pool for other
// KBs while still giving one KB solid parallelism.
wikiInflightDefault = 4
// wikiInflightTTL is how long a reserved slot survives without renewal.