feat(worker): replace aggregate asynq concurrency with per-pool governance
Split the monolithic upstream worker budget into explicit core, post-process, enrichment, maintenance, and shared elastic pools, expose live cluster capacity on the runtime dashboard, and retire the legacy asynq.concurrency setting.
This commit is contained in:
@@ -626,7 +626,7 @@ func (s *ImageMultimodalService) enqueueKnowledgePostProcessTask(ctx context.Con
|
||||
}
|
||||
|
||||
task := asynq.NewTask(types.TypeKnowledgePostProcess, payloadBytes,
|
||||
asynq.Queue(types.QueueDefault), asynq.MaxRetry(3), asynq.Timeout(30*time.Minute))
|
||||
knowledgePostProcessTaskOptions()...)
|
||||
if _, err := s.taskEnqueuer.Enqueue(task); err != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Failed to enqueue post process task for %s: %v", payload.KnowledgeID, err)
|
||||
} else {
|
||||
|
||||
@@ -186,8 +186,9 @@ func (h *HousekeepingService) runSweep(ctx context.Context) {
|
||||
if queueSkipped > 0 {
|
||||
// Visibility into "stale span heartbeat but tasks still queued"
|
||||
// — i.e. backpressure, not a stuck row. Persistent counts here
|
||||
// mean the queue is the bottleneck (raise WEKNORA_ASYNQ_CONCURRENCY
|
||||
// or document_process_timeout), not that housekeeping misfires.
|
||||
// mean the queue is the bottleneck (raise the matching per-pool or
|
||||
// shared asynq concurrency, or document_process_timeout), not that
|
||||
// housekeeping misfires.
|
||||
logger.Infof(ctx,
|
||||
"[Housekeeping] %d candidate(s) skipped — tasks still queued (backpressure, not stuck)",
|
||||
queueSkipped)
|
||||
|
||||
@@ -126,6 +126,12 @@ func (f fakeTaskInspector) QueueStats(
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
func (f fakeTaskInspector) WorkerServerStats(
|
||||
_ context.Context,
|
||||
) ([]types.WorkerServerStat, bool, error) {
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
func newHousekeepingSvcForTest(db *gorm.DB) *HousekeepingService {
|
||||
return newHousekeepingSvcWithInspector(db, fakeTaskInspector{})
|
||||
}
|
||||
|
||||
@@ -675,7 +675,7 @@ func (s *knowledgeService) processChunks(ctx context.Context,
|
||||
payloadBytes, err := json.Marshal(postProcessPayload)
|
||||
if err == nil {
|
||||
task := asynq.NewTask(types.TypeKnowledgePostProcess, payloadBytes,
|
||||
asynq.Queue(types.QueueDefault), asynq.MaxRetry(3), asynq.Timeout(30*time.Minute))
|
||||
knowledgePostProcessTaskOptions()...)
|
||||
if _, err := s.task.Enqueue(task); err != nil {
|
||||
logger.Errorf(ctx, "Failed to enqueue knowledge post process task: %v", err)
|
||||
} else {
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/hibiken/asynq"
|
||||
@@ -15,3 +17,11 @@ func documentProcessTaskOptions(cfg *config.Config, extra ...asynq.Option) []asy
|
||||
opts = append(opts, extra...)
|
||||
return opts
|
||||
}
|
||||
|
||||
func knowledgePostProcessTaskOptions() []asynq.Option {
|
||||
return []asynq.Option{
|
||||
asynq.Queue(types.QueuePostProcess),
|
||||
asynq.MaxRetry(3),
|
||||
asynq.Timeout(30 * time.Minute),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -79,3 +79,12 @@ func TestDocumentProcessTaskOptions_extraMaxRetry(t *testing.T) {
|
||||
require.NotNil(t, maxRetry)
|
||||
assert.Equal(t, 3, *maxRetry)
|
||||
}
|
||||
|
||||
func TestKnowledgePostProcessTaskOptionsUseDedicatedQueue(t *testing.T) {
|
||||
t.Parallel()
|
||||
queue, timeout, maxRetry := parseDocumentProcessOpts(t, knowledgePostProcessTaskOptions())
|
||||
assert.Equal(t, types.QueuePostProcess, queue)
|
||||
assert.Equal(t, 30*time.Minute, timeout)
|
||||
require.NotNil(t, maxRetry)
|
||||
assert.Equal(t, 3, *maxRetry)
|
||||
}
|
||||
|
||||
@@ -183,24 +183,50 @@ var registry = map[string]settingSpec{
|
||||
"用于兼容旧版本「创建租户即下发默认 API Key」的行为(属于破坏性变更的回退开关)。" +
|
||||
"每次创建租户时实时读取,修改后立即生效。默认 false(不自动创建,需通过 API Key 管理显式创建)。",
|
||||
},
|
||||
// asynq.concurrency is the total per-process upstream worker budget.
|
||||
// It is divided deterministically across independent core, enrichment,
|
||||
// and maintenance servers. Read once at startup; changing it requires a
|
||||
// process restart. Mirrors WEKNORA_ASYNQ_CONCURRENCY (default 32).
|
||||
"asynq.concurrency": {
|
||||
"asynq.core_concurrency": {
|
||||
Type: "int",
|
||||
EnvName: "WEKNORA_ASYNQ_CONCURRENCY",
|
||||
Default: int64(types.DefaultUpstreamWorkerConcurrency),
|
||||
EnvName: "WEKNORA_ASYNQ_CORE_CONCURRENCY",
|
||||
Default: int64(types.DefaultCoreWorkerConcurrency),
|
||||
Category: "worker",
|
||||
RequiresRestart: true,
|
||||
Description: "上游任务的每实例 worker 总并发预算(不含 Wiki)。系统会按核心解析 1/2、" +
|
||||
"内容富化 3/8、维护与同步为剩余容量,拆分为相互隔离的 worker 池。" +
|
||||
"最小值为 3;修改后需重启服务进程方可生效。",
|
||||
Description: "文档解析、手工重解析等核心任务的每实例保底并发。可额外使用共享弹性池;修改后需重启。",
|
||||
},
|
||||
"asynq.postprocess_concurrency": {
|
||||
Type: "int",
|
||||
EnvName: "WEKNORA_ASYNQ_POSTPROCESS_CONCURRENCY",
|
||||
Default: int64(types.DefaultPostProcessWorkerConcurrency),
|
||||
Category: "worker",
|
||||
RequiresRestart: true,
|
||||
Description: "解析完成后的轻量编排与富化扇出专用并发,避免被长时间文档解析阻塞;修改后需重启。",
|
||||
},
|
||||
"asynq.enrichment_concurrency": {
|
||||
Type: "int",
|
||||
EnvName: "WEKNORA_ASYNQ_ENRICHMENT_CONCURRENCY",
|
||||
Default: int64(types.DefaultEnrichmentWorkerConcurrency),
|
||||
Category: "worker",
|
||||
RequiresRestart: true,
|
||||
Description: "摘要、图片、图谱和问题生成的每实例保底并发。可额外使用共享弹性池;修改后需重启。",
|
||||
},
|
||||
"asynq.maintenance_concurrency": {
|
||||
Type: "int",
|
||||
EnvName: "WEKNORA_ASYNQ_MAINTENANCE_CONCURRENCY",
|
||||
Default: int64(types.DefaultMaintenanceWorkerConcurrency),
|
||||
Category: "worker",
|
||||
RequiresRestart: true,
|
||||
Description: "数据源同步、批处理、移动和删除清理的每实例保底并发,与用户面流水线硬隔离;修改后需重启。",
|
||||
},
|
||||
"asynq.shared_concurrency": {
|
||||
Type: "int",
|
||||
EnvName: "WEKNORA_ASYNQ_SHARED_CONCURRENCY",
|
||||
Default: int64(types.DefaultSharedWorkerConcurrency),
|
||||
Category: "worker",
|
||||
RequiresRestart: true,
|
||||
Description: "核心解析与内容富化共用的每实例弹性并发。空闲容量由有积压的一侧借用;修改后需重启。",
|
||||
},
|
||||
// asynq.wiki_concurrency is the size of the DEDICATED wiki worker pool,
|
||||
// separate from asynq.concurrency. Read once when the wiki asynq server
|
||||
// separate from the upstream pools. Read once when the wiki asynq server
|
||||
// starts — changing it in the UI requires a process restart. Mirrors
|
||||
// WEKNORA_WIKI_ASYNQ_CONCURRENCY (default 16).
|
||||
// WEKNORA_WIKI_ASYNQ_CONCURRENCY (default 8).
|
||||
"asynq.wiki_concurrency": {
|
||||
Type: "int",
|
||||
EnvName: "WEKNORA_WIKI_ASYNQ_CONCURRENCY",
|
||||
@@ -669,6 +695,10 @@ func (s *systemSettingService) List(ctx context.Context) ([]*types.SystemSetting
|
||||
for _, row := range rows {
|
||||
byKey[row.Key] = row
|
||||
}
|
||||
// asynq.concurrency was the old fixed-ratio aggregate. Keeping an old DB
|
||||
// row visible would suggest it still controls runtime capacity, so retire it
|
||||
// explicitly while preserving all genuinely unknown rows for diagnostics.
|
||||
delete(byKey, "asynq.concurrency")
|
||||
|
||||
keys := make([]string, 0, len(registry))
|
||||
for key := range registry {
|
||||
@@ -1228,17 +1258,15 @@ func encodeForType(declared string, rawValue any) (types.JSON, error) {
|
||||
// 400 body verbatim).
|
||||
func validateRegistryEntry(key string, rawValue any) error {
|
||||
switch key {
|
||||
case "asynq.concurrency", "asynq.wiki_concurrency":
|
||||
case "asynq.core_concurrency", "asynq.postprocess_concurrency",
|
||||
"asynq.enrichment_concurrency", "asynq.maintenance_concurrency",
|
||||
"asynq.shared_concurrency", "asynq.wiki_concurrency":
|
||||
n, err := coerceToPositiveInt64(rawValue)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
minimum := int64(1)
|
||||
if key == "asynq.concurrency" {
|
||||
minimum = 3
|
||||
}
|
||||
if n < minimum {
|
||||
return fmt.Errorf("concurrency must be at least %d", minimum)
|
||||
if n < 1 {
|
||||
return errors.New("concurrency must be at least 1")
|
||||
}
|
||||
case "ssrf.whitelist":
|
||||
// Coerce into the same shape encodeForType produced. We don't
|
||||
|
||||
@@ -9,9 +9,12 @@ func TestValidateWorkerConcurrencyMinimums(t *testing.T) {
|
||||
value any
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "upstream below one worker per pool", key: "asynq.concurrency", value: 2, wantErr: true},
|
||||
{name: "upstream minimum", key: "asynq.concurrency", value: 3},
|
||||
{name: "upstream normal", key: "asynq.concurrency", value: 32},
|
||||
{name: "core zero", key: "asynq.core_concurrency", value: 0, wantErr: true},
|
||||
{name: "core minimum", key: "asynq.core_concurrency", value: 1},
|
||||
{name: "postprocess minimum", key: "asynq.postprocess_concurrency", value: 1},
|
||||
{name: "enrichment minimum", key: "asynq.enrichment_concurrency", value: 1},
|
||||
{name: "maintenance minimum", key: "asynq.maintenance_concurrency", value: 1},
|
||||
{name: "shared minimum", key: "asynq.shared_concurrency", value: 1},
|
||||
{name: "wiki zero", key: "asynq.wiki_concurrency", value: 0, wantErr: true},
|
||||
{name: "wiki minimum", key: "asynq.wiki_concurrency", value: 1},
|
||||
}
|
||||
|
||||
@@ -79,7 +79,7 @@ const (
|
||||
// wikiInflightPrefix keys the per-KB slot set: wiki:inflight:{kbID}.
|
||||
wikiInflightPrefix = "wiki:inflight:"
|
||||
// wikiInflightDefault is the fallback cap when WikiConfig.IngestMaxInflight
|
||||
// is unset. 4 leaves headroom in a default 16-worker wiki pool for other
|
||||
// is unset. 4 leaves half of the default 8-worker wiki pool for other
|
||||
// KBs while still giving one KB solid parallelism.
|
||||
wikiInflightDefault = 4
|
||||
// wikiInflightTTL is how long a reserved slot survives without renewal.
|
||||
|
||||
Reference in New Issue
Block a user