feat(task-queue): split upstream workers into isolated core/enrichment/maintenance pools
Route summary/sync/maintenance tasks to dedicated queues and servers so parsing, enrichment, and batch work no longer share one concurrency budget, while extending the runtime dashboard and settings validation to reflect the new topology.
This commit is contained in:
@@ -256,7 +256,7 @@ func (h *KnowledgeHandler) enqueueKnowledgeListDelete(
|
||||
return "", fmt.Errorf("marshal payload: %w", err)
|
||||
}
|
||||
task := asynq.NewTask(types.TypeKnowledgeListDelete, payloadBytes,
|
||||
asynq.Queue("low"), asynq.MaxRetry(3))
|
||||
asynq.Queue(types.QueueMaintenance), asynq.MaxRetry(3), asynq.Timeout(2*time.Hour))
|
||||
info, err := h.asynqClient.Enqueue(task)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("enqueue task: %w", err)
|
||||
@@ -280,7 +280,7 @@ func (h *KnowledgeHandler) enqueueKnowledgeListReparse(
|
||||
return "", fmt.Errorf("marshal payload: %w", err)
|
||||
}
|
||||
task := asynq.NewTask(types.TypeKnowledgeListReparse, payloadBytes,
|
||||
asynq.Queue("low"), asynq.MaxRetry(3))
|
||||
asynq.Queue(types.QueueMaintenance), asynq.MaxRetry(3), asynq.Timeout(time.Hour))
|
||||
info, err := h.asynqClient.Enqueue(task)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("enqueue task: %w", err)
|
||||
@@ -2186,7 +2186,8 @@ func (h *KnowledgeHandler) MoveKnowledge(c *gin.Context) {
|
||||
|
||||
// Enqueue move task
|
||||
task := asynq.NewTask(types.TypeKnowledgeMove, payloadBytes,
|
||||
asynq.TaskID(taskID), asynq.Queue("default"), asynq.MaxRetry(3))
|
||||
asynq.TaskID(taskID), asynq.Queue(types.QueueMaintenance),
|
||||
asynq.MaxRetry(3), asynq.Timeout(2*time.Hour))
|
||||
info, err := h.asynqClient.Enqueue(task)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "MoveKnowledge: failed to enqueue task: %v", err)
|
||||
|
||||
@@ -1083,7 +1083,8 @@ func (h *KnowledgeBaseHandler) CopyKnowledgeBase(c *gin.Context) {
|
||||
|
||||
// Enqueue KB clone task to Asynq
|
||||
task := asynq.NewTask(types.TypeKBClone, payloadBytes,
|
||||
asynq.TaskID(taskID), asynq.Queue("default"), asynq.MaxRetry(3))
|
||||
asynq.TaskID(taskID), asynq.Queue(types.QueueMaintenance),
|
||||
asynq.MaxRetry(3), asynq.Timeout(2*time.Hour))
|
||||
info, err := h.asynqClient.Enqueue(task)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to enqueue KB clone task: %v", err)
|
||||
|
||||
+45
-33
@@ -1384,26 +1384,24 @@ func (h *SystemHandler) ListSystemAdmins(c *gin.Context) {
|
||||
// RuntimeQueuesResponse is the payload for the SystemAdmin runtime queue
|
||||
// dashboard. `available` is false in Lite mode (no Redis/asynq) so the
|
||||
// UI can render an "unavailable in this deployment" state instead of an
|
||||
// empty table. ParseConcurrency / WikiConcurrency are the *configured*
|
||||
// worker-pool sizes (resolved DB > ENV > default); they are static
|
||||
// capacity, not the live active-worker count (see QueueStat.Active for
|
||||
// what's actually running).
|
||||
type RuntimeQueuesResponse struct {
|
||||
Available bool `json:"available"`
|
||||
ParseConcurrency int `json:"parse_concurrency"`
|
||||
WikiConcurrency int `json:"wiki_concurrency"`
|
||||
Queues []types.QueueStat `json:"queues"`
|
||||
Timestamp int64 `json:"timestamp"`
|
||||
// empty table. Pool concurrency is the configured per-process capacity
|
||||
// (resolved DB > ENV > default), not live busy-worker count or cluster-wide
|
||||
// capacity (see QueueStat.Active for currently running tasks).
|
||||
type RuntimeWorkerPool struct {
|
||||
Name string `json:"name"`
|
||||
Concurrency int `json:"concurrency"`
|
||||
QueueCount int `json:"queue_count"`
|
||||
}
|
||||
|
||||
// defaultParseConcurrency / defaultWikiConcurrency mirror the unexported
|
||||
// router constants (defaultAsynqConcurrency / defaultWikiAsynqConcurrency).
|
||||
// Duplicated here rather than exported from router to avoid an import
|
||||
// cycle (router already imports handler); kept in sync by convention.
|
||||
const (
|
||||
defaultParseConcurrency = 32
|
||||
defaultWikiConcurrency = 16
|
||||
)
|
||||
type RuntimeQueuesResponse struct {
|
||||
Available bool `json:"available"`
|
||||
UpstreamConcurrency int `json:"upstream_concurrency"`
|
||||
ParseConcurrency int `json:"parse_concurrency"` // compatibility alias for upstream_concurrency
|
||||
WikiConcurrency int `json:"wiki_concurrency"` // compatibility field
|
||||
Pools []RuntimeWorkerPool `json:"pools"`
|
||||
Queues []types.QueueStat `json:"queues"`
|
||||
Timestamp int64 `json:"timestamp"`
|
||||
}
|
||||
|
||||
// GetRuntimeQueues godoc
|
||||
// @Summary 获取解析任务队列运行时状态
|
||||
@@ -1414,10 +1412,36 @@ const (
|
||||
// @Router /system/admin/runtime/queues [get]
|
||||
func (h *SystemHandler) GetRuntimeQueues(c *gin.Context) {
|
||||
ctx := logger.CloneContext(c.Request.Context())
|
||||
upstreamConcurrency := types.DefaultUpstreamWorkerConcurrency
|
||||
wikiConcurrency := types.DefaultWikiWorkerConcurrency
|
||||
if h.systemSettingSvc != nil {
|
||||
if configured := h.systemSettingSvc.GetInt(
|
||||
ctx, "asynq.concurrency", "WEKNORA_ASYNQ_CONCURRENCY", types.DefaultUpstreamWorkerConcurrency,
|
||||
); configured > 0 {
|
||||
upstreamConcurrency = int(configured)
|
||||
}
|
||||
if configured := h.systemSettingSvc.GetInt(
|
||||
ctx, "asynq.wiki_concurrency", "WEKNORA_WIKI_ASYNQ_CONCURRENCY", types.DefaultWikiWorkerConcurrency,
|
||||
); configured > 0 {
|
||||
wikiConcurrency = int(configured)
|
||||
}
|
||||
}
|
||||
allocation := types.AllocateWorkerPoolConcurrency(upstreamConcurrency)
|
||||
queueCounts := make(map[string]int)
|
||||
for _, definition := range types.QueueDefinitions() {
|
||||
queueCounts[definition.Pool]++
|
||||
}
|
||||
resp := RuntimeQueuesResponse{
|
||||
ParseConcurrency: defaultParseConcurrency,
|
||||
WikiConcurrency: defaultWikiConcurrency,
|
||||
Timestamp: time.Now().Unix(),
|
||||
UpstreamConcurrency: allocation.Total,
|
||||
ParseConcurrency: allocation.Total,
|
||||
WikiConcurrency: wikiConcurrency,
|
||||
Pools: []RuntimeWorkerPool{
|
||||
{Name: types.WorkerPoolCore, Concurrency: allocation.Core, QueueCount: queueCounts[types.WorkerPoolCore]},
|
||||
{Name: types.WorkerPoolEnrichment, Concurrency: allocation.Enrichment, QueueCount: queueCounts[types.WorkerPoolEnrichment]},
|
||||
{Name: types.WorkerPoolMaintenance, Concurrency: allocation.Maintenance, QueueCount: queueCounts[types.WorkerPoolMaintenance]},
|
||||
{Name: types.WorkerPoolWiki, Concurrency: wikiConcurrency, QueueCount: queueCounts[types.WorkerPoolWiki]},
|
||||
},
|
||||
Timestamp: time.Now().Unix(),
|
||||
}
|
||||
|
||||
if h.taskInspector != nil {
|
||||
@@ -1436,18 +1460,6 @@ func (h *SystemHandler) GetRuntimeQueues(c *gin.Context) {
|
||||
resp.Queues = []types.QueueStat{}
|
||||
}
|
||||
|
||||
// Resolve configured worker-pool sizes (DB > ENV > default). These
|
||||
// are the same keys NewParseAsynqServer / NewWikiAsynqServer read at
|
||||
// startup, so operators see the value that will apply on next restart.
|
||||
if h.systemSettingSvc != nil {
|
||||
resp.ParseConcurrency = int(h.systemSettingSvc.GetInt(
|
||||
ctx, "asynq.concurrency", "WEKNORA_ASYNQ_CONCURRENCY", defaultParseConcurrency,
|
||||
))
|
||||
resp.WikiConcurrency = int(h.systemSettingSvc.GetInt(
|
||||
ctx, "asynq.wiki_concurrency", "WEKNORA_WIKI_ASYNQ_CONCURRENCY", defaultWikiConcurrency,
|
||||
))
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, resp)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
type runtimeTestSettings struct{}
|
||||
|
||||
func (runtimeTestSettings) GetInt(_ context.Context, key, _ string, def int64) int64 {
|
||||
switch key {
|
||||
case "asynq.concurrency":
|
||||
return 32
|
||||
case "asynq.wiki_concurrency":
|
||||
return 16
|
||||
default:
|
||||
return def
|
||||
}
|
||||
}
|
||||
func (runtimeTestSettings) GetString(_ context.Context, _, _, def string) string { return def }
|
||||
func (runtimeTestSettings) GetBool(_ context.Context, _, _ string, def bool) bool { return def }
|
||||
func (runtimeTestSettings) GetStringList(_ context.Context, _, _ string, def []string) []string {
|
||||
return def
|
||||
}
|
||||
func (runtimeTestSettings) List(context.Context) ([]*types.SystemSetting, error) { return nil, nil }
|
||||
func (runtimeTestSettings) Get(context.Context, string) (*types.SystemSetting, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (runtimeTestSettings) Update(context.Context, string, any) (*types.SystemSetting, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (runtimeTestSettings) Reset(context.Context, string) error { return nil }
|
||||
func (runtimeTestSettings) SubscribeRedis(context.Context) error { return nil }
|
||||
|
||||
type runtimeInvalidSettings struct{ runtimeTestSettings }
|
||||
|
||||
func (runtimeInvalidSettings) GetInt(_ context.Context, _ string, _ string, _ int64) int64 {
|
||||
return 0
|
||||
}
|
||||
|
||||
type runtimeTestInspector struct{}
|
||||
|
||||
func (runtimeTestInspector) CancelTasksForKnowledge(context.Context, string) (int, int, error) {
|
||||
return 0, 0, nil
|
||||
}
|
||||
func (runtimeTestInspector) HasQueuedTasksForKnowledge(context.Context, string) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
func (runtimeTestInspector) QueueStats(context.Context) ([]types.QueueStat, bool, error) {
|
||||
return []types.QueueStat{}, true, nil
|
||||
}
|
||||
|
||||
func TestGetRuntimeQueuesReportsIsolatedPoolCapacity(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
handler := &SystemHandler{
|
||||
systemSettingSvc: runtimeTestSettings{},
|
||||
taskInspector: runtimeTestInspector{},
|
||||
}
|
||||
recorder := httptest.NewRecorder()
|
||||
ctx, _ := gin.CreateTestContext(recorder)
|
||||
ctx.Request = httptest.NewRequest(http.MethodGet, "/api/v1/system/admin/runtime/queues", nil)
|
||||
|
||||
handler.GetRuntimeQueues(ctx)
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d; body=%s", recorder.Code, http.StatusOK, recorder.Body.String())
|
||||
}
|
||||
|
||||
var response RuntimeQueuesResponse
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if !response.Available {
|
||||
t.Fatal("queue inspection should be available")
|
||||
}
|
||||
if response.UpstreamConcurrency != 32 || response.ParseConcurrency != 32 {
|
||||
t.Fatalf("upstream compatibility values are wrong: %+v", response)
|
||||
}
|
||||
want := map[string]struct {
|
||||
concurrency int
|
||||
queueCount int
|
||||
}{
|
||||
types.WorkerPoolCore: {16, 1},
|
||||
types.WorkerPoolEnrichment: {12, 4},
|
||||
types.WorkerPoolMaintenance: {4, 2},
|
||||
types.WorkerPoolWiki: {16, 1},
|
||||
}
|
||||
if len(response.Pools) != len(want) {
|
||||
t.Fatalf("pool count = %d, want %d", len(response.Pools), len(want))
|
||||
}
|
||||
for _, pool := range response.Pools {
|
||||
expected, ok := want[pool.Name]
|
||||
if !ok {
|
||||
t.Fatalf("unexpected pool %q", pool.Name)
|
||||
}
|
||||
if pool.Concurrency != expected.concurrency || pool.QueueCount != expected.queueCount {
|
||||
t.Fatalf("pool %q = %+v, want concurrency=%d queue_count=%d",
|
||||
pool.Name, pool, expected.concurrency, expected.queueCount)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetRuntimeQueuesFallsBackFromInvalidHistoricalConcurrency(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
handler := &SystemHandler{
|
||||
systemSettingSvc: runtimeInvalidSettings{},
|
||||
taskInspector: runtimeTestInspector{},
|
||||
}
|
||||
recorder := httptest.NewRecorder()
|
||||
ctx, _ := gin.CreateTestContext(recorder)
|
||||
ctx.Request = httptest.NewRequest(http.MethodGet, "/api/v1/system/admin/runtime/queues", nil)
|
||||
|
||||
handler.GetRuntimeQueues(ctx)
|
||||
|
||||
var response RuntimeQueuesResponse
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if response.UpstreamConcurrency != types.DefaultUpstreamWorkerConcurrency ||
|
||||
response.WikiConcurrency != types.DefaultWikiWorkerConcurrency {
|
||||
t.Fatalf("invalid stored values should use worker defaults: %+v", response)
|
||||
}
|
||||
for _, pool := range response.Pools {
|
||||
if pool.Concurrency < 1 {
|
||||
t.Fatalf("pool %q reported non-positive concurrency: %+v", pool.Name, pool)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user