feat(wiki): dedicated worker pool with concurrent claim-based batching

Isolate the wiki generation pipeline from the document parse pipeline so
the two can no longer starve each other during concurrent uploads.

- Dedicated asynq worker pool for QueueWiki (NewWikiAsynqServer) with its
  own concurrency budget (WEKNORA_WIKI_ASYNQ_CONCURRENCY / asynq.wiki_
  concurrency, default 16), split from the parse pool (NewParseAsynqServer).
  Both run the same mux over disjoint queue sets. Replaces the old
  weighted-lottery approach where wiki sat on the low queue (weight 1).
- Task queue: add ClaimBatch (SELECT ... FOR UPDATE SKIP LOCKED + claimed_at)
  and ReleaseByIDs so multiple concurrent consumers pull disjoint rows
  without an external per-KB lock; PeekBatch retained for Lite mode.
- New TypeWikiFinalize debounced KB-global finalize task (index rebuild /
  dead-link cleanup / cross-linking), registered on both Redis and Lite paths.
- Two-level wiki ingest concurrency: IngestMaxInflight caps concurrent
  batches per KB; Map/Reduce fan-out sized within a batch. New config knob
  + system setting + swagger docs.
This commit is contained in:
wizardchen
2026-07-09 13:27:47 +08:00
committed by lyingbug
parent 7642117b70
commit 70e2e85fa6
17 changed files with 1282 additions and 377 deletions
@@ -5,10 +5,12 @@ import (
"errors"
"fmt"
"strconv"
"time"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
// taskPendingOpsRepository implements interfaces.TaskPendingOpsRepository.
@@ -71,6 +73,74 @@ func (r *taskPendingOpsRepository) PeekBatch(
return ops, nil
}
// ClaimBatch atomically claims up to `limit` eligible rows for the tuple.
// Eligibility = unclaimed (claimed_at IS NULL) OR stale claim
// (claimed_at < staleBefore). The whole thing runs in one transaction:
// we first lock the candidate ids (FOR UPDATE SKIP LOCKED on Postgres so
// concurrent claimers slide past each other's rows), stamp claimed_at,
// then read the claimed rows back. SKIP LOCKED is Postgres-only; on other
// dialects (SQLite, used by unit tests / Lite) writes are serialized by
// the engine so the plain SELECT+UPDATE is already race-free within the
// single-writer model.
func (r *taskPendingOpsRepository) ClaimBatch(
ctx context.Context,
taskType, scope, scopeID string,
limit int,
staleBefore time.Time,
) ([]*types.TaskPendingOp, error) {
if limit <= 0 {
limit = 1
}
if limit > 1000 {
limit = 1000
}
now := time.Now()
var claimed []*types.TaskPendingOp
err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
sel := tx.Model(&types.TaskPendingOp{}).
Where("task_type = ? AND scope = ? AND scope_id = ?", taskType, scope, scopeID).
// Parentheses are explicit so the OR binds tightly and doesn't
// leak past the tuple filter (GORM ANDs Where clauses but does
// not parenthesize raw fragments).
Where("(claimed_at IS NULL OR claimed_at < ?)", staleBefore).
Order("id ASC").
Limit(limit)
if tx.Dialector.Name() == "postgres" {
sel = sel.Clauses(clause.Locking{Strength: "UPDATE", Options: "SKIP LOCKED"})
}
var ids []int64
if err := sel.Pluck("id", &ids).Error; err != nil {
return err
}
if len(ids) == 0 {
return nil
}
if err := tx.Model(&types.TaskPendingOp{}).
Where("id IN ?", ids).
Update("claimed_at", now).Error; err != nil {
return err
}
return tx.Where("id IN ?", ids).Order("id ASC").Find(&claimed).Error
})
if err != nil {
return nil, err
}
return claimed, nil
}
// ReleaseByIDs clears claimed_at for the given rows, returning them to the
// unclaimed pool. Empty input is a no-op. Setting claimed_at back to NULL
// on a row that was never claimed is harmless.
func (r *taskPendingOpsRepository) ReleaseByIDs(ctx context.Context, ids []int64) error {
if len(ids) == 0 {
return nil
}
return r.db.WithContext(ctx).
Model(&types.TaskPendingOp{}).
Where("id IN ?", ids).
Update("claimed_at", nil).Error
}
// DeleteByIDs removes the given rows in one statement. Empty input is a
// no-op so the caller can invoke unconditionally at the end of a batch.
func (r *taskPendingOpsRepository) DeleteByIDs(ctx context.Context, ids []int64) error {