From 70e2e85fa6a1983c49ff4a4382d3060e118007e6 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Thu, 9 Jul 2026 13:27:47 +0800 Subject: [PATCH] feat(wiki): dedicated worker pool with concurrent claim-based batching Isolate the wiki generation pipeline from the document parse pipeline so the two can no longer starve each other during concurrent uploads. - Dedicated asynq worker pool for QueueWiki (NewWikiAsynqServer) with its own concurrency budget (WEKNORA_WIKI_ASYNQ_CONCURRENCY / asynq.wiki_ concurrency, default 16), split from the parse pool (NewParseAsynqServer). Both run the same mux over disjoint queue sets. Replaces the old weighted-lottery approach where wiki sat on the low queue (weight 1). - Task queue: add ClaimBatch (SELECT ... FOR UPDATE SKIP LOCKED + claimed_at) and ReleaseByIDs so multiple concurrent consumers pull disjoint rows without an external per-KB lock; PeekBatch retained for Lite mode. - New TypeWikiFinalize debounced KB-global finalize task (index rebuild / dead-link cleanup / cross-linking), registered on both Redis and Lite paths. - Two-level wiki ingest concurrency: IngestMaxInflight caps concurrent batches per KB; Map/Reduce fan-out sized within a batch. New config knob + system setting + swagger docs. --- .env.example | 20 +- docker-compose.yml | 1 + docs/docs.go | 10 +- docs/swagger.json | 10 +- docs/swagger.yaml | 33 +- internal/application/repository/task_queue.go | 70 ++ .../application/repository/task_queue_test.go | 92 +++ .../application/service/system_setting.go | 16 +- internal/application/service/wiki_ingest.go | 437 +++++++++- .../application/service/wiki_ingest_batch.go | 748 +++++++++++------- internal/container/container.go | 8 +- internal/router/sync_task.go | 1 + internal/router/task.go | 110 ++- internal/router/task_inspector.go | 1 + internal/types/interfaces/task_queue.go | 35 +- internal/types/task.go | 17 +- internal/types/wiki_page.go | 50 +- 17 files changed, 1282 insertions(+), 377 deletions(-) diff --git a/.env.example b/.env.example index b8aea97a..39fb2acc 100644 --- a/.env.example +++ b/.env.example @@ -708,13 +708,23 @@ DOCREADER_TRANSPORT=grpc # WEKNORA_DOCUMENT_PROCESS_TIMEOUT=2h # --- Async pipeline tuning (optional) ----------------------------------------- -# Worker pool size for the asynq server. Default 32 — raise it on machines -# that handle many concurrent uploads (default Go runtime.NumCPU() under- -# provisions for the I/O-bound document pipeline). Can also be set in the -# management UI under Settings → System settings (asynq.concurrency); -# UI changes require a process restart. +# Worker pool size for the PARSE asynq server (document parse / post-process / +# embedding / multimodal / graph / question — everything except wiki). Default +# 32 — raise it on machines that handle many concurrent uploads (default Go +# runtime.NumCPU() under-provisions for the I/O-bound document pipeline). Can +# also be set in the management UI under Settings → System settings +# (asynq.concurrency); UI changes require a process restart. # WEKNORA_ASYNQ_CONCURRENCY=32 +# Worker pool size for the DEDICATED wiki asynq server. The wiki generation +# pipeline runs on its own pool with its own concurrency budget, hard-isolated +# from the parse pool above: heavy document parsing can never starve wiki +# generation during upload storms, and a wiki burst can never starve +# user-facing parsing. Default 16. Wiki work is synthesis-LLM-bound, so size +# this to your LLM provider's concurrency budget. Also settable in the UI +# (asynq.wiki_concurrency); requires a process restart. +# WEKNORA_WIKI_ASYNQ_CONCURRENCY=16 + # Read/write timeout (in milliseconds) the asynq client uses against Redis. # Default 500ms (writes scale to 1000ms). Bump if you see "i/o timeout" # errors during heavy multimodal counter DECRs or large batch uploads. diff --git a/docker-compose.yml b/docker-compose.yml index 698f5843..d2955a23 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -193,6 +193,7 @@ services: - WEKNORA_TENANT_ENABLE_RBAC=${WEKNORA_TENANT_ENABLE_RBAC:-} - WEKNORA_TENANT_MAX_OWNED_PER_USER=${WEKNORA_TENANT_MAX_OWNED_PER_USER:-} - WEKNORA_ASYNQ_CONCURRENCY=${WEKNORA_ASYNQ_CONCURRENCY:-32} + - WEKNORA_WIKI_ASYNQ_CONCURRENCY=${WEKNORA_WIKI_ASYNQ_CONCURRENCY:-16} - APK_MIRROR_ARG=${APK_MIRROR_ARG:-} - WEKNORA_BOOTSTRAP_SYSTEM_ADMIN_EMAIL=${WEKNORA_BOOTSTRAP_SYSTEM_ADMIN_EMAIL:-} depends_on: diff --git a/docs/docs.go b/docs/docs.go index 4c2c2b74..b6e4eda2 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -18084,15 +18084,19 @@ const docTemplate = `{ ] }, "ingest_batch_size": { - "description": "IngestBatchSize controls how many pending ops a single batch\nprocesses before scheduling a follow-up. 0 falls back to the\nhard-coded default (5). Operators on large KBs (4w+ docs) can\nraise this to 10–20 to amortize the lock-acquire / index-rebuild\noverhead across more documents per round.", + "description": "IngestBatchSize controls how many pending ops a single batch claims and\nprocesses before scheduling a follow-up. 0 falls back to the hard-coded\ndefault (5). Larger batches amortize per-batch setup and let more docs\nshare the batch-internal Map/Reduce fan-out; smaller batches spread a\nKB's backlog across more concurrent batches (finer scheduling grain).", "type": "integer" }, "ingest_map_parallel": { - "description": "IngestMapParallel sets the errgroup limit for the Map phase\n(per-document extraction + summary + chunk citation). 0 falls\nback to 10. Bound by the LLM provider's concurrency limit and\nthe worker's outbound HTTP pool.", + "description": "IngestMapParallel sets the errgroup limit for the Map phase\n(per-document extraction + summary + chunk citation) WITHIN one batch.\n0 falls back to 10. Bound by the LLM provider's concurrency limit and\nthe worker's outbound HTTP pool. Remember it multiplies with the number\nof concurrent batches (IngestMaxInflight).", + "type": "integer" + }, + "ingest_max_inflight": { + "description": "IngestMaxInflight caps how many ingest batches for THIS KB may run\nconcurrently in the shared wiki worker pool (standard/Redis mode\nonly). 0 falls back to the hard-coded default (4). Since Phase 3\nremoved the exclusive per-KB lock, one KB's backlog could otherwise\nmonopolize the whole pool during a bulk import and starve other KBs;\nthis knob trades a single KB's peak throughput for cross-KB fairness.\nSet it \u003e= the wiki pool size to effectively disable the cap.", "type": "integer" }, "ingest_reduce_parallel": { - "description": "IngestReduceParallel sets the errgroup limit for the Reduce phase\n(per-slug page write). 0 falls back to 10. Bound by the same\nLLM concurrency / HTTP pool considerations as the Map phase,\nplus DB connection pool size.", + "description": "IngestReduceParallel sets the errgroup limit for the Reduce phase\n(per-slug page write) WITHIN one batch. 0 falls back to 10. Bound by the\nsame LLM concurrency / HTTP pool considerations as the Map phase, plus\nDB connection pool size. Same multiplier caveat as IngestMapParallel.", "type": "integer" }, "max_pages_per_ingest": { diff --git a/docs/swagger.json b/docs/swagger.json index f25c42c3..b8d76f93 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -18077,15 +18077,19 @@ ] }, "ingest_batch_size": { - "description": "IngestBatchSize controls how many pending ops a single batch\nprocesses before scheduling a follow-up. 0 falls back to the\nhard-coded default (5). Operators on large KBs (4w+ docs) can\nraise this to 10–20 to amortize the lock-acquire / index-rebuild\noverhead across more documents per round.", + "description": "IngestBatchSize controls how many pending ops a single batch claims and\nprocesses before scheduling a follow-up. 0 falls back to the hard-coded\ndefault (5). Larger batches amortize per-batch setup and let more docs\nshare the batch-internal Map/Reduce fan-out; smaller batches spread a\nKB's backlog across more concurrent batches (finer scheduling grain).", "type": "integer" }, "ingest_map_parallel": { - "description": "IngestMapParallel sets the errgroup limit for the Map phase\n(per-document extraction + summary + chunk citation). 0 falls\nback to 10. Bound by the LLM provider's concurrency limit and\nthe worker's outbound HTTP pool.", + "description": "IngestMapParallel sets the errgroup limit for the Map phase\n(per-document extraction + summary + chunk citation) WITHIN one batch.\n0 falls back to 10. Bound by the LLM provider's concurrency limit and\nthe worker's outbound HTTP pool. Remember it multiplies with the number\nof concurrent batches (IngestMaxInflight).", + "type": "integer" + }, + "ingest_max_inflight": { + "description": "IngestMaxInflight caps how many ingest batches for THIS KB may run\nconcurrently in the shared wiki worker pool (standard/Redis mode\nonly). 0 falls back to the hard-coded default (4). Since Phase 3\nremoved the exclusive per-KB lock, one KB's backlog could otherwise\nmonopolize the whole pool during a bulk import and starve other KBs;\nthis knob trades a single KB's peak throughput for cross-KB fairness.\nSet it \u003e= the wiki pool size to effectively disable the cap.", "type": "integer" }, "ingest_reduce_parallel": { - "description": "IngestReduceParallel sets the errgroup limit for the Reduce phase\n(per-slug page write). 0 falls back to 10. Bound by the same\nLLM concurrency / HTTP pool considerations as the Map phase,\nplus DB connection pool size.", + "description": "IngestReduceParallel sets the errgroup limit for the Reduce phase\n(per-slug page write) WITHIN one batch. 0 falls back to 10. Bound by the\nsame LLM concurrency / HTTP pool considerations as the Map phase, plus\nDB connection pool size. Same multiplier caveat as IngestMapParallel.", "type": "integer" }, "max_pages_per_ingest": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index ba66134b..fdc1c5ea 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -3480,25 +3480,36 @@ definitions: per document. Empty / unknown value is treated as WikiExtractionStandard. ingest_batch_size: description: |- - IngestBatchSize controls how many pending ops a single batch - processes before scheduling a follow-up. 0 falls back to the - hard-coded default (5). Operators on large KBs (4w+ docs) can - raise this to 10–20 to amortize the lock-acquire / index-rebuild - overhead across more documents per round. + IngestBatchSize controls how many pending ops a single batch claims and + processes before scheduling a follow-up. 0 falls back to the hard-coded + default (5). Larger batches amortize per-batch setup and let more docs + share the batch-internal Map/Reduce fan-out; smaller batches spread a + KB's backlog across more concurrent batches (finer scheduling grain). type: integer ingest_map_parallel: description: |- IngestMapParallel sets the errgroup limit for the Map phase - (per-document extraction + summary + chunk citation). 0 falls - back to 10. Bound by the LLM provider's concurrency limit and - the worker's outbound HTTP pool. + (per-document extraction + summary + chunk citation) WITHIN one batch. + 0 falls back to 10. Bound by the LLM provider's concurrency limit and + the worker's outbound HTTP pool. Remember it multiplies with the number + of concurrent batches (IngestMaxInflight). + type: integer + ingest_max_inflight: + description: |- + IngestMaxInflight caps how many ingest batches for THIS KB may run + concurrently in the shared wiki worker pool (standard/Redis mode + only). 0 falls back to the hard-coded default (4). Since Phase 3 + removed the exclusive per-KB lock, one KB's backlog could otherwise + monopolize the whole pool during a bulk import and starve other KBs; + this knob trades a single KB's peak throughput for cross-KB fairness. + Set it >= the wiki pool size to effectively disable the cap. type: integer ingest_reduce_parallel: description: |- IngestReduceParallel sets the errgroup limit for the Reduce phase - (per-slug page write). 0 falls back to 10. Bound by the same - LLM concurrency / HTTP pool considerations as the Map phase, - plus DB connection pool size. + (per-slug page write) WITHIN one batch. 0 falls back to 10. Bound by the + same LLM concurrency / HTTP pool considerations as the Map phase, plus + DB connection pool size. Same multiplier caveat as IngestMapParallel. type: integer max_pages_per_ingest: description: MaxPagesPerIngest limits pages created/updated per ingest operation diff --git a/internal/application/repository/task_queue.go b/internal/application/repository/task_queue.go index d16e11f8..c9b63c8d 100644 --- a/internal/application/repository/task_queue.go +++ b/internal/application/repository/task_queue.go @@ -5,10 +5,12 @@ import ( "errors" "fmt" "strconv" + "time" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" "gorm.io/gorm" + "gorm.io/gorm/clause" ) // taskPendingOpsRepository implements interfaces.TaskPendingOpsRepository. @@ -71,6 +73,74 @@ func (r *taskPendingOpsRepository) PeekBatch( return ops, nil } +// ClaimBatch atomically claims up to `limit` eligible rows for the tuple. +// Eligibility = unclaimed (claimed_at IS NULL) OR stale claim +// (claimed_at < staleBefore). The whole thing runs in one transaction: +// we first lock the candidate ids (FOR UPDATE SKIP LOCKED on Postgres so +// concurrent claimers slide past each other's rows), stamp claimed_at, +// then read the claimed rows back. SKIP LOCKED is Postgres-only; on other +// dialects (SQLite, used by unit tests / Lite) writes are serialized by +// the engine so the plain SELECT+UPDATE is already race-free within the +// single-writer model. +func (r *taskPendingOpsRepository) ClaimBatch( + ctx context.Context, + taskType, scope, scopeID string, + limit int, + staleBefore time.Time, +) ([]*types.TaskPendingOp, error) { + if limit <= 0 { + limit = 1 + } + if limit > 1000 { + limit = 1000 + } + now := time.Now() + var claimed []*types.TaskPendingOp + err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + sel := tx.Model(&types.TaskPendingOp{}). + Where("task_type = ? AND scope = ? AND scope_id = ?", taskType, scope, scopeID). + // Parentheses are explicit so the OR binds tightly and doesn't + // leak past the tuple filter (GORM ANDs Where clauses but does + // not parenthesize raw fragments). + Where("(claimed_at IS NULL OR claimed_at < ?)", staleBefore). + Order("id ASC"). + Limit(limit) + if tx.Dialector.Name() == "postgres" { + sel = sel.Clauses(clause.Locking{Strength: "UPDATE", Options: "SKIP LOCKED"}) + } + var ids []int64 + if err := sel.Pluck("id", &ids).Error; err != nil { + return err + } + if len(ids) == 0 { + return nil + } + if err := tx.Model(&types.TaskPendingOp{}). + Where("id IN ?", ids). + Update("claimed_at", now).Error; err != nil { + return err + } + return tx.Where("id IN ?", ids).Order("id ASC").Find(&claimed).Error + }) + if err != nil { + return nil, err + } + return claimed, nil +} + +// ReleaseByIDs clears claimed_at for the given rows, returning them to the +// unclaimed pool. Empty input is a no-op. Setting claimed_at back to NULL +// on a row that was never claimed is harmless. +func (r *taskPendingOpsRepository) ReleaseByIDs(ctx context.Context, ids []int64) error { + if len(ids) == 0 { + return nil + } + return r.db.WithContext(ctx). + Model(&types.TaskPendingOp{}). + Where("id IN ?", ids). + Update("claimed_at", nil).Error +} + // DeleteByIDs removes the given rows in one statement. Empty input is a // no-op so the caller can invoke unconditionally at the end of a batch. func (r *taskPendingOpsRepository) DeleteByIDs(ctx context.Context, ids []int64) error { diff --git a/internal/application/repository/task_queue_test.go b/internal/application/repository/task_queue_test.go index dc3f1406..3a45f380 100644 --- a/internal/application/repository/task_queue_test.go +++ b/internal/application/repository/task_queue_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "testing" + "time" "github.com/Tencent/WeKnora/internal/types" "github.com/stretchr/testify/assert" @@ -270,6 +271,97 @@ func TestTaskPendingOps_DeleteByDedupKey_Filters(t *testing.T) { assert.Equal(t, "k2", rows[0].DedupKey) } +// TestTaskPendingOps_ClaimBatch_MarksAndReturnsDisjoint verifies that +// ClaimBatch returns rows in FIFO order, stamps claimed_at, and that a +// second claim does NOT re-return the already-claimed rows (the disjoint +// property concurrent consumers rely on). +func TestTaskPendingOps_ClaimBatch_MarksAndReturnsDisjoint(t *testing.T) { + db := setupTaskQueueTestDB(t) + repo := NewTaskPendingOpsRepository(db) + ctx := context.Background() + + for _, k := range []string{"k1", "k2", "k3"} { + require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "ingest", k, nil))) + } + // Unrelated tuple must never be claimed. + require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "other", "ingest", "z", nil))) + + stale := time.Now().Add(-time.Hour) + + first, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 2, stale) + require.NoError(t, err) + require.Len(t, first, 2) + assert.Equal(t, "k1", first[0].DedupKey) + assert.Equal(t, "k2", first[1].DedupKey) + assert.NotNil(t, first[0].ClaimedAt, "claimed_at should be stamped") + + // Second claim skips the two already-claimed rows and returns the last. + second, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 10, stale) + require.NoError(t, err) + require.Len(t, second, 1) + assert.Equal(t, "k3", second[0].DedupKey) + + // Nothing left to claim. + third, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 10, stale) + require.NoError(t, err) + assert.Len(t, third, 0) +} + +// TestTaskPendingOps_ClaimBatch_ReclaimsStale verifies a claim older than +// staleBefore is re-claimable (crash recovery), while a fresh claim is not. +func TestTaskPendingOps_ClaimBatch_ReclaimsStale(t *testing.T) { + db := setupTaskQueueTestDB(t) + repo := NewTaskPendingOpsRepository(db) + ctx := context.Background() + + require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "ingest", "k1", nil))) + + // Claim it now. + got, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 10, time.Now().Add(-time.Hour)) + require.NoError(t, err) + require.Len(t, got, 1) + + // A stale threshold in the past — the fresh claim is NOT stale yet. + got, err = repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 10, time.Now().Add(-time.Hour)) + require.NoError(t, err) + assert.Len(t, got, 0, "fresh claim must not be re-claimed") + + // A stale threshold in the future makes every existing claim stale. + got, err = repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 10, time.Now().Add(time.Hour)) + require.NoError(t, err) + require.Len(t, got, 1, "stale claim must be recoverable") + assert.Equal(t, "k1", got[0].DedupKey) +} + +// TestTaskPendingOps_ReleaseByIDs_ReturnsToPool verifies a released row +// becomes immediately claimable again without waiting for staleness. +func TestTaskPendingOps_ReleaseByIDs_ReturnsToPool(t *testing.T) { + db := setupTaskQueueTestDB(t) + repo := NewTaskPendingOpsRepository(db) + ctx := context.Background() + + op := makePendingOp("wiki:ingest", "knowledge_base", "kb", "ingest", "k1", nil) + require.NoError(t, repo.Enqueue(ctx, op)) + + stale := time.Now().Add(-time.Hour) + got, err := repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 10, stale) + require.NoError(t, err) + require.Len(t, got, 1) + + // Claimed → not re-claimable. + got, err = repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 10, stale) + require.NoError(t, err) + require.Len(t, got, 0) + + // Release → immediately claimable again. + require.NoError(t, repo.ReleaseByIDs(ctx, nil)) // no-op tolerated + require.NoError(t, repo.ReleaseByIDs(ctx, []int64{op.ID})) + got, err = repo.ClaimBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 10, stale) + require.NoError(t, err) + require.Len(t, got, 1) + assert.Equal(t, "k1", got[0].DedupKey) +} + // ---------------- TaskDeadLetterRepository ---------------- func makeDeadLetter(taskType, scope, scopeID, relatedID, lastErr string) *types.TaskDeadLetter { diff --git a/internal/application/service/system_setting.go b/internal/application/service/system_setting.go index ea58b068..2bffa081 100644 --- a/internal/application/service/system_setting.go +++ b/internal/application/service/system_setting.go @@ -175,10 +175,24 @@ var registry = map[string]settingSpec{ Default: int64(32), Category: "worker", RequiresRestart: true, - Description: "异步任务 worker 并发数(asynq 线程池大小)。" + + Description: "文档解析池的 worker 并发数(asynq 线程池大小,不含 Wiki)。" + "文档解析、嵌入等任务多为 I/O 等待,适当提高可缩短批量上传排队时间。" + "修改后需重启服务进程方可生效。", }, + // asynq.wiki_concurrency is the size of the DEDICATED wiki worker pool, + // separate from asynq.concurrency. Read once when the wiki asynq server + // starts — changing it in the UI requires a process restart. Mirrors + // WEKNORA_WIKI_ASYNQ_CONCURRENCY (default 16). + "asynq.wiki_concurrency": { + Type: "int", + EnvName: "WEKNORA_WIKI_ASYNQ_CONCURRENCY", + Default: int64(16), + Category: "worker", + RequiresRestart: true, + Description: "Wiki 生成专用池的 worker 并发数(与文档解析池相互隔离)。" + + "Wiki 生成以合成大模型调用为主,独立并发预算可避免上传高峰期被解析任务饿死," + + "同时不会因 Wiki 洪峰拖慢用户面解析。修改后需重启服务进程方可生效。", + }, } // systemSettingService wires the repository, audit log, and (P2) diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go index 60aa2e38..c5ae1e0c 100644 --- a/internal/application/service/wiki_ingest.go +++ b/internal/application/service/wiki_ingest.go @@ -20,31 +20,97 @@ import ( "github.com/Tencent/WeKnora/internal/tracing/langfuse" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/google/uuid" "github.com/hibiken/asynq" "github.com/redis/go-redis/v9" ) -// ErrWikiIngestConcurrent is returned by the wiki ingest handler when another -// batch is already running for the same KB (i.e. the `wiki:active:` -// Redis lock is held). The asynq server's RetryDelayFunc uses errors.Is on -// this sentinel to apply a short, fixed retry delay instead of asynq's default -// exponential backoff — otherwise a freshly orphaned lock (e.g. from a crash -// or restart) would force newcomers to wait minutes even after the lock -// naturally expires. +// ErrWikiIngestConcurrent is returned by the wiki ingest handler in Lite mode +// when another batch is already running for the same KB (the in-process +// liteLocks guard is held). The asynq/sync executor's RetryDelayFunc uses +// errors.Is on this sentinel to apply a short, fixed retry delay instead of +// exponential backoff, so the deferred batch retries promptly once the active +// one releases. Standard (Redis) mode no longer takes an exclusive per-KB +// lock (Phase 3) and never returns this. var ErrWikiIngestConcurrent = errors.New("concurrent wiki task active") const ( // maxContentForWiki limits the document content sent to LLM for wiki generation maxContentForWiki = 32768 - // wikiActiveKeyPrefix is the Redis key for the "batch in progress" flag. - // Key format: wiki:active:{kbID} → "1" with TTL. Prevents concurrent batches. - wikiActiveKeyPrefix = "wiki:active:" + // --- Phase 3: concurrent per-KB batches (standard/Redis mode) --------- + // + // Phase 3 removed the exclusive per-KB Redis "batch in progress" lock + // (wiki:active:). Standard mode now allows concurrent batches for + // one KB, kept safe by row claiming (claimPendingList) plus per-slug + // reduce locks (withSlugLock). Lite mode still serializes per KB via the + // in-process liteLocks map. + // + // wikiClaimStaleAfter is how long a claimed-but-undrained ingest row + // waits before another worker may re-claim it. It MUST exceed the asynq + // task Timeout for wiki:ingest (60m) so a still-running batch's rows are + // never stolen mid-flight — only genuinely crashed/abandoned claims are + // recovered. This preserves the pre-claim crash behaviour where a dead + // batch's undeleted rows were simply re-peeked by the next trigger. + wikiClaimStaleAfter = 90 * time.Minute + + // wikiSlugLockPrefix guards read-modify-write on a single shared wiki + // page (entity/concept/summary/index) so two concurrent batches for the + // same KB can't lost-update the same slug. Key: wiki:slug:{kbID}:{slug}. + wikiSlugLockPrefix = "wiki:slug:" + // wikiSlugLockTTL bounds the per-slug lock so a crashed reducer can't + // wedge a hot page forever. Comfortably longer than a single reduce + // (one LLM modify call). + wikiSlugLockTTL = 5 * time.Minute + // wikiSlugLockWait / Poll bound how long a reduce goroutine blocks + // waiting for a contended slug before falling back to a best-effort + // skip (matching the pre-existing reduce-failure semantics). + wikiSlugLockWait = 2 * time.Minute + wikiSlugLockPoll = 50 * time.Millisecond + + // --- Phase 4: per-KB in-flight cap (standard/Redis mode) -------------- + // + // Since Phase 3 lets many batches for one KB run at once, a single KB's + // bulk import could otherwise occupy the entire wiki pool and starve + // other KBs. Each running batch reserves a slot in a per-KB Redis sorted + // set (score = expiry). ZREMRANGEBYSCORE purges slots left behind by a + // crashed worker, so the cap is self-healing without an explicit lock. + // + // wikiInflightPrefix keys the per-KB slot set: wiki:inflight:{kbID}. + wikiInflightPrefix = "wiki:inflight:" + // wikiInflightDefault is the fallback cap when WikiConfig.IngestMaxInflight + // is unset. 4 leaves headroom in a default 16-worker wiki pool for other + // KBs while still giving one KB solid parallelism. + wikiInflightDefault = 4 + // wikiInflightTTL is how long a reserved slot survives without renewal. + // MUST exceed wikiInflightRenew by a comfortable margin so a single + // missed renew (GC pause, Redis blip) doesn't drop a live batch's slot. + wikiInflightTTL = 90 * time.Second + // wikiInflightRenew is how often a running batch bumps its slot's expiry. + wikiInflightRenew = 30 * time.Second + // wikiInflightBackoff delays the follow-up trigger when a batch is turned + // away by the cap, so it retries after a slot plausibly frees. + wikiInflightBackoff = 10 * time.Second // wikiIngestDelay is how long to wait after a document is added before // the batch task fires. Debounces rapid uploads. wikiIngestDelay = 30 * time.Second + // wikiFollowUpDelay is the normal light debounce before a follow-up + // batch drains the remaining pending rows. Standard mode already fans a + // KB's backlog across concurrent claiming batches, so this is just a + // short breather, not a lock-release wait. + wikiFollowUpDelay = 5 * time.Second + + // wikiRateLimitBackoff is the (much longer) follow-up delay used when a + // batch's failures were caused by upstream rate limiting (HTTP 429 / + // quota). Retrying a rate-limited document at the normal 5s cadence just + // throws more requests at an already-saturated rpm budget — every retry + // re-issues extract + classify + summary calls for the doc — which keeps + // the limiter tripped and drags the whole KB's ingest out. Backing off + // gives the per-minute window time to reset before we try again. + wikiRateLimitBackoff = 60 * time.Second + // wikiMaxDocsPerBatch limits how many documents a single batch processes. // Prevents unbounded execution time. Remaining ops stay in // task_pending_ops and are picked up by the follow-up task. @@ -75,20 +141,6 @@ const ( // exceed the longest plausible ingest run (LLM extraction + reduce). wikiDeletedTTL = 1 * time.Hour - // wikiActiveLockTTL is the TTL for the per-KB "batch in progress" flag. - // Kept short (relative to total batch runtime) so that if the owning - // process crashes without running its `defer Del`, the orphaned lock - // expires quickly and newcomers aren't blocked. A periodic renew - // (wikiActiveLockRenew) keeps the lock alive while the handler is - // genuinely still running. - wikiActiveLockTTL = 60 * time.Second - - // wikiActiveLockRenew is how often the in-flight handler bumps the TTL. - // Must be comfortably shorter than wikiActiveLockTTL so a single missed - // tick (GC pause, Redis blip) doesn't let the lock slip out from under a - // live handler. - wikiActiveLockRenew = 20 * time.Second - // wikiLLMMaxAttempts is the total attempt count (initial + retries) for // every LLM call routed through generateWithTemplate. 3 was chosen to // absorb transient 504/timeouts from upstream gateways without @@ -108,8 +160,65 @@ const ( // wikiTaskScope is the scope used by both pending ops and dead letters. // Wiki ingest is per-KB, so every op is scoped to a knowledge_base. wikiTaskScope = types.TaskScopeKnowledgeBase + + // --- Finalize lane (Phase 1: debounced KB-global convergence) --------- + // + // The KB-global convergence steps (index-intro rebuild, dead-link + // cleanup, cross-link injection) used to run at the tail of EVERY + // 5-doc ingest batch. On a bulk import that meant O(docs/batchSize) + // index rebuilds — each an LLM call plus an index-page write. They are + // now deferred to a debounced per-KB wiki:finalize task: each ingest + // batch records what changed into a separate lane of task_pending_ops + // (task_type=wiki:finalize) and schedules a coalesced trigger, so a + // burst of N documents rebuilds the index ONCE at the end. + + // wikiFinalizeTaskType is the task_pending_ops lane for finalize work. + // Keyed by (task_type, scope, scope_id) like the ingest lane, but with a + // distinct task_type so PeekBatch never mixes the two. + wikiFinalizeTaskType = "wiki:finalize" + + // wikiFinalizeOpSlug rows carry one affected page slug (+ its fresh title + // when this batch wrote it) for the dead-link / cross-link passes. + wikiFinalizeOpSlug = "slug" + // wikiFinalizeOpChange rows carry a doc-level add/remove change entry for + // the index-intro change description. + wikiFinalizeOpChange = "change" + + wikiFinalizeAdded = "added" + wikiFinalizeRemoved = "removed" + + // wikiFinalizeDelay debounces the finalize trigger so multiple ingest + // batches within the window coalesce into a single index rebuild. + wikiFinalizeDelay = 20 * time.Second + + // wikiFinalizeMaxRows caps how many finalize rows one run drains. A run + // that hits the cap reschedules itself for the remainder. + wikiFinalizeMaxRows = 5000 + + // wikiFinalizeLockTTL / Renew guard against two finalize runs for the + // same KB writing the index page concurrently (belt-and-suspenders on + // top of the asynq.TaskID coalescing). + wikiFinalizeLockTTL = 60 * time.Second + wikiFinalizeLockRenew = 20 * time.Second ) +// wikiFinalizeChange is a doc-level add/remove entry for the index-intro +// change description, persisted as a finalize-lane row payload. +type wikiFinalizeChange struct { + Action string `json:"action"` // wikiFinalizeAdded | wikiFinalizeRemoved + DocTitle string `json:"doc_title,omitempty"` + DocSummary string `json:"doc_summary,omitempty"` +} + +// wikiFinalizeRow is the JSON payload of a task_pending_ops row in the +// finalize lane. Exactly one of {Slug, Change} is set, distinguished by the +// row's Op column (wikiFinalizeOpSlug / wikiFinalizeOpChange). +type wikiFinalizeRow struct { + Slug string `json:"slug,omitempty"` + Title string `json:"title,omitempty"` + Change *wikiFinalizeChange `json:"change,omitempty"` +} + // WikiDeletedTombstoneKey returns the Redis key used to mark a knowledge as // recently deleted, so wiki_ingest tasks in flight can short-circuit. Exposed // so knowledgeService.cleanupWikiOnKnowledgeDelete can write the same key @@ -212,6 +321,10 @@ type wikiIngestService struct { // liteLocks provides per-KB mutual exclusion in Lite mode (no Redis). // Keys are kbID strings; values are unused (presence = locked). liteLocks sync.Map + // liteFinalizeLocks is the Lite-mode counterpart of the Redis + // wiki:finalize:active: lock, keeping two finalize runs for the + // same KB from overlapping when there is no Redis. + liteFinalizeLocks sync.Map } // NewWikiIngestService creates a new wiki ingest service @@ -343,7 +456,7 @@ func EnqueueWikiIngest( triggerBytes, _ := json.Marshal(trigger) t := asynq.NewTask(types.TypeWikiIngest, triggerBytes, - asynq.Queue("low"), + asynq.Queue(types.QueueWiki), asynq.MaxRetry(wikiIngestMaxRetry), asynq.Timeout(60*time.Minute), asynq.ProcessIn(wikiIngestDelay), @@ -401,7 +514,7 @@ func EnqueueWikiRetract( langfuse.InjectTracing(ctx, &trigger) triggerBytes, _ := json.Marshal(trigger) t := asynq.NewTask(types.TypeWikiIngest, triggerBytes, - asynq.Queue("low"), + asynq.Queue(types.QueueWiki), asynq.MaxRetry(wikiIngestMaxRetry), asynq.Timeout(60*time.Minute), asynq.ProcessIn(5*time.Second), // Retract can trigger the batch quickly @@ -411,11 +524,95 @@ func EnqueueWikiRetract( } } -// Handle implements interfaces.TaskHandler for asynq task processing. -// Wiki ingest tasks are debounced via asynq.Unique + ProcessIn, so at most -// one ingest task runs per KB at a time. No distributed lock needed. +// Handle implements interfaces.TaskHandler for asynq task processing. The +// wiki service owns two task types, dispatched here by type so a single +// registered handler covers both the ingest batch and the debounced +// KB-global finalize pass. func (s *wikiIngestService) Handle(ctx context.Context, t *asynq.Task) error { - return s.ProcessWikiIngest(ctx, t) + switch t.Type() { + case types.TypeWikiFinalize: + return s.ProcessWikiFinalize(ctx, t) + default: + return s.ProcessWikiIngest(ctx, t) + } +} + +// enqueueFinalize persists this batch's KB-global convergence work into the +// finalize lane of task_pending_ops and schedules a debounced trigger. One +// "slug" row per affected page (carrying its fresh title when this batch wrote +// it, for cross-linking) plus one "change" row per doc added/removed (for the +// index-intro change description). +func (s *wikiIngestService) enqueueFinalize( + ctx context.Context, + payload WikiIngestPayload, + affectedSlugs []string, + freshTitleBySlug map[string]string, + changes []wikiFinalizeChange, +) { + if s.pendingRepo == nil { + return + } + for _, slug := range affectedSlugs { + row := wikiFinalizeRow{Slug: slug, Title: freshTitleBySlug[slug]} + b, err := json.Marshal(row) + if err != nil { + continue + } + if err := s.pendingRepo.Enqueue(ctx, &types.TaskPendingOp{ + TenantID: payload.TenantID, + TaskType: wikiFinalizeTaskType, + Scope: wikiTaskScope, + ScopeID: payload.KnowledgeBaseID, + Op: wikiFinalizeOpSlug, + DedupKey: slug, + Payload: b, + }); err != nil { + logger.Warnf(ctx, "wiki finalize: enqueue slug row for %s failed: %v", slug, err) + } + } + for i := range changes { + row := wikiFinalizeRow{Change: &changes[i]} + b, err := json.Marshal(row) + if err != nil { + continue + } + if err := s.pendingRepo.Enqueue(ctx, &types.TaskPendingOp{ + TenantID: payload.TenantID, + TaskType: wikiFinalizeTaskType, + Scope: wikiTaskScope, + ScopeID: payload.KnowledgeBaseID, + Op: wikiFinalizeOpChange, + DedupKey: "", + Payload: b, + }); err != nil { + logger.Warnf(ctx, "wiki finalize: enqueue change row failed: %v", err) + } + } + s.scheduleFinalize(ctx, payload) +} + +// scheduleFinalize enqueues a debounced, coalesced KB-global finalize trigger. +// asynq.TaskID ("wiki-finalize-") makes concurrent schedules within the +// debounce window collapse into one pending task; the conflict error is the +// expected coalescing signal, not a failure. In Lite mode the sync executor +// ignores TaskID, so finalize simply runs once per batch (acceptable at the +// small scale Lite mode targets). +func (s *wikiIngestService) scheduleFinalize(ctx context.Context, payload WikiIngestPayload) { + langfuse.InjectTracing(ctx, &payload) + b, _ := json.Marshal(payload) + t := asynq.NewTask(types.TypeWikiFinalize, b, + asynq.Queue(types.QueueWiki), + asynq.MaxRetry(wikiIngestMaxRetry), + asynq.Timeout(30*time.Minute), + asynq.ProcessIn(wikiFinalizeDelay), + asynq.TaskID("wiki-finalize-"+payload.KnowledgeBaseID), + ) + if _, err := s.task.Enqueue(t); err != nil { + if errors.Is(err, asynq.ErrTaskIDConflict) || errors.Is(err, asynq.ErrDuplicateTask) { + return // a finalize is already scheduled/running for this KB — coalesced + } + logger.Warnf(ctx, "wiki finalize: schedule trigger failed: %v", err) + } } // peekPendingList loads up to `limit` ops from task_pending_ops for @@ -441,6 +638,178 @@ func (s *wikiIngestService) peekPendingList(ctx context.Context, kbID string, li logger.Warnf(ctx, "wiki ingest: failed to peek pending list: %v", err) return nil, nil } + return s.decodePendingRows(ctx, rows) +} + +// claimPendingList is the standard-mode (Redis) counterpart of +// peekPendingList: it atomically CLAIMS up to `limit` ops (marks +// claimed_at) so concurrent batches for the same KB — allowed since Phase 3 +// removed the exclusive per-KB lock — pull DISJOINT documents instead of +// double-processing. Stale claims (older than wikiClaimStaleAfter, i.e. from +// a crashed worker) are recovered. Dedup / peekedIDs semantics match +// peekPendingList; the returned peekedIDs are the claimed rows that the +// caller must DeleteByIDs on success or ReleaseByIDs to retry. +func (s *wikiIngestService) claimPendingList(ctx context.Context, kbID string, limit int) (ops []WikiPendingOp, peekedIDs []int64) { + if s.pendingRepo == nil { + return nil, nil + } + if limit <= 0 { + limit = wikiMaxDocsPerBatch + } + rows, err := s.pendingRepo.ClaimBatch(ctx, wikiTaskType, wikiTaskScope, kbID, limit, + time.Now().Add(-wikiClaimStaleAfter)) + if err != nil { + logger.Warnf(ctx, "wiki ingest: failed to claim pending list: %v", err) + return nil, nil + } + return s.decodePendingRows(ctx, rows) +} + +// withSlugLock serializes read-modify-write on one shared wiki page across +// concurrent batches. Phase 3 removed the exclusive per-KB batch lock, so +// two batches for the same KB can both produce updates for a shared +// entity/concept slug; without this lock their GetPageBySlug→UpdatePage +// cycles would race and lose one contribution. +// +// Redis mode only: in Lite mode a single process runs one batch per KB at a +// time (liteLocks), so same-slug contention cannot occur and fn runs +// directly. Returns (false, nil) if the lock could not be acquired within +// wikiSlugLockWait — the caller then treats the slug like a best-effort +// reduce miss (matching pre-existing reduce-failure handling). On a Redis +// error we fail OPEN (run fn unlocked): a rare lost-update on a shared page +// is tolerated by the finalize/dead-link passes, whereas silently dropping +// the update is strictly worse. +func (s *wikiIngestService) withSlugLock(ctx context.Context, kbID, slug string, fn func() error) (bool, error) { + if s.redisClient == nil { + return true, fn() + } + key := wikiSlugLockPrefix + kbID + ":" + slug + deadline := time.Now().Add(wikiSlugLockWait) + for { + ok, rerr := s.redisClient.SetNX(ctx, key, "1", wikiSlugLockTTL).Result() + if rerr != nil { + logger.Warnf(ctx, "wiki reduce: slug lock SetNX failed for %s: %v (running unlocked)", slug, rerr) + return true, fn() + } + if ok { + break + } + if time.Now().After(deadline) { + return false, nil + } + select { + case <-ctx.Done(): + return false, ctx.Err() + case <-time.After(wikiSlugLockPoll): + } + } + defer s.redisClient.Del(context.Background(), key) + return true, fn() +} + +// wikiInflightReserveScript atomically enforces the per-KB in-flight cap. +// It purges expired slots (crash recovery), counts live slots, and adds the +// caller's slot iff under the cap. Returning 1 = reserved, 0 = at cap. +// Keeping purge+count+add in one Lua call closes the check-then-act race two +// concurrent reservers would otherwise have. +const wikiInflightReserveScript = ` +local now = tonumber(ARGV[1]) +local expiry = tonumber(ARGV[2]) +local maxInflight = tonumber(ARGV[3]) +local token = ARGV[4] +local ttl = tonumber(ARGV[5]) +redis.call('ZREMRANGEBYSCORE', KEYS[1], 0, now) +if redis.call('ZCARD', KEYS[1]) >= maxInflight then + return 0 +end +redis.call('ZADD', KEYS[1], expiry, token) +redis.call('PEXPIRE', KEYS[1], ttl) +return 1 +` + +// reserveInflightSlot claims one of KB's concurrent-batch slots (Phase 4, +// standard/Redis mode). Returns (release, true) when granted — release() MUST +// run when the batch finishes; (nil, false) when the KB is already at +// maxInflight, so the caller should reschedule and bail. A background renew +// keeps the slot alive for the batch's duration; a crashed batch's slot simply +// expires (wikiInflightTTL) and is purged by the next reserver. Lite mode has +// no shared-pool contention (liteLocks already serialize per KB), so it always +// grants a no-op slot. Fails OPEN on a Redis error: a blip must not halt wiki +// generation, and the pool size still bounds total work. +func (s *wikiIngestService) reserveInflightSlot(ctx context.Context, kbID string, maxInflight int) (func(), bool) { + if s.redisClient == nil || maxInflight <= 0 { + return func() {}, true + } + key := wikiInflightPrefix + kbID + token := uuid.New().String() + now := time.Now() + res, err := s.redisClient.Eval(ctx, wikiInflightReserveScript, + []string{key}, + now.UnixMilli(), + now.Add(wikiInflightTTL).UnixMilli(), + maxInflight, + token, + wikiInflightTTL.Milliseconds(), + ).Int() + if err != nil { + logger.Warnf(ctx, "wiki ingest: inflight reserve failed for KB %s: %v (running uncapped)", kbID, err) + return func() {}, true + } + if res == 0 { + return nil, false + } + + renewCtx, cancel := context.WithCancel(context.Background()) + go func() { + ticker := time.NewTicker(wikiInflightRenew) + defer ticker.Stop() + for { + select { + case <-renewCtx.Done(): + return + case <-ticker.C: + exp := float64(time.Now().Add(wikiInflightTTL).UnixMilli()) + s.redisClient.ZAdd(context.Background(), key, redis.Z{Score: exp, Member: token}) + s.redisClient.PExpire(context.Background(), key, wikiInflightTTL) + } + } + }() + return func() { + cancel() + s.redisClient.ZRem(context.Background(), key, token) + }, true +} + +// scheduleCappedRetry enqueues a single coalesced follow-up trigger after a +// batch was turned away by the in-flight cap. asynq.TaskID collapses all +// turned-away triggers for one KB into a single pending retry (no thundering +// herd), and the running batches that hold the slots also chain their own +// follow-ups on completion, so the turned-away rows are guaranteed to drain +// once a slot frees. +func (s *wikiIngestService) scheduleCappedRetry(ctx context.Context, payload WikiIngestPayload) { + langfuse.InjectTracing(ctx, &payload) + b, _ := json.Marshal(payload) + t := asynq.NewTask(types.TypeWikiIngest, b, + asynq.Queue(types.QueueWiki), + asynq.MaxRetry(wikiIngestMaxRetry), + asynq.Timeout(60*time.Minute), + asynq.ProcessIn(wikiInflightBackoff), + asynq.TaskID("wiki-ingest-capped-"+payload.KnowledgeBaseID), + ) + if _, err := s.task.Enqueue(t); err != nil { + if errors.Is(err, asynq.ErrTaskIDConflict) || errors.Is(err, asynq.ErrDuplicateTask) { + return // a capped retry is already pending for this KB — coalesced + } + logger.Warnf(ctx, "wiki ingest: capped-retry enqueue failed: %v", err) + } +} + +// decodePendingRows converts raw task_pending_ops rows into WikiPendingOps, +// applying the last-write-wins dedup by knowledge_id. peekedIDs carries the +// db ids of EVERY row (including dedup-collapsed ones) so the caller can +// drain them all at trim time. Shared by peekPendingList (no claim) and +// claimPendingList (claimed rows). +func (s *wikiIngestService) decodePendingRows(ctx context.Context, rows []*types.TaskPendingOp) (ops []WikiPendingOp, peekedIDs []int64) { if len(rows) == 0 { return nil, nil } @@ -562,6 +931,14 @@ func (s *wikiIngestService) requeueFailedOps(ctx context.Context, payload WikiIn continue } if count <= wikiMaxFailRetries { + // Release the claim so the row is immediately eligible for the + // next trigger's ClaimBatch instead of waiting out + // wikiClaimStaleAfter. No-op in Lite mode (row was peeked, never + // claimed). ReleaseByIDs preserves fail_count, so the retry + // budget still counts down. + if err := s.pendingRepo.ReleaseByIDs(ctx, []int64{op.dbID}); err != nil { + logger.Warnf(ctx, "wiki ingest: failed to release claim for retry id=%d: %v", op.dbID, err) + } logger.Infof(ctx, "wiki ingest: re-queued failed op %s (%s) for retry (attempt %d/%d)", op.KnowledgeID, op.DocTitle, count, wikiMaxFailRetries) continue } diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go index a78aacc5..b9f0edd6 100644 --- a/internal/application/service/wiki_ingest_batch.go +++ b/internal/application/service/wiki_ingest_batch.go @@ -23,10 +23,19 @@ import ( // still pending ops in task_pending_ops for this KB. Returns true when // a follow-up was scheduled. // -// We use a short ProcessIn (5s) so the active-batch lock has time to -// release before the next worker tries to acquire it; otherwise we'd -// just bounce on ErrWikiIngestConcurrent and burn an asynq retry slot. -func (s *wikiIngestService) scheduleFollowUp(ctx context.Context, payload WikiIngestPayload) bool { +// Post-Phase-3 this only backstops the case where a batch drained its +// claimed window but more rows remain and no other trigger is pending +// (e.g. steady trickle of uploads). Standard mode already fans a KB's +// backlog across concurrent claiming batches, so the short delay is +// normally just a light debounce rather than a lock-release wait. +// +// `delay` is the ProcessIn before the follow-up fires. Callers pass +// wikiFollowUpDelay for the normal case and wikiRateLimitBackoff when the +// batch tripped an upstream rate limit — the released failed rows are +// eligible immediately, but nothing claims them until a trigger fires, so +// stretching the follow-up interval is what actually paces retries down +// during a 429 storm. +func (s *wikiIngestService) scheduleFollowUp(ctx context.Context, payload WikiIngestPayload, delay time.Duration) bool { if s.pendingRepo == nil { return false } @@ -35,15 +44,15 @@ func (s *wikiIngestService) scheduleFollowUp(ctx context.Context, payload WikiIn return false } - logger.Infof(ctx, "wiki ingest: %d more documents pending for KB %s, scheduling follow-up", count, payload.KnowledgeBaseID) + logger.Infof(ctx, "wiki ingest: %d more documents pending for KB %s, scheduling follow-up in %s", count, payload.KnowledgeBaseID, delay) langfuse.InjectTracing(ctx, &payload) payloadBytes, _ := json.Marshal(payload) t := asynq.NewTask(types.TypeWikiIngest, payloadBytes, - asynq.Queue("low"), + asynq.Queue(types.QueueWiki), asynq.MaxRetry(wikiIngestMaxRetry), asynq.Timeout(60*time.Minute), - asynq.ProcessIn(5*time.Second), // short delay — active flag will be released by then + asynq.ProcessIn(delay), // debounce (or rate-limit backoff) before draining the remainder ) if _, err := s.task.Enqueue(t); err != nil { logger.Warnf(ctx, "wiki ingest: follow-up enqueue failed: %v", err) @@ -52,205 +61,17 @@ func (s *wikiIngestService) scheduleFollowUp(ctx context.Context, payload WikiIn return true } -func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task) error { - taskStartedAt := time.Now() - retryCount, _ := asynq.GetRetryCount(ctx) - maxRetry, _ := asynq.GetMaxRetry(ctx) - - var payload WikiIngestPayload - exitStatus := "success" - mode := "redis" - lockAcquired := false - pendingOpsCount := 0 - ingestOps := 0 - retractOps := 0 - ingestSucceeded := 0 - ingestFailed := 0 - retractHandled := 0 - indexRebuildAttempted := false - indexRebuildSucceeded := false - followUpScheduled := false - totalPagesAffected := 0 - docPreview := make([]string, 0, 6) - // Tunables resolved from KB.WikiConfig once we've loaded the KB. - // Captured up here so the deferred stats log can observe them - // regardless of which exit path we took. - loggedBatchSize := 0 - loggedMapPar := 0 - loggedReducePar := 0 - - defer func() { - logger.Infof( - ctx, - "wiki ingest stats: kb=%s tenant=%d retry=%d/%d status=%s elapsed=%s mode=%s lock_acquired=%v pending_ops=%d ops(ingest=%d,retract=%d) ingest(success=%d,failed=%d) retract_handled=%d pages(total=%d) index(rebuild_attempted=%v,rebuild_succeeded=%v) followup=%v tunables(batch=%d,map_par=%d,reduce_par=%d) preview=%s", - payload.KnowledgeBaseID, - payload.TenantID, - retryCount, - maxRetry, - exitStatus, - time.Since(taskStartedAt).Round(time.Millisecond), - mode, - lockAcquired, - pendingOpsCount, - ingestOps, - retractOps, - ingestSucceeded, - ingestFailed, - retractHandled, - totalPagesAffected, - indexRebuildAttempted, - indexRebuildSucceeded, - followUpScheduled, - loggedBatchSize, - loggedMapPar, - loggedReducePar, - previewStringSlice(docPreview, 6), - ) - }() - - if err := json.Unmarshal(t.Payload(), &payload); err != nil { - exitStatus = "invalid_payload" - return fmt.Errorf("wiki ingest: unmarshal payload: %w", err) - } - - // Inject context - ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID) - if payload.Language != "" { - ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language) - } - - // Try to acquire the "active batch" flag (non-blocking). - // - // TTL is intentionally short (wikiActiveLockTTL ≈ 60s) so that if the - // owning process dies without releasing the lock (crash, kill -9, - // container restart), the orphaned key expires within ~1 minute and new - // tasks aren't starved. A renew goroutine keeps the lock alive while - // the handler is genuinely running. - if s.redisClient != nil { - activeKey := wikiActiveKeyPrefix + payload.KnowledgeBaseID - acquired, err := s.redisClient.SetNX(ctx, activeKey, "1", wikiActiveLockTTL).Result() - if err != nil { - logger.Warnf(ctx, "wiki ingest: redis SetNX failed: %v", err) - } else if !acquired { - exitStatus = "active_lock_conflict" - // If task_pending_ops is already empty for this KB, the active - // batch will drain whatever was queued. Returning nil avoids - // burning through the retry budget on tasks that would be - // no-ops when they eventually acquire the lock. If rows still - // remain, retry so we don't miss them in case the active - // batch drained its peek before our op landed. - n, nErr := s.pendingRepo.PendingCount(ctx, wikiTaskType, wikiTaskScope, payload.KnowledgeBaseID) - if nErr != nil { - logger.Warnf(ctx, "wiki ingest: failed to read pending count during lock conflict for KB %s: %v", payload.KnowledgeBaseID, nErr) - logger.Infof(ctx, "wiki ingest: another batch active for KB %s, deferring to asynq retry", payload.KnowledgeBaseID) - return ErrWikiIngestConcurrent - } - if n == 0 { - exitStatus = "active_lock_conflict_empty" - logger.Infof(ctx, "wiki ingest: concurrent batch active for KB %s, pending queue empty — skipping", payload.KnowledgeBaseID) - return nil - } - logger.Infof(ctx, "wiki ingest: another batch active for KB %s, deferring to asynq retry", payload.KnowledgeBaseID) - return ErrWikiIngestConcurrent - } - lockAcquired = acquired - - lockCtx, cancelLock := context.WithCancel(context.Background()) - defer func() { - cancelLock() - s.redisClient.Del(context.Background(), activeKey) - }() - - go func() { - ticker := time.NewTicker(wikiActiveLockRenew) - defer ticker.Stop() - for { - select { - case <-lockCtx.Done(): - return - case <-ticker.C: - s.redisClient.Expire(context.Background(), activeKey, wikiActiveLockTTL) - } - } - }() - } else { - mode = "lite" - // In-process mutual exclusion: mirrors the Redis SetNX lock above. - if _, loaded := s.liteLocks.LoadOrStore(payload.KnowledgeBaseID, struct{}{}); loaded { - exitStatus = "active_lock_conflict" - logger.Infof(ctx, "wiki ingest: another batch active for KB %s (lite lock), deferring to asynq retry", payload.KnowledgeBaseID) - return ErrWikiIngestConcurrent - } - lockAcquired = true - defer s.liteLocks.Delete(payload.KnowledgeBaseID) - } - - kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID) - if err != nil { - exitStatus = "get_kb_failed" - return fmt.Errorf("wiki ingest: get KB: %w", err) - } - if !kb.IsWikiEnabled() { - exitStatus = "kb_not_wiki_enabled" - return fmt.Errorf("wiki ingest: KB %s is not wiki type", kb.ID) - } - - var synthesisModelID string - if kb.WikiConfig != nil { - synthesisModelID = kb.WikiConfig.SynthesisModelID - } - if synthesisModelID == "" { - synthesisModelID = kb.SummaryModelID - } - if synthesisModelID == "" { - exitStatus = "missing_synthesis_model" - return fmt.Errorf("wiki ingest: no synthesis model configured for KB %s", kb.ID) - } - chatModel, err := s.modelService.GetChatModel(ctx, synthesisModelID) - if err != nil { - exitStatus = "get_chat_model_failed" - return fmt.Errorf("wiki ingest: get chat model: %w", err) - } - - // Resolve per-KB tunables once. WikiConfig.IngestBatchSize / - // IngestMapParallel / IngestReduceParallel let operators on - // 4w-document KBs raise the throughput knob (more docs per batch + - // more concurrent LLM calls) without a code deploy. Zero falls back - // to the historical defaults so existing KBs see no behaviour - // change until they opt in. - batchSize := kb.WikiConfig.IngestBatchSizeOrDefault(wikiMaxDocsPerBatch) - mapParallel := kb.WikiConfig.IngestMapParallelOrDefault(10) - reduceParallel := kb.WikiConfig.IngestReduceParallelOrDefault(10) - loggedBatchSize = batchSize - loggedMapPar = mapParallel - loggedReducePar = reduceParallel - - lang := types.LanguageNameFromContext(ctx) - - pendingOps, peekedIDs := s.peekPendingList(ctx, payload.KnowledgeBaseID, batchSize) - pendingOpsCount = len(pendingOps) - if len(pendingOps) == 0 { - exitStatus = "no_pending_ops" - logger.Infof(ctx, "wiki ingest: no pending operations for KB %s", payload.KnowledgeBaseID) - return nil - } - - logger.Infof(ctx, "wiki ingest: batch processing %d ops for KB %s", len(pendingOps), payload.KnowledgeBaseID) - - // Resolve extraction granularity once per batch. Historical rows with - // empty/unknown values fall back to Standard via Normalize(). Failures - // to load the KB (unlikely since we're already acting on it) also - // degrade gracefully to Standard. - granularity := types.WikiExtractionStandard - if kb, kbErr := s.kbService.GetKnowledgeBaseByID(ctx, payload.KnowledgeBaseID); kbErr == nil && kb != nil && kb.WikiConfig != nil { - granularity = kb.WikiConfig.ExtractionGranularity.Normalize() - } - - // Build the per-batch lazy fetchers. These replace the legacy - // pre-batch ListAllPages dump: instead of pulling ~100MB of rows - // up front (and walking them several more times during the batch), - // callers pay only for the slugs / knowledge ids they actually - // reach for. Cache hits keep repeat lookups within the batch free. +// newWikiBatchContext builds the per-run lazy fetchers used by both the ingest +// batch and the debounced finalize task. These replace the legacy pre-batch +// ListAllPages dump: instead of pulling ~100MB of rows up front (and walking +// them several more times), callers pay only for the slugs / knowledge ids they +// actually reach for. Cache hits keep repeat lookups within a single run free. +// The cache is per-call (goroutine-safe via fetchMu), so each task gets a fresh, +// isolated view. +func (s *wikiIngestService) newWikiBatchContext( + kbID string, + granularity types.WikiExtractionGranularity, +) *WikiBatchContext { var ( fetchMu sync.Mutex slugTitleCache = make(map[string]string) // slug -> title; "" = known-missing @@ -270,7 +91,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task fetchMu.Unlock() if len(need) > 0 { - pages, err := s.wikiService.ListBySlugs(ctx, payload.KnowledgeBaseID, need) + pages, err := s.wikiService.ListBySlugs(ctx, kbID, need) if err != nil { logger.Warnf(ctx, "wiki ingest: ListBySlugs(%d slugs) failed: %v", len(need), err) } @@ -318,7 +139,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task fetchMu.Unlock() if len(need) > 0 { - contents, err := s.wikiService.ListSummariesByKnowledgeIDs(ctx, payload.KnowledgeBaseID, need) + contents, err := s.wikiService.ListSummariesByKnowledgeIDs(ctx, kbID, need) if err != nil { logger.Warnf(ctx, "wiki ingest: ListSummariesByKnowledgeIDs(%d kids) failed: %v", len(need), err) } @@ -344,7 +165,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task return out } - batchCtx := &WikiBatchContext{ + return &WikiBatchContext{ SlugTitle: func(ctx context.Context, slug string) string { m := resolveSlugs(ctx, []string{slug}) return m[slug] @@ -356,13 +177,194 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task }, ExtractionGranularity: granularity, } +} + +func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task) error { + taskStartedAt := time.Now() + retryCount, _ := asynq.GetRetryCount(ctx) + maxRetry, _ := asynq.GetMaxRetry(ctx) + + var payload WikiIngestPayload + exitStatus := "success" + mode := "redis" + pendingOpsCount := 0 + ingestOps := 0 + retractOps := 0 + ingestSucceeded := 0 + ingestFailed := 0 + retractHandled := 0 + followUpScheduled := false + totalPagesAffected := 0 + docPreview := make([]string, 0, 6) + // Tunables resolved from KB.WikiConfig once we've loaded the KB. + // Captured up here so the deferred stats log can observe them + // regardless of which exit path we took. (Index-intro rebuild moved to + // the debounced wiki:finalize task in Phase 1, so it's logged there, not + // here; the exclusive per-KB lock was removed in Phase 3.) + loggedBatchSize := 0 + loggedMapPar := 0 + loggedReducePar := 0 + loggedMaxInflight := 0 + + defer func() { + logger.Infof( + ctx, + "wiki ingest stats: kb=%s tenant=%d retry=%d/%d status=%s elapsed=%s mode=%s pending_ops=%d ops(ingest=%d,retract=%d) ingest(success=%d,failed=%d) retract_handled=%d pages(total=%d) followup=%v tunables(batch=%d,map_par=%d,reduce_par=%d,max_inflight=%d) preview=%s", + payload.KnowledgeBaseID, + payload.TenantID, + retryCount, + maxRetry, + exitStatus, + time.Since(taskStartedAt).Round(time.Millisecond), + mode, + pendingOpsCount, + ingestOps, + retractOps, + ingestSucceeded, + ingestFailed, + retractHandled, + totalPagesAffected, + followUpScheduled, + loggedBatchSize, + loggedMapPar, + loggedReducePar, + loggedMaxInflight, + previewStringSlice(docPreview, 6), + ) + }() + + if err := json.Unmarshal(t.Payload(), &payload); err != nil { + exitStatus = "invalid_payload" + return fmt.Errorf("wiki ingest: unmarshal payload: %w", err) + } + + // Inject context + ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID) + if payload.Language != "" { + ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language) + } + + // Concurrency model (Phase 3): + // + // - Standard (Redis) mode: NO exclusive per-KB lock. Multiple batches + // for the same KB may run at once, each claiming a DISJOINT set of + // rows via claimPendingList (below). This lets one KB's backlog fan + // out across the whole wiki worker pool instead of draining one + // 5-doc batch at a time. Same-slug reduce safety is provided by + // withSlugLock, not by a batch-wide lock. + // + // - Lite mode (no Redis, single process): keep the in-process + // liteLocks guard so exactly one batch per KB runs at a time. Lite + // targets small/local scale where serial-per-KB is simplest and the + // claiming machinery (which needs FOR UPDATE SKIP LOCKED) buys + // nothing. + if s.redisClient == nil { + mode = "lite" + if _, loaded := s.liteLocks.LoadOrStore(payload.KnowledgeBaseID, struct{}{}); loaded { + exitStatus = "active_lock_conflict" + logger.Infof(ctx, "wiki ingest: another batch active for KB %s (lite lock), deferring to asynq retry", payload.KnowledgeBaseID) + return ErrWikiIngestConcurrent + } + defer s.liteLocks.Delete(payload.KnowledgeBaseID) + } + + kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID) + if err != nil { + exitStatus = "get_kb_failed" + return fmt.Errorf("wiki ingest: get KB: %w", err) + } + if !kb.IsWikiEnabled() { + exitStatus = "kb_not_wiki_enabled" + return fmt.Errorf("wiki ingest: KB %s is not wiki type", kb.ID) + } + + var synthesisModelID string + if kb.WikiConfig != nil { + synthesisModelID = kb.WikiConfig.SynthesisModelID + } + if synthesisModelID == "" { + synthesisModelID = kb.SummaryModelID + } + if synthesisModelID == "" { + exitStatus = "missing_synthesis_model" + return fmt.Errorf("wiki ingest: no synthesis model configured for KB %s", kb.ID) + } + chatModel, err := s.modelService.GetChatModel(ctx, synthesisModelID) + if err != nil { + exitStatus = "get_chat_model_failed" + return fmt.Errorf("wiki ingest: get chat model: %w", err) + } + + // Resolve per-KB tunables once. WikiConfig.IngestBatchSize / + // IngestMapParallel / IngestReduceParallel let operators on + // 4w-document KBs raise the throughput knob (more docs per batch + + // more concurrent LLM calls) without a code deploy. Zero falls back + // to the historical defaults so existing KBs see no behaviour + // change until they opt in. + batchSize := kb.WikiConfig.IngestBatchSizeOrDefault(wikiMaxDocsPerBatch) + mapParallel := kb.WikiConfig.IngestMapParallelOrDefault(10) + reduceParallel := kb.WikiConfig.IngestReduceParallelOrDefault(10) + loggedBatchSize = batchSize + loggedMapPar = mapParallel + loggedReducePar = reduceParallel + + lang := types.LanguageNameFromContext(ctx) + + // Per-KB in-flight cap (Phase 4, standard mode): keep one KB's bulk + // import from monopolizing the shared wiki pool. If the KB is already at + // its cap we schedule a coalesced retry and bail without claiming, so the + // rows stay available for whichever running batch frees a slot first. + maxInflight := kb.WikiConfig.IngestMaxInflightOrDefault(wikiInflightDefault) + loggedMaxInflight = maxInflight + releaseSlot, gotSlot := s.reserveInflightSlot(ctx, payload.KnowledgeBaseID, maxInflight) + if !gotSlot { + exitStatus = "inflight_cap" + logger.Infof(ctx, "wiki ingest: KB %s at in-flight cap (%d), rescheduling", payload.KnowledgeBaseID, maxInflight) + s.scheduleCappedRetry(ctx, payload) + return nil + } + defer releaseSlot() + + // Standard mode claims rows (marks claimed_at, disjoint across concurrent + // batches); Lite mode peeks under its in-process lock. + var pendingOps []WikiPendingOp + var peekedIDs []int64 + if s.redisClient != nil { + pendingOps, peekedIDs = s.claimPendingList(ctx, payload.KnowledgeBaseID, batchSize) + } else { + pendingOps, peekedIDs = s.peekPendingList(ctx, payload.KnowledgeBaseID, batchSize) + } + pendingOpsCount = len(pendingOps) + if len(pendingOps) == 0 { + exitStatus = "no_pending_ops" + logger.Infof(ctx, "wiki ingest: no pending operations for KB %s", payload.KnowledgeBaseID) + return nil + } + + logger.Infof(ctx, "wiki ingest: batch processing %d ops for KB %s", len(pendingOps), payload.KnowledgeBaseID) + + // Resolve extraction granularity once per batch. Historical rows with + // empty/unknown values fall back to Standard via Normalize(). Failures + // to load the KB (unlikely since we're already acting on it) also + // degrade gracefully to Standard. + granularity := types.WikiExtractionStandard + if kb, kbErr := s.kbService.GetKnowledgeBaseByID(ctx, payload.KnowledgeBaseID); kbErr == nil && kb != nil && kb.WikiConfig != nil { + granularity = kb.WikiConfig.ExtractionGranularity.Normalize() + } + + batchCtx := s.newWikiBatchContext(payload.KnowledgeBaseID, granularity) // 1. MAP PHASE (Parallel extraction and generation of updates) var mapMu sync.Mutex var failedOps []WikiPendingOp slugUpdates := make(map[string][]SlugUpdate) var docResults []*docIngestResult - var retractChangeDesc strings.Builder + // rateLimited flips true when any map/reduce LLM failure looks like an + // upstream 429/quota trip. It steers the follow-up scheduler onto the + // longer wikiRateLimitBackoff so retries don't keep hammering an already + // saturated rpm budget. Guarded by mapMu (map phase) / reduceMu (reduce + // phase) — both are held when written. + rateLimited := false eg, mapCtx := errgroup.WithContext(ctx) eg.SetLimit(mapParallel) // Map phase limit (configurable via WikiConfig) @@ -416,7 +418,6 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task retractOps++ retractHandled++ docPreview = append(docPreview, fmt.Sprintf("retract[%s]: %s (%d slugs)", previewText(op.KnowledgeID, 24), previewText(op.DocTitle, 48), len(slugSet))) - fmt.Fprintf(&retractChangeDesc, "\n%s\n%s\n\n\n", op.DocTitle, op.DocSummary) for slug := range slugSet { slugUpdates[slug] = append(slugUpdates[slug], SlugUpdate{ @@ -443,6 +444,9 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task mapMu.Lock() ingestFailed++ failedOps = append(failedOps, op) + if isLikelyRateLimitError(err) { + rateLimited = true + } mapMu.Unlock() logger.Warnf(mapCtx, "wiki ingest: failed to map knowledge %s: %v", op.KnowledgeID, err) return nil // Don't fail the whole batch @@ -524,9 +528,37 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task slug := slug updates := updates egReduce.Go(func() error { - changed, affectedType, additionFailed, err := s.reduceSlugUpdates(reduceCtx, chatModel, payload.KnowledgeBaseID, slug, updates, payload.TenantID, batchCtx, kidToWikiSpan) - if err != nil { - logger.Warnf(reduceCtx, "wiki ingest: reduce failed for slug %s: %v", slug, err) + var ( + changed bool + affectedType string + additionFailed bool + reduceErr error + ) + // Serialize same-slug read-modify-write across concurrent batches + // (standard mode). runs fn directly in Lite mode. + acquired, lockErr := s.withSlugLock(reduceCtx, payload.KnowledgeBaseID, slug, func() error { + changed, affectedType, additionFailed, reduceErr = s.reduceSlugUpdates( + reduceCtx, chatModel, payload.KnowledgeBaseID, slug, updates, payload.TenantID, batchCtx, kidToWikiSpan) + return reduceErr + }) + if lockErr != nil { + // ctx cancelled (batch timeout / shutdown) — stop quietly. + return nil + } + if !acquired { + // Contended slug we couldn't get in time. Best-effort skip, + // matching reduce-failure semantics: the page keeps its prior + // content and the finalize/dead-link passes reconcile later. + logger.Warnf(reduceCtx, "wiki ingest: slug %s busy > %s, deferring update", slug, wikiSlugLockWait) + return nil + } + if reduceErr != nil { + logger.Warnf(reduceCtx, "wiki ingest: reduce failed for slug %s: %v", slug, reduceErr) + if isLikelyRateLimitError(reduceErr) { + reduceMu.Lock() + rateLimited = true + reduceMu.Unlock() + } } if changed { reduceMu.Lock() @@ -612,74 +644,55 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task } } - // Build change description for the Index Intro LLM prompt - var changeDesc strings.Builder - if len(docResults) > 0 { - for _, r := range docResults { - fmt.Fprintf(&changeDesc, "\n%s\n%s\n\n\n", r.DocTitle, r.Summary) - } - } - if retractChangeDesc.Len() > 0 { - changeDesc.WriteString(retractChangeDesc.String()) - } - - // Rebuild index page - if changeDesc.Len() > 0 { - indexRebuildAttempted = true - logger.Infof(ctx, "wiki ingest: rebuilding index page") - if err := s.rebuildIndexPage(ctx, chatModel, payload, changeDesc.String(), lang); err != nil { - logger.Warnf(ctx, "wiki ingest: rebuild index failed: %v", err) - docPreview = append(docPreview, fmt.Sprintf("index_change=%s", previewText(changeDesc.String(), 160))) - } else { - indexRebuildSucceeded = true - docPreview = append(docPreview, fmt.Sprintf("index_change=%s", previewText(changeDesc.String(), 160))) - } - } - - // Clean dead [[slug]] references whenever ANY page was touched this - // batch (not just retracts). Reduce-phase failures can leave stale - // references in pages we just rewrote (e.g. summary pages cite - // failed entity slugs); sanitizeDeadSummaryLinks above handles the - // well-known summary case, and this pass is the safety net for the - // long tail (cross-doc citations, prior batches' lingering refs). - // Dead-link cleanup: scoped to this batch's affected pages so the - // pass scales with batch size, not with KB size. The lint - // AutoFix path takes care of long-tail cleanup across the whole - // KB out-of-band. + // Publish freshly-generated pages immediately (NOT deferred to finalize): + // users should see a document's wiki pages as soon as their content is + // written, not after the debounce window. This is a cheap status flip. if len(allPagesAffected) > 0 { - logger.Infof(ctx, "wiki ingest: cleaning dead links") - s.cleanDeadLinks(ctx, payload.KnowledgeBaseID, allPagesAffected, batchCtx) - } - - if len(allPagesAffected) > 0 { - // Build the freshRefs set: every (slug, title) pair this batch - // successfully wrote, minus any that landed in failedAdditionSlugs. - // These are the "newly-mentionable" pages — links to them will - // not have appeared in older content yet, so injectCrossLinks - // targets exactly the affected pages with this fresh ref set. - freshRefs := make([]linkRef, 0, len(docResults)*4) - for _, dr := range docResults { - if dr == nil { - continue - } - for _, p := range dr.Pages { - if p.Slug == "" || p.Title == "" { - continue - } - if _, bad := failedAdditionSlugs[p.Slug]; bad { - continue - } - freshRefs = append(freshRefs, linkRef{slug: p.Slug, matchText: p.Title}) - } - } - - logger.Infof(ctx, "wiki ingest: injecting cross links") - s.injectCrossLinks(ctx, payload.KnowledgeBaseID, allPagesAffected, freshRefs, batchCtx) - logger.Infof(ctx, "wiki ingest: publishing draft pages") s.publishDraftPages(ctx, payload.KnowledgeBaseID, allPagesAffected) } + // Defer KB-global convergence (index-intro rebuild + dead-link cleanup + + // cross-link injection) to a debounced per-KB wiki:finalize task instead of + // running it at the tail of every 5-doc batch. We record what changed into + // the finalize lane and schedule a coalesced trigger; a burst of N documents + // then rebuilds the index ONCE. See ProcessWikiFinalize. + // + // freshTitleBySlug carries the (slug → title) pairs this batch successfully + // wrote (minus reduce-phase failures) so finalize's cross-link pass can + // linkify mentions of the new pages. + freshTitleBySlug := make(map[string]string, len(docResults)*4) + for _, dr := range docResults { + if dr == nil { + continue + } + for _, p := range dr.Pages { + if p.Slug == "" || p.Title == "" { + continue + } + if _, bad := failedAdditionSlugs[p.Slug]; bad { + continue + } + freshTitleBySlug[p.Slug] = p.Title + } + } + if len(allPagesAffected) > 0 || len(docResults) > 0 { + changes := make([]wikiFinalizeChange, 0, len(docResults)+len(pendingOps)) + for _, r := range docResults { + changes = append(changes, wikiFinalizeChange{ + Action: wikiFinalizeAdded, DocTitle: r.DocTitle, DocSummary: r.Summary, + }) + } + for _, op := range pendingOps { + if op.Op == WikiOpRetract { + changes = append(changes, wikiFinalizeChange{ + Action: wikiFinalizeRemoved, DocTitle: op.DocTitle, DocSummary: op.DocSummary, + }) + } + } + s.enqueueFinalize(ctx, payload, allPagesAffected, freshTitleBySlug, changes) + } + // Close postprocess.wiki spans for every successfully-mapped doc. // Span duration now spans map + reduce + index rebuild + cleanup + // cross-link injection + publish, matching the wall-clock window @@ -765,7 +778,186 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task logger.Infof(ctx, "wiki ingest: batch completed for KB %s, %d ops, %d pages affected", payload.KnowledgeBaseID, len(pendingOps), len(allPagesAffected)) - followUpScheduled = s.scheduleFollowUp(ctx, payload) + // Pace the follow-up: on a rate-limit trip, back off so the per-minute + // window can reset instead of retrying the failed docs immediately. + followUpDelay := wikiFollowUpDelay + if rateLimited { + followUpDelay = wikiRateLimitBackoff + logger.Warnf(ctx, "wiki ingest: KB %s hit upstream rate limiting, backing off follow-up to %s", payload.KnowledgeBaseID, followUpDelay) + } + followUpScheduled = s.scheduleFollowUp(ctx, payload, followUpDelay) + return nil +} + +// ProcessWikiFinalize runs the debounced, per-KB KB-global convergence pass: +// index-intro rebuild, dead-link cleanup, and cross-link injection. It drains +// the finalize lane of task_pending_ops (written by ProcessWikiIngest via +// enqueueFinalize) so a burst of documents rebuilds the index ONCE instead of +// once per 5-doc batch. +func (s *wikiIngestService) ProcessWikiFinalize(ctx context.Context, t *asynq.Task) error { + startedAt := time.Now() + var payload WikiIngestPayload + if err := json.Unmarshal(t.Payload(), &payload); err != nil { + return fmt.Errorf("wiki finalize: unmarshal payload: %w", err) + } + ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID) + if payload.Language != "" { + ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language) + } + if s.pendingRepo == nil { + return nil + } + + // Per-KB finalize lock, separate from the ingest active lock so finalize + // and ingest batches never block each other. Coalescing via asynq.TaskID + // already keeps at most one finalize pending per KB; this guards the + // reschedule-overlap window against a concurrent index-page write. + if s.redisClient != nil { + key := "wiki:finalize:active:" + payload.KnowledgeBaseID + acquired, err := s.redisClient.SetNX(ctx, key, "1", wikiFinalizeLockTTL).Result() + if err != nil { + logger.Warnf(ctx, "wiki finalize: SetNX failed: %v", err) + } else if !acquired { + // Another finalize is running; it will drain the lane and + // reschedule if rows remain. Safe to no-op. + return nil + } + lockCtx, cancelLock := context.WithCancel(context.Background()) + defer func() { + cancelLock() + s.redisClient.Del(context.Background(), key) + }() + go func() { + ticker := time.NewTicker(wikiFinalizeLockRenew) + defer ticker.Stop() + for { + select { + case <-lockCtx.Done(): + return + case <-ticker.C: + s.redisClient.Expire(context.Background(), key, wikiFinalizeLockTTL) + } + } + }() + } else { + if _, loaded := s.liteFinalizeLocks.LoadOrStore(payload.KnowledgeBaseID, struct{}{}); loaded { + return nil + } + defer s.liteFinalizeLocks.Delete(payload.KnowledgeBaseID) + } + + rows, err := s.pendingRepo.PeekBatch(ctx, wikiFinalizeTaskType, wikiTaskScope, payload.KnowledgeBaseID, wikiFinalizeMaxRows) + if err != nil { + return fmt.Errorf("wiki finalize: peek: %w", err) + } + if len(rows) == 0 { + return nil + } + + kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID) + if err != nil { + return fmt.Errorf("wiki finalize: get KB: %w", err) + } + + // Aggregate the drained rows into: affected slugs (dedup), fresh cross-link + // refs, and the index-intro change description. We collect ids up front so + // we can drain the lane even on the KB-disabled short-circuit below. + ids := make([]int64, 0, len(rows)) + affectedSet := make(map[string]struct{}, len(rows)) + var affectedSlugs []string + var freshRefs []linkRef + var changeDesc strings.Builder + for _, r := range rows { + ids = append(ids, r.ID) + if len(r.Payload) == 0 { + continue + } + var row wikiFinalizeRow + if err := json.Unmarshal(r.Payload, &row); err != nil { + logger.Warnf(ctx, "wiki finalize: unmarshal row id=%d failed: %v", r.ID, err) + continue + } + if row.Change != nil { + if row.Change.Action == wikiFinalizeRemoved { + fmt.Fprintf(&changeDesc, "\n%s\n%s\n\n\n", row.Change.DocTitle, row.Change.DocSummary) + } else { + fmt.Fprintf(&changeDesc, "\n%s\n%s\n\n\n", row.Change.DocTitle, row.Change.DocSummary) + } + continue + } + if row.Slug != "" { + if _, ok := affectedSet[row.Slug]; !ok { + affectedSet[row.Slug] = struct{}{} + affectedSlugs = append(affectedSlugs, row.Slug) + } + if row.Title != "" { + freshRefs = append(freshRefs, linkRef{slug: row.Slug, matchText: row.Title}) + } + } + } + + // KB flipped away from wiki (deleted / type change) — drain the lane so the + // rows don't accumulate, then stop. + if !kb.IsWikiEnabled() { + s.trimPendingList(ctx, ids) + return nil + } + + synthesisModelID := "" + if kb.WikiConfig != nil { + synthesisModelID = kb.WikiConfig.SynthesisModelID + } + if synthesisModelID == "" { + synthesisModelID = kb.SummaryModelID + } + if synthesisModelID == "" { + // No model to rebuild the index with; still run the pure-text passes, + // then drain. Missing model is a config gap, not a transient error. + logger.Warnf(ctx, "wiki finalize: no synthesis model for KB %s, skipping index rebuild", payload.KnowledgeBaseID) + } + + granularity := types.WikiExtractionStandard + if kb.WikiConfig != nil { + granularity = kb.WikiConfig.ExtractionGranularity.Normalize() + } + batchCtx := s.newWikiBatchContext(payload.KnowledgeBaseID, granularity) + lang := types.LanguageNameFromContext(ctx) + + indexRebuilt := false + if changeDesc.Len() > 0 && synthesisModelID != "" { + chatModel, mErr := s.modelService.GetChatModel(ctx, synthesisModelID) + if mErr != nil { + logger.Warnf(ctx, "wiki finalize: get chat model failed: %v", mErr) + } else if err := s.rebuildIndexPage(ctx, chatModel, payload, changeDesc.String(), lang); err != nil { + logger.Warnf(ctx, "wiki finalize: rebuild index failed: %v", err) + } else { + indexRebuilt = true + } + } + + if len(affectedSlugs) > 0 { + s.cleanDeadLinks(ctx, payload.KnowledgeBaseID, affectedSlugs, batchCtx) + s.injectCrossLinks(ctx, payload.KnowledgeBaseID, affectedSlugs, freshRefs, batchCtx) + } + + // Drain the processed rows. Best-effort convergence mirrors the legacy + // in-batch behaviour: a failed index rebuild is logged (not retried), + // so we delete regardless to avoid re-doing the whole pass forever. + s.trimPendingList(ctx, ids) + + // If more finalize rows landed while we were working, reschedule so they + // get their own convergence pass. + rescheduled := false + if n, cErr := s.pendingRepo.PendingCount(ctx, wikiFinalizeTaskType, wikiTaskScope, payload.KnowledgeBaseID); cErr == nil && n > 0 { + s.scheduleFinalize(ctx, payload) + rescheduled = true + } + + logger.Infof(ctx, + "wiki finalize: kb=%s rows=%d affected_slugs=%d index_rebuilt=%v rescheduled=%v elapsed=%s", + payload.KnowledgeBaseID, len(rows), len(affectedSlugs), indexRebuilt, rescheduled, + time.Since(startedAt).Round(time.Millisecond), + ) return nil } diff --git a/internal/container/container.go b/internal/container/container.go index 0f71b57a..66da1bfb 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -263,7 +263,13 @@ func BuildContainer(container *dig.Container) *dig.Container { redisAvailable := os.Getenv("REDIS_ADDR") != "" if redisAvailable { must(container.Provide(router.NewAsyncqClient, dig.As(new(interfaces.TaskEnqueuer)))) - must(container.Provide(router.NewAsynqServer)) + // Two independent asynq worker pools with separate concurrency budgets: + // the parse pool (all queues except QueueWiki) and the wiki pool + // (QueueWiki only). Hard capacity isolation so upstream parsing and the + // downstream wiki pipeline can't starve each other during concurrent + // uploads. Both run the same mux (see router.RunAsynqServer). + must(container.Provide(router.NewParseAsynqServer, dig.Name("parseAsynqServer"))) + must(container.Provide(router.NewWikiAsynqServer, dig.Name("wikiAsynqServer"))) // Asynq inspector for cancel-by-knowledge-id (best-effort // dequeue of pending/scheduled/retry tasks + active-task cancel). must(container.Provide(router.NewAsynqInspector)) diff --git a/internal/router/sync_task.go b/internal/router/sync_task.go index f8c6981c..848bedbd 100644 --- a/internal/router/sync_task.go +++ b/internal/router/sync_task.go @@ -146,5 +146,6 @@ func RegisterSyncHandlers(params SyncTaskParams) { params.Executor.RegisterHandler(types.TypeKnowledgePostProcess, params.KnowledgePostProcess.Handle) params.Executor.RegisterHandler(types.TypeDataSourceSync, params.DataSourceService.ProcessSync) params.Executor.RegisterHandler(types.TypeWikiIngest, params.WikiIngest.Handle) + params.Executor.RegisterHandler(types.TypeWikiFinalize, params.WikiIngest.Handle) logger.Infof(context.Background(), "[SyncTask] All task handlers registered (Lite mode, no Redis)") } diff --git a/internal/router/task.go b/internal/router/task.go index daf31d53..0060a3ff 100644 --- a/internal/router/task.go +++ b/internal/router/task.go @@ -24,7 +24,15 @@ import ( type AsynqTaskParams struct { dig.In - Server *asynq.Server + // ParseServer drains every queue EXCEPT QueueWiki: the user-facing + // document parsing / post-processing / enrichment pipeline. WikiServer + // drains QueueWiki only. Two independent worker pools give hard capacity + // isolation between the upstream parse pipeline and the downstream wiki + // pipeline (see NewParseAsynqServer / NewWikiAsynqServer). The same mux is + // run on both; asynq routes by the queue each server subscribes to, and + // queue↔task-type is 1:1 by convention, so no task is processed twice. + ParseServer *asynq.Server `name:"parseAsynqServer"` + WikiServer *asynq.Server `name:"wikiAsynqServer"` KnowledgeService interfaces.KnowledgeService KnowledgeBaseService interfaces.KnowledgeBaseService TagService interfaces.KnowledgeTagService @@ -122,8 +130,36 @@ func asynqRetryDelayFunc(n int, e error, t *asynq.Task) time.Duration { // not on local CPU). const defaultAsynqConcurrency = 32 -func NewAsynqServer(svc interfaces.SystemSettingService) *asynq.Server { +// defaultWikiAsynqConcurrency is the worker pool size for the dedicated wiki +// pool (QueueWiki) when WEKNORA_WIKI_ASYNQ_CONCURRENCY is unset. Kept +// independent from the parse pool so the two can be sized separately for the +// host / LLM-provider budget. The wiki pipeline is LLM-bound (synthesis model +// calls dominate), so a moderate default avoids overrunning upstream provider +// concurrency while still letting many documents' wiki generation run in +// parallel across knowledge bases. +const defaultWikiAsynqConcurrency = 16 + +// newAsynqServer builds an asynq server bound to a specific queue set and +// concurrency. Shared by the parse and wiki pools so both inherit the same +// Redis options and retry-delay policy. +func newAsynqServer(concurrency int, queues map[string]int) *asynq.Server { opt := getAsynqRedisClientOpt() + return asynq.NewServer( + opt, + asynq.Config{ + Concurrency: concurrency, + Queues: queues, + RetryDelayFunc: asynqRetryDelayFunc, + }, + ) +} + +// NewParseAsynqServer builds the upstream pool: every queue EXCEPT QueueWiki. +// This is the user-facing document parse / post-process / enrichment pipeline. +// Concurrency comes from WEKNORA_ASYNQ_CONCURRENCY (default 32), unchanged from +// the historical single-pool behaviour so existing deployments see the same +// parse throughput. +func NewParseAsynqServer(svc interfaces.SystemSettingService) *asynq.Server { concurrency := defaultAsynqConcurrency if svc != nil { n := svc.GetInt(context.Background(), "asynq.concurrency", "WEKNORA_ASYNQ_CONCURRENCY", defaultAsynqConcurrency) @@ -131,24 +167,35 @@ func NewAsynqServer(svc interfaces.SystemSettingService) *asynq.Server { concurrency = int(n) } } - log.Printf("asynq server starting with concurrency=%d redis_op_timeout=%dms", + log.Printf("asynq parse-pool server starting with concurrency=%d redis_op_timeout=%dms", concurrency, readRedisOpTimeoutMs()) - srv := asynq.NewServer( - opt, - asynq.Config{ - Concurrency: concurrency, - Queues: map[string]int{ - types.QueueCritical: 6, // Highest priority queue - types.QueueDefault: 3, // Default priority queue - types.QueueLow: 1, // Lowest priority queue - types.QueueMultimodal: 1, // Isolated lane for high-volume slow VLM image tasks - types.QueueGraph: 1, // Isolated lane for high-volume slow graph-extraction tasks - types.QueueQuestion: 1, // Isolated lane for high-volume slow question-generation tasks - }, - RetryDelayFunc: asynqRetryDelayFunc, - }, - ) - return srv + return newAsynqServer(concurrency, map[string]int{ + types.QueueCritical: 6, // Highest priority queue + types.QueueDefault: 3, // Default priority queue + types.QueueLow: 1, // Lowest priority queue + types.QueueMultimodal: 1, // Isolated lane for high-volume slow VLM image tasks + types.QueueGraph: 1, // Isolated lane for high-volume slow graph-extraction tasks + types.QueueQuestion: 1, // Isolated lane for high-volume slow question-generation tasks + }) +} + +// NewWikiAsynqServer builds the dedicated wiki pool: QueueWiki only. Runs the +// same mux as the parse pool but only ever pulls wiki tasks, so its concurrency +// budget (WEKNORA_WIKI_ASYNQ_CONCURRENCY, default 16) is spent exclusively on +// wiki generation. This is the hard capacity isolation that prevents the parse +// pipeline from starving wiki (and vice-versa) during concurrent uploads. +func NewWikiAsynqServer(svc interfaces.SystemSettingService) *asynq.Server { + concurrency := defaultWikiAsynqConcurrency + if svc != nil { + n := svc.GetInt(context.Background(), "asynq.wiki_concurrency", "WEKNORA_WIKI_ASYNQ_CONCURRENCY", defaultWikiAsynqConcurrency) + if n > 0 { + concurrency = int(n) + } + } + log.Printf("asynq wiki-pool server starting with concurrency=%d", concurrency) + return newAsynqServer(concurrency, map[string]int{ + types.QueueWiki: 1, + }) } func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux { @@ -225,15 +272,26 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux { // Register data source sync handler mux.HandleFunc(types.TypeDataSourceSync, params.DataSourceService.ProcessSync) - // Register wiki ingest handler + // Register wiki ingest handler + the debounced KB-global finalize handler. + // Both route to the same dispatch (WikiIngest.Handle switches on task type) + // and both land on QueueWiki, so the dedicated wiki pool serves them. mux.HandleFunc(types.TypeWikiIngest, params.WikiIngest.Handle) + mux.HandleFunc(types.TypeWikiFinalize, params.WikiIngest.Handle) - go func() { - // Start the server - if err := params.Server.Run(mux); err != nil { - log.Fatalf("could not run server: %v", err) - } - }() + // Run the same mux on both pools. Each server subscribes to a disjoint set + // of queues (parse pool = everything except QueueWiki; wiki pool = + // QueueWiki only), so a given task is dequeued by exactly one pool. This is + // how the two pools get independent concurrency budgets while sharing one + // handler registration. + runPool := func(name string, srv *asynq.Server) { + go func() { + if err := srv.Run(mux); err != nil { + log.Fatalf("could not run %s asynq server: %v", name, err) + } + }() + } + runPool("parse-pool", params.ParseServer) + runPool("wiki-pool", params.WikiServer) return mux } diff --git a/internal/router/task_inspector.go b/internal/router/task_inspector.go index 3ff4188b..c46f8088 100644 --- a/internal/router/task_inspector.go +++ b/internal/router/task_inspector.go @@ -57,6 +57,7 @@ var queuesScanned = []string{ types.QueueMultimodal, types.QueueGraph, types.QueueQuestion, + types.QueueWiki, } // taskTypesForKnowledgeCancel lists every asynq task type that carries diff --git a/internal/types/interfaces/task_queue.go b/internal/types/interfaces/task_queue.go index 1ffea0f9..1018c95e 100644 --- a/internal/types/interfaces/task_queue.go +++ b/internal/types/interfaces/task_queue.go @@ -2,6 +2,7 @@ package interfaces import ( "context" + "time" "github.com/Tencent/WeKnora/internal/types" ) @@ -13,11 +14,15 @@ import ( // tuple is the only routing primitive the repository understands; // deduplication, batching, and retry policy live in the consumer. // -// Concurrency model in this revision: PeekBatch does NOT take row -// locks. Consumers are expected to enforce per-(scope_id) serialization -// out-of-band (wiki ingest does this via Redis SetNX wiki:active:). -// The reserved `claimed_at` column lets a future revision adopt -// pessimistic locking without a schema change. +// Concurrency model: two consumption primitives coexist. +// - PeekBatch does NOT take row locks; consumers enforce per-(scope_id) +// serialization out-of-band (e.g. an external Redis lock). This is the +// original primitive and is still used by single-process (Lite) mode. +// - ClaimBatch DOES take row locks (SELECT ... FOR UPDATE SKIP LOCKED on +// Postgres) and marks rows with claimed_at, so multiple concurrent +// consumers of the same tuple pull DISJOINT rows without an external +// lock. This is what lets the wiki pipeline drop its exclusive per-KB +// batch lock and spread one KB's backlog across the whole worker pool. type TaskPendingOpsRepository interface { // Enqueue inserts a single op. The caller fills in TenantID, TaskType, // Scope, ScopeID, Op, DedupKey, Payload; ID, FailCount, EnqueuedAt @@ -30,6 +35,26 @@ type TaskPendingOpsRepository interface { // IncrFailCount and leave them for the next pass). PeekBatch(ctx context.Context, taskType, scope, scopeID string, limit int) ([]*types.TaskPendingOp, error) + // ClaimBatch atomically claims up to `limit` rows for the tuple, + // ordered by id ASC (FIFO), and returns them with claimed_at set to + // now. A row is eligible when it is unclaimed (claimed_at IS NULL) or + // its claim is stale (claimed_at < staleBefore) — the latter recovers + // rows abandoned by a crashed worker. On Postgres the selection uses + // FOR UPDATE SKIP LOCKED so concurrent claimers never block on or + // double-claim each other's rows. + // + // Claimed rows are NOT removed: the consumer must DeleteByIDs on + // success, or ReleaseByIDs to hand a still-retryable row back to the + // pool (otherwise it stays claimed until staleBefore elapses). + ClaimBatch(ctx context.Context, taskType, scope, scopeID string, limit int, staleBefore time.Time) ([]*types.TaskPendingOp, error) + + // ReleaseByIDs clears claimed_at (back to NULL) for the given rows so + // a claimed-but-not-consumed row becomes immediately eligible for the + // next ClaimBatch. No-op for empty input. Used by the wiki retry path + // to re-queue a transiently-failed op without waiting for its claim to + // go stale. Harmless on rows that were never claimed (Lite mode). + ReleaseByIDs(ctx context.Context, ids []int64) error + // DeleteByIDs removes the given rows. No-op for empty input. Used // to consume a successfully-processed batch, and to drop ops that // have been moved to task_dead_letters. diff --git a/internal/types/task.go b/internal/types/task.go index c01fb530..efe3632c 100644 --- a/internal/types/task.go +++ b/internal/types/task.go @@ -1,8 +1,8 @@ package types -// Asynq queue names. MUST stay in sync with the Queues weight map in -// router.NewAsynqServer — a task enqueued to a queue that the server does not -// list will never be consumed. +// Asynq queue names. MUST stay in sync with the Queues weight maps in +// router.NewParseAsynqServer / router.NewWikiAsynqServer — a task enqueued to a +// queue that no server subscribes to will never be consumed. const ( QueueCritical = "critical" QueueDefault = "default" @@ -21,6 +21,16 @@ const ( // question batches from starving the lightweight tasks in the low queue // (summary, deletes, wiki ingest). QueueQuestion = "question" + // QueueWiki is the dedicated lane for the Wiki ingest pipeline. Unlike the + // other queues above (which all share a single asynq worker pool), this + // queue is consumed by a SEPARATE asynq server (router.NewWikiAsynqServer) + // with its own concurrency budget. Hard capacity isolation: heavy document + // parsing in the upstream pool can never starve wiki generation, and a + // wiki-generation burst can never starve user-facing parsing. Sizing the + // two pools independently replaces the old weighted-lottery approach where + // wiki sat on the low queue (weight 1) and got ~1/13 of the schedule + // during upload storms. + QueueWiki = "wiki" ) const ( @@ -41,6 +51,7 @@ const ( TypeManualProcess = "manual:process" // 手工知识更新任务(cleanup + 重新索引) TypeDataSourceSync = "datasource:sync" // 数据源同步任务 TypeWikiIngest = "wiki:ingest" // Wiki 页面同步任务 + TypeWikiFinalize = "wiki:finalize" // Wiki KB 级收尾任务(防抖:索引重建/死链清理/交叉链接) ) // ExtractChunkPayload represents the extract chunk task payload diff --git a/internal/types/wiki_page.go b/internal/types/wiki_page.go index 7f0ac6c1..90943b24 100644 --- a/internal/types/wiki_page.go +++ b/internal/types/wiki_page.go @@ -348,24 +348,43 @@ type WikiConfig struct { // per document. Empty / unknown value is treated as WikiExtractionStandard. ExtractionGranularity WikiExtractionGranularity `yaml:"extraction_granularity" json:"extraction_granularity,omitempty"` - // IngestBatchSize controls how many pending ops a single batch - // processes before scheduling a follow-up. 0 falls back to the - // hard-coded default (5). Operators on large KBs (4w+ docs) can - // raise this to 10–20 to amortize the lock-acquire / index-rebuild - // overhead across more documents per round. + // Wiki ingest concurrency is two-level: + // 1. batch-level: multiple batches per KB run concurrently in the wiki + // worker pool, capped by IngestMaxInflight (below). + // 2. batch-internal: within one batch, Map and Reduce fan out with + // errgroups sized by IngestMapParallel / IngestReduceParallel. + // Effective peak in-flight LLM calls for one KB ≈ + // IngestMaxInflight × max(IngestMapParallel, IngestReduceParallel), + // further bounded by the wiki pool size (WEKNORA_WIKI_ASYNQ_CONCURRENCY). + + // IngestBatchSize controls how many pending ops a single batch claims and + // processes before scheduling a follow-up. 0 falls back to the hard-coded + // default (5). Larger batches amortize per-batch setup and let more docs + // share the batch-internal Map/Reduce fan-out; smaller batches spread a + // KB's backlog across more concurrent batches (finer scheduling grain). IngestBatchSize int `yaml:"ingest_batch_size" json:"ingest_batch_size,omitempty"` // IngestMapParallel sets the errgroup limit for the Map phase - // (per-document extraction + summary + chunk citation). 0 falls - // back to 10. Bound by the LLM provider's concurrency limit and - // the worker's outbound HTTP pool. + // (per-document extraction + summary + chunk citation) WITHIN one batch. + // 0 falls back to 10. Bound by the LLM provider's concurrency limit and + // the worker's outbound HTTP pool. Remember it multiplies with the number + // of concurrent batches (IngestMaxInflight). IngestMapParallel int `yaml:"ingest_map_parallel" json:"ingest_map_parallel,omitempty"` // IngestReduceParallel sets the errgroup limit for the Reduce phase - // (per-slug page write). 0 falls back to 10. Bound by the same - // LLM concurrency / HTTP pool considerations as the Map phase, - // plus DB connection pool size. + // (per-slug page write) WITHIN one batch. 0 falls back to 10. Bound by the + // same LLM concurrency / HTTP pool considerations as the Map phase, plus + // DB connection pool size. Same multiplier caveat as IngestMapParallel. IngestReduceParallel int `yaml:"ingest_reduce_parallel" json:"ingest_reduce_parallel,omitempty"` + + // IngestMaxInflight caps how many ingest batches for THIS KB may run + // concurrently in the shared wiki worker pool (standard/Redis mode + // only). 0 falls back to the hard-coded default (4). Since Phase 3 + // removed the exclusive per-KB lock, one KB's backlog could otherwise + // monopolize the whole pool during a bulk import and starve other KBs; + // this knob trades a single KB's peak throughput for cross-KB fairness. + // Set it >= the wiki pool size to effectively disable the cap. + IngestMaxInflight int `yaml:"ingest_max_inflight" json:"ingest_max_inflight,omitempty"` } // IngestBatchSizeOrDefault returns IngestBatchSize when set (> 0), @@ -396,6 +415,15 @@ func (c *WikiConfig) IngestReduceParallelOrDefault(fallback int) int { return c.IngestReduceParallel } +// IngestMaxInflightOrDefault returns IngestMaxInflight when set, +// otherwise the hard-coded fallback. +func (c *WikiConfig) IngestMaxInflightOrDefault(fallback int) int { + if c == nil || c.IngestMaxInflight <= 0 { + return fallback + } + return c.IngestMaxInflight +} + // Value implements the driver.Valuer interface func (c WikiConfig) Value() (driver.Value, error) { return json.Marshal(c)