fix(wiki): Lite ingest lock, failed-op requeue, sync task retry parity

- Add per-KB sync.Map lock in Lite mode to mirror Redis SetNX concurrency.
- Requeue failed wiki ops in Lite by enqueueing asynq tasks with delay/tracing.
- Teach SyncTaskExecutor to honor ProcessIn delay and MaxRetry like Redis asynq.
This commit is contained in:
wizardchen
2026-04-28 18:08:20 +08:00
committed by lyingbug
parent 48c032ccbe
commit 1f03741462
3 changed files with 104 additions and 22 deletions
+45 -14
View File
@@ -7,6 +7,7 @@ import (
"fmt"
"sort"
"strings"
"sync"
"text/template"
"time"
@@ -139,6 +140,9 @@ type wikiIngestService struct {
modelService interfaces.ModelService
task interfaces.TaskEnqueuer
redisClient *redis.Client // nil in Lite mode (no Redis)
// liteLocks provides per-KB mutual exclusion in Lite mode (no Redis).
// Keys are kbID strings; values are unused (presence = locked).
liteLocks sync.Map
}
// NewWikiIngestService creates a new wiki ingest service
@@ -336,25 +340,52 @@ func (s *wikiIngestService) trimPendingList(ctx context.Context, kbID string, co
}
}
// requeueFailedOps appends failed operations back to the pending list so they
// are retried in the next follow-up batch. Called after trimPendingList has
// already removed the consumed batch head.
func (s *wikiIngestService) requeueFailedOps(ctx context.Context, kbID string, ops []WikiPendingOp) {
if s.redisClient == nil {
// requeueFailedOps re-enqueues failed operations for retry.
//
// Redis mode: appends ops back to the pending list tail so the next follow-up
// batch picks them up.
//
// Lite mode (no Redis): enqueues a new asynq task per failed op with a short
// delay, since there is no shared pending list to append to.
func (s *wikiIngestService) requeueFailedOps(ctx context.Context, payload WikiIngestPayload, ops []WikiPendingOp) {
if s.redisClient != nil {
pendingKey := wikiPendingKeyPrefix + payload.KnowledgeBaseID
for _, op := range ops {
data, err := json.Marshal(op)
if err != nil {
logger.Warnf(ctx, "wiki ingest: failed to marshal op for requeue: %v", err)
continue
}
if err := s.redisClient.RPush(ctx, pendingKey, string(data)).Err(); err != nil {
logger.Warnf(ctx, "wiki ingest: failed to requeue op %s: %v", op.KnowledgeID, err)
continue
}
logger.Infof(ctx, "wiki ingest: re-queued failed op %s (%s) for retry", op.KnowledgeID, op.DocTitle)
}
return
}
pendingKey := wikiPendingKeyPrefix + kbID
// Lite mode: re-enqueue each failed op as a new asynq task.
for _, op := range ops {
data, err := json.Marshal(op)
if err != nil {
logger.Warnf(ctx, "wiki ingest: failed to marshal op for requeue: %v", err)
retryPayload := WikiIngestPayload{
TenantID: payload.TenantID,
KnowledgeBaseID: payload.KnowledgeBaseID,
Language: op.Language,
LiteOps: []WikiPendingOp{op},
}
langfuse.InjectTracing(ctx, &retryPayload)
payloadBytes, _ := json.Marshal(retryPayload)
t := asynq.NewTask(types.TypeWikiIngest, payloadBytes,
asynq.Queue("low"),
asynq.MaxRetry(10),
asynq.Timeout(60*time.Minute),
asynq.ProcessIn(wikiIngestDelay),
)
if _, err := s.task.Enqueue(t); err != nil {
logger.Warnf(ctx, "wiki ingest: failed to requeue lite op %s: %v", op.KnowledgeID, err)
continue
}
if err := s.redisClient.RPush(ctx, pendingKey, string(data)).Err(); err != nil {
logger.Warnf(ctx, "wiki ingest: failed to requeue op %s: %v", op.KnowledgeID, err)
continue
}
logger.Infof(ctx, "wiki ingest: re-queued failed op %s (%s) for retry", op.KnowledgeID, op.DocTitle)
logger.Infof(ctx, "wiki ingest: re-queued failed lite op %s (%s) for retry", op.KnowledgeID, op.DocTitle)
}
}
@@ -142,6 +142,14 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
}()
} else {
mode = "lite"
// In-process mutual exclusion: mirrors the Redis SetNX lock above.
if _, loaded := s.liteLocks.LoadOrStore(payload.KnowledgeBaseID, struct{}{}); loaded {
exitStatus = "active_lock_conflict"
logger.Infof(ctx, "wiki ingest: another batch active for KB %s (lite lock), deferring to asynq retry", payload.KnowledgeBaseID)
return ErrWikiIngestConcurrent
}
lockAcquired = true
defer s.liteLocks.Delete(payload.KnowledgeBaseID)
}
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
@@ -419,7 +427,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
// This must happen after trim: trim removes the consumed batch head, then
// requeue appends the failed items to the tail for a future attempt.
if len(failedOps) > 0 {
s.requeueFailedOps(ctx, payload.KnowledgeBaseID, failedOps)
s.requeueFailedOps(ctx, payload, failedOps)
}
logger.Infof(ctx, "wiki ingest: batch completed for KB %s, %d ops, %d pages affected", payload.KnowledgeBaseID, len(pendingOps), len(allPagesAffected))
+50 -7
View File
@@ -36,7 +36,8 @@ func (e *SyncTaskExecutor) RegisterHandler(pattern string, handler func(context.
// Enqueue satisfies interfaces.TaskEnqueuer.
// Instead of queuing to Redis, it dispatches the task to a goroutine.
func (e *SyncTaskExecutor) Enqueue(task *asynq.Task, _ ...asynq.Option) (*asynq.TaskInfo, error) {
// Supports ProcessIn (delay) and MaxRetry options for parity with asynq.
func (e *SyncTaskExecutor) Enqueue(task *asynq.Task, opts ...asynq.Option) (*asynq.TaskInfo, error) {
e.mu.RLock()
handler, ok := e.handlers[task.Type()]
e.mu.RUnlock()
@@ -45,6 +46,28 @@ func (e *SyncTaskExecutor) Enqueue(task *asynq.Task, _ ...asynq.Option) (*asynq.
return nil, fmt.Errorf("sync task executor: no handler registered for type %q", task.Type())
}
var delay time.Duration
maxRetry := 25 // asynq default
maxRetrySet := false
for _, opt := range opts {
switch opt.Type() {
case asynq.ProcessInOpt:
if d, ok := opt.Value().(time.Duration); ok {
delay = d
}
case asynq.MaxRetryOpt:
if n, ok := opt.Value().(int); ok {
maxRetry = n
maxRetrySet = true
}
}
}
// Callers that explicitly pass MaxRetry(0) want no retries.
// Without the flag we can't distinguish "not set" from "set to 0".
if maxRetrySet && maxRetry < 0 {
maxRetry = 0
}
taskID := uuid.New().String()
info := &asynq.TaskInfo{
ID: taskID,
@@ -53,16 +76,36 @@ func (e *SyncTaskExecutor) Enqueue(task *asynq.Task, _ ...asynq.Option) (*asynq.
}
go func() {
if delay > 0 {
time.Sleep(delay)
}
ctx := context.Background()
start := time.Now()
logger.Infof(ctx, "[SyncTask] Executing task type=%s id=%s", task.Type(), taskID)
if err := handler(ctx, task); err != nil {
logger.Errorf(ctx, "[SyncTask] Task failed type=%s id=%s elapsed=%v err=%v",
task.Type(), taskID, time.Since(start), err)
} else {
logger.Infof(ctx, "[SyncTask] Task completed type=%s id=%s elapsed=%v",
task.Type(), taskID, time.Since(start))
var lastErr error
for attempt := 0; attempt <= maxRetry; attempt++ {
if attempt > 0 {
backoff := time.Duration(attempt) * 5 * time.Second
if backoff > 30*time.Second {
backoff = 30 * time.Second
}
logger.Infof(ctx, "[SyncTask] Retrying task type=%s id=%s attempt=%d/%d backoff=%s",
task.Type(), taskID, attempt, maxRetry, backoff)
time.Sleep(backoff)
}
lastErr = handler(ctx, task)
if lastErr == nil {
logger.Infof(ctx, "[SyncTask] Task completed type=%s id=%s elapsed=%v",
task.Type(), taskID, time.Since(start))
return
}
}
logger.Errorf(ctx, "[SyncTask] Task failed (exhausted retries) type=%s id=%s elapsed=%v err=%v",
task.Type(), taskID, time.Since(start), lastErr)
}()
return info, nil