fix(wiki): Lite ingest lock, failed-op requeue, sync task retry parity
- Add per-KB sync.Map lock in Lite mode to mirror Redis SetNX concurrency. - Requeue failed wiki ops in Lite by enqueueing asynq tasks with delay/tracing. - Teach SyncTaskExecutor to honor ProcessIn delay and MaxRetry like Redis asynq.
This commit is contained in:
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
@@ -139,6 +140,9 @@ type wikiIngestService struct {
|
||||
modelService interfaces.ModelService
|
||||
task interfaces.TaskEnqueuer
|
||||
redisClient *redis.Client // nil in Lite mode (no Redis)
|
||||
// liteLocks provides per-KB mutual exclusion in Lite mode (no Redis).
|
||||
// Keys are kbID strings; values are unused (presence = locked).
|
||||
liteLocks sync.Map
|
||||
}
|
||||
|
||||
// NewWikiIngestService creates a new wiki ingest service
|
||||
@@ -336,25 +340,52 @@ func (s *wikiIngestService) trimPendingList(ctx context.Context, kbID string, co
|
||||
}
|
||||
}
|
||||
|
||||
// requeueFailedOps appends failed operations back to the pending list so they
|
||||
// are retried in the next follow-up batch. Called after trimPendingList has
|
||||
// already removed the consumed batch head.
|
||||
func (s *wikiIngestService) requeueFailedOps(ctx context.Context, kbID string, ops []WikiPendingOp) {
|
||||
if s.redisClient == nil {
|
||||
// requeueFailedOps re-enqueues failed operations for retry.
|
||||
//
|
||||
// Redis mode: appends ops back to the pending list tail so the next follow-up
|
||||
// batch picks them up.
|
||||
//
|
||||
// Lite mode (no Redis): enqueues a new asynq task per failed op with a short
|
||||
// delay, since there is no shared pending list to append to.
|
||||
func (s *wikiIngestService) requeueFailedOps(ctx context.Context, payload WikiIngestPayload, ops []WikiPendingOp) {
|
||||
if s.redisClient != nil {
|
||||
pendingKey := wikiPendingKeyPrefix + payload.KnowledgeBaseID
|
||||
for _, op := range ops {
|
||||
data, err := json.Marshal(op)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: failed to marshal op for requeue: %v", err)
|
||||
continue
|
||||
}
|
||||
if err := s.redisClient.RPush(ctx, pendingKey, string(data)).Err(); err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: failed to requeue op %s: %v", op.KnowledgeID, err)
|
||||
continue
|
||||
}
|
||||
logger.Infof(ctx, "wiki ingest: re-queued failed op %s (%s) for retry", op.KnowledgeID, op.DocTitle)
|
||||
}
|
||||
return
|
||||
}
|
||||
pendingKey := wikiPendingKeyPrefix + kbID
|
||||
|
||||
// Lite mode: re-enqueue each failed op as a new asynq task.
|
||||
for _, op := range ops {
|
||||
data, err := json.Marshal(op)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: failed to marshal op for requeue: %v", err)
|
||||
retryPayload := WikiIngestPayload{
|
||||
TenantID: payload.TenantID,
|
||||
KnowledgeBaseID: payload.KnowledgeBaseID,
|
||||
Language: op.Language,
|
||||
LiteOps: []WikiPendingOp{op},
|
||||
}
|
||||
langfuse.InjectTracing(ctx, &retryPayload)
|
||||
payloadBytes, _ := json.Marshal(retryPayload)
|
||||
t := asynq.NewTask(types.TypeWikiIngest, payloadBytes,
|
||||
asynq.Queue("low"),
|
||||
asynq.MaxRetry(10),
|
||||
asynq.Timeout(60*time.Minute),
|
||||
asynq.ProcessIn(wikiIngestDelay),
|
||||
)
|
||||
if _, err := s.task.Enqueue(t); err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: failed to requeue lite op %s: %v", op.KnowledgeID, err)
|
||||
continue
|
||||
}
|
||||
if err := s.redisClient.RPush(ctx, pendingKey, string(data)).Err(); err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: failed to requeue op %s: %v", op.KnowledgeID, err)
|
||||
continue
|
||||
}
|
||||
logger.Infof(ctx, "wiki ingest: re-queued failed op %s (%s) for retry", op.KnowledgeID, op.DocTitle)
|
||||
logger.Infof(ctx, "wiki ingest: re-queued failed lite op %s (%s) for retry", op.KnowledgeID, op.DocTitle)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -142,6 +142,14 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
}()
|
||||
} else {
|
||||
mode = "lite"
|
||||
// In-process mutual exclusion: mirrors the Redis SetNX lock above.
|
||||
if _, loaded := s.liteLocks.LoadOrStore(payload.KnowledgeBaseID, struct{}{}); loaded {
|
||||
exitStatus = "active_lock_conflict"
|
||||
logger.Infof(ctx, "wiki ingest: another batch active for KB %s (lite lock), deferring to asynq retry", payload.KnowledgeBaseID)
|
||||
return ErrWikiIngestConcurrent
|
||||
}
|
||||
lockAcquired = true
|
||||
defer s.liteLocks.Delete(payload.KnowledgeBaseID)
|
||||
}
|
||||
|
||||
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
|
||||
@@ -419,7 +427,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
// This must happen after trim: trim removes the consumed batch head, then
|
||||
// requeue appends the failed items to the tail for a future attempt.
|
||||
if len(failedOps) > 0 {
|
||||
s.requeueFailedOps(ctx, payload.KnowledgeBaseID, failedOps)
|
||||
s.requeueFailedOps(ctx, payload, failedOps)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "wiki ingest: batch completed for KB %s, %d ops, %d pages affected", payload.KnowledgeBaseID, len(pendingOps), len(allPagesAffected))
|
||||
|
||||
@@ -36,7 +36,8 @@ func (e *SyncTaskExecutor) RegisterHandler(pattern string, handler func(context.
|
||||
|
||||
// Enqueue satisfies interfaces.TaskEnqueuer.
|
||||
// Instead of queuing to Redis, it dispatches the task to a goroutine.
|
||||
func (e *SyncTaskExecutor) Enqueue(task *asynq.Task, _ ...asynq.Option) (*asynq.TaskInfo, error) {
|
||||
// Supports ProcessIn (delay) and MaxRetry options for parity with asynq.
|
||||
func (e *SyncTaskExecutor) Enqueue(task *asynq.Task, opts ...asynq.Option) (*asynq.TaskInfo, error) {
|
||||
e.mu.RLock()
|
||||
handler, ok := e.handlers[task.Type()]
|
||||
e.mu.RUnlock()
|
||||
@@ -45,6 +46,28 @@ func (e *SyncTaskExecutor) Enqueue(task *asynq.Task, _ ...asynq.Option) (*asynq.
|
||||
return nil, fmt.Errorf("sync task executor: no handler registered for type %q", task.Type())
|
||||
}
|
||||
|
||||
var delay time.Duration
|
||||
maxRetry := 25 // asynq default
|
||||
maxRetrySet := false
|
||||
for _, opt := range opts {
|
||||
switch opt.Type() {
|
||||
case asynq.ProcessInOpt:
|
||||
if d, ok := opt.Value().(time.Duration); ok {
|
||||
delay = d
|
||||
}
|
||||
case asynq.MaxRetryOpt:
|
||||
if n, ok := opt.Value().(int); ok {
|
||||
maxRetry = n
|
||||
maxRetrySet = true
|
||||
}
|
||||
}
|
||||
}
|
||||
// Callers that explicitly pass MaxRetry(0) want no retries.
|
||||
// Without the flag we can't distinguish "not set" from "set to 0".
|
||||
if maxRetrySet && maxRetry < 0 {
|
||||
maxRetry = 0
|
||||
}
|
||||
|
||||
taskID := uuid.New().String()
|
||||
info := &asynq.TaskInfo{
|
||||
ID: taskID,
|
||||
@@ -53,16 +76,36 @@ func (e *SyncTaskExecutor) Enqueue(task *asynq.Task, _ ...asynq.Option) (*asynq.
|
||||
}
|
||||
|
||||
go func() {
|
||||
if delay > 0 {
|
||||
time.Sleep(delay)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
start := time.Now()
|
||||
logger.Infof(ctx, "[SyncTask] Executing task type=%s id=%s", task.Type(), taskID)
|
||||
if err := handler(ctx, task); err != nil {
|
||||
logger.Errorf(ctx, "[SyncTask] Task failed type=%s id=%s elapsed=%v err=%v",
|
||||
task.Type(), taskID, time.Since(start), err)
|
||||
} else {
|
||||
logger.Infof(ctx, "[SyncTask] Task completed type=%s id=%s elapsed=%v",
|
||||
task.Type(), taskID, time.Since(start))
|
||||
|
||||
var lastErr error
|
||||
for attempt := 0; attempt <= maxRetry; attempt++ {
|
||||
if attempt > 0 {
|
||||
backoff := time.Duration(attempt) * 5 * time.Second
|
||||
if backoff > 30*time.Second {
|
||||
backoff = 30 * time.Second
|
||||
}
|
||||
logger.Infof(ctx, "[SyncTask] Retrying task type=%s id=%s attempt=%d/%d backoff=%s",
|
||||
task.Type(), taskID, attempt, maxRetry, backoff)
|
||||
time.Sleep(backoff)
|
||||
}
|
||||
|
||||
lastErr = handler(ctx, task)
|
||||
if lastErr == nil {
|
||||
logger.Infof(ctx, "[SyncTask] Task completed type=%s id=%s elapsed=%v",
|
||||
task.Type(), taskID, time.Since(start))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
logger.Errorf(ctx, "[SyncTask] Task failed (exhausted retries) type=%s id=%s elapsed=%v err=%v",
|
||||
task.Type(), taskID, time.Since(start), lastErr)
|
||||
}()
|
||||
|
||||
return info, nil
|
||||
|
||||
Reference in New Issue
Block a user