fix(knowledge): prevent documents from getting stuck in "processing"

Several failure modes left Knowledge.parse_status pinned at "processing"
forever, with no signal to users beyond a permanent spinner. This commit
addresses the root causes and adds a safety net.

- Asynq worker pool: explicit Concurrency (default 16, env-tunable via
  WEKNORA_ASYNQ_CONCURRENCY) so batch uploads don't queue behind a
  CPU-count-sized worker pool. Redis op timeouts raised to 500ms/1000ms
  (WEKNORA_REDIS_OP_TIMEOUT_MS) to absorb bursty multimodal counter ops.

- DocReader RPC: cap each call with WEKNORA_DOCREADER_CALL_TIMEOUT
  (default 30m). Without this, a hung docreader pinned a worker for the
  full DocumentProcessTimeout window.

- ImageMultimodal: finalize-on-last-attempt semantics. A permanently
  failing single image no longer strands the parent — the asynq retry
  is allowed to run, but on the final attempt we count the image
  regardless of outcome. Redis DECR errors fall back to enqueuing the
  post-process task instead of returning silently.

- Dead-letter callback: when DocumentProcess / KnowledgePostProcess /
  ManualProcess exhausts retries, immediately mark the corresponding
  Knowledge as failed with the last error. This surfaces the failure
  in the UI without waiting for the housekeeping sweep.

- HousekeepingService: 5-minute cron that flips knowledge rows stuck
  in "processing" past DocumentProcessTimeout + 10m to failed, plus
  summary rows stuck > 1h. Catches anything the other safety nets
  miss (worker SIGKILL mid-handler, etc.). Disable with
  WEKNORA_HOUSEKEEPING_ENABLED=false.

- Distributed startup recovery: previously the post-restart sweep was
  skipped whenever REDIS_ADDR was set, even though Asynq does not
  reschedule the task that was actively running on the dead instance.
  Now the sweep runs in distributed mode too, but only against rows
  older than 30 minutes to avoid racing peer instances.
This commit is contained in:
wizardchen
2026-05-27 13:17:08 +08:00
committed by lyingbug
parent ae4ec0cf06
commit 3ae3ea97c5
8 changed files with 536 additions and 65 deletions
@@ -2340,7 +2340,7 @@ func (s *knowledgeService) convert(
req.FileType = fileType
}
result, err := reader.Read(ctx, req)
result, err := s.callDocReaderWithTimeout(ctx, reader, req)
if err != nil {
return s.failKnowledge(ctx, knowledge, isLastRetry, "document read failed: %v", err)
}
@@ -2356,7 +2356,42 @@ func (s *knowledgeService) convert(
return result, nil
}
// resolveDocReader returns the appropriate DocReader for the given engine.
// callDocReaderWithTimeout wraps the DocReader RPC in a child context whose
// deadline is min(parent_deadline, DocReaderCallTimeout). Without this cap,
// a hung docreader (network partition, GC pause, OCR runaway) silently
// burns the whole DocumentProcessTimeout budget and pins a worker for hours
// — the #1 cause of "knowledge stuck in processing" reports.
//
// On timeout we annotate the error so retries / dead-letter consumers can
// distinguish "docreader was slow" from "docreader returned an error".
func (s *knowledgeService) callDocReaderWithTimeout(
ctx context.Context, reader interfaces.DocReader, req *types.ReadRequest,
) (*types.ReadResult, error) {
timeout := 30 * time.Minute
if s.config != nil && s.config.KnowledgeBase != nil && s.config.KnowledgeBase.DocReaderCallTimeout > 0 {
timeout = s.config.KnowledgeBase.DocReaderCallTimeout
}
callCtx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
start := time.Now()
result, err := reader.Read(callCtx, req)
elapsed := time.Since(start)
if err != nil {
// Promote DeadlineExceeded into a clearer message; retain underlying
// error via %w so errors.Is(callCtx.Err(), context.DeadlineExceeded)
// still works for upstream classification.
if errors.Is(callCtx.Err(), context.DeadlineExceeded) && !errors.Is(ctx.Err(), context.DeadlineExceeded) {
logger.Errorf(ctx, "[convert] docreader call timed out after %s (limit %s) for %q",
elapsed, timeout, req.FileName)
return nil, fmt.Errorf("docreader call timeout after %s: %w", timeout, err)
}
return nil, err
}
logger.Infof(ctx, "[convert] docreader call ok in %s for %q", elapsed, req.FileName)
return result, nil
}
// Returns nil when the required service is unavailable.
func (s *knowledgeService) resolveDocReader(ctx context.Context, engine, fileType string, isURL bool, overrides map[string]string) interfaces.DocReader {
switch engine {