fix(knowledge): prevent documents from getting stuck in "processing"
Several failure modes left Knowledge.parse_status pinned at "processing" forever, with no signal to users beyond a permanent spinner. This commit addresses the root causes and adds a safety net. - Asynq worker pool: explicit Concurrency (default 16, env-tunable via WEKNORA_ASYNQ_CONCURRENCY) so batch uploads don't queue behind a CPU-count-sized worker pool. Redis op timeouts raised to 500ms/1000ms (WEKNORA_REDIS_OP_TIMEOUT_MS) to absorb bursty multimodal counter ops. - DocReader RPC: cap each call with WEKNORA_DOCREADER_CALL_TIMEOUT (default 30m). Without this, a hung docreader pinned a worker for the full DocumentProcessTimeout window. - ImageMultimodal: finalize-on-last-attempt semantics. A permanently failing single image no longer strands the parent — the asynq retry is allowed to run, but on the final attempt we count the image regardless of outcome. Redis DECR errors fall back to enqueuing the post-process task instead of returning silently. - Dead-letter callback: when DocumentProcess / KnowledgePostProcess / ManualProcess exhausts retries, immediately mark the corresponding Knowledge as failed with the last error. This surfaces the failure in the UI without waiting for the housekeeping sweep. - HousekeepingService: 5-minute cron that flips knowledge rows stuck in "processing" past DocumentProcessTimeout + 10m to failed, plus summary rows stuck > 1h. Catches anything the other safety nets miss (worker SIGKILL mid-handler, etc.). Disable with WEKNORA_HOUSEKEEPING_ENABLED=false. - Distributed startup recovery: previously the post-restart sweep was skipped whenever REDIS_ADDR was set, even though Asynq does not reschedule the task that was actively running on the dead instance. Now the sweep runs in distributed mode too, but only against rows older than 30 minutes to avoid racing peer instances.
This commit is contained in:
@@ -2340,7 +2340,7 @@ func (s *knowledgeService) convert(
|
||||
req.FileType = fileType
|
||||
}
|
||||
|
||||
result, err := reader.Read(ctx, req)
|
||||
result, err := s.callDocReaderWithTimeout(ctx, reader, req)
|
||||
if err != nil {
|
||||
return s.failKnowledge(ctx, knowledge, isLastRetry, "document read failed: %v", err)
|
||||
}
|
||||
@@ -2356,7 +2356,42 @@ func (s *knowledgeService) convert(
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// resolveDocReader returns the appropriate DocReader for the given engine.
|
||||
// callDocReaderWithTimeout wraps the DocReader RPC in a child context whose
|
||||
// deadline is min(parent_deadline, DocReaderCallTimeout). Without this cap,
|
||||
// a hung docreader (network partition, GC pause, OCR runaway) silently
|
||||
// burns the whole DocumentProcessTimeout budget and pins a worker for hours
|
||||
// — the #1 cause of "knowledge stuck in processing" reports.
|
||||
//
|
||||
// On timeout we annotate the error so retries / dead-letter consumers can
|
||||
// distinguish "docreader was slow" from "docreader returned an error".
|
||||
func (s *knowledgeService) callDocReaderWithTimeout(
|
||||
ctx context.Context, reader interfaces.DocReader, req *types.ReadRequest,
|
||||
) (*types.ReadResult, error) {
|
||||
timeout := 30 * time.Minute
|
||||
if s.config != nil && s.config.KnowledgeBase != nil && s.config.KnowledgeBase.DocReaderCallTimeout > 0 {
|
||||
timeout = s.config.KnowledgeBase.DocReaderCallTimeout
|
||||
}
|
||||
callCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
start := time.Now()
|
||||
result, err := reader.Read(callCtx, req)
|
||||
elapsed := time.Since(start)
|
||||
if err != nil {
|
||||
// Promote DeadlineExceeded into a clearer message; retain underlying
|
||||
// error via %w so errors.Is(callCtx.Err(), context.DeadlineExceeded)
|
||||
// still works for upstream classification.
|
||||
if errors.Is(callCtx.Err(), context.DeadlineExceeded) && !errors.Is(ctx.Err(), context.DeadlineExceeded) {
|
||||
logger.Errorf(ctx, "[convert] docreader call timed out after %s (limit %s) for %q",
|
||||
elapsed, timeout, req.FileName)
|
||||
return nil, fmt.Errorf("docreader call timeout after %s: %w", timeout, err)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
logger.Infof(ctx, "[convert] docreader call ok in %s for %q", elapsed, req.FileName)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// Returns nil when the required service is unavailable.
|
||||
func (s *knowledgeService) resolveDocReader(ctx context.Context, engine, fileType string, isURL bool, overrides map[string]string) interfaces.DocReader {
|
||||
switch engine {
|
||||
|
||||
Reference in New Issue
Block a user