fix(knowledge): prevent documents from getting stuck in "processing"
Several failure modes left Knowledge.parse_status pinned at "processing" forever, with no signal to users beyond a permanent spinner. This commit addresses the root causes and adds a safety net. - Asynq worker pool: explicit Concurrency (default 16, env-tunable via WEKNORA_ASYNQ_CONCURRENCY) so batch uploads don't queue behind a CPU-count-sized worker pool. Redis op timeouts raised to 500ms/1000ms (WEKNORA_REDIS_OP_TIMEOUT_MS) to absorb bursty multimodal counter ops. - DocReader RPC: cap each call with WEKNORA_DOCREADER_CALL_TIMEOUT (default 30m). Without this, a hung docreader pinned a worker for the full DocumentProcessTimeout window. - ImageMultimodal: finalize-on-last-attempt semantics. A permanently failing single image no longer strands the parent — the asynq retry is allowed to run, but on the final attempt we count the image regardless of outcome. Redis DECR errors fall back to enqueuing the post-process task instead of returning silently. - Dead-letter callback: when DocumentProcess / KnowledgePostProcess / ManualProcess exhausts retries, immediately mark the corresponding Knowledge as failed with the last error. This surfaces the failure in the UI without waiting for the housekeeping sweep. - HousekeepingService: 5-minute cron that flips knowledge rows stuck in "processing" past DocumentProcessTimeout + 10m to failed, plus summary rows stuck > 1h. Catches anything the other safety nets miss (worker SIGKILL mid-handler, etc.). Disable with WEKNORA_HOUSEKEEPING_ENABLED=false. - Distributed startup recovery: previously the post-restart sweep was skipped whenever REDIS_ADDR was set, even though Asynq does not reschedule the task that was actively running on the dead instance. Now the sweep runs in distributed mode too, but only against rows older than 30 minutes to avoid racing peer instances.
This commit is contained in:
@@ -624,3 +624,23 @@ DOCREADER_TRANSPORT=grpc
|
||||
|
||||
# Document processing task timeout. Large files may need more than Asynq's default 30m.
|
||||
# WEKNORA_DOCUMENT_PROCESS_TIMEOUT=2h
|
||||
|
||||
# --- Async pipeline tuning (optional) -----------------------------------------
|
||||
# Worker pool size for the asynq server. Default 16 — raise it on machines
|
||||
# that handle many concurrent uploads (default Go runtime.NumCPU() under-
|
||||
# provisions for the I/O-bound document pipeline).
|
||||
# WEKNORA_ASYNQ_CONCURRENCY=16
|
||||
|
||||
# Read/write timeout (in milliseconds) the asynq client uses against Redis.
|
||||
# Default 500ms (writes scale to 1000ms). Bump if you see "i/o timeout"
|
||||
# errors during heavy multimodal counter DECRs or large batch uploads.
|
||||
# WEKNORA_REDIS_OP_TIMEOUT_MS=500
|
||||
|
||||
# Maximum wall time for a single DocReader call. Caps a hung docreader
|
||||
# (network partition, OCR runaway) so it cannot pin a worker for hours.
|
||||
# Default 30m. Must be shorter than WEKNORA_DOCUMENT_PROCESS_TIMEOUT.
|
||||
# WEKNORA_DOCREADER_CALL_TIMEOUT=30m
|
||||
|
||||
# Background sweep that recovers knowledge stuck in "processing" past
|
||||
# DocumentProcessTimeout + 10m. Set to "false" to opt out (not recommended).
|
||||
# WEKNORA_HOUSEKEEPING_ENABLED=true
|
||||
|
||||
@@ -115,20 +115,36 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) e
|
||||
ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language)
|
||||
}
|
||||
|
||||
// finalize-once semantics: on success we always decrement the parent's
|
||||
// pending counter. On failure we only decrement when this is the last
|
||||
// asynq retry, so a permanently-failing single image cannot leave the
|
||||
// parent knowledge stuck in "processing" forever — which was the #1
|
||||
// cause of "stuck parsing" reports. Intermediate retries skip finalize
|
||||
// so we don't double-count and prematurely trigger post-process.
|
||||
var handleErr error
|
||||
defer func() {
|
||||
if handleErr == nil || isFinalAsynqAttempt(ctx) {
|
||||
s.checkAndFinalizeAllImages(ctx, payload)
|
||||
} else {
|
||||
logger.Infof(ctx,
|
||||
"[ImageMultimodal] Skip finalize on retryable error for %s (will count on last attempt)",
|
||||
payload.ImageURL)
|
||||
}
|
||||
}()
|
||||
|
||||
vlmModel, err := s.resolveVLM(ctx, payload.KnowledgeBaseID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolve VLM: %w", err)
|
||||
handleErr = fmt.Errorf("resolve VLM: %w", err)
|
||||
return handleErr
|
||||
}
|
||||
|
||||
// Read image bytes. A provider:// URL must be resolved via FileService —
|
||||
// it must NEVER be handed to the HTTP downloader (which would fail with
|
||||
// "unsupported URL scheme"). On unrecoverable read failure for a single
|
||||
// image, skip it and still trigger finalize so the parent knowledge
|
||||
// doesn't get stuck in "processing" forever (see issue #1282).
|
||||
// image, skip it (deferred finalize will count it).
|
||||
imgBytes, readErr := s.readImageBytes(ctx, payload)
|
||||
if readErr != nil {
|
||||
logger.Errorf(ctx, "[ImageMultimodal] Skip unreadable image %s: %v", payload.ImageURL, readErr)
|
||||
s.checkAndFinalizeAllImages(ctx, payload)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -203,13 +219,14 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) e
|
||||
}
|
||||
|
||||
if len(newChunks) == 0 {
|
||||
s.checkAndFinalizeAllImages(ctx, payload)
|
||||
// Deferred finalize will count this image on success.
|
||||
return nil
|
||||
}
|
||||
|
||||
// Persist chunks
|
||||
if err := s.chunkService.CreateChunks(ctx, newChunks); err != nil {
|
||||
return fmt.Errorf("create multimodal chunks: %w", err)
|
||||
handleErr = fmt.Errorf("create multimodal chunks: %w", err)
|
||||
return handleErr
|
||||
}
|
||||
for _, c := range newChunks {
|
||||
logger.Infof(ctx, "[ImageMultimodal] Created %s chunk %s for image %s, len=%d",
|
||||
@@ -225,11 +242,32 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) e
|
||||
// have real textual content (caption/OCR), we can generate questions.
|
||||
// Note: for documents with multiple images (e.g. PDFs), we also wait until
|
||||
// all images are processed before triggering summary/question generation.
|
||||
s.checkAndFinalizeAllImages(ctx, payload)
|
||||
|
||||
// Deferred finalize handles the parent knowledge counter.
|
||||
return nil
|
||||
}
|
||||
|
||||
// isFinalAsynqAttempt reports whether the current task context belongs to the
|
||||
// last retry attempt before asynq archives the task as a dead-letter. We use
|
||||
// this to flip multimodal finalize semantics: during normal retries we skip
|
||||
// counter decrement (the retry might still succeed), but on the final attempt
|
||||
// we count the image regardless of outcome so a permanently-failing image
|
||||
// cannot pin the parent knowledge in "processing" forever.
|
||||
//
|
||||
// Returns false when the values are unavailable (e.g. when the handler is
|
||||
// invoked outside an asynq worker, as in unit tests). Treating that case as
|
||||
// "not final" keeps test ergonomics — tests should drive finalize explicitly.
|
||||
func isFinalAsynqAttempt(ctx context.Context) bool {
|
||||
retried, ok := asynq.GetRetryCount(ctx)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
maxRetry, ok := asynq.GetMaxRetry(ctx)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
return retried >= maxRetry
|
||||
}
|
||||
|
||||
// indexChunks indexes the newly created multimodal chunks into the retrieval engine
|
||||
// so they can participate in semantic search.
|
||||
func (s *ImageMultimodalService) indexChunks(ctx context.Context, payload types.ImageMultimodalPayload, chunks []*types.Chunk) {
|
||||
@@ -433,7 +471,16 @@ func (s *ImageMultimodalService) checkAndFinalizeAllImages(ctx context.Context,
|
||||
|
||||
pendingCount, err := s.redisClient.Decr(ctx, redisKey).Result()
|
||||
if err != nil && err != redis.Nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Failed to decrement pending count for %s: %v", payload.KnowledgeID, err)
|
||||
// Redis hiccup must not strand the parent knowledge. Best-effort:
|
||||
// enqueue post-process anyway. KnowledgePostProcess is idempotent
|
||||
// (it transitions parse_status processing → completed under a row
|
||||
// guard), so a duplicate triggered by a sibling image is harmless.
|
||||
// The alternative — silently returning — is what produced the
|
||||
// "permanently stuck" reports we are fixing here.
|
||||
logger.Warnf(ctx,
|
||||
"[ImageMultimodal] Decrement failed for %s (%v); fallback-enqueueing post-process",
|
||||
payload.KnowledgeID, err)
|
||||
s.enqueueKnowledgePostProcessTask(ctx, payload)
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
// Package service: knowledge housekeeping.
|
||||
//
|
||||
// HousekeepingService periodically scans for knowledge rows that have been
|
||||
// stuck in "processing" longer than any reasonable execution window and
|
||||
// marks them as failed. This is the safety net that catches anything the
|
||||
// other defences (asynq retry, dead-letter callback, image_multimodal
|
||||
// finalize-on-last-attempt) miss — for example:
|
||||
//
|
||||
// - Worker process killed mid-handler before any defer could run.
|
||||
// - DocReader call genuinely exceeding DocReaderCallTimeout AND the
|
||||
// worker subsequently being lost before retry kicks in.
|
||||
// - Multimodal Redis counter set to N but ALL N image tasks failing in
|
||||
// ways that bypass finalize (extremely rare; defence-in-depth here).
|
||||
//
|
||||
// Without this sweep, a single unlucky failure mode can leave a knowledge
|
||||
// row in "processing" forever — invisible to users except as a permanent
|
||||
// spinner. With this sweep the worst-case latency from stall to user-
|
||||
// visible failure is bounded to ~1 stale-threshold + 1 sweep interval.
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/robfig/cron/v3"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// HousekeepingService runs background sweeps to recover stuck rows.
|
||||
type HousekeepingService struct {
|
||||
db *gorm.DB
|
||||
cfg *config.Config
|
||||
cron *cron.Cron
|
||||
|
||||
mu sync.Mutex
|
||||
started bool
|
||||
}
|
||||
|
||||
// NewHousekeepingService constructs a HousekeepingService. It does NOT start
|
||||
// the cron — call Start in the application bootstrap so a misconfigured
|
||||
// cron schedule cannot prevent the rest of the service from coming up.
|
||||
func NewHousekeepingService(db *gorm.DB, cfg *config.Config) *HousekeepingService {
|
||||
return &HousekeepingService{
|
||||
db: db,
|
||||
cfg: cfg,
|
||||
cron: cron.New(cron.WithSeconds(), cron.WithChain(
|
||||
cron.Recover(cron.DefaultLogger),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
// Start registers the sweep schedule and begins the background runner.
|
||||
// Idempotent — repeated calls are a no-op so wiring code can call Start
|
||||
// without coordinating ordering.
|
||||
func (h *HousekeepingService) Start(ctx context.Context) error {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if h.started {
|
||||
return nil
|
||||
}
|
||||
if !housekeepingEnabled() {
|
||||
logger.Infof(ctx, "[Housekeeping] disabled via WEKNORA_HOUSEKEEPING_ENABLED=false")
|
||||
return nil
|
||||
}
|
||||
// Every 5 minutes — frequent enough that user-visible recovery latency
|
||||
// is acceptable, infrequent enough that the SQL sweep is invisible to
|
||||
// query load even on large knowledge tables.
|
||||
if _, err := h.cron.AddFunc("0 */5 * * * *", func() {
|
||||
// Use Background so a cancelled bootstrap ctx doesn't stop sweeps.
|
||||
h.runSweep(context.Background())
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
h.cron.Start()
|
||||
h.started = true
|
||||
logger.Infof(ctx, "[Housekeeping] started with 5-minute sweep")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop halts the cron and waits for in-flight sweeps to finish.
|
||||
func (h *HousekeepingService) Stop() {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if !h.started {
|
||||
return
|
||||
}
|
||||
c := h.cron.Stop()
|
||||
<-c.Done()
|
||||
h.started = false
|
||||
}
|
||||
|
||||
// runSweep is exported on the type for testability — tests can drive a
|
||||
// single sweep without waiting for the cron tick.
|
||||
func (h *HousekeepingService) runSweep(ctx context.Context) {
|
||||
threshold := h.staleThreshold()
|
||||
cutoff := time.Now().Add(-threshold)
|
||||
|
||||
// Sweep A: knowledge stuck in "processing".
|
||||
resKnowledge := h.db.WithContext(ctx).Model(&types.Knowledge{}).
|
||||
Where("parse_status = ? AND updated_at < ?", types.ParseStatusProcessing, cutoff).
|
||||
Updates(map[string]interface{}{
|
||||
"parse_status": types.ParseStatusFailed,
|
||||
"error_message": "task stuck in processing > " + threshold.String() + ", recovered by housekeeping",
|
||||
})
|
||||
if resKnowledge.Error != nil {
|
||||
logger.Warnf(ctx, "[Housekeeping] knowledge sweep failed: %v", resKnowledge.Error)
|
||||
} else if resKnowledge.RowsAffected > 0 {
|
||||
logger.Infof(ctx, "[Housekeeping] recovered %d stuck knowledge rows (threshold=%s)",
|
||||
resKnowledge.RowsAffected, threshold)
|
||||
}
|
||||
|
||||
// Sweep B: knowledge summary stuck. Summary is post-parse; threshold
|
||||
// is shorter because summary tasks are bounded by a single LLM call.
|
||||
summaryCutoff := time.Now().Add(-1 * time.Hour)
|
||||
resSummary := h.db.WithContext(ctx).Model(&types.Knowledge{}).
|
||||
Where("summary_status = ? AND updated_at < ?", types.SummaryStatusProcessing, summaryCutoff).
|
||||
Update("summary_status", types.SummaryStatusFailed)
|
||||
if resSummary.Error != nil {
|
||||
logger.Warnf(ctx, "[Housekeeping] summary sweep failed: %v", resSummary.Error)
|
||||
} else if resSummary.RowsAffected > 0 {
|
||||
logger.Infof(ctx, "[Housekeeping] recovered %d stuck summary rows", resSummary.RowsAffected)
|
||||
}
|
||||
}
|
||||
|
||||
// staleThreshold returns how long a "processing" row may sit untouched
|
||||
// before housekeeping treats it as orphaned. The floor is 1 hour so that a
|
||||
// genuinely slow large-PDF parse cannot be killed mid-flight; the ceiling
|
||||
// scales with the operator-configured DocumentProcessTimeout plus 10 minute
|
||||
// buffer to absorb scheduling jitter.
|
||||
func (h *HousekeepingService) staleThreshold() time.Duration {
|
||||
base := 1 * time.Hour
|
||||
if h.cfg != nil && h.cfg.KnowledgeBase != nil && h.cfg.KnowledgeBase.DocumentProcessTimeout > base {
|
||||
base = h.cfg.KnowledgeBase.DocumentProcessTimeout
|
||||
}
|
||||
return base + 10*time.Minute
|
||||
}
|
||||
|
||||
func housekeepingEnabled() bool {
|
||||
// Default-on: missing/empty env enables the sweep. Operators must
|
||||
// explicitly set "false" to opt out, matching the plan's commitment
|
||||
// that no env change is required for the safety net to engage.
|
||||
v := strings.TrimSpace(os.Getenv("WEKNORA_HOUSEKEEPING_ENABLED"))
|
||||
if v == "" {
|
||||
return true
|
||||
}
|
||||
switch strings.ToLower(v) {
|
||||
case "0", "false", "off", "no":
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -2340,7 +2340,7 @@ func (s *knowledgeService) convert(
|
||||
req.FileType = fileType
|
||||
}
|
||||
|
||||
result, err := reader.Read(ctx, req)
|
||||
result, err := s.callDocReaderWithTimeout(ctx, reader, req)
|
||||
if err != nil {
|
||||
return s.failKnowledge(ctx, knowledge, isLastRetry, "document read failed: %v", err)
|
||||
}
|
||||
@@ -2356,7 +2356,42 @@ func (s *knowledgeService) convert(
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// resolveDocReader returns the appropriate DocReader for the given engine.
|
||||
// callDocReaderWithTimeout wraps the DocReader RPC in a child context whose
|
||||
// deadline is min(parent_deadline, DocReaderCallTimeout). Without this cap,
|
||||
// a hung docreader (network partition, GC pause, OCR runaway) silently
|
||||
// burns the whole DocumentProcessTimeout budget and pins a worker for hours
|
||||
// — the #1 cause of "knowledge stuck in processing" reports.
|
||||
//
|
||||
// On timeout we annotate the error so retries / dead-letter consumers can
|
||||
// distinguish "docreader was slow" from "docreader returned an error".
|
||||
func (s *knowledgeService) callDocReaderWithTimeout(
|
||||
ctx context.Context, reader interfaces.DocReader, req *types.ReadRequest,
|
||||
) (*types.ReadResult, error) {
|
||||
timeout := 30 * time.Minute
|
||||
if s.config != nil && s.config.KnowledgeBase != nil && s.config.KnowledgeBase.DocReaderCallTimeout > 0 {
|
||||
timeout = s.config.KnowledgeBase.DocReaderCallTimeout
|
||||
}
|
||||
callCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
start := time.Now()
|
||||
result, err := reader.Read(callCtx, req)
|
||||
elapsed := time.Since(start)
|
||||
if err != nil {
|
||||
// Promote DeadlineExceeded into a clearer message; retain underlying
|
||||
// error via %w so errors.Is(callCtx.Err(), context.DeadlineExceeded)
|
||||
// still works for upstream classification.
|
||||
if errors.Is(callCtx.Err(), context.DeadlineExceeded) && !errors.Is(ctx.Err(), context.DeadlineExceeded) {
|
||||
logger.Errorf(ctx, "[convert] docreader call timed out after %s (limit %s) for %q",
|
||||
elapsed, timeout, req.FileName)
|
||||
return nil, fmt.Errorf("docreader call timeout after %s: %w", timeout, err)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
logger.Infof(ctx, "[convert] docreader call ok in %s for %q", elapsed, req.FileName)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// Returns nil when the required service is unavailable.
|
||||
func (s *knowledgeService) resolveDocReader(ctx context.Context, engine, fileType string, isURL bool, overrides map[string]string) interfaces.DocReader {
|
||||
switch engine {
|
||||
|
||||
@@ -168,7 +168,13 @@ type KnowledgeBaseConfig struct {
|
||||
SplitMarkers []string `yaml:"split_markers" json:"split_markers"`
|
||||
KeepSeparator bool `yaml:"keep_separator" json:"keep_separator"`
|
||||
ImageProcessing *ImageProcessingConfig `yaml:"image_processing" json:"image_processing"`
|
||||
DocumentProcessTimeout time.Duration `yaml:"document_process_timeout" json:"document_process_timeout"`
|
||||
DocumentProcessTimeout time.Duration `yaml:"document_process_timeout" json:"document_process_timeout"`
|
||||
// DocReaderCallTimeout caps a single DocReader RPC. Without this the
|
||||
// gRPC call inherits the asynq task context (whole DocumentProcessTimeout,
|
||||
// default 2h+), so a hung docreader would block a worker for hours and
|
||||
// leave knowledge in "processing". Default 30 minutes is generous enough
|
||||
// for OCR-heavy large PDFs while ensuring forward progress.
|
||||
DocReaderCallTimeout time.Duration `yaml:"docreader_call_timeout" json:"docreader_call_timeout"`
|
||||
}
|
||||
|
||||
// ImageProcessingConfig 图像处理配置
|
||||
@@ -716,6 +722,14 @@ func applyKnowledgeBaseEnvOverrides(cfg *Config) {
|
||||
cfg.KnowledgeBase.DocumentProcessTimeout = d
|
||||
}
|
||||
}
|
||||
if cfg.KnowledgeBase.DocReaderCallTimeout <= 0 {
|
||||
cfg.KnowledgeBase.DocReaderCallTimeout = 30 * time.Minute
|
||||
}
|
||||
if value := strings.TrimSpace(os.Getenv("WEKNORA_DOCREADER_CALL_TIMEOUT")); value != "" {
|
||||
if d, err := time.ParseDuration(value); err == nil && d > 0 {
|
||||
cfg.KnowledgeBase.DocReaderCallTimeout = d
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func applyAgentEnvOverrides(cfg *Config) {
|
||||
|
||||
@@ -271,6 +271,9 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
||||
logger.Debugf(ctx, "[Container] Data source sync framework registered")
|
||||
must(container.Invoke(startAuditLogRetention))
|
||||
logger.Debugf(ctx, "[Container] Audit log retention runner registered")
|
||||
must(container.Provide(service.NewHousekeepingService))
|
||||
must(container.Invoke(startHousekeepingService))
|
||||
logger.Debugf(ctx, "[Container] Knowledge housekeeping runner registered")
|
||||
must(container.Provide(chatpipeline.NewEventManager))
|
||||
must(container.Invoke(chatpipeline.NewPluginSearch))
|
||||
must(container.Invoke(chatpipeline.NewPluginRerank))
|
||||
@@ -623,49 +626,74 @@ func syncSequences(db *gorm.DB) {
|
||||
}
|
||||
|
||||
// resetPendingTasks resets the state of any knowledge items or sync logs stuck in processing
|
||||
// due to an unexpected application restart when using in-memory queues (Lite mode).
|
||||
// due to an unexpected application restart.
|
||||
//
|
||||
// In Lite mode (no REDIS_ADDR) every queued task lived in process memory, so
|
||||
// any "processing" row at startup is definitively orphaned and must be marked
|
||||
// failed wholesale. In distributed mode (Asynq on Redis) the active queue
|
||||
// survives restart, but the *currently executing* task on the dead instance
|
||||
// is lost — Asynq won't reschedule it until at-least-once retry kicks in,
|
||||
// which can take minutes or never (e.g. if the deadline has passed). To bound
|
||||
// the worst case we mark only "long-stale" rows failed: anything that hasn't
|
||||
// been touched for 30 minutes is well past any reasonable in-flight window.
|
||||
// Newer rows are left alone so we don't race a peer instance that's mid-process.
|
||||
func resetPendingTasks(db *gorm.DB) {
|
||||
if os.Getenv("REDIS_ADDR") != "" {
|
||||
return // Distributed queue (Asynq) will handle its own retries
|
||||
distributed := os.Getenv("REDIS_ADDR") != ""
|
||||
|
||||
knowledgeQuery := db.Model(&types.Knowledge{}).
|
||||
Where("parse_status IN ?", []string{types.ParseStatusPending, types.ParseStatusProcessing, types.ParseStatusDeleting})
|
||||
summaryQuery := db.Model(&types.Knowledge{}).
|
||||
Where("summary_status IN ?", []string{types.SummaryStatusPending, types.SummaryStatusProcessing})
|
||||
syncQuery := db.Model(&types.SyncLog{}).
|
||||
Where("status IN ?", []string{types.SyncLogStatusRunning, "pending"})
|
||||
|
||||
if distributed {
|
||||
// 30-minute stale threshold — comfortably longer than any individual
|
||||
// stage's expected duration (DocReader 30m, embedding seconds, etc.)
|
||||
// while short enough that operators don't wait hours after a crash.
|
||||
staleCutoff := time.Now().Add(-30 * time.Minute)
|
||||
knowledgeQuery = knowledgeQuery.Where("updated_at < ?", staleCutoff)
|
||||
summaryQuery = summaryQuery.Where("updated_at < ?", staleCutoff)
|
||||
syncQuery = syncQuery.Where("start_time < ?", staleCutoff)
|
||||
}
|
||||
|
||||
// 1. Reset knowledge parsing tasks
|
||||
result := db.Model(&types.Knowledge{}).
|
||||
Where("parse_status IN ?", []string{types.ParseStatusPending, types.ParseStatusProcessing, types.ParseStatusDeleting}).
|
||||
Updates(map[string]interface{}{
|
||||
"parse_status": types.ParseStatusFailed,
|
||||
"error_message": "Task interrupted due to application restart",
|
||||
})
|
||||
result := knowledgeQuery.Updates(map[string]interface{}{
|
||||
"parse_status": types.ParseStatusFailed,
|
||||
"error_message": "Task interrupted due to application restart",
|
||||
})
|
||||
if result.Error != nil {
|
||||
logger.Warnf(context.Background(), "Failed to reset pending knowledge tasks: %v", result.Error)
|
||||
} else if result.RowsAffected > 0 {
|
||||
logger.Infof(context.Background(), "Reset %d stuck knowledge parsing tasks to failed state", result.RowsAffected)
|
||||
logger.Infof(context.Background(),
|
||||
"Reset %d stuck knowledge parsing tasks to failed state (distributed=%v)",
|
||||
result.RowsAffected, distributed)
|
||||
}
|
||||
|
||||
// 2. Reset knowledge summary tasks
|
||||
resultSummary := db.Model(&types.Knowledge{}).
|
||||
Where("summary_status IN ?", []string{types.SummaryStatusPending, types.SummaryStatusProcessing}).
|
||||
Updates(map[string]interface{}{
|
||||
"summary_status": types.SummaryStatusFailed,
|
||||
})
|
||||
resultSummary := summaryQuery.Updates(map[string]interface{}{
|
||||
"summary_status": types.SummaryStatusFailed,
|
||||
})
|
||||
if resultSummary.Error != nil {
|
||||
logger.Warnf(context.Background(), "Failed to reset pending summary tasks: %v", resultSummary.Error)
|
||||
} else if resultSummary.RowsAffected > 0 {
|
||||
logger.Infof(context.Background(), "Reset %d stuck summary generation tasks to failed state", resultSummary.RowsAffected)
|
||||
logger.Infof(context.Background(),
|
||||
"Reset %d stuck summary generation tasks to failed state (distributed=%v)",
|
||||
resultSummary.RowsAffected, distributed)
|
||||
}
|
||||
|
||||
// 3. Reset data source sync tasks
|
||||
resultSync := db.Model(&types.SyncLog{}).
|
||||
Where("status IN ?", []string{types.SyncLogStatusRunning, "pending"}).
|
||||
Updates(map[string]interface{}{
|
||||
"status": types.SyncLogStatusFailed,
|
||||
"error_message": "Sync interrupted due to application restart",
|
||||
"end_time": time.Now(),
|
||||
})
|
||||
resultSync := syncQuery.Updates(map[string]interface{}{
|
||||
"status": types.SyncLogStatusFailed,
|
||||
"error_message": "Sync interrupted due to application restart",
|
||||
"end_time": time.Now(),
|
||||
})
|
||||
if resultSync.Error != nil {
|
||||
logger.Warnf(context.Background(), "Failed to reset pending data source sync tasks: %v", resultSync.Error)
|
||||
} else if resultSync.RowsAffected > 0 {
|
||||
logger.Infof(context.Background(), "Reset %d stuck data source sync tasks to failed state", resultSync.RowsAffected)
|
||||
logger.Infof(context.Background(),
|
||||
"Reset %d stuck data source sync tasks to failed state (distributed=%v)",
|
||||
resultSync.RowsAffected, distributed)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1359,6 +1387,24 @@ func startDataSourceScheduler(scheduler *datasource.Scheduler, cleaner interface
|
||||
})
|
||||
}
|
||||
|
||||
// startHousekeepingService starts the knowledge housekeeping cron and registers
|
||||
// cleanup. This is the safety net that recovers any knowledge stuck in
|
||||
// "processing" past a configurable threshold (see HousekeepingService for
|
||||
// rationale). Best-effort: a startup error is logged but does NOT abort the
|
||||
// container — the rest of the system stays usable.
|
||||
func startHousekeepingService(svc *service.HousekeepingService, cleaner interfaces.ResourceCleaner) {
|
||||
if svc == nil {
|
||||
return
|
||||
}
|
||||
if err := svc.Start(context.Background()); err != nil {
|
||||
logger.Warnf(context.Background(), "[Container] housekeeping start failed: %v", err)
|
||||
}
|
||||
cleaner.RegisterWithName("KnowledgeHousekeeping", func() error {
|
||||
svc.Stop()
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// startAuditLogRetention spins up the daily audit_logs purge sweep
|
||||
// and registers shutdown cleanup. Mirrors the data-source-scheduler
|
||||
// pattern: container init kicks the goroutine, ResourceCleaner stops
|
||||
|
||||
@@ -46,7 +46,22 @@ import (
|
||||
//
|
||||
// This middleware should be installed once on the asynq mux, BEFORE
|
||||
// other middlewares that may transform errors. See router/task.go.
|
||||
func Middleware(repo interfaces.TaskDeadLetterRepository) asynq.MiddlewareFunc {
|
||||
// OnDeadLetter is an optional callback invoked AFTER the dead-letter row has
|
||||
// been recorded. It exists so callers can flip user-visible state alongside
|
||||
// the bookkeeping row — e.g. mark a Knowledge as failed when its document
|
||||
// processing task is permanently archived. Without this hook the only signal
|
||||
// to the UI is the timestamp on the knowledge row going stale, which is what
|
||||
// produced "stuck in processing forever" reports.
|
||||
//
|
||||
// The callback receives the raw asynq Task and the final error. It runs with
|
||||
// context.Background() (NOT the task ctx) so a cancelled task ctx during
|
||||
// shutdown doesn't also drop the state update. Errors are logged and
|
||||
// swallowed; the callback must NEVER alter the original task error.
|
||||
type OnDeadLetter func(ctx context.Context, t *asynq.Task, taskErr error)
|
||||
|
||||
// MiddlewareWithCallback is the extended form of Middleware. The callback may
|
||||
// be nil, in which case behaviour matches Middleware exactly.
|
||||
func MiddlewareWithCallback(repo interfaces.TaskDeadLetterRepository, cb OnDeadLetter) asynq.MiddlewareFunc {
|
||||
return func(next asynq.Handler) asynq.Handler {
|
||||
return asynq.HandlerFunc(func(ctx context.Context, t *asynq.Task) error {
|
||||
err := next.ProcessTask(ctx, t)
|
||||
@@ -56,39 +71,47 @@ func Middleware(repo interfaces.TaskDeadLetterRepository) asynq.MiddlewareFunc {
|
||||
if !isFinalAttempt(ctx) {
|
||||
return err
|
||||
}
|
||||
if repo == nil {
|
||||
return err
|
||||
}
|
||||
// Capture the actual attempt count from asynq so the
|
||||
// dead-letter row reflects how many tries were made
|
||||
// before the task was archived. On the final attempt
|
||||
// asynq's counter satisfies retried == maxRetry, so the
|
||||
// total attempt count is retried + 1 (initial try +
|
||||
// retries). Both helpers can fail outside a worker ctx;
|
||||
// in that case we record 0 so it's clear the count
|
||||
// wasn't observable.
|
||||
attempts := 0
|
||||
if retried, ok := asynq.GetRetryCount(ctx); ok {
|
||||
attempts = retried + 1
|
||||
}
|
||||
dl := buildDeadLetter(t, err, attempts)
|
||||
if dl == nil {
|
||||
return err
|
||||
if repo != nil {
|
||||
dl := buildDeadLetter(t, err, attempts)
|
||||
if dl != nil {
|
||||
if insertErr := repo.Insert(context.Background(), dl); insertErr != nil {
|
||||
logger.Warnf(ctx,
|
||||
"asynq dead-letter: failed to record %s task: %v (original task error: %v)",
|
||||
t.Type(), insertErr, err,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
// context.Background() so a cancelled task ctx (e.g. server
|
||||
// shutdown signaling cancellation right as a task fails)
|
||||
// doesn't also drop the dead-letter record.
|
||||
if insertErr := repo.Insert(context.Background(), dl); insertErr != nil {
|
||||
logger.Warnf(ctx,
|
||||
"asynq dead-letter: failed to record %s task: %v (original task error: %v)",
|
||||
t.Type(), insertErr, err,
|
||||
)
|
||||
if cb != nil {
|
||||
// Wrap callback so a panic in user code doesn't escape
|
||||
// into asynq's worker goroutine.
|
||||
safeInvokeCallback(ctx, t, err, cb)
|
||||
}
|
||||
return err
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func safeInvokeCallback(ctx context.Context, t *asynq.Task, taskErr error, cb OnDeadLetter) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
logger.Warnf(ctx, "asynq dead-letter callback panicked for %s: %v", t.Type(), r)
|
||||
}
|
||||
}()
|
||||
cb(context.Background(), t, taskErr)
|
||||
}
|
||||
|
||||
// Middleware preserves the original signature for callers that don't need
|
||||
// the dead-letter state-linkback hook. Equivalent to MiddlewareWithCallback
|
||||
// with a nil callback.
|
||||
func Middleware(repo interfaces.TaskDeadLetterRepository) asynq.MiddlewareFunc {
|
||||
return MiddlewareWithCallback(repo, nil)
|
||||
}
|
||||
|
||||
// isFinalAttempt reports whether the just-finished handler invocation
|
||||
// was the LAST one asynq will run before archiving the task. Asynq
|
||||
// increments retry_count *after* a successful retry, so on the final
|
||||
|
||||
+135
-6
@@ -1,13 +1,17 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/application/service"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/middleware/asynqdl"
|
||||
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
@@ -32,6 +36,24 @@ type AsynqTaskParams struct {
|
||||
DeadLetterRepo interfaces.TaskDeadLetterRepository
|
||||
}
|
||||
|
||||
// defaultRedisOpTimeout is the previous hard-coded read timeout. The 100ms
|
||||
// floor was tight enough to cause spurious i/o timeout errors during bursty
|
||||
// workloads (large batch uploads, multimodal counter DECRs under load), so we
|
||||
// raise the default to 500ms while still allowing operators to tune via env.
|
||||
const defaultRedisOpTimeoutMs = 500
|
||||
|
||||
// readRedisOpTimeoutMs reads WEKNORA_REDIS_OP_TIMEOUT_MS, falling back to
|
||||
// defaultRedisOpTimeoutMs on missing/invalid input. Kept as a separate helper
|
||||
// so both ReadTimeout and WriteTimeout share the same source of truth.
|
||||
func readRedisOpTimeoutMs() int {
|
||||
if v := strings.TrimSpace(os.Getenv("WEKNORA_REDIS_OP_TIMEOUT_MS")); v != "" {
|
||||
if parsed, err := strconv.Atoi(v); err == nil && parsed > 0 {
|
||||
return parsed
|
||||
}
|
||||
}
|
||||
return defaultRedisOpTimeoutMs
|
||||
}
|
||||
|
||||
func getAsynqRedisClientOpt() *asynq.RedisClientOpt {
|
||||
db := 0
|
||||
if dbStr := os.Getenv("REDIS_DB"); dbStr != "" {
|
||||
@@ -39,12 +61,16 @@ func getAsynqRedisClientOpt() *asynq.RedisClientOpt {
|
||||
db = parsed
|
||||
}
|
||||
}
|
||||
timeoutMs := readRedisOpTimeoutMs()
|
||||
opt := &asynq.RedisClientOpt{
|
||||
Addr: os.Getenv("REDIS_ADDR"),
|
||||
Username: os.Getenv("REDIS_USERNAME"),
|
||||
Password: os.Getenv("REDIS_PASSWORD"),
|
||||
ReadTimeout: 100 * time.Millisecond,
|
||||
WriteTimeout: 200 * time.Millisecond,
|
||||
Addr: os.Getenv("REDIS_ADDR"),
|
||||
Username: os.Getenv("REDIS_USERNAME"),
|
||||
Password: os.Getenv("REDIS_PASSWORD"),
|
||||
ReadTimeout: time.Duration(timeoutMs) * time.Millisecond,
|
||||
// Writes are typically more sensitive to congestion than reads
|
||||
// (RESP pipelining, BRPOPLPUSH on Asynq dequeue), so we keep
|
||||
// WriteTimeout slightly larger to absorb head-of-line stalls.
|
||||
WriteTimeout: time.Duration(timeoutMs*2) * time.Millisecond,
|
||||
DB: db,
|
||||
}
|
||||
return opt
|
||||
@@ -83,11 +109,34 @@ func asynqRetryDelayFunc(n int, e error, t *asynq.Task) time.Duration {
|
||||
return asynq.DefaultRetryDelayFunc(n, e, t)
|
||||
}
|
||||
|
||||
// defaultAsynqConcurrency is the worker pool size used when
|
||||
// WEKNORA_ASYNQ_CONCURRENCY is unset. The asynq library defaults to
|
||||
// runtime.NumCPU(), which under-provisions during batch document uploads:
|
||||
// a single 4-core container can only process 4 documents in parallel even
|
||||
// when 100 are queued, so the queue wait time eats into each task's
|
||||
// DocumentProcessTimeout budget. 16 is a safer default for the I/O-bound
|
||||
// nature of doc parsing (most time is spent in DocReader / embedding RPCs,
|
||||
// not on local CPU).
|
||||
const defaultAsynqConcurrency = 16
|
||||
|
||||
func readAsynqConcurrency() int {
|
||||
if v := strings.TrimSpace(os.Getenv("WEKNORA_ASYNQ_CONCURRENCY")); v != "" {
|
||||
if parsed, err := strconv.Atoi(v); err == nil && parsed > 0 {
|
||||
return parsed
|
||||
}
|
||||
}
|
||||
return defaultAsynqConcurrency
|
||||
}
|
||||
|
||||
func NewAsynqServer() *asynq.Server {
|
||||
opt := getAsynqRedisClientOpt()
|
||||
concurrency := readAsynqConcurrency()
|
||||
log.Printf("asynq server starting with concurrency=%d redis_op_timeout=%dms",
|
||||
concurrency, readRedisOpTimeoutMs())
|
||||
srv := asynq.NewServer(
|
||||
opt,
|
||||
asynq.Config{
|
||||
Concurrency: concurrency,
|
||||
Queues: map[string]int{
|
||||
"critical": 6, // Highest priority queue
|
||||
"default": 3, // Default priority queue
|
||||
@@ -110,7 +159,14 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux {
|
||||
// failures by task type, scope, or tenant without scraping logs.
|
||||
// Best-effort: a DB failure is logged and swallowed; the original task
|
||||
// error always propagates upstream to asynq for retry/archival.
|
||||
mux.Use(asynqdl.Middleware(params.DeadLetterRepo))
|
||||
//
|
||||
// The callback flips Knowledge.parse_status to "failed" the moment a
|
||||
// document-related task exhausts its retry budget. Without this hook,
|
||||
// a permanently-failing task left its parent knowledge stranded in
|
||||
// "processing" until housekeeping cron caught it minutes later — the
|
||||
// UI signal users actually see.
|
||||
knowledgeFailer := newDeadLetterKnowledgeFailer(params.KnowledgeService)
|
||||
mux.Use(asynqdl.MiddlewareWithCallback(params.DeadLetterRepo, knowledgeFailer))
|
||||
|
||||
// Install Langfuse middleware BEFORE handler registration so every task
|
||||
// type is automatically wrapped. When Langfuse is disabled the middleware
|
||||
@@ -174,3 +230,76 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux {
|
||||
}()
|
||||
return mux
|
||||
}
|
||||
|
||||
// deadLetterKnowledgePayload extracts only the field we need from any
|
||||
// document-related asynq payload. Kept narrow so we don't accidentally
|
||||
// depend on the full payload schema and survive future field churn.
|
||||
type deadLetterKnowledgePayload struct {
|
||||
KnowledgeID string `json:"knowledge_id,omitempty"`
|
||||
}
|
||||
|
||||
// taskTypesAffectingKnowledgeStatus enumerates the asynq task types whose
|
||||
// dead-letter event should flip the parent Knowledge to "failed". Only
|
||||
// terminal task types are listed here:
|
||||
//
|
||||
// - TypeDocumentProcess: the entry point of the parsing pipeline.
|
||||
// - TypeImageMultimodal: a single image hitting dead-letter would have
|
||||
// been counted by isFinalAsynqAttempt (see image_multimodal.go), so
|
||||
// the parent might still complete via remaining images. We DO NOT mark
|
||||
// the parent failed for this case — finalize-on-last-attempt already
|
||||
// ensures progress.
|
||||
// - TypeKnowledgePostProcess: terminal stage; failure here strands the
|
||||
// knowledge in "processing".
|
||||
// - TypeManualProcess: same shape as DocumentProcess for re-indexing.
|
||||
//
|
||||
// Question/Summary generation are NOT included: they run after parse_status
|
||||
// has already become "completed" and have their own status fields.
|
||||
var taskTypesAffectingKnowledgeStatus = map[string]struct{}{
|
||||
types.TypeDocumentProcess: {},
|
||||
types.TypeKnowledgePostProcess: {},
|
||||
types.TypeManualProcess: {},
|
||||
}
|
||||
|
||||
// newDeadLetterKnowledgeFailer returns the callback wired into the asynq
|
||||
// dead-letter middleware. When a document-related task exhausts its retry
|
||||
// budget, this callback marks the corresponding Knowledge row as failed so
|
||||
// the UI surfaces the error instead of a perpetual spinner.
|
||||
//
|
||||
// All work is best-effort: missing payload, missing knowledge_id, or DB
|
||||
// errors are logged and swallowed. The dead-letter record is the source of
|
||||
// truth — this is purely a UX shortcut so users don't wait for the
|
||||
// housekeeping cron's next sweep.
|
||||
func newDeadLetterKnowledgeFailer(ks interfaces.KnowledgeService) asynqdl.OnDeadLetter {
|
||||
if ks == nil {
|
||||
return nil
|
||||
}
|
||||
repo := ks.GetRepository()
|
||||
if repo == nil {
|
||||
return nil
|
||||
}
|
||||
return func(ctx context.Context, t *asynq.Task, taskErr error) {
|
||||
if t == nil {
|
||||
return
|
||||
}
|
||||
if _, ok := taskTypesAffectingKnowledgeStatus[t.Type()]; !ok {
|
||||
return
|
||||
}
|
||||
var probe deadLetterKnowledgePayload
|
||||
if err := json.Unmarshal(t.Payload(), &probe); err != nil || probe.KnowledgeID == "" {
|
||||
return
|
||||
}
|
||||
errMsg := "task " + t.Type() + " exhausted retries: " + taskErr.Error()
|
||||
// 8KB is the same cap the dead-letter row uses for last_error.
|
||||
if len(errMsg) > 8192 {
|
||||
errMsg = errMsg[:8192]
|
||||
}
|
||||
if err := repo.UpdateKnowledgeColumn(ctx, probe.KnowledgeID, "parse_status", types.ParseStatusFailed); err != nil {
|
||||
logger.Warnf(ctx, "dead-letter callback: failed to update parse_status for %s: %v", probe.KnowledgeID, err)
|
||||
return
|
||||
}
|
||||
if err := repo.UpdateKnowledgeColumn(ctx, probe.KnowledgeID, "error_message", errMsg); err != nil {
|
||||
logger.Warnf(ctx, "dead-letter callback: failed to update error_message for %s: %v", probe.KnowledgeID, err)
|
||||
}
|
||||
logger.Infof(ctx, "dead-letter callback: marked knowledge %s as failed (task=%s)", probe.KnowledgeID, t.Type())
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user