From ace84bc757741f9babac8c96c18cf39b6e3bd811 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Tue, 14 Jul 2026 17:20:22 +0800 Subject: [PATCH] fix(knowledge): recover tasks safely after server restart Stop failing distributed-mode knowledge rows on startup; let housekeeping decide using span activity and Asynq queue state. In Lite mode, resume wiki work from durable task_pending_ops and surface SERVER_RESTART errors in the processing timeline with i18n support. --- frontend/src/i18n/locales/en-US.ts | 2 + frontend/src/i18n/locales/ko-KR.ts | 2 + frontend/src/i18n/locales/ru-RU.ts | 2 + frontend/src/i18n/locales/zh-CN.ts | 2 + .../service/knowledge_housekeeping.go | 10 +- .../service/knowledge_housekeeping_test.go | 34 +++ internal/container/container.go | 5 + .../container/recover_pending_wiki_tasks.go | 87 ++++++++ internal/container/reset_pending_tasks.go | 175 ++++++++++------ .../container/reset_pending_tasks_test.go | 198 +++++++++++++++++- internal/handler/knowledge.go | 9 +- internal/handler/knowledge_spans_test.go | 10 + 12 files changed, 462 insertions(+), 74 deletions(-) create mode 100644 internal/container/recover_pending_wiki_tasks.go diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index 41a95b2c..48580bde 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -884,6 +884,8 @@ export default { MULTIMODAL_ALL_FAILED_SUGGESTION: 'Verify the multimodal model configuration.', TASK_TIMEOUT: 'Task exceeded maximum runtime', TASK_TIMEOUT_SUGGESTION: 'The task ran longer than allowed. Retry or contact support.', + SERVER_RESTART: 'Task interrupted by server restart', + SERVER_RESTART_SUGGESTION: 'Redis-backed tasks retry from the queue; recoverable Wiki work also resumes in Lite mode. Retry if this task remains failed.', UPSTREAM_FAILED: 'Stopped because an upstream stage failed', UPSTREAM_FAILED_SUGGESTION: 'A previous stage failed and prevented this step from running.', UNKNOWN: 'Unknown error', diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts index e3f786b1..c55fdbf9 100755 --- a/frontend/src/i18n/locales/ko-KR.ts +++ b/frontend/src/i18n/locales/ko-KR.ts @@ -888,6 +888,8 @@ export default { MULTIMODAL_ALL_FAILED_SUGGESTION: "멀티모달 모델 구성을 확인하세요.", TASK_TIMEOUT: "작업이 최대 실행 시간을 초과함", TASK_TIMEOUT_SUGGESTION: "작업이 허용된 시간보다 오래 실행되었습니다. 재시도하거나 지원팀에 문의하세요.", + SERVER_RESTART: "서버 재시작으로 작업이 중단됨", + SERVER_RESTART_SUGGESTION: "Redis 모드 작업은 큐에서 재시도되고, Lite 모드의 복구 가능한 Wiki 작업도 자동으로 재개됩니다. 계속 실패하면 다시 시도하세요.", UPSTREAM_FAILED: "상위 단계 실패로 중단됨", UPSTREAM_FAILED_SUGGESTION: "이전 단계가 실패하여 이 단계가 실행되지 않았습니다.", UNKNOWN: "알 수 없는 오류", diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index f4aa25a5..2fa99311 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -803,6 +803,8 @@ export default { MULTIMODAL_ALL_FAILED_SUGGESTION: 'Проверьте конфигурацию мультимодальной модели.', TASK_TIMEOUT: 'Задача превысила максимальное время выполнения', TASK_TIMEOUT_SUGGESTION: 'Задача выполнялась дольше допустимого. Повторите или обратитесь в поддержку.', + SERVER_RESTART: 'Задача прервана перезапуском сервера', + SERVER_RESTART_SUGGESTION: 'Задачи с Redis повторяются из очереди, а восстанавливаемая обработка Wiki возобновляется и в Lite-режиме. Повторите попытку, если ошибка осталась.', UPSTREAM_FAILED: 'Остановлено из-за ошибки на предыдущем этапе', UPSTREAM_FAILED_SUGGESTION: 'Предыдущий этап завершился с ошибкой и заблокировал выполнение.', UNKNOWN: 'Неизвестная ошибка', diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index 15a8cbe3..9c67662b 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -886,6 +886,8 @@ export default { MULTIMODAL_ALL_FAILED_SUGGESTION: "请检查多模态模型配置。", TASK_TIMEOUT: "任务超过最大运行时间", TASK_TIMEOUT_SUGGESTION: "任务运行时间超出限制,请重试或联系支持人员。", + SERVER_RESTART: "服务重启导致任务中断", + SERVER_RESTART_SUGGESTION: "Redis 模式会由任务队列继续重试;Lite 模式中可恢复的 Wiki 任务会自动续跑。若仍失败,请点击重试。", UPSTREAM_FAILED: "上游阶段失败已中止", UPSTREAM_FAILED_SUGGESTION: "前置阶段失败,导致本步骤无法执行。", UNKNOWN: "未知错误", diff --git a/internal/application/service/knowledge_housekeeping.go b/internal/application/service/knowledge_housekeeping.go index 02995b63..95abd863 100644 --- a/internal/application/service/knowledge_housekeeping.go +++ b/internal/application/service/knowledge_housekeeping.go @@ -112,7 +112,7 @@ func (h *HousekeepingService) runSweep(ctx context.Context) { threshold := h.staleThreshold() cutoff := time.Now().Add(-threshold) - // Sweep A: knowledge stuck in "processing". + // Sweep A: knowledge stuck in "pending", "processing", or "finalizing". // // Two-stage check is critical here: knowledge.updated_at advances // only at parse_status transitions, but a long stage (DocReader on @@ -127,7 +127,9 @@ func (h *HousekeepingService) runSweep(ctx context.Context) { // Knowledge rows with no spans at all (lite mode, in-flight tasks // from before this code shipped) fall back to the simple // updated_at check — they have no heartbeat to consult. - // Include 'finalizing' alongside 'processing': finalizing rows still + // Include 'pending' so a task whose enqueue was lost is eventually + // recovered too; filterOutQueued below protects legitimately backlogged + // tasks. Include 'finalizing' alongside 'processing': finalizing rows still // consume LLM compute via enrichment subtasks (summary/question/graph), // and the same stall modes (subtask worker dies, retry budget exhausted // without decrementing the counter) leave the row hanging just as @@ -136,7 +138,7 @@ func (h *HousekeepingService) runSweep(ctx context.Context) { var candidates []types.Knowledge if err := h.db.WithContext(ctx). Where("parse_status IN ? AND updated_at < ?", - []string{types.ParseStatusProcessing, types.ParseStatusFinalizing}, cutoff). + []string{types.ParseStatusPending, types.ParseStatusProcessing, types.ParseStatusFinalizing}, cutoff). Find(&candidates).Error; err != nil { logger.Warnf(ctx, "[Housekeeping] knowledge candidate query failed: %v", err) return @@ -162,7 +164,7 @@ func (h *HousekeepingService) runSweep(ctx context.Context) { } res := h.db.WithContext(ctx).Model(&types.Knowledge{}). Where("id IN ? AND parse_status IN ?", stuckIDs, - []string{types.ParseStatusProcessing, types.ParseStatusFinalizing}). + []string{types.ParseStatusPending, types.ParseStatusProcessing, types.ParseStatusFinalizing}). Updates(map[string]interface{}{ "parse_status": types.ParseStatusFailed, "error_message": "task stuck in processing > " + threshold.String() + ", recovered by housekeeping", diff --git a/internal/application/service/knowledge_housekeeping_test.go b/internal/application/service/knowledge_housekeeping_test.go index f2c483c7..9c290df2 100644 --- a/internal/application/service/knowledge_housekeeping_test.go +++ b/internal/application/service/knowledge_housekeeping_test.go @@ -165,6 +165,40 @@ func TestHousekeeping_RecoversAbandoned(t *testing.T) { assert.Contains(t, errMsg, "stuck in processing") } +func TestHousekeeping_RecoversPendingTaskMissingFromQueue(t *testing.T) { + db := setupHousekeepingDB(t) + svc := newHousekeepingSvcForTest(db) + stale := time.Now().Add(-3 * time.Hour) + insertKnowledge(t, db, "kid-pending-orphan", types.ParseStatusPending, stale) + + svc.runSweep(context.Background()) + + var status string + require.NoError(t, db.Raw( + `SELECT parse_status FROM knowledges WHERE id = ?`, "kid-pending-orphan", + ).Row().Scan(&status)) + assert.Equal(t, types.ParseStatusFailed, status, + "a stale pending row with no queue task must not remain pending forever") +} + +func TestHousekeeping_PreservesPendingTaskStillQueued(t *testing.T) { + db := setupHousekeepingDB(t) + svc := newHousekeepingSvcWithInspector(db, fakeTaskInspector{ + queued: map[string]bool{"kid-pending-queued": true}, + }) + stale := time.Now().Add(-3 * time.Hour) + insertKnowledge(t, db, "kid-pending-queued", types.ParseStatusPending, stale) + + svc.runSweep(context.Background()) + + var status string + require.NoError(t, db.Raw( + `SELECT parse_status FROM knowledges WHERE id = ?`, "kid-pending-queued", + ).Row().Scan(&status)) + assert.Equal(t, types.ParseStatusPending, status, + "backlogged pending work remains owned by the durable queue") +} + // TestHousekeeping_NoFalseKill_ActiveSpan is the regression test for // the "long DocReader silently runs longer than DocumentProcessTimeout" // scenario the user flagged. A knowledge whose knowledge.updated_at diff --git a/internal/container/container.go b/internal/container/container.go index cfdb69d6..e5db4aff 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -390,6 +390,11 @@ func BuildContainer(container *dig.Container) *dig.Container { } else { must(container.Invoke(router.RegisterSyncHandlers)) } + // Wiki operation rows are durable, while their wake-up triggers may be + // lost across a process restart (always in Lite mode, and in Redis mode if + // persistence succeeded immediately before trigger enqueue failed). Re-arm + // them only after the matching handlers are ready. + must(container.Invoke(recoverPendingWikiTasks)) logger.Infof(ctx, "[Container] Container initialization completed successfully") return container diff --git a/internal/container/recover_pending_wiki_tasks.go b/internal/container/recover_pending_wiki_tasks.go new file mode 100644 index 00000000..6d9d529a --- /dev/null +++ b/internal/container/recover_pending_wiki_tasks.go @@ -0,0 +1,87 @@ +package container + +import ( + "context" + "encoding/json" + "errors" + "time" + + "github.com/Tencent/WeKnora/internal/application/service" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/hibiken/asynq" + "gorm.io/gorm" +) + +// pendingWikiScope is the minimum routing information needed to recreate an +// ephemeral wiki trigger from the durable task_pending_ops queue. +type pendingWikiScope struct { + TenantID uint64 `gorm:"column:tenant_id"` + TaskType string `gorm:"column:task_type"` + ScopeID string `gorm:"column:scope_id"` +} + +// recoverPendingWikiTasks recreates one trigger per pending wiki queue lane. +// +// task_pending_ops is deliberately durable, but the trigger that wakes its +// consumer is not durable in Lite mode (SyncTaskExecutor is process-local) and +// may also be absent after an interrupted Redis enqueue. Running this after all +// handlers are registered closes that gap. Duplicate triggers are harmless: +// ingest claims/peeks disjoint rows and finalize coalesces its pending lane. +func recoverPendingWikiTasks(db *gorm.DB, task interfaces.TaskEnqueuer) { + if db == nil || task == nil { + return + } + ctx := context.Background() + var scopes []pendingWikiScope + if err := db.WithContext(ctx). + Model(&types.TaskPendingOp{}). + Distinct("tenant_id", "task_type", "scope_id"). + Where("scope = ? AND task_type IN ?", types.TaskScopeKnowledgeBase, + []string{types.TypeWikiIngest, types.TypeWikiFinalize}). + Find(&scopes).Error; err != nil { + logger.Warnf(ctx, "[WikiRecovery] failed to list pending queues: %v", err) + return + } + + recovered := 0 + for _, scope := range scopes { + if scope.ScopeID == "" { + continue + } + payload, err := json.Marshal(service.WikiIngestPayload{ + TenantID: scope.TenantID, + KnowledgeBaseID: scope.ScopeID, + }) + if err != nil { + logger.Warnf(ctx, "[WikiRecovery] marshal trigger for KB %s failed: %v", scope.ScopeID, err) + continue + } + opts := []asynq.Option{ + asynq.Queue(types.QueueWiki), + asynq.MaxRetry(10), // keep aligned with the wiki ingest retry policy + asynq.Timeout(60 * time.Minute), + } + if scope.TaskType == types.TypeWikiFinalize { + opts[2] = asynq.Timeout(30 * time.Minute) + // Match scheduleFinalize so simultaneous replica startups collapse + // into the same per-KB finalize trigger. + opts = append(opts, asynq.TaskID("wiki-finalize-"+scope.ScopeID)) + } + trigger := asynq.NewTask(scope.TaskType, payload, opts...) + if _, err := task.Enqueue(trigger); err != nil { + if errors.Is(err, asynq.ErrTaskIDConflict) || errors.Is(err, asynq.ErrDuplicateTask) { + recovered++ + continue + } + logger.Warnf(ctx, "[WikiRecovery] enqueue %s trigger for KB %s failed: %v", + scope.TaskType, scope.ScopeID, err) + continue + } + recovered++ + } + if recovered > 0 { + logger.Infof(ctx, "[WikiRecovery] recreated %d trigger(s) from durable pending queues", recovered) + } +} diff --git a/internal/container/reset_pending_tasks.go b/internal/container/reset_pending_tasks.go index 2d4c2c2b..349f9e49 100644 --- a/internal/container/reset_pending_tasks.go +++ b/internal/container/reset_pending_tasks.go @@ -13,18 +13,22 @@ import ( const resetPendingStaleWindow = 30 * time.Minute +const restartInterruptedMessage = "Task interrupted due to application restart" + // resetPendingTasks resets the state of any knowledge items or sync logs stuck in processing // due to an unexpected application restart. // -// In Lite mode (no REDIS_ADDR) every queued task lived in process memory, so -// any "processing" row at startup is definitively orphaned and must be marked -// failed wholesale. In distributed mode (Asynq on Redis) the active queue -// survives restart, but the *currently executing* task on the dead instance -// is lost — Asynq won't reschedule it until at-least-once retry kicks in, -// which can take minutes or never (e.g. if the deadline has passed). To bound -// the worst case we mark only "long-stale" rows failed: anything that hasn't -// been touched for 30 minutes is well past any reasonable in-flight window. -// Newer rows are left alone so we don't race a peer instance that's mid-process. +// In Lite mode (no REDIS_ADDR) normal queued tasks live in process memory, so +// a "processing" row at startup is orphaned unless it has a durable wiki op +// that can be re-triggered. +// +// Distributed mode is intentionally different: Asynq persists queued/retry +// tasks, and another replica may still be executing the same knowledge. A +// startup hook cannot safely distinguish an orphan from a backlogged task that +// has not opened a span yet. HousekeepingService owns that decision because it +// checks BOTH recent span activity and the real Asynq queue. Consequently this +// hook never resets knowledge/summary rows in distributed mode; it only keeps +// the separate sync-log cleanup below. func resetPendingTasks(db *gorm.DB) { distributed := os.Getenv("REDIS_ADDR") != "" ctx := context.Background() @@ -35,22 +39,64 @@ func resetPendingTasks(db *gorm.DB) { staleCutoff = time.Now().Add(-resetPendingStaleWindow) } - // Cancel orphaned trace spans for knowledge rows we are about to mark - // failed. resetPendingTasks does not touch asynq queues; this only - // prevents the UI from showing duplicate running postprocess.* - // subspans when a later retry also opens fresh spans. + // Resolve Lite-mode orphaned knowledge rows first. A finalizing row whose + // ONLY remaining slot is backed by a durable wiki op is excluded and resumed + // by recoverPendingWikiTasks after handlers are registered. Rows with other + // outstanding in-memory subtasks still fail: the wiki op cannot recover them. var stuckKnowledge []types.Knowledge - if err := stuckKnowledgeParseQuery(db, distributed, staleCutoff). - Select("id").Find(&stuckKnowledge).Error; err != nil { - logger.Warnf(ctx, "resetPendingTasks: list stuck knowledge failed: %v", err) - } else { - for _, k := range stuckKnowledge { + if !distributed { + if err := stuckKnowledgeParseQuery(db). + Select("id").Find(&stuckKnowledge).Error; err != nil { + logger.Warnf(ctx, "resetPendingTasks: list stuck knowledge failed: %v", err) + } + } + + // 1. Reset knowledge parsing tasks (including finalizing rows whose + // enrichment subtasks were lost with the process). + // Update by the resolved ids rather than reusing the GORM chain after + // Find() (which makes PostgreSQL emit an invalid UPDATE ... FROM self). + stuckIDs := knowledgeIDs(stuckKnowledge) + var resetErr error + var resetCount int64 + if len(stuckIDs) > 0 { + // Rebuild the query instead of reusing the chain after Find(); reusing it + // makes PostgreSQL emit an invalid UPDATE ... FROM self statement. + result := stuckKnowledgeParseQuery(db). + Where("id IN ?", stuckIDs). + Updates(map[string]interface{}{ + "parse_status": types.ParseStatusFailed, + "error_message": restartInterruptedMessage, + "pending_subtasks_count": 0, + }) + resetErr = result.Error + resetCount = result.RowsAffected + } + if resetErr != nil { + logger.Warnf(context.Background(), "Failed to reset pending knowledge tasks: %v", resetErr) + } else if resetCount > 0 { + logger.Infof(context.Background(), + "Reset %d stuck knowledge parsing tasks to failed state (distributed=%v)", + resetCount, distributed) + + // Cancel orphaned trace spans only after the owning knowledge rows are + // terminal. This prevents the UI from showing duplicate running + // postprocess.* subspans when a later manual retry opens fresh spans. + // Re-read the successfully reset ids so a row whose status changed in + // the small SELECT/UPDATE gap is not accidentally cancelled. + var resetKnowledge []types.Knowledge + if err := db.Select("id"). + Where("id IN ? AND parse_status = ? AND error_message = ?", + stuckIDs, types.ParseStatusFailed, restartInterruptedMessage). + Find(&resetKnowledge).Error; err != nil { + logger.Warnf(ctx, "resetPendingTasks: list reset knowledge failed: %v", err) + } + for _, k := range resetKnowledge { attempt, err := spanRepo.LatestAttempt(ctx, k.ID) if err != nil || attempt <= 0 { continue } if n, err := spanRepo.CancelAllOpenSpans(ctx, k.ID, attempt, - "SERVER_RESTART", "task interrupted due to application restart"); err != nil { + "SERVER_RESTART", restartInterruptedMessage); err != nil { logger.Warnf(ctx, "resetPendingTasks: cancel spans for %s failed: %v", k.ID, err) } else if n > 0 { logger.Infof(ctx, "resetPendingTasks: cancelled %d open span(s) for knowledge %s attempt %d", @@ -59,33 +105,19 @@ func resetPendingTasks(db *gorm.DB) { } } - // 1. Reset knowledge parsing tasks (including finalizing rows whose - // enrichment subtasks were lost with the process). - // Fresh query — reusing the *gorm.DB chain after Find() makes GORM emit - // UPDATE ... FROM knowledges which PostgreSQL rejects (SQLSTATE 42712). - result := stuckKnowledgeParseQuery(db, distributed, staleCutoff).Updates(map[string]interface{}{ - "parse_status": types.ParseStatusFailed, - "error_message": "Task interrupted due to application restart", - "pending_subtasks_count": 0, - }) - if result.Error != nil { - logger.Warnf(context.Background(), "Failed to reset pending knowledge tasks: %v", result.Error) - } else if result.RowsAffected > 0 { - logger.Infof(context.Background(), - "Reset %d stuck knowledge parsing tasks to failed state (distributed=%v)", - result.RowsAffected, distributed) - } - - // 2. Reset knowledge summary tasks - resultSummary := stuckKnowledgeSummaryQuery(db, distributed, staleCutoff).Updates(map[string]interface{}{ - "summary_status": types.SummaryStatusFailed, - }) - if resultSummary.Error != nil { - logger.Warnf(context.Background(), "Failed to reset pending summary tasks: %v", resultSummary.Error) - } else if resultSummary.RowsAffected > 0 { - logger.Infof(context.Background(), - "Reset %d stuck summary generation tasks to failed state (distributed=%v)", - resultSummary.RowsAffected, distributed) + // 2. Lite summary tasks are process-local too. Distributed summary tasks + // stay in Asynq and must not be failed merely because this replica started. + if !distributed { + resultSummary := stuckKnowledgeSummaryQuery(db).Updates(map[string]interface{}{ + "summary_status": types.SummaryStatusFailed, + }) + if resultSummary.Error != nil { + logger.Warnf(context.Background(), "Failed to reset pending summary tasks: %v", resultSummary.Error) + } else if resultSummary.RowsAffected > 0 { + logger.Infof(context.Background(), + "Reset %d stuck summary generation tasks to failed state (distributed=false)", + resultSummary.RowsAffected) + } } // 3. Reset data source sync tasks @@ -104,27 +136,27 @@ func resetPendingTasks(db *gorm.DB) { } } -func stuckKnowledgeParseQuery(db *gorm.DB, distributed bool, staleCutoff time.Time) *gorm.DB { +func stuckKnowledgeParseQuery(db *gorm.DB) *gorm.DB { q := db.Model(&types.Knowledge{}). - Where("parse_status IN ?", []string{ - types.ParseStatusPending, - types.ParseStatusProcessing, - types.ParseStatusFinalizing, - types.ParseStatusDeleting, - }) - if distributed { - q = q.Where("updated_at < ?", staleCutoff) - } + Where("parse_status IN ?", resettableParseStatuses()). + // Wiki ingest ops are persisted independently from the task trigger. + // When wiki owns the only outstanding slot, keeping this finalizing row + // alive lets startup recreate the trigger and finish cleanly. A count + // above one means at least one non-wiki in-memory subtask was also lost. + Where(`NOT (parse_status = ? AND pending_subtasks_count = 1 AND EXISTS ( + SELECT 1 FROM task_pending_ops + WHERE task_pending_ops.task_type = ? + AND task_pending_ops.scope = ? + AND task_pending_ops.dedup_key = knowledges.id + AND task_pending_ops.op = ? + ))`, types.ParseStatusFinalizing, types.TypeWikiIngest, + types.TaskScopeKnowledgeBase, "ingest") return q } -func stuckKnowledgeSummaryQuery(db *gorm.DB, distributed bool, staleCutoff time.Time) *gorm.DB { - q := db.Model(&types.Knowledge{}). +func stuckKnowledgeSummaryQuery(db *gorm.DB) *gorm.DB { + return db.Model(&types.Knowledge{}). Where("summary_status IN ?", []string{types.SummaryStatusPending, types.SummaryStatusProcessing}) - if distributed { - q = q.Where("updated_at < ?", staleCutoff) - } - return q } func stuckSyncLogQuery(db *gorm.DB, distributed bool, staleCutoff time.Time) *gorm.DB { @@ -135,3 +167,22 @@ func stuckSyncLogQuery(db *gorm.DB, distributed bool, staleCutoff time.Time) *go } return q } + +func resettableParseStatuses() []string { + return []string{ + types.ParseStatusPending, + types.ParseStatusProcessing, + types.ParseStatusFinalizing, + types.ParseStatusDeleting, + } +} + +func knowledgeIDs(rows []types.Knowledge) []string { + ids := make([]string, 0, len(rows)) + for _, row := range rows { + if row.ID != "" { + ids = append(ids, row.ID) + } + } + return ids +} diff --git a/internal/container/reset_pending_tasks_test.go b/internal/container/reset_pending_tasks_test.go index 97128dd5..5c4b2aeb 100644 --- a/internal/container/reset_pending_tasks_test.go +++ b/internal/container/reset_pending_tasks_test.go @@ -1,11 +1,14 @@ package container import ( + "encoding/json" "os" "testing" "time" + "github.com/Tencent/WeKnora/internal/application/service" "github.com/Tencent/WeKnora/internal/types" + "github.com/hibiken/asynq" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gorm.io/driver/sqlite" @@ -38,12 +41,51 @@ CREATE TABLE IF NOT EXISTS sync_logs ( ); ` +const resetPendingSpansDDL = ` +CREATE TABLE IF NOT EXISTS knowledge_processing_spans ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + knowledge_id VARCHAR(64) NOT NULL, + attempt INTEGER NOT NULL DEFAULT 1, + span_id VARCHAR(64) NOT NULL, + parent_span_id VARCHAR(64), + name VARCHAR(255) NOT NULL, + kind VARCHAR(16) NOT NULL, + status VARCHAR(16) NOT NULL, + error_code VARCHAR(64), + error_message TEXT, + started_at DATETIME, + finished_at DATETIME, + duration_ms INTEGER, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, + UNIQUE (knowledge_id, attempt, span_id) +); +` + +const resetPendingOpsDDL = ` +CREATE TABLE IF NOT EXISTS task_pending_ops ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + tenant_id INTEGER NOT NULL DEFAULT 0, + task_type VARCHAR(64) NOT NULL, + scope VARCHAR(32) NOT NULL, + scope_id VARCHAR(64) NOT NULL, + op VARCHAR(32) NOT NULL, + dedup_key VARCHAR(128) NOT NULL DEFAULT '', + payload TEXT NOT NULL DEFAULT '{}', + fail_count INTEGER NOT NULL DEFAULT 0, + enqueued_at DATETIME DEFAULT CURRENT_TIMESTAMP, + claimed_at DATETIME +); +` + func setupResetPendingDB(t *testing.T) *gorm.DB { t.Helper() db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) require.NoError(t, err) require.NoError(t, db.Exec(resetPendingKnowledgeDDL).Error) require.NoError(t, db.Exec(resetPendingSyncLogDDL).Error) + require.NoError(t, db.Exec(resetPendingSpansDDL).Error) + require.NoError(t, db.Exec(resetPendingOpsDDL).Error) return db } @@ -55,7 +97,7 @@ func TestResetPendingTasks_KnowledgeFindThenUpdate(t *testing.T) { "k-stuck", types.ParseStatusProcessing, stale, ).Error) - t.Setenv("REDIS_ADDR", "redis:6379") + os.Unsetenv("REDIS_ADDR") resetPendingTasks(db) var status, errMsg string @@ -84,6 +126,107 @@ func TestResetPendingTasks_KnowledgeFreshInDistributedMode(t *testing.T) { assert.Equal(t, types.ParseStatusProcessing, status) } +func TestResetPendingTasks_DistributedModePreservesEveryStage(t *testing.T) { + cases := []struct { + name string + parseStatus string + }{ + {"pending", types.ParseStatusPending}, + {"docreader_chunking_embedding", types.ParseStatusProcessing}, + {"multimodal", types.ParseStatusProcessing}, + {"postprocess", types.ParseStatusFinalizing}, + {"wiki", types.ParseStatusFinalizing}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + db := setupResetPendingDB(t) + stale := time.Now().Add(-2 * time.Hour) + require.NoError(t, db.Exec( + `INSERT INTO knowledges (id, parse_status, updated_at) VALUES (?, ?, ?)`, + "k-active-span", tc.parseStatus, stale, + ).Error) + + t.Setenv("REDIS_ADDR", "redis:6379") + resetPendingTasks(db) + + var status string + require.NoError(t, db.Raw( + `SELECT parse_status FROM knowledges WHERE id = ?`, "k-active-span", + ).Row().Scan(&status)) + assert.Equal(t, tc.parseStatus, status, + "distributed tasks belong to Asynq/housekeeping, not startup reset") + }) + } +} + +func TestResetPendingTasks_DistributedSummaryTaskSurvivesRestart(t *testing.T) { + db := setupResetPendingDB(t) + stale := time.Now().Add(-2 * time.Hour) + require.NoError(t, db.Exec( + `INSERT INTO knowledges (id, parse_status, summary_status, updated_at) + VALUES (?, ?, ?, ?)`, + "k-summary", types.ParseStatusCompleted, types.SummaryStatusProcessing, stale, + ).Error) + + t.Setenv("REDIS_ADDR", "redis:6379") + resetPendingTasks(db) + + var status string + require.NoError(t, db.Raw( + `SELECT summary_status FROM knowledges WHERE id = ?`, "k-summary", + ).Row().Scan(&status)) + assert.Equal(t, types.SummaryStatusProcessing, status) +} + +func TestResetPendingTasks_DurableWikiOpSurvivesLiteRestart(t *testing.T) { + db := setupResetPendingDB(t) + require.NoError(t, db.Exec( + `INSERT INTO knowledges (id, parse_status, pending_subtasks_count) + VALUES (?, ?, 1)`, "k-wiki", types.ParseStatusFinalizing, + ).Error) + require.NoError(t, db.Exec( + `INSERT INTO task_pending_ops + (tenant_id, task_type, scope, scope_id, op, dedup_key, payload) + VALUES (7, ?, ?, 'kb-wiki', 'ingest', 'k-wiki', '{}')`, + types.TypeWikiIngest, types.TaskScopeKnowledgeBase, + ).Error) + + os.Unsetenv("REDIS_ADDR") + resetPendingTasks(db) + + var status string + var pending int + require.NoError(t, db.Raw( + `SELECT parse_status, pending_subtasks_count FROM knowledges WHERE id = ?`, "k-wiki", + ).Row().Scan(&status, &pending)) + assert.Equal(t, types.ParseStatusFinalizing, status) + assert.Equal(t, 1, pending, "resumed wiki worker still owns its finalizing slot") +} + +func TestResetPendingTasks_LiteWikiDoesNotHideOtherLostSubtasks(t *testing.T) { + db := setupResetPendingDB(t) + require.NoError(t, db.Exec( + `INSERT INTO knowledges (id, parse_status, pending_subtasks_count) + VALUES (?, ?, 2)`, "k-wiki-plus-summary", types.ParseStatusFinalizing, + ).Error) + require.NoError(t, db.Exec( + `INSERT INTO task_pending_ops + (tenant_id, task_type, scope, scope_id, op, dedup_key, payload) + VALUES (7, ?, ?, 'kb-wiki', 'ingest', 'k-wiki-plus-summary', '{}')`, + types.TypeWikiIngest, types.TaskScopeKnowledgeBase, + ).Error) + + os.Unsetenv("REDIS_ADDR") + resetPendingTasks(db) + + var status string + require.NoError(t, db.Raw( + `SELECT parse_status FROM knowledges WHERE id = ?`, "k-wiki-plus-summary", + ).Row().Scan(&status)) + assert.Equal(t, types.ParseStatusFailed, status, + "a durable wiki op cannot recover another lost in-memory subtask") +} + func TestResetPendingTasks_SyncLogStaleRunning(t *testing.T) { db := setupResetPendingDB(t) stale := time.Now().Add(-2 * time.Hour) @@ -129,17 +272,60 @@ func TestStuckKnowledgeParseQuery_ReuseAfterFindDoesNotBreakUpdate(t *testing.T) "k-reuse", types.ParseStatusProcessing, stale, ).Error) - distributed := true - staleCutoff := time.Now().Add(-resetPendingStaleWindow) - var rows []types.Knowledge - q := stuckKnowledgeParseQuery(db, distributed, staleCutoff) + q := stuckKnowledgeParseQuery(db) require.NoError(t, q.Select("id").Find(&rows).Error) require.Len(t, rows, 1) - result := stuckKnowledgeParseQuery(db, distributed, staleCutoff).Updates(map[string]interface{}{ + result := stuckKnowledgeParseQuery(db).Updates(map[string]interface{}{ "parse_status": types.ParseStatusFailed, }) require.NoError(t, result.Error) assert.Equal(t, int64(1), result.RowsAffected) } + +type recordingTaskEnqueuer struct { + tasks []*asynq.Task +} + +func (r *recordingTaskEnqueuer) Enqueue(task *asynq.Task, _ ...asynq.Option) (*asynq.TaskInfo, error) { + r.tasks = append(r.tasks, task) + return &asynq.TaskInfo{ID: "test", Type: task.Type()}, nil +} + +func TestRecoverPendingWikiTasks_RecreatesOneTriggerPerLaneAndKB(t *testing.T) { + db := setupResetPendingDB(t) + rows := []struct { + tenantID uint64 + taskType string + kbID string + dedup string + }{ + {7, types.TypeWikiIngest, "kb-a", "k-1"}, + {7, types.TypeWikiIngest, "kb-a", "k-2"}, // same lane: one trigger + {7, types.TypeWikiFinalize, "kb-a", "slug-a"}, + {8, types.TypeWikiIngest, "kb-b", "k-3"}, + } + for _, row := range rows { + require.NoError(t, db.Exec( + `INSERT INTO task_pending_ops + (tenant_id, task_type, scope, scope_id, op, dedup_key, payload) + VALUES (?, ?, ?, ?, 'ingest', ?, '{}')`, + row.tenantID, row.taskType, types.TaskScopeKnowledgeBase, row.kbID, row.dedup, + ).Error) + } + + recorder := &recordingTaskEnqueuer{} + recoverPendingWikiTasks(db, recorder) + require.Len(t, recorder.tasks, 3) + + seen := map[string]service.WikiIngestPayload{} + for _, task := range recorder.tasks { + var payload service.WikiIngestPayload + require.NoError(t, json.Unmarshal(task.Payload(), &payload)) + seen[task.Type()+":"+payload.KnowledgeBaseID] = payload + } + assert.Equal(t, uint64(7), seen[types.TypeWikiIngest+":kb-a"].TenantID) + assert.Equal(t, uint64(7), seen[types.TypeWikiFinalize+":kb-a"].TenantID) + assert.Equal(t, uint64(8), seen[types.TypeWikiIngest+":kb-b"].TenantID) +} diff --git a/internal/handler/knowledge.go b/internal/handler/knowledge.go index 15cbbf8b..da43e962 100644 --- a/internal/handler/knowledge.go +++ b/internal/handler/knowledge.go @@ -761,12 +761,17 @@ func knowledgeSpansLastError( if currentAttempt != latestAttempt || parseStatus != types.ParseStatusFailed || knowledgeErrorMessage == "" { return nil } + errorCode := "UNKNOWN" + if strings.EqualFold(strings.TrimSpace(knowledgeErrorMessage), + "Task interrupted due to application restart") { + errorCode = "SERVER_RESTART" + } return gin.H{ "stage": "knowledge_processing", - "code": "UNKNOWN", + "code": errorCode, "message": knowledgeErrorMessage, "name": "knowledge_processing", - "error_code": "UNKNOWN", + "error_code": errorCode, "error_message": knowledgeErrorMessage, "finished_at": knowledgeUpdatedAt, } diff --git a/internal/handler/knowledge_spans_test.go b/internal/handler/knowledge_spans_test.go index be31d985..0976794d 100644 --- a/internal/handler/knowledge_spans_test.go +++ b/internal/handler/knowledge_spans_test.go @@ -150,6 +150,16 @@ func TestKnowledgeSpansLastError_RecoveryFallbackUsesKnowledgeMessage(t *testing a.Equal(now, got["finished_at"]) } +func TestKnowledgeSpansLastError_ClassifiesServerRestart(t *testing.T) { + now := time.Now() + got := knowledgeSpansLastError(2, 2, types.ParseStatusFailed, + "Task interrupted due to application restart", now, nil) + a := assert.New(t) + a.NotNil(got) + a.Equal("SERVER_RESTART", got["error_code"]) + a.Equal("SERVER_RESTART", got["code"]) +} + func TestKnowledgeSpansLastError_SkipsRecoveryFallbackForHistoricalAttempt(t *testing.T) { now := time.Now() got := knowledgeSpansLastError(1, 2, types.ParseStatusFailed, "wiki ingest timed out", now, nil)