fix(knowledge): recover tasks safely after server restart

Stop failing distributed-mode knowledge rows on startup; let housekeeping
decide using span activity and Asynq queue state. In Lite mode, resume
wiki work from durable task_pending_ops and surface SERVER_RESTART errors
in the processing timeline with i18n support.
This commit is contained in:
wizardchen
2026-07-14 17:20:22 +08:00
committed by lyingbug
parent 28ebcf2525
commit ace84bc757
12 changed files with 462 additions and 74 deletions
+2
View File
@@ -884,6 +884,8 @@ export default {
MULTIMODAL_ALL_FAILED_SUGGESTION: 'Verify the multimodal model configuration.',
TASK_TIMEOUT: 'Task exceeded maximum runtime',
TASK_TIMEOUT_SUGGESTION: 'The task ran longer than allowed. Retry or contact support.',
SERVER_RESTART: 'Task interrupted by server restart',
SERVER_RESTART_SUGGESTION: 'Redis-backed tasks retry from the queue; recoverable Wiki work also resumes in Lite mode. Retry if this task remains failed.',
UPSTREAM_FAILED: 'Stopped because an upstream stage failed',
UPSTREAM_FAILED_SUGGESTION: 'A previous stage failed and prevented this step from running.',
UNKNOWN: 'Unknown error',
+2
View File
@@ -888,6 +888,8 @@ export default {
MULTIMODAL_ALL_FAILED_SUGGESTION: "멀티모달 모델 구성을 확인하세요.",
TASK_TIMEOUT: "작업이 최대 실행 시간을 초과함",
TASK_TIMEOUT_SUGGESTION: "작업이 허용된 시간보다 오래 실행되었습니다. 재시도하거나 지원팀에 문의하세요.",
SERVER_RESTART: "서버 재시작으로 작업이 중단됨",
SERVER_RESTART_SUGGESTION: "Redis 모드 작업은 큐에서 재시도되고, Lite 모드의 복구 가능한 Wiki 작업도 자동으로 재개됩니다. 계속 실패하면 다시 시도하세요.",
UPSTREAM_FAILED: "상위 단계 실패로 중단됨",
UPSTREAM_FAILED_SUGGESTION: "이전 단계가 실패하여 이 단계가 실행되지 않았습니다.",
UNKNOWN: "알 수 없는 오류",
+2
View File
@@ -803,6 +803,8 @@ export default {
MULTIMODAL_ALL_FAILED_SUGGESTION: 'Проверьте конфигурацию мультимодальной модели.',
TASK_TIMEOUT: 'Задача превысила максимальное время выполнения',
TASK_TIMEOUT_SUGGESTION: 'Задача выполнялась дольше допустимого. Повторите или обратитесь в поддержку.',
SERVER_RESTART: 'Задача прервана перезапуском сервера',
SERVER_RESTART_SUGGESTION: 'Задачи с Redis повторяются из очереди, а восстанавливаемая обработка Wiki возобновляется и в Lite-режиме. Повторите попытку, если ошибка осталась.',
UPSTREAM_FAILED: 'Остановлено из-за ошибки на предыдущем этапе',
UPSTREAM_FAILED_SUGGESTION: 'Предыдущий этап завершился с ошибкой и заблокировал выполнение.',
UNKNOWN: 'Неизвестная ошибка',
+2
View File
@@ -886,6 +886,8 @@ export default {
MULTIMODAL_ALL_FAILED_SUGGESTION: "请检查多模态模型配置。",
TASK_TIMEOUT: "任务超过最大运行时间",
TASK_TIMEOUT_SUGGESTION: "任务运行时间超出限制,请重试或联系支持人员。",
SERVER_RESTART: "服务重启导致任务中断",
SERVER_RESTART_SUGGESTION: "Redis 模式会由任务队列继续重试;Lite 模式中可恢复的 Wiki 任务会自动续跑。若仍失败,请点击重试。",
UPSTREAM_FAILED: "上游阶段失败已中止",
UPSTREAM_FAILED_SUGGESTION: "前置阶段失败,导致本步骤无法执行。",
UNKNOWN: "未知错误",
@@ -112,7 +112,7 @@ func (h *HousekeepingService) runSweep(ctx context.Context) {
threshold := h.staleThreshold()
cutoff := time.Now().Add(-threshold)
// Sweep A: knowledge stuck in "processing".
// Sweep A: knowledge stuck in "pending", "processing", or "finalizing".
//
// Two-stage check is critical here: knowledge.updated_at advances
// only at parse_status transitions, but a long stage (DocReader on
@@ -127,7 +127,9 @@ func (h *HousekeepingService) runSweep(ctx context.Context) {
// Knowledge rows with no spans at all (lite mode, in-flight tasks
// from before this code shipped) fall back to the simple
// updated_at check — they have no heartbeat to consult.
// Include 'finalizing' alongside 'processing': finalizing rows still
// Include 'pending' so a task whose enqueue was lost is eventually
// recovered too; filterOutQueued below protects legitimately backlogged
// tasks. Include 'finalizing' alongside 'processing': finalizing rows still
// consume LLM compute via enrichment subtasks (summary/question/graph),
// and the same stall modes (subtask worker dies, retry budget exhausted
// without decrementing the counter) leave the row hanging just as
@@ -136,7 +138,7 @@ func (h *HousekeepingService) runSweep(ctx context.Context) {
var candidates []types.Knowledge
if err := h.db.WithContext(ctx).
Where("parse_status IN ? AND updated_at < ?",
[]string{types.ParseStatusProcessing, types.ParseStatusFinalizing}, cutoff).
[]string{types.ParseStatusPending, types.ParseStatusProcessing, types.ParseStatusFinalizing}, cutoff).
Find(&candidates).Error; err != nil {
logger.Warnf(ctx, "[Housekeeping] knowledge candidate query failed: %v", err)
return
@@ -162,7 +164,7 @@ func (h *HousekeepingService) runSweep(ctx context.Context) {
}
res := h.db.WithContext(ctx).Model(&types.Knowledge{}).
Where("id IN ? AND parse_status IN ?", stuckIDs,
[]string{types.ParseStatusProcessing, types.ParseStatusFinalizing}).
[]string{types.ParseStatusPending, types.ParseStatusProcessing, types.ParseStatusFinalizing}).
Updates(map[string]interface{}{
"parse_status": types.ParseStatusFailed,
"error_message": "task stuck in processing > " + threshold.String() + ", recovered by housekeeping",
@@ -165,6 +165,40 @@ func TestHousekeeping_RecoversAbandoned(t *testing.T) {
assert.Contains(t, errMsg, "stuck in processing")
}
func TestHousekeeping_RecoversPendingTaskMissingFromQueue(t *testing.T) {
db := setupHousekeepingDB(t)
svc := newHousekeepingSvcForTest(db)
stale := time.Now().Add(-3 * time.Hour)
insertKnowledge(t, db, "kid-pending-orphan", types.ParseStatusPending, stale)
svc.runSweep(context.Background())
var status string
require.NoError(t, db.Raw(
`SELECT parse_status FROM knowledges WHERE id = ?`, "kid-pending-orphan",
).Row().Scan(&status))
assert.Equal(t, types.ParseStatusFailed, status,
"a stale pending row with no queue task must not remain pending forever")
}
func TestHousekeeping_PreservesPendingTaskStillQueued(t *testing.T) {
db := setupHousekeepingDB(t)
svc := newHousekeepingSvcWithInspector(db, fakeTaskInspector{
queued: map[string]bool{"kid-pending-queued": true},
})
stale := time.Now().Add(-3 * time.Hour)
insertKnowledge(t, db, "kid-pending-queued", types.ParseStatusPending, stale)
svc.runSweep(context.Background())
var status string
require.NoError(t, db.Raw(
`SELECT parse_status FROM knowledges WHERE id = ?`, "kid-pending-queued",
).Row().Scan(&status))
assert.Equal(t, types.ParseStatusPending, status,
"backlogged pending work remains owned by the durable queue")
}
// TestHousekeeping_NoFalseKill_ActiveSpan is the regression test for
// the "long DocReader silently runs longer than DocumentProcessTimeout"
// scenario the user flagged. A knowledge whose knowledge.updated_at
+5
View File
@@ -390,6 +390,11 @@ func BuildContainer(container *dig.Container) *dig.Container {
} else {
must(container.Invoke(router.RegisterSyncHandlers))
}
// Wiki operation rows are durable, while their wake-up triggers may be
// lost across a process restart (always in Lite mode, and in Redis mode if
// persistence succeeded immediately before trigger enqueue failed). Re-arm
// them only after the matching handlers are ready.
must(container.Invoke(recoverPendingWikiTasks))
logger.Infof(ctx, "[Container] Container initialization completed successfully")
return container
@@ -0,0 +1,87 @@
package container
import (
"context"
"encoding/json"
"errors"
"time"
"github.com/Tencent/WeKnora/internal/application/service"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/hibiken/asynq"
"gorm.io/gorm"
)
// pendingWikiScope is the minimum routing information needed to recreate an
// ephemeral wiki trigger from the durable task_pending_ops queue.
type pendingWikiScope struct {
TenantID uint64 `gorm:"column:tenant_id"`
TaskType string `gorm:"column:task_type"`
ScopeID string `gorm:"column:scope_id"`
}
// recoverPendingWikiTasks recreates one trigger per pending wiki queue lane.
//
// task_pending_ops is deliberately durable, but the trigger that wakes its
// consumer is not durable in Lite mode (SyncTaskExecutor is process-local) and
// may also be absent after an interrupted Redis enqueue. Running this after all
// handlers are registered closes that gap. Duplicate triggers are harmless:
// ingest claims/peeks disjoint rows and finalize coalesces its pending lane.
func recoverPendingWikiTasks(db *gorm.DB, task interfaces.TaskEnqueuer) {
if db == nil || task == nil {
return
}
ctx := context.Background()
var scopes []pendingWikiScope
if err := db.WithContext(ctx).
Model(&types.TaskPendingOp{}).
Distinct("tenant_id", "task_type", "scope_id").
Where("scope = ? AND task_type IN ?", types.TaskScopeKnowledgeBase,
[]string{types.TypeWikiIngest, types.TypeWikiFinalize}).
Find(&scopes).Error; err != nil {
logger.Warnf(ctx, "[WikiRecovery] failed to list pending queues: %v", err)
return
}
recovered := 0
for _, scope := range scopes {
if scope.ScopeID == "" {
continue
}
payload, err := json.Marshal(service.WikiIngestPayload{
TenantID: scope.TenantID,
KnowledgeBaseID: scope.ScopeID,
})
if err != nil {
logger.Warnf(ctx, "[WikiRecovery] marshal trigger for KB %s failed: %v", scope.ScopeID, err)
continue
}
opts := []asynq.Option{
asynq.Queue(types.QueueWiki),
asynq.MaxRetry(10), // keep aligned with the wiki ingest retry policy
asynq.Timeout(60 * time.Minute),
}
if scope.TaskType == types.TypeWikiFinalize {
opts[2] = asynq.Timeout(30 * time.Minute)
// Match scheduleFinalize so simultaneous replica startups collapse
// into the same per-KB finalize trigger.
opts = append(opts, asynq.TaskID("wiki-finalize-"+scope.ScopeID))
}
trigger := asynq.NewTask(scope.TaskType, payload, opts...)
if _, err := task.Enqueue(trigger); err != nil {
if errors.Is(err, asynq.ErrTaskIDConflict) || errors.Is(err, asynq.ErrDuplicateTask) {
recovered++
continue
}
logger.Warnf(ctx, "[WikiRecovery] enqueue %s trigger for KB %s failed: %v",
scope.TaskType, scope.ScopeID, err)
continue
}
recovered++
}
if recovered > 0 {
logger.Infof(ctx, "[WikiRecovery] recreated %d trigger(s) from durable pending queues", recovered)
}
}
+113 -62
View File
@@ -13,18 +13,22 @@ import (
const resetPendingStaleWindow = 30 * time.Minute
const restartInterruptedMessage = "Task interrupted due to application restart"
// resetPendingTasks resets the state of any knowledge items or sync logs stuck in processing
// due to an unexpected application restart.
//
// In Lite mode (no REDIS_ADDR) every queued task lived in process memory, so
// any "processing" row at startup is definitively orphaned and must be marked
// failed wholesale. In distributed mode (Asynq on Redis) the active queue
// survives restart, but the *currently executing* task on the dead instance
// is lost — Asynq won't reschedule it until at-least-once retry kicks in,
// which can take minutes or never (e.g. if the deadline has passed). To bound
// the worst case we mark only "long-stale" rows failed: anything that hasn't
// been touched for 30 minutes is well past any reasonable in-flight window.
// Newer rows are left alone so we don't race a peer instance that's mid-process.
// In Lite mode (no REDIS_ADDR) normal queued tasks live in process memory, so
// a "processing" row at startup is orphaned unless it has a durable wiki op
// that can be re-triggered.
//
// Distributed mode is intentionally different: Asynq persists queued/retry
// tasks, and another replica may still be executing the same knowledge. A
// startup hook cannot safely distinguish an orphan from a backlogged task that
// has not opened a span yet. HousekeepingService owns that decision because it
// checks BOTH recent span activity and the real Asynq queue. Consequently this
// hook never resets knowledge/summary rows in distributed mode; it only keeps
// the separate sync-log cleanup below.
func resetPendingTasks(db *gorm.DB) {
distributed := os.Getenv("REDIS_ADDR") != ""
ctx := context.Background()
@@ -35,22 +39,64 @@ func resetPendingTasks(db *gorm.DB) {
staleCutoff = time.Now().Add(-resetPendingStaleWindow)
}
// Cancel orphaned trace spans for knowledge rows we are about to mark
// failed. resetPendingTasks does not touch asynq queues; this only
// prevents the UI from showing duplicate running postprocess.*
// subspans when a later retry also opens fresh spans.
// Resolve Lite-mode orphaned knowledge rows first. A finalizing row whose
// ONLY remaining slot is backed by a durable wiki op is excluded and resumed
// by recoverPendingWikiTasks after handlers are registered. Rows with other
// outstanding in-memory subtasks still fail: the wiki op cannot recover them.
var stuckKnowledge []types.Knowledge
if err := stuckKnowledgeParseQuery(db, distributed, staleCutoff).
Select("id").Find(&stuckKnowledge).Error; err != nil {
logger.Warnf(ctx, "resetPendingTasks: list stuck knowledge failed: %v", err)
} else {
for _, k := range stuckKnowledge {
if !distributed {
if err := stuckKnowledgeParseQuery(db).
Select("id").Find(&stuckKnowledge).Error; err != nil {
logger.Warnf(ctx, "resetPendingTasks: list stuck knowledge failed: %v", err)
}
}
// 1. Reset knowledge parsing tasks (including finalizing rows whose
// enrichment subtasks were lost with the process).
// Update by the resolved ids rather than reusing the GORM chain after
// Find() (which makes PostgreSQL emit an invalid UPDATE ... FROM self).
stuckIDs := knowledgeIDs(stuckKnowledge)
var resetErr error
var resetCount int64
if len(stuckIDs) > 0 {
// Rebuild the query instead of reusing the chain after Find(); reusing it
// makes PostgreSQL emit an invalid UPDATE ... FROM self statement.
result := stuckKnowledgeParseQuery(db).
Where("id IN ?", stuckIDs).
Updates(map[string]interface{}{
"parse_status": types.ParseStatusFailed,
"error_message": restartInterruptedMessage,
"pending_subtasks_count": 0,
})
resetErr = result.Error
resetCount = result.RowsAffected
}
if resetErr != nil {
logger.Warnf(context.Background(), "Failed to reset pending knowledge tasks: %v", resetErr)
} else if resetCount > 0 {
logger.Infof(context.Background(),
"Reset %d stuck knowledge parsing tasks to failed state (distributed=%v)",
resetCount, distributed)
// Cancel orphaned trace spans only after the owning knowledge rows are
// terminal. This prevents the UI from showing duplicate running
// postprocess.* subspans when a later manual retry opens fresh spans.
// Re-read the successfully reset ids so a row whose status changed in
// the small SELECT/UPDATE gap is not accidentally cancelled.
var resetKnowledge []types.Knowledge
if err := db.Select("id").
Where("id IN ? AND parse_status = ? AND error_message = ?",
stuckIDs, types.ParseStatusFailed, restartInterruptedMessage).
Find(&resetKnowledge).Error; err != nil {
logger.Warnf(ctx, "resetPendingTasks: list reset knowledge failed: %v", err)
}
for _, k := range resetKnowledge {
attempt, err := spanRepo.LatestAttempt(ctx, k.ID)
if err != nil || attempt <= 0 {
continue
}
if n, err := spanRepo.CancelAllOpenSpans(ctx, k.ID, attempt,
"SERVER_RESTART", "task interrupted due to application restart"); err != nil {
"SERVER_RESTART", restartInterruptedMessage); err != nil {
logger.Warnf(ctx, "resetPendingTasks: cancel spans for %s failed: %v", k.ID, err)
} else if n > 0 {
logger.Infof(ctx, "resetPendingTasks: cancelled %d open span(s) for knowledge %s attempt %d",
@@ -59,33 +105,19 @@ func resetPendingTasks(db *gorm.DB) {
}
}
// 1. Reset knowledge parsing tasks (including finalizing rows whose
// enrichment subtasks were lost with the process).
// Fresh query — reusing the *gorm.DB chain after Find() makes GORM emit
// UPDATE ... FROM knowledges which PostgreSQL rejects (SQLSTATE 42712).
result := stuckKnowledgeParseQuery(db, distributed, staleCutoff).Updates(map[string]interface{}{
"parse_status": types.ParseStatusFailed,
"error_message": "Task interrupted due to application restart",
"pending_subtasks_count": 0,
})
if result.Error != nil {
logger.Warnf(context.Background(), "Failed to reset pending knowledge tasks: %v", result.Error)
} else if result.RowsAffected > 0 {
logger.Infof(context.Background(),
"Reset %d stuck knowledge parsing tasks to failed state (distributed=%v)",
result.RowsAffected, distributed)
}
// 2. Reset knowledge summary tasks
resultSummary := stuckKnowledgeSummaryQuery(db, distributed, staleCutoff).Updates(map[string]interface{}{
"summary_status": types.SummaryStatusFailed,
})
if resultSummary.Error != nil {
logger.Warnf(context.Background(), "Failed to reset pending summary tasks: %v", resultSummary.Error)
} else if resultSummary.RowsAffected > 0 {
logger.Infof(context.Background(),
"Reset %d stuck summary generation tasks to failed state (distributed=%v)",
resultSummary.RowsAffected, distributed)
// 2. Lite summary tasks are process-local too. Distributed summary tasks
// stay in Asynq and must not be failed merely because this replica started.
if !distributed {
resultSummary := stuckKnowledgeSummaryQuery(db).Updates(map[string]interface{}{
"summary_status": types.SummaryStatusFailed,
})
if resultSummary.Error != nil {
logger.Warnf(context.Background(), "Failed to reset pending summary tasks: %v", resultSummary.Error)
} else if resultSummary.RowsAffected > 0 {
logger.Infof(context.Background(),
"Reset %d stuck summary generation tasks to failed state (distributed=false)",
resultSummary.RowsAffected)
}
}
// 3. Reset data source sync tasks
@@ -104,27 +136,27 @@ func resetPendingTasks(db *gorm.DB) {
}
}
func stuckKnowledgeParseQuery(db *gorm.DB, distributed bool, staleCutoff time.Time) *gorm.DB {
func stuckKnowledgeParseQuery(db *gorm.DB) *gorm.DB {
q := db.Model(&types.Knowledge{}).
Where("parse_status IN ?", []string{
types.ParseStatusPending,
types.ParseStatusProcessing,
types.ParseStatusFinalizing,
types.ParseStatusDeleting,
})
if distributed {
q = q.Where("updated_at < ?", staleCutoff)
}
Where("parse_status IN ?", resettableParseStatuses()).
// Wiki ingest ops are persisted independently from the task trigger.
// When wiki owns the only outstanding slot, keeping this finalizing row
// alive lets startup recreate the trigger and finish cleanly. A count
// above one means at least one non-wiki in-memory subtask was also lost.
Where(`NOT (parse_status = ? AND pending_subtasks_count = 1 AND EXISTS (
SELECT 1 FROM task_pending_ops
WHERE task_pending_ops.task_type = ?
AND task_pending_ops.scope = ?
AND task_pending_ops.dedup_key = knowledges.id
AND task_pending_ops.op = ?
))`, types.ParseStatusFinalizing, types.TypeWikiIngest,
types.TaskScopeKnowledgeBase, "ingest")
return q
}
func stuckKnowledgeSummaryQuery(db *gorm.DB, distributed bool, staleCutoff time.Time) *gorm.DB {
q := db.Model(&types.Knowledge{}).
func stuckKnowledgeSummaryQuery(db *gorm.DB) *gorm.DB {
return db.Model(&types.Knowledge{}).
Where("summary_status IN ?", []string{types.SummaryStatusPending, types.SummaryStatusProcessing})
if distributed {
q = q.Where("updated_at < ?", staleCutoff)
}
return q
}
func stuckSyncLogQuery(db *gorm.DB, distributed bool, staleCutoff time.Time) *gorm.DB {
@@ -135,3 +167,22 @@ func stuckSyncLogQuery(db *gorm.DB, distributed bool, staleCutoff time.Time) *go
}
return q
}
func resettableParseStatuses() []string {
return []string{
types.ParseStatusPending,
types.ParseStatusProcessing,
types.ParseStatusFinalizing,
types.ParseStatusDeleting,
}
}
func knowledgeIDs(rows []types.Knowledge) []string {
ids := make([]string, 0, len(rows))
for _, row := range rows {
if row.ID != "" {
ids = append(ids, row.ID)
}
}
return ids
}
+192 -6
View File
@@ -1,11 +1,14 @@
package container
import (
"encoding/json"
"os"
"testing"
"time"
"github.com/Tencent/WeKnora/internal/application/service"
"github.com/Tencent/WeKnora/internal/types"
"github.com/hibiken/asynq"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/driver/sqlite"
@@ -38,12 +41,51 @@ CREATE TABLE IF NOT EXISTS sync_logs (
);
`
const resetPendingSpansDDL = `
CREATE TABLE IF NOT EXISTS knowledge_processing_spans (
id INTEGER PRIMARY KEY AUTOINCREMENT,
knowledge_id VARCHAR(64) NOT NULL,
attempt INTEGER NOT NULL DEFAULT 1,
span_id VARCHAR(64) NOT NULL,
parent_span_id VARCHAR(64),
name VARCHAR(255) NOT NULL,
kind VARCHAR(16) NOT NULL,
status VARCHAR(16) NOT NULL,
error_code VARCHAR(64),
error_message TEXT,
started_at DATETIME,
finished_at DATETIME,
duration_ms INTEGER,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
UNIQUE (knowledge_id, attempt, span_id)
);
`
const resetPendingOpsDDL = `
CREATE TABLE IF NOT EXISTS task_pending_ops (
id INTEGER PRIMARY KEY AUTOINCREMENT,
tenant_id INTEGER NOT NULL DEFAULT 0,
task_type VARCHAR(64) NOT NULL,
scope VARCHAR(32) NOT NULL,
scope_id VARCHAR(64) NOT NULL,
op VARCHAR(32) NOT NULL,
dedup_key VARCHAR(128) NOT NULL DEFAULT '',
payload TEXT NOT NULL DEFAULT '{}',
fail_count INTEGER NOT NULL DEFAULT 0,
enqueued_at DATETIME DEFAULT CURRENT_TIMESTAMP,
claimed_at DATETIME
);
`
func setupResetPendingDB(t *testing.T) *gorm.DB {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
require.NoError(t, err)
require.NoError(t, db.Exec(resetPendingKnowledgeDDL).Error)
require.NoError(t, db.Exec(resetPendingSyncLogDDL).Error)
require.NoError(t, db.Exec(resetPendingSpansDDL).Error)
require.NoError(t, db.Exec(resetPendingOpsDDL).Error)
return db
}
@@ -55,7 +97,7 @@ func TestResetPendingTasks_KnowledgeFindThenUpdate(t *testing.T) {
"k-stuck", types.ParseStatusProcessing, stale,
).Error)
t.Setenv("REDIS_ADDR", "redis:6379")
os.Unsetenv("REDIS_ADDR")
resetPendingTasks(db)
var status, errMsg string
@@ -84,6 +126,107 @@ func TestResetPendingTasks_KnowledgeFreshInDistributedMode(t *testing.T) {
assert.Equal(t, types.ParseStatusProcessing, status)
}
func TestResetPendingTasks_DistributedModePreservesEveryStage(t *testing.T) {
cases := []struct {
name string
parseStatus string
}{
{"pending", types.ParseStatusPending},
{"docreader_chunking_embedding", types.ParseStatusProcessing},
{"multimodal", types.ParseStatusProcessing},
{"postprocess", types.ParseStatusFinalizing},
{"wiki", types.ParseStatusFinalizing},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
db := setupResetPendingDB(t)
stale := time.Now().Add(-2 * time.Hour)
require.NoError(t, db.Exec(
`INSERT INTO knowledges (id, parse_status, updated_at) VALUES (?, ?, ?)`,
"k-active-span", tc.parseStatus, stale,
).Error)
t.Setenv("REDIS_ADDR", "redis:6379")
resetPendingTasks(db)
var status string
require.NoError(t, db.Raw(
`SELECT parse_status FROM knowledges WHERE id = ?`, "k-active-span",
).Row().Scan(&status))
assert.Equal(t, tc.parseStatus, status,
"distributed tasks belong to Asynq/housekeeping, not startup reset")
})
}
}
func TestResetPendingTasks_DistributedSummaryTaskSurvivesRestart(t *testing.T) {
db := setupResetPendingDB(t)
stale := time.Now().Add(-2 * time.Hour)
require.NoError(t, db.Exec(
`INSERT INTO knowledges (id, parse_status, summary_status, updated_at)
VALUES (?, ?, ?, ?)`,
"k-summary", types.ParseStatusCompleted, types.SummaryStatusProcessing, stale,
).Error)
t.Setenv("REDIS_ADDR", "redis:6379")
resetPendingTasks(db)
var status string
require.NoError(t, db.Raw(
`SELECT summary_status FROM knowledges WHERE id = ?`, "k-summary",
).Row().Scan(&status))
assert.Equal(t, types.SummaryStatusProcessing, status)
}
func TestResetPendingTasks_DurableWikiOpSurvivesLiteRestart(t *testing.T) {
db := setupResetPendingDB(t)
require.NoError(t, db.Exec(
`INSERT INTO knowledges (id, parse_status, pending_subtasks_count)
VALUES (?, ?, 1)`, "k-wiki", types.ParseStatusFinalizing,
).Error)
require.NoError(t, db.Exec(
`INSERT INTO task_pending_ops
(tenant_id, task_type, scope, scope_id, op, dedup_key, payload)
VALUES (7, ?, ?, 'kb-wiki', 'ingest', 'k-wiki', '{}')`,
types.TypeWikiIngest, types.TaskScopeKnowledgeBase,
).Error)
os.Unsetenv("REDIS_ADDR")
resetPendingTasks(db)
var status string
var pending int
require.NoError(t, db.Raw(
`SELECT parse_status, pending_subtasks_count FROM knowledges WHERE id = ?`, "k-wiki",
).Row().Scan(&status, &pending))
assert.Equal(t, types.ParseStatusFinalizing, status)
assert.Equal(t, 1, pending, "resumed wiki worker still owns its finalizing slot")
}
func TestResetPendingTasks_LiteWikiDoesNotHideOtherLostSubtasks(t *testing.T) {
db := setupResetPendingDB(t)
require.NoError(t, db.Exec(
`INSERT INTO knowledges (id, parse_status, pending_subtasks_count)
VALUES (?, ?, 2)`, "k-wiki-plus-summary", types.ParseStatusFinalizing,
).Error)
require.NoError(t, db.Exec(
`INSERT INTO task_pending_ops
(tenant_id, task_type, scope, scope_id, op, dedup_key, payload)
VALUES (7, ?, ?, 'kb-wiki', 'ingest', 'k-wiki-plus-summary', '{}')`,
types.TypeWikiIngest, types.TaskScopeKnowledgeBase,
).Error)
os.Unsetenv("REDIS_ADDR")
resetPendingTasks(db)
var status string
require.NoError(t, db.Raw(
`SELECT parse_status FROM knowledges WHERE id = ?`, "k-wiki-plus-summary",
).Row().Scan(&status))
assert.Equal(t, types.ParseStatusFailed, status,
"a durable wiki op cannot recover another lost in-memory subtask")
}
func TestResetPendingTasks_SyncLogStaleRunning(t *testing.T) {
db := setupResetPendingDB(t)
stale := time.Now().Add(-2 * time.Hour)
@@ -129,17 +272,60 @@ func TestStuckKnowledgeParseQuery_ReuseAfterFindDoesNotBreakUpdate(t *testing.T)
"k-reuse", types.ParseStatusProcessing, stale,
).Error)
distributed := true
staleCutoff := time.Now().Add(-resetPendingStaleWindow)
var rows []types.Knowledge
q := stuckKnowledgeParseQuery(db, distributed, staleCutoff)
q := stuckKnowledgeParseQuery(db)
require.NoError(t, q.Select("id").Find(&rows).Error)
require.Len(t, rows, 1)
result := stuckKnowledgeParseQuery(db, distributed, staleCutoff).Updates(map[string]interface{}{
result := stuckKnowledgeParseQuery(db).Updates(map[string]interface{}{
"parse_status": types.ParseStatusFailed,
})
require.NoError(t, result.Error)
assert.Equal(t, int64(1), result.RowsAffected)
}
type recordingTaskEnqueuer struct {
tasks []*asynq.Task
}
func (r *recordingTaskEnqueuer) Enqueue(task *asynq.Task, _ ...asynq.Option) (*asynq.TaskInfo, error) {
r.tasks = append(r.tasks, task)
return &asynq.TaskInfo{ID: "test", Type: task.Type()}, nil
}
func TestRecoverPendingWikiTasks_RecreatesOneTriggerPerLaneAndKB(t *testing.T) {
db := setupResetPendingDB(t)
rows := []struct {
tenantID uint64
taskType string
kbID string
dedup string
}{
{7, types.TypeWikiIngest, "kb-a", "k-1"},
{7, types.TypeWikiIngest, "kb-a", "k-2"}, // same lane: one trigger
{7, types.TypeWikiFinalize, "kb-a", "slug-a"},
{8, types.TypeWikiIngest, "kb-b", "k-3"},
}
for _, row := range rows {
require.NoError(t, db.Exec(
`INSERT INTO task_pending_ops
(tenant_id, task_type, scope, scope_id, op, dedup_key, payload)
VALUES (?, ?, ?, ?, 'ingest', ?, '{}')`,
row.tenantID, row.taskType, types.TaskScopeKnowledgeBase, row.kbID, row.dedup,
).Error)
}
recorder := &recordingTaskEnqueuer{}
recoverPendingWikiTasks(db, recorder)
require.Len(t, recorder.tasks, 3)
seen := map[string]service.WikiIngestPayload{}
for _, task := range recorder.tasks {
var payload service.WikiIngestPayload
require.NoError(t, json.Unmarshal(task.Payload(), &payload))
seen[task.Type()+":"+payload.KnowledgeBaseID] = payload
}
assert.Equal(t, uint64(7), seen[types.TypeWikiIngest+":kb-a"].TenantID)
assert.Equal(t, uint64(7), seen[types.TypeWikiFinalize+":kb-a"].TenantID)
assert.Equal(t, uint64(8), seen[types.TypeWikiIngest+":kb-b"].TenantID)
}
+7 -2
View File
@@ -761,12 +761,17 @@ func knowledgeSpansLastError(
if currentAttempt != latestAttempt || parseStatus != types.ParseStatusFailed || knowledgeErrorMessage == "" {
return nil
}
errorCode := "UNKNOWN"
if strings.EqualFold(strings.TrimSpace(knowledgeErrorMessage),
"Task interrupted due to application restart") {
errorCode = "SERVER_RESTART"
}
return gin.H{
"stage": "knowledge_processing",
"code": "UNKNOWN",
"code": errorCode,
"message": knowledgeErrorMessage,
"name": "knowledge_processing",
"error_code": "UNKNOWN",
"error_code": errorCode,
"error_message": knowledgeErrorMessage,
"finished_at": knowledgeUpdatedAt,
}
+10
View File
@@ -150,6 +150,16 @@ func TestKnowledgeSpansLastError_RecoveryFallbackUsesKnowledgeMessage(t *testing
a.Equal(now, got["finished_at"])
}
func TestKnowledgeSpansLastError_ClassifiesServerRestart(t *testing.T) {
now := time.Now()
got := knowledgeSpansLastError(2, 2, types.ParseStatusFailed,
"Task interrupted due to application restart", now, nil)
a := assert.New(t)
a.NotNil(got)
a.Equal("SERVER_RESTART", got["error_code"])
a.Equal("SERVER_RESTART", got["code"])
}
func TestKnowledgeSpansLastError_SkipsRecoveryFallbackForHistoricalAttempt(t *testing.T) {
now := time.Now()
got := knowledgeSpansLastError(1, 2, types.ParseStatusFailed, "wiki ingest timed out", now, nil)