fix(knowledge): recover tasks safely after server restart
Stop failing distributed-mode knowledge rows on startup; let housekeeping decide using span activity and Asynq queue state. In Lite mode, resume wiki work from durable task_pending_ops and surface SERVER_RESTART errors in the processing timeline with i18n support.
This commit is contained in:
@@ -884,6 +884,8 @@ export default {
|
||||
MULTIMODAL_ALL_FAILED_SUGGESTION: 'Verify the multimodal model configuration.',
|
||||
TASK_TIMEOUT: 'Task exceeded maximum runtime',
|
||||
TASK_TIMEOUT_SUGGESTION: 'The task ran longer than allowed. Retry or contact support.',
|
||||
SERVER_RESTART: 'Task interrupted by server restart',
|
||||
SERVER_RESTART_SUGGESTION: 'Redis-backed tasks retry from the queue; recoverable Wiki work also resumes in Lite mode. Retry if this task remains failed.',
|
||||
UPSTREAM_FAILED: 'Stopped because an upstream stage failed',
|
||||
UPSTREAM_FAILED_SUGGESTION: 'A previous stage failed and prevented this step from running.',
|
||||
UNKNOWN: 'Unknown error',
|
||||
|
||||
@@ -888,6 +888,8 @@ export default {
|
||||
MULTIMODAL_ALL_FAILED_SUGGESTION: "멀티모달 모델 구성을 확인하세요.",
|
||||
TASK_TIMEOUT: "작업이 최대 실행 시간을 초과함",
|
||||
TASK_TIMEOUT_SUGGESTION: "작업이 허용된 시간보다 오래 실행되었습니다. 재시도하거나 지원팀에 문의하세요.",
|
||||
SERVER_RESTART: "서버 재시작으로 작업이 중단됨",
|
||||
SERVER_RESTART_SUGGESTION: "Redis 모드 작업은 큐에서 재시도되고, Lite 모드의 복구 가능한 Wiki 작업도 자동으로 재개됩니다. 계속 실패하면 다시 시도하세요.",
|
||||
UPSTREAM_FAILED: "상위 단계 실패로 중단됨",
|
||||
UPSTREAM_FAILED_SUGGESTION: "이전 단계가 실패하여 이 단계가 실행되지 않았습니다.",
|
||||
UNKNOWN: "알 수 없는 오류",
|
||||
|
||||
@@ -803,6 +803,8 @@ export default {
|
||||
MULTIMODAL_ALL_FAILED_SUGGESTION: 'Проверьте конфигурацию мультимодальной модели.',
|
||||
TASK_TIMEOUT: 'Задача превысила максимальное время выполнения',
|
||||
TASK_TIMEOUT_SUGGESTION: 'Задача выполнялась дольше допустимого. Повторите или обратитесь в поддержку.',
|
||||
SERVER_RESTART: 'Задача прервана перезапуском сервера',
|
||||
SERVER_RESTART_SUGGESTION: 'Задачи с Redis повторяются из очереди, а восстанавливаемая обработка Wiki возобновляется и в Lite-режиме. Повторите попытку, если ошибка осталась.',
|
||||
UPSTREAM_FAILED: 'Остановлено из-за ошибки на предыдущем этапе',
|
||||
UPSTREAM_FAILED_SUGGESTION: 'Предыдущий этап завершился с ошибкой и заблокировал выполнение.',
|
||||
UNKNOWN: 'Неизвестная ошибка',
|
||||
|
||||
@@ -886,6 +886,8 @@ export default {
|
||||
MULTIMODAL_ALL_FAILED_SUGGESTION: "请检查多模态模型配置。",
|
||||
TASK_TIMEOUT: "任务超过最大运行时间",
|
||||
TASK_TIMEOUT_SUGGESTION: "任务运行时间超出限制,请重试或联系支持人员。",
|
||||
SERVER_RESTART: "服务重启导致任务中断",
|
||||
SERVER_RESTART_SUGGESTION: "Redis 模式会由任务队列继续重试;Lite 模式中可恢复的 Wiki 任务会自动续跑。若仍失败,请点击重试。",
|
||||
UPSTREAM_FAILED: "上游阶段失败已中止",
|
||||
UPSTREAM_FAILED_SUGGESTION: "前置阶段失败,导致本步骤无法执行。",
|
||||
UNKNOWN: "未知错误",
|
||||
|
||||
@@ -112,7 +112,7 @@ func (h *HousekeepingService) runSweep(ctx context.Context) {
|
||||
threshold := h.staleThreshold()
|
||||
cutoff := time.Now().Add(-threshold)
|
||||
|
||||
// Sweep A: knowledge stuck in "processing".
|
||||
// Sweep A: knowledge stuck in "pending", "processing", or "finalizing".
|
||||
//
|
||||
// Two-stage check is critical here: knowledge.updated_at advances
|
||||
// only at parse_status transitions, but a long stage (DocReader on
|
||||
@@ -127,7 +127,9 @@ func (h *HousekeepingService) runSweep(ctx context.Context) {
|
||||
// Knowledge rows with no spans at all (lite mode, in-flight tasks
|
||||
// from before this code shipped) fall back to the simple
|
||||
// updated_at check — they have no heartbeat to consult.
|
||||
// Include 'finalizing' alongside 'processing': finalizing rows still
|
||||
// Include 'pending' so a task whose enqueue was lost is eventually
|
||||
// recovered too; filterOutQueued below protects legitimately backlogged
|
||||
// tasks. Include 'finalizing' alongside 'processing': finalizing rows still
|
||||
// consume LLM compute via enrichment subtasks (summary/question/graph),
|
||||
// and the same stall modes (subtask worker dies, retry budget exhausted
|
||||
// without decrementing the counter) leave the row hanging just as
|
||||
@@ -136,7 +138,7 @@ func (h *HousekeepingService) runSweep(ctx context.Context) {
|
||||
var candidates []types.Knowledge
|
||||
if err := h.db.WithContext(ctx).
|
||||
Where("parse_status IN ? AND updated_at < ?",
|
||||
[]string{types.ParseStatusProcessing, types.ParseStatusFinalizing}, cutoff).
|
||||
[]string{types.ParseStatusPending, types.ParseStatusProcessing, types.ParseStatusFinalizing}, cutoff).
|
||||
Find(&candidates).Error; err != nil {
|
||||
logger.Warnf(ctx, "[Housekeeping] knowledge candidate query failed: %v", err)
|
||||
return
|
||||
@@ -162,7 +164,7 @@ func (h *HousekeepingService) runSweep(ctx context.Context) {
|
||||
}
|
||||
res := h.db.WithContext(ctx).Model(&types.Knowledge{}).
|
||||
Where("id IN ? AND parse_status IN ?", stuckIDs,
|
||||
[]string{types.ParseStatusProcessing, types.ParseStatusFinalizing}).
|
||||
[]string{types.ParseStatusPending, types.ParseStatusProcessing, types.ParseStatusFinalizing}).
|
||||
Updates(map[string]interface{}{
|
||||
"parse_status": types.ParseStatusFailed,
|
||||
"error_message": "task stuck in processing > " + threshold.String() + ", recovered by housekeeping",
|
||||
|
||||
@@ -165,6 +165,40 @@ func TestHousekeeping_RecoversAbandoned(t *testing.T) {
|
||||
assert.Contains(t, errMsg, "stuck in processing")
|
||||
}
|
||||
|
||||
func TestHousekeeping_RecoversPendingTaskMissingFromQueue(t *testing.T) {
|
||||
db := setupHousekeepingDB(t)
|
||||
svc := newHousekeepingSvcForTest(db)
|
||||
stale := time.Now().Add(-3 * time.Hour)
|
||||
insertKnowledge(t, db, "kid-pending-orphan", types.ParseStatusPending, stale)
|
||||
|
||||
svc.runSweep(context.Background())
|
||||
|
||||
var status string
|
||||
require.NoError(t, db.Raw(
|
||||
`SELECT parse_status FROM knowledges WHERE id = ?`, "kid-pending-orphan",
|
||||
).Row().Scan(&status))
|
||||
assert.Equal(t, types.ParseStatusFailed, status,
|
||||
"a stale pending row with no queue task must not remain pending forever")
|
||||
}
|
||||
|
||||
func TestHousekeeping_PreservesPendingTaskStillQueued(t *testing.T) {
|
||||
db := setupHousekeepingDB(t)
|
||||
svc := newHousekeepingSvcWithInspector(db, fakeTaskInspector{
|
||||
queued: map[string]bool{"kid-pending-queued": true},
|
||||
})
|
||||
stale := time.Now().Add(-3 * time.Hour)
|
||||
insertKnowledge(t, db, "kid-pending-queued", types.ParseStatusPending, stale)
|
||||
|
||||
svc.runSweep(context.Background())
|
||||
|
||||
var status string
|
||||
require.NoError(t, db.Raw(
|
||||
`SELECT parse_status FROM knowledges WHERE id = ?`, "kid-pending-queued",
|
||||
).Row().Scan(&status))
|
||||
assert.Equal(t, types.ParseStatusPending, status,
|
||||
"backlogged pending work remains owned by the durable queue")
|
||||
}
|
||||
|
||||
// TestHousekeeping_NoFalseKill_ActiveSpan is the regression test for
|
||||
// the "long DocReader silently runs longer than DocumentProcessTimeout"
|
||||
// scenario the user flagged. A knowledge whose knowledge.updated_at
|
||||
|
||||
@@ -390,6 +390,11 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
||||
} else {
|
||||
must(container.Invoke(router.RegisterSyncHandlers))
|
||||
}
|
||||
// Wiki operation rows are durable, while their wake-up triggers may be
|
||||
// lost across a process restart (always in Lite mode, and in Redis mode if
|
||||
// persistence succeeded immediately before trigger enqueue failed). Re-arm
|
||||
// them only after the matching handlers are ready.
|
||||
must(container.Invoke(recoverPendingWikiTasks))
|
||||
|
||||
logger.Infof(ctx, "[Container] Container initialization completed successfully")
|
||||
return container
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
package container
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/application/service"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/hibiken/asynq"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// pendingWikiScope is the minimum routing information needed to recreate an
|
||||
// ephemeral wiki trigger from the durable task_pending_ops queue.
|
||||
type pendingWikiScope struct {
|
||||
TenantID uint64 `gorm:"column:tenant_id"`
|
||||
TaskType string `gorm:"column:task_type"`
|
||||
ScopeID string `gorm:"column:scope_id"`
|
||||
}
|
||||
|
||||
// recoverPendingWikiTasks recreates one trigger per pending wiki queue lane.
|
||||
//
|
||||
// task_pending_ops is deliberately durable, but the trigger that wakes its
|
||||
// consumer is not durable in Lite mode (SyncTaskExecutor is process-local) and
|
||||
// may also be absent after an interrupted Redis enqueue. Running this after all
|
||||
// handlers are registered closes that gap. Duplicate triggers are harmless:
|
||||
// ingest claims/peeks disjoint rows and finalize coalesces its pending lane.
|
||||
func recoverPendingWikiTasks(db *gorm.DB, task interfaces.TaskEnqueuer) {
|
||||
if db == nil || task == nil {
|
||||
return
|
||||
}
|
||||
ctx := context.Background()
|
||||
var scopes []pendingWikiScope
|
||||
if err := db.WithContext(ctx).
|
||||
Model(&types.TaskPendingOp{}).
|
||||
Distinct("tenant_id", "task_type", "scope_id").
|
||||
Where("scope = ? AND task_type IN ?", types.TaskScopeKnowledgeBase,
|
||||
[]string{types.TypeWikiIngest, types.TypeWikiFinalize}).
|
||||
Find(&scopes).Error; err != nil {
|
||||
logger.Warnf(ctx, "[WikiRecovery] failed to list pending queues: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
recovered := 0
|
||||
for _, scope := range scopes {
|
||||
if scope.ScopeID == "" {
|
||||
continue
|
||||
}
|
||||
payload, err := json.Marshal(service.WikiIngestPayload{
|
||||
TenantID: scope.TenantID,
|
||||
KnowledgeBaseID: scope.ScopeID,
|
||||
})
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[WikiRecovery] marshal trigger for KB %s failed: %v", scope.ScopeID, err)
|
||||
continue
|
||||
}
|
||||
opts := []asynq.Option{
|
||||
asynq.Queue(types.QueueWiki),
|
||||
asynq.MaxRetry(10), // keep aligned with the wiki ingest retry policy
|
||||
asynq.Timeout(60 * time.Minute),
|
||||
}
|
||||
if scope.TaskType == types.TypeWikiFinalize {
|
||||
opts[2] = asynq.Timeout(30 * time.Minute)
|
||||
// Match scheduleFinalize so simultaneous replica startups collapse
|
||||
// into the same per-KB finalize trigger.
|
||||
opts = append(opts, asynq.TaskID("wiki-finalize-"+scope.ScopeID))
|
||||
}
|
||||
trigger := asynq.NewTask(scope.TaskType, payload, opts...)
|
||||
if _, err := task.Enqueue(trigger); err != nil {
|
||||
if errors.Is(err, asynq.ErrTaskIDConflict) || errors.Is(err, asynq.ErrDuplicateTask) {
|
||||
recovered++
|
||||
continue
|
||||
}
|
||||
logger.Warnf(ctx, "[WikiRecovery] enqueue %s trigger for KB %s failed: %v",
|
||||
scope.TaskType, scope.ScopeID, err)
|
||||
continue
|
||||
}
|
||||
recovered++
|
||||
}
|
||||
if recovered > 0 {
|
||||
logger.Infof(ctx, "[WikiRecovery] recreated %d trigger(s) from durable pending queues", recovered)
|
||||
}
|
||||
}
|
||||
@@ -13,18 +13,22 @@ import (
|
||||
|
||||
const resetPendingStaleWindow = 30 * time.Minute
|
||||
|
||||
const restartInterruptedMessage = "Task interrupted due to application restart"
|
||||
|
||||
// resetPendingTasks resets the state of any knowledge items or sync logs stuck in processing
|
||||
// due to an unexpected application restart.
|
||||
//
|
||||
// In Lite mode (no REDIS_ADDR) every queued task lived in process memory, so
|
||||
// any "processing" row at startup is definitively orphaned and must be marked
|
||||
// failed wholesale. In distributed mode (Asynq on Redis) the active queue
|
||||
// survives restart, but the *currently executing* task on the dead instance
|
||||
// is lost — Asynq won't reschedule it until at-least-once retry kicks in,
|
||||
// which can take minutes or never (e.g. if the deadline has passed). To bound
|
||||
// the worst case we mark only "long-stale" rows failed: anything that hasn't
|
||||
// been touched for 30 minutes is well past any reasonable in-flight window.
|
||||
// Newer rows are left alone so we don't race a peer instance that's mid-process.
|
||||
// In Lite mode (no REDIS_ADDR) normal queued tasks live in process memory, so
|
||||
// a "processing" row at startup is orphaned unless it has a durable wiki op
|
||||
// that can be re-triggered.
|
||||
//
|
||||
// Distributed mode is intentionally different: Asynq persists queued/retry
|
||||
// tasks, and another replica may still be executing the same knowledge. A
|
||||
// startup hook cannot safely distinguish an orphan from a backlogged task that
|
||||
// has not opened a span yet. HousekeepingService owns that decision because it
|
||||
// checks BOTH recent span activity and the real Asynq queue. Consequently this
|
||||
// hook never resets knowledge/summary rows in distributed mode; it only keeps
|
||||
// the separate sync-log cleanup below.
|
||||
func resetPendingTasks(db *gorm.DB) {
|
||||
distributed := os.Getenv("REDIS_ADDR") != ""
|
||||
ctx := context.Background()
|
||||
@@ -35,22 +39,64 @@ func resetPendingTasks(db *gorm.DB) {
|
||||
staleCutoff = time.Now().Add(-resetPendingStaleWindow)
|
||||
}
|
||||
|
||||
// Cancel orphaned trace spans for knowledge rows we are about to mark
|
||||
// failed. resetPendingTasks does not touch asynq queues; this only
|
||||
// prevents the UI from showing duplicate running postprocess.*
|
||||
// subspans when a later retry also opens fresh spans.
|
||||
// Resolve Lite-mode orphaned knowledge rows first. A finalizing row whose
|
||||
// ONLY remaining slot is backed by a durable wiki op is excluded and resumed
|
||||
// by recoverPendingWikiTasks after handlers are registered. Rows with other
|
||||
// outstanding in-memory subtasks still fail: the wiki op cannot recover them.
|
||||
var stuckKnowledge []types.Knowledge
|
||||
if err := stuckKnowledgeParseQuery(db, distributed, staleCutoff).
|
||||
Select("id").Find(&stuckKnowledge).Error; err != nil {
|
||||
logger.Warnf(ctx, "resetPendingTasks: list stuck knowledge failed: %v", err)
|
||||
} else {
|
||||
for _, k := range stuckKnowledge {
|
||||
if !distributed {
|
||||
if err := stuckKnowledgeParseQuery(db).
|
||||
Select("id").Find(&stuckKnowledge).Error; err != nil {
|
||||
logger.Warnf(ctx, "resetPendingTasks: list stuck knowledge failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1. Reset knowledge parsing tasks (including finalizing rows whose
|
||||
// enrichment subtasks were lost with the process).
|
||||
// Update by the resolved ids rather than reusing the GORM chain after
|
||||
// Find() (which makes PostgreSQL emit an invalid UPDATE ... FROM self).
|
||||
stuckIDs := knowledgeIDs(stuckKnowledge)
|
||||
var resetErr error
|
||||
var resetCount int64
|
||||
if len(stuckIDs) > 0 {
|
||||
// Rebuild the query instead of reusing the chain after Find(); reusing it
|
||||
// makes PostgreSQL emit an invalid UPDATE ... FROM self statement.
|
||||
result := stuckKnowledgeParseQuery(db).
|
||||
Where("id IN ?", stuckIDs).
|
||||
Updates(map[string]interface{}{
|
||||
"parse_status": types.ParseStatusFailed,
|
||||
"error_message": restartInterruptedMessage,
|
||||
"pending_subtasks_count": 0,
|
||||
})
|
||||
resetErr = result.Error
|
||||
resetCount = result.RowsAffected
|
||||
}
|
||||
if resetErr != nil {
|
||||
logger.Warnf(context.Background(), "Failed to reset pending knowledge tasks: %v", resetErr)
|
||||
} else if resetCount > 0 {
|
||||
logger.Infof(context.Background(),
|
||||
"Reset %d stuck knowledge parsing tasks to failed state (distributed=%v)",
|
||||
resetCount, distributed)
|
||||
|
||||
// Cancel orphaned trace spans only after the owning knowledge rows are
|
||||
// terminal. This prevents the UI from showing duplicate running
|
||||
// postprocess.* subspans when a later manual retry opens fresh spans.
|
||||
// Re-read the successfully reset ids so a row whose status changed in
|
||||
// the small SELECT/UPDATE gap is not accidentally cancelled.
|
||||
var resetKnowledge []types.Knowledge
|
||||
if err := db.Select("id").
|
||||
Where("id IN ? AND parse_status = ? AND error_message = ?",
|
||||
stuckIDs, types.ParseStatusFailed, restartInterruptedMessage).
|
||||
Find(&resetKnowledge).Error; err != nil {
|
||||
logger.Warnf(ctx, "resetPendingTasks: list reset knowledge failed: %v", err)
|
||||
}
|
||||
for _, k := range resetKnowledge {
|
||||
attempt, err := spanRepo.LatestAttempt(ctx, k.ID)
|
||||
if err != nil || attempt <= 0 {
|
||||
continue
|
||||
}
|
||||
if n, err := spanRepo.CancelAllOpenSpans(ctx, k.ID, attempt,
|
||||
"SERVER_RESTART", "task interrupted due to application restart"); err != nil {
|
||||
"SERVER_RESTART", restartInterruptedMessage); err != nil {
|
||||
logger.Warnf(ctx, "resetPendingTasks: cancel spans for %s failed: %v", k.ID, err)
|
||||
} else if n > 0 {
|
||||
logger.Infof(ctx, "resetPendingTasks: cancelled %d open span(s) for knowledge %s attempt %d",
|
||||
@@ -59,33 +105,19 @@ func resetPendingTasks(db *gorm.DB) {
|
||||
}
|
||||
}
|
||||
|
||||
// 1. Reset knowledge parsing tasks (including finalizing rows whose
|
||||
// enrichment subtasks were lost with the process).
|
||||
// Fresh query — reusing the *gorm.DB chain after Find() makes GORM emit
|
||||
// UPDATE ... FROM knowledges which PostgreSQL rejects (SQLSTATE 42712).
|
||||
result := stuckKnowledgeParseQuery(db, distributed, staleCutoff).Updates(map[string]interface{}{
|
||||
"parse_status": types.ParseStatusFailed,
|
||||
"error_message": "Task interrupted due to application restart",
|
||||
"pending_subtasks_count": 0,
|
||||
})
|
||||
if result.Error != nil {
|
||||
logger.Warnf(context.Background(), "Failed to reset pending knowledge tasks: %v", result.Error)
|
||||
} else if result.RowsAffected > 0 {
|
||||
logger.Infof(context.Background(),
|
||||
"Reset %d stuck knowledge parsing tasks to failed state (distributed=%v)",
|
||||
result.RowsAffected, distributed)
|
||||
}
|
||||
|
||||
// 2. Reset knowledge summary tasks
|
||||
resultSummary := stuckKnowledgeSummaryQuery(db, distributed, staleCutoff).Updates(map[string]interface{}{
|
||||
"summary_status": types.SummaryStatusFailed,
|
||||
})
|
||||
if resultSummary.Error != nil {
|
||||
logger.Warnf(context.Background(), "Failed to reset pending summary tasks: %v", resultSummary.Error)
|
||||
} else if resultSummary.RowsAffected > 0 {
|
||||
logger.Infof(context.Background(),
|
||||
"Reset %d stuck summary generation tasks to failed state (distributed=%v)",
|
||||
resultSummary.RowsAffected, distributed)
|
||||
// 2. Lite summary tasks are process-local too. Distributed summary tasks
|
||||
// stay in Asynq and must not be failed merely because this replica started.
|
||||
if !distributed {
|
||||
resultSummary := stuckKnowledgeSummaryQuery(db).Updates(map[string]interface{}{
|
||||
"summary_status": types.SummaryStatusFailed,
|
||||
})
|
||||
if resultSummary.Error != nil {
|
||||
logger.Warnf(context.Background(), "Failed to reset pending summary tasks: %v", resultSummary.Error)
|
||||
} else if resultSummary.RowsAffected > 0 {
|
||||
logger.Infof(context.Background(),
|
||||
"Reset %d stuck summary generation tasks to failed state (distributed=false)",
|
||||
resultSummary.RowsAffected)
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Reset data source sync tasks
|
||||
@@ -104,27 +136,27 @@ func resetPendingTasks(db *gorm.DB) {
|
||||
}
|
||||
}
|
||||
|
||||
func stuckKnowledgeParseQuery(db *gorm.DB, distributed bool, staleCutoff time.Time) *gorm.DB {
|
||||
func stuckKnowledgeParseQuery(db *gorm.DB) *gorm.DB {
|
||||
q := db.Model(&types.Knowledge{}).
|
||||
Where("parse_status IN ?", []string{
|
||||
types.ParseStatusPending,
|
||||
types.ParseStatusProcessing,
|
||||
types.ParseStatusFinalizing,
|
||||
types.ParseStatusDeleting,
|
||||
})
|
||||
if distributed {
|
||||
q = q.Where("updated_at < ?", staleCutoff)
|
||||
}
|
||||
Where("parse_status IN ?", resettableParseStatuses()).
|
||||
// Wiki ingest ops are persisted independently from the task trigger.
|
||||
// When wiki owns the only outstanding slot, keeping this finalizing row
|
||||
// alive lets startup recreate the trigger and finish cleanly. A count
|
||||
// above one means at least one non-wiki in-memory subtask was also lost.
|
||||
Where(`NOT (parse_status = ? AND pending_subtasks_count = 1 AND EXISTS (
|
||||
SELECT 1 FROM task_pending_ops
|
||||
WHERE task_pending_ops.task_type = ?
|
||||
AND task_pending_ops.scope = ?
|
||||
AND task_pending_ops.dedup_key = knowledges.id
|
||||
AND task_pending_ops.op = ?
|
||||
))`, types.ParseStatusFinalizing, types.TypeWikiIngest,
|
||||
types.TaskScopeKnowledgeBase, "ingest")
|
||||
return q
|
||||
}
|
||||
|
||||
func stuckKnowledgeSummaryQuery(db *gorm.DB, distributed bool, staleCutoff time.Time) *gorm.DB {
|
||||
q := db.Model(&types.Knowledge{}).
|
||||
func stuckKnowledgeSummaryQuery(db *gorm.DB) *gorm.DB {
|
||||
return db.Model(&types.Knowledge{}).
|
||||
Where("summary_status IN ?", []string{types.SummaryStatusPending, types.SummaryStatusProcessing})
|
||||
if distributed {
|
||||
q = q.Where("updated_at < ?", staleCutoff)
|
||||
}
|
||||
return q
|
||||
}
|
||||
|
||||
func stuckSyncLogQuery(db *gorm.DB, distributed bool, staleCutoff time.Time) *gorm.DB {
|
||||
@@ -135,3 +167,22 @@ func stuckSyncLogQuery(db *gorm.DB, distributed bool, staleCutoff time.Time) *go
|
||||
}
|
||||
return q
|
||||
}
|
||||
|
||||
func resettableParseStatuses() []string {
|
||||
return []string{
|
||||
types.ParseStatusPending,
|
||||
types.ParseStatusProcessing,
|
||||
types.ParseStatusFinalizing,
|
||||
types.ParseStatusDeleting,
|
||||
}
|
||||
}
|
||||
|
||||
func knowledgeIDs(rows []types.Knowledge) []string {
|
||||
ids := make([]string, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
if row.ID != "" {
|
||||
ids = append(ids, row.ID)
|
||||
}
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
package container
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/application/service"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/driver/sqlite"
|
||||
@@ -38,12 +41,51 @@ CREATE TABLE IF NOT EXISTS sync_logs (
|
||||
);
|
||||
`
|
||||
|
||||
const resetPendingSpansDDL = `
|
||||
CREATE TABLE IF NOT EXISTS knowledge_processing_spans (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
knowledge_id VARCHAR(64) NOT NULL,
|
||||
attempt INTEGER NOT NULL DEFAULT 1,
|
||||
span_id VARCHAR(64) NOT NULL,
|
||||
parent_span_id VARCHAR(64),
|
||||
name VARCHAR(255) NOT NULL,
|
||||
kind VARCHAR(16) NOT NULL,
|
||||
status VARCHAR(16) NOT NULL,
|
||||
error_code VARCHAR(64),
|
||||
error_message TEXT,
|
||||
started_at DATETIME,
|
||||
finished_at DATETIME,
|
||||
duration_ms INTEGER,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE (knowledge_id, attempt, span_id)
|
||||
);
|
||||
`
|
||||
|
||||
const resetPendingOpsDDL = `
|
||||
CREATE TABLE IF NOT EXISTS task_pending_ops (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
tenant_id INTEGER NOT NULL DEFAULT 0,
|
||||
task_type VARCHAR(64) NOT NULL,
|
||||
scope VARCHAR(32) NOT NULL,
|
||||
scope_id VARCHAR(64) NOT NULL,
|
||||
op VARCHAR(32) NOT NULL,
|
||||
dedup_key VARCHAR(128) NOT NULL DEFAULT '',
|
||||
payload TEXT NOT NULL DEFAULT '{}',
|
||||
fail_count INTEGER NOT NULL DEFAULT 0,
|
||||
enqueued_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
claimed_at DATETIME
|
||||
);
|
||||
`
|
||||
|
||||
func setupResetPendingDB(t *testing.T) *gorm.DB {
|
||||
t.Helper()
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, db.Exec(resetPendingKnowledgeDDL).Error)
|
||||
require.NoError(t, db.Exec(resetPendingSyncLogDDL).Error)
|
||||
require.NoError(t, db.Exec(resetPendingSpansDDL).Error)
|
||||
require.NoError(t, db.Exec(resetPendingOpsDDL).Error)
|
||||
return db
|
||||
}
|
||||
|
||||
@@ -55,7 +97,7 @@ func TestResetPendingTasks_KnowledgeFindThenUpdate(t *testing.T) {
|
||||
"k-stuck", types.ParseStatusProcessing, stale,
|
||||
).Error)
|
||||
|
||||
t.Setenv("REDIS_ADDR", "redis:6379")
|
||||
os.Unsetenv("REDIS_ADDR")
|
||||
resetPendingTasks(db)
|
||||
|
||||
var status, errMsg string
|
||||
@@ -84,6 +126,107 @@ func TestResetPendingTasks_KnowledgeFreshInDistributedMode(t *testing.T) {
|
||||
assert.Equal(t, types.ParseStatusProcessing, status)
|
||||
}
|
||||
|
||||
func TestResetPendingTasks_DistributedModePreservesEveryStage(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
parseStatus string
|
||||
}{
|
||||
{"pending", types.ParseStatusPending},
|
||||
{"docreader_chunking_embedding", types.ParseStatusProcessing},
|
||||
{"multimodal", types.ParseStatusProcessing},
|
||||
{"postprocess", types.ParseStatusFinalizing},
|
||||
{"wiki", types.ParseStatusFinalizing},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
db := setupResetPendingDB(t)
|
||||
stale := time.Now().Add(-2 * time.Hour)
|
||||
require.NoError(t, db.Exec(
|
||||
`INSERT INTO knowledges (id, parse_status, updated_at) VALUES (?, ?, ?)`,
|
||||
"k-active-span", tc.parseStatus, stale,
|
||||
).Error)
|
||||
|
||||
t.Setenv("REDIS_ADDR", "redis:6379")
|
||||
resetPendingTasks(db)
|
||||
|
||||
var status string
|
||||
require.NoError(t, db.Raw(
|
||||
`SELECT parse_status FROM knowledges WHERE id = ?`, "k-active-span",
|
||||
).Row().Scan(&status))
|
||||
assert.Equal(t, tc.parseStatus, status,
|
||||
"distributed tasks belong to Asynq/housekeeping, not startup reset")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResetPendingTasks_DistributedSummaryTaskSurvivesRestart(t *testing.T) {
|
||||
db := setupResetPendingDB(t)
|
||||
stale := time.Now().Add(-2 * time.Hour)
|
||||
require.NoError(t, db.Exec(
|
||||
`INSERT INTO knowledges (id, parse_status, summary_status, updated_at)
|
||||
VALUES (?, ?, ?, ?)`,
|
||||
"k-summary", types.ParseStatusCompleted, types.SummaryStatusProcessing, stale,
|
||||
).Error)
|
||||
|
||||
t.Setenv("REDIS_ADDR", "redis:6379")
|
||||
resetPendingTasks(db)
|
||||
|
||||
var status string
|
||||
require.NoError(t, db.Raw(
|
||||
`SELECT summary_status FROM knowledges WHERE id = ?`, "k-summary",
|
||||
).Row().Scan(&status))
|
||||
assert.Equal(t, types.SummaryStatusProcessing, status)
|
||||
}
|
||||
|
||||
func TestResetPendingTasks_DurableWikiOpSurvivesLiteRestart(t *testing.T) {
|
||||
db := setupResetPendingDB(t)
|
||||
require.NoError(t, db.Exec(
|
||||
`INSERT INTO knowledges (id, parse_status, pending_subtasks_count)
|
||||
VALUES (?, ?, 1)`, "k-wiki", types.ParseStatusFinalizing,
|
||||
).Error)
|
||||
require.NoError(t, db.Exec(
|
||||
`INSERT INTO task_pending_ops
|
||||
(tenant_id, task_type, scope, scope_id, op, dedup_key, payload)
|
||||
VALUES (7, ?, ?, 'kb-wiki', 'ingest', 'k-wiki', '{}')`,
|
||||
types.TypeWikiIngest, types.TaskScopeKnowledgeBase,
|
||||
).Error)
|
||||
|
||||
os.Unsetenv("REDIS_ADDR")
|
||||
resetPendingTasks(db)
|
||||
|
||||
var status string
|
||||
var pending int
|
||||
require.NoError(t, db.Raw(
|
||||
`SELECT parse_status, pending_subtasks_count FROM knowledges WHERE id = ?`, "k-wiki",
|
||||
).Row().Scan(&status, &pending))
|
||||
assert.Equal(t, types.ParseStatusFinalizing, status)
|
||||
assert.Equal(t, 1, pending, "resumed wiki worker still owns its finalizing slot")
|
||||
}
|
||||
|
||||
func TestResetPendingTasks_LiteWikiDoesNotHideOtherLostSubtasks(t *testing.T) {
|
||||
db := setupResetPendingDB(t)
|
||||
require.NoError(t, db.Exec(
|
||||
`INSERT INTO knowledges (id, parse_status, pending_subtasks_count)
|
||||
VALUES (?, ?, 2)`, "k-wiki-plus-summary", types.ParseStatusFinalizing,
|
||||
).Error)
|
||||
require.NoError(t, db.Exec(
|
||||
`INSERT INTO task_pending_ops
|
||||
(tenant_id, task_type, scope, scope_id, op, dedup_key, payload)
|
||||
VALUES (7, ?, ?, 'kb-wiki', 'ingest', 'k-wiki-plus-summary', '{}')`,
|
||||
types.TypeWikiIngest, types.TaskScopeKnowledgeBase,
|
||||
).Error)
|
||||
|
||||
os.Unsetenv("REDIS_ADDR")
|
||||
resetPendingTasks(db)
|
||||
|
||||
var status string
|
||||
require.NoError(t, db.Raw(
|
||||
`SELECT parse_status FROM knowledges WHERE id = ?`, "k-wiki-plus-summary",
|
||||
).Row().Scan(&status))
|
||||
assert.Equal(t, types.ParseStatusFailed, status,
|
||||
"a durable wiki op cannot recover another lost in-memory subtask")
|
||||
}
|
||||
|
||||
func TestResetPendingTasks_SyncLogStaleRunning(t *testing.T) {
|
||||
db := setupResetPendingDB(t)
|
||||
stale := time.Now().Add(-2 * time.Hour)
|
||||
@@ -129,17 +272,60 @@ func TestStuckKnowledgeParseQuery_ReuseAfterFindDoesNotBreakUpdate(t *testing.T)
|
||||
"k-reuse", types.ParseStatusProcessing, stale,
|
||||
).Error)
|
||||
|
||||
distributed := true
|
||||
staleCutoff := time.Now().Add(-resetPendingStaleWindow)
|
||||
|
||||
var rows []types.Knowledge
|
||||
q := stuckKnowledgeParseQuery(db, distributed, staleCutoff)
|
||||
q := stuckKnowledgeParseQuery(db)
|
||||
require.NoError(t, q.Select("id").Find(&rows).Error)
|
||||
require.Len(t, rows, 1)
|
||||
|
||||
result := stuckKnowledgeParseQuery(db, distributed, staleCutoff).Updates(map[string]interface{}{
|
||||
result := stuckKnowledgeParseQuery(db).Updates(map[string]interface{}{
|
||||
"parse_status": types.ParseStatusFailed,
|
||||
})
|
||||
require.NoError(t, result.Error)
|
||||
assert.Equal(t, int64(1), result.RowsAffected)
|
||||
}
|
||||
|
||||
type recordingTaskEnqueuer struct {
|
||||
tasks []*asynq.Task
|
||||
}
|
||||
|
||||
func (r *recordingTaskEnqueuer) Enqueue(task *asynq.Task, _ ...asynq.Option) (*asynq.TaskInfo, error) {
|
||||
r.tasks = append(r.tasks, task)
|
||||
return &asynq.TaskInfo{ID: "test", Type: task.Type()}, nil
|
||||
}
|
||||
|
||||
func TestRecoverPendingWikiTasks_RecreatesOneTriggerPerLaneAndKB(t *testing.T) {
|
||||
db := setupResetPendingDB(t)
|
||||
rows := []struct {
|
||||
tenantID uint64
|
||||
taskType string
|
||||
kbID string
|
||||
dedup string
|
||||
}{
|
||||
{7, types.TypeWikiIngest, "kb-a", "k-1"},
|
||||
{7, types.TypeWikiIngest, "kb-a", "k-2"}, // same lane: one trigger
|
||||
{7, types.TypeWikiFinalize, "kb-a", "slug-a"},
|
||||
{8, types.TypeWikiIngest, "kb-b", "k-3"},
|
||||
}
|
||||
for _, row := range rows {
|
||||
require.NoError(t, db.Exec(
|
||||
`INSERT INTO task_pending_ops
|
||||
(tenant_id, task_type, scope, scope_id, op, dedup_key, payload)
|
||||
VALUES (?, ?, ?, ?, 'ingest', ?, '{}')`,
|
||||
row.tenantID, row.taskType, types.TaskScopeKnowledgeBase, row.kbID, row.dedup,
|
||||
).Error)
|
||||
}
|
||||
|
||||
recorder := &recordingTaskEnqueuer{}
|
||||
recoverPendingWikiTasks(db, recorder)
|
||||
require.Len(t, recorder.tasks, 3)
|
||||
|
||||
seen := map[string]service.WikiIngestPayload{}
|
||||
for _, task := range recorder.tasks {
|
||||
var payload service.WikiIngestPayload
|
||||
require.NoError(t, json.Unmarshal(task.Payload(), &payload))
|
||||
seen[task.Type()+":"+payload.KnowledgeBaseID] = payload
|
||||
}
|
||||
assert.Equal(t, uint64(7), seen[types.TypeWikiIngest+":kb-a"].TenantID)
|
||||
assert.Equal(t, uint64(7), seen[types.TypeWikiFinalize+":kb-a"].TenantID)
|
||||
assert.Equal(t, uint64(8), seen[types.TypeWikiIngest+":kb-b"].TenantID)
|
||||
}
|
||||
|
||||
@@ -761,12 +761,17 @@ func knowledgeSpansLastError(
|
||||
if currentAttempt != latestAttempt || parseStatus != types.ParseStatusFailed || knowledgeErrorMessage == "" {
|
||||
return nil
|
||||
}
|
||||
errorCode := "UNKNOWN"
|
||||
if strings.EqualFold(strings.TrimSpace(knowledgeErrorMessage),
|
||||
"Task interrupted due to application restart") {
|
||||
errorCode = "SERVER_RESTART"
|
||||
}
|
||||
return gin.H{
|
||||
"stage": "knowledge_processing",
|
||||
"code": "UNKNOWN",
|
||||
"code": errorCode,
|
||||
"message": knowledgeErrorMessage,
|
||||
"name": "knowledge_processing",
|
||||
"error_code": "UNKNOWN",
|
||||
"error_code": errorCode,
|
||||
"error_message": knowledgeErrorMessage,
|
||||
"finished_at": knowledgeUpdatedAt,
|
||||
}
|
||||
|
||||
@@ -150,6 +150,16 @@ func TestKnowledgeSpansLastError_RecoveryFallbackUsesKnowledgeMessage(t *testing
|
||||
a.Equal(now, got["finished_at"])
|
||||
}
|
||||
|
||||
func TestKnowledgeSpansLastError_ClassifiesServerRestart(t *testing.T) {
|
||||
now := time.Now()
|
||||
got := knowledgeSpansLastError(2, 2, types.ParseStatusFailed,
|
||||
"Task interrupted due to application restart", now, nil)
|
||||
a := assert.New(t)
|
||||
a.NotNil(got)
|
||||
a.Equal("SERVER_RESTART", got["error_code"])
|
||||
a.Equal("SERVER_RESTART", got["code"])
|
||||
}
|
||||
|
||||
func TestKnowledgeSpansLastError_SkipsRecoveryFallbackForHistoricalAttempt(t *testing.T) {
|
||||
now := time.Now()
|
||||
got := knowledgeSpansLastError(1, 2, types.ParseStatusFailed, "wiki ingest timed out", now, nil)
|
||||
|
||||
Reference in New Issue
Block a user