Files
Enterprise-Knowledge-Base/internal/container/recover_pending_wiki_tasks.go
T
wizardchen ace84bc757 fix(knowledge): recover tasks safely after server restart
Stop failing distributed-mode knowledge rows on startup; let housekeeping
decide using span activity and Asynq queue state. In Lite mode, resume
wiki work from durable task_pending_ops and surface SERVER_RESTART errors
in the processing timeline with i18n support.
2026-07-14 17:55:26 +08:00

88 lines
2.9 KiB
Go

package container
import (
"context"
"encoding/json"
"errors"
"time"
"github.com/Tencent/WeKnora/internal/application/service"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/hibiken/asynq"
"gorm.io/gorm"
)
// pendingWikiScope is the minimum routing information needed to recreate an
// ephemeral wiki trigger from the durable task_pending_ops queue.
type pendingWikiScope struct {
TenantID uint64 `gorm:"column:tenant_id"`
TaskType string `gorm:"column:task_type"`
ScopeID string `gorm:"column:scope_id"`
}
// recoverPendingWikiTasks recreates one trigger per pending wiki queue lane.
//
// task_pending_ops is deliberately durable, but the trigger that wakes its
// consumer is not durable in Lite mode (SyncTaskExecutor is process-local) and
// may also be absent after an interrupted Redis enqueue. Running this after all
// handlers are registered closes that gap. Duplicate triggers are harmless:
// ingest claims/peeks disjoint rows and finalize coalesces its pending lane.
func recoverPendingWikiTasks(db *gorm.DB, task interfaces.TaskEnqueuer) {
if db == nil || task == nil {
return
}
ctx := context.Background()
var scopes []pendingWikiScope
if err := db.WithContext(ctx).
Model(&types.TaskPendingOp{}).
Distinct("tenant_id", "task_type", "scope_id").
Where("scope = ? AND task_type IN ?", types.TaskScopeKnowledgeBase,
[]string{types.TypeWikiIngest, types.TypeWikiFinalize}).
Find(&scopes).Error; err != nil {
logger.Warnf(ctx, "[WikiRecovery] failed to list pending queues: %v", err)
return
}
recovered := 0
for _, scope := range scopes {
if scope.ScopeID == "" {
continue
}
payload, err := json.Marshal(service.WikiIngestPayload{
TenantID: scope.TenantID,
KnowledgeBaseID: scope.ScopeID,
})
if err != nil {
logger.Warnf(ctx, "[WikiRecovery] marshal trigger for KB %s failed: %v", scope.ScopeID, err)
continue
}
opts := []asynq.Option{
asynq.Queue(types.QueueWiki),
asynq.MaxRetry(10), // keep aligned with the wiki ingest retry policy
asynq.Timeout(60 * time.Minute),
}
if scope.TaskType == types.TypeWikiFinalize {
opts[2] = asynq.Timeout(30 * time.Minute)
// Match scheduleFinalize so simultaneous replica startups collapse
// into the same per-KB finalize trigger.
opts = append(opts, asynq.TaskID("wiki-finalize-"+scope.ScopeID))
}
trigger := asynq.NewTask(scope.TaskType, payload, opts...)
if _, err := task.Enqueue(trigger); err != nil {
if errors.Is(err, asynq.ErrTaskIDConflict) || errors.Is(err, asynq.ErrDuplicateTask) {
recovered++
continue
}
logger.Warnf(ctx, "[WikiRecovery] enqueue %s trigger for KB %s failed: %v",
scope.TaskType, scope.ScopeID, err)
continue
}
recovered++
}
if recovered > 0 {
logger.Infof(ctx, "[WikiRecovery] recreated %d trigger(s) from durable pending queues", recovered)
}
}