diff --git a/internal/application/service/evaluation.go b/internal/application/service/evaluation.go index cf3c01fe..d6d1cb92 100644 --- a/internal/application/service/evaluation.go +++ b/internal/application/service/evaluation.go @@ -12,7 +12,7 @@ import ( "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" - "github.com/google/uuid" + "github.com/Tencent/WeKnora/internal/utils" "golang.org/x/sync/errgroup" ) @@ -253,7 +253,7 @@ func (e *EvaluationService) Evaluation(ctx context.Context, // Create evaluation task with unique ID logger.Info(ctx, "Creating evaluation task") - taskID := uuid.New().String() + taskID := utils.GenerateTaskID("evaluation", tenantID, datasetID) logger.Infof(ctx, "Generated task ID: %s", taskID) // Prepare evaluation detail with all parameters diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index e9fa35e6..5d6530e4 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -2736,10 +2736,10 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context, tenantID := ctx.Value(types.TenantIDContextKey).(uint64) - // 使用传入的TaskID,如果没传则生成UUID + // 使用传入的TaskID,如果没传则生成增强的TaskID taskID := payload.TaskID if taskID == "" { - taskID = uuid.New().String() + taskID = secutils.GenerateTaskID("faq_import", tenantID, kbID) } var knowledgeID string @@ -2810,7 +2810,7 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context, return "", fmt.Errorf("failed to marshal task payload: %w", err) } - task := asynq.NewTask(types.TypeFAQImport, payloadBytes, asynq.Queue("default"), asynq.MaxRetry(5)) + task := asynq.NewTask(types.TypeFAQImport, payloadBytes, asynq.TaskID(taskID), asynq.Queue("default"), asynq.MaxRetry(5)) info, err := s.task.Enqueue(task) if err != nil { logger.Errorf(ctx, "Failed to enqueue FAQ import task: %v", err) @@ -2818,6 +2818,12 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context, } logger.Infof(ctx, "Enqueued FAQ import task: id=%s queue=%s task_id=%s dry_run=%v", info.ID, info.Queue, taskID, payload.DryRun) + // 设置 running key 标记任务正在进行 + if err := s.setRunningFAQImportTaskID(ctx, kbID, taskID); err != nil { + logger.Errorf(ctx, "Failed to set running FAQ import task ID: %v", err) + // 这个错误不影响任务执行,只是会影响重复提交检查 + } + return taskID, nil } @@ -5961,10 +5967,18 @@ func (s *knowledgeService) finalizeFAQValidation(ctx context.Context, payload *t } } + // 使用 updateFAQImportProgressStatus 来确保正确清理 running key + // 但是需要先保存其他字段,因为 updateFAQImportProgressStatus 不会保存所有字段 if err := s.saveFAQImportProgress(ctx, progress); err != nil { logger.Warnf(ctx, "Failed to save final FAQ import progress: %v", err) } + // 然后调用状态更新来清理 running key + if err := s.updateFAQImportProgressStatus(ctx, payload.TaskID, types.FAQImportStatusCompleted, + 100, originalTotalEntries, originalTotalEntries, progress.Message, ""); err != nil { + logger.Warnf(ctx, "Failed to update final FAQ import status: %v", err) + } + logger.Infof(ctx, "FAQ task completed: %s, dry_run=%v, success: %d, failed: %d", payload.TaskID, payload.DryRun, progress.SuccessCount, progress.FailedCount) diff --git a/internal/handler/knowledgebase.go b/internal/handler/knowledgebase.go index f98c0717..abbb6aaa 100644 --- a/internal/handler/knowledgebase.go +++ b/internal/handler/knowledgebase.go @@ -10,9 +10,9 @@ import ( "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/Tencent/WeKnora/internal/utils" secutils "github.com/Tencent/WeKnora/internal/utils" "github.com/gin-gonic/gin" - "github.com/google/uuid" "github.com/hibiken/asynq" ) @@ -375,7 +375,7 @@ func (h *KnowledgeBaseHandler) CopyKnowledgeBase(c *gin.Context) { // Generate task ID if not provided taskID := req.TaskID if taskID == "" { - taskID = uuid.New().String() + taskID = utils.GenerateTaskID("kb_clone", tenantID.(uint64), req.SourceID) } // Create KB clone payload @@ -394,7 +394,8 @@ func (h *KnowledgeBaseHandler) CopyKnowledgeBase(c *gin.Context) { } // Enqueue KB clone task to Asynq - task := asynq.NewTask(types.TypeKBClone, payloadBytes, asynq.Queue("default"), asynq.MaxRetry(3)) + task := asynq.NewTask(types.TypeKBClone, payloadBytes, + asynq.TaskID(taskID), asynq.Queue("default"), asynq.MaxRetry(3)) info, err := h.asynqClient.Enqueue(task) if err != nil { logger.Errorf(ctx, "Failed to enqueue KB clone task: %v", err) diff --git a/internal/utils/debug.go b/internal/utils/debug.go new file mode 100644 index 00000000..581df76a --- /dev/null +++ b/internal/utils/debug.go @@ -0,0 +1,90 @@ +package utils + +import ( + "context" + "fmt" + "time" + + "github.com/redis/go-redis/v9" +) + +// CleanupStaleRunningTasks 清理可能残留的running task keys +// 这是一个调试和维护工具,可以用来清理因异常情况导致的残留running keys +func CleanupStaleRunningTasks(ctx context.Context, redisClient *redis.Client, keyPrefix string, maxAge time.Duration) (int, error) { + // 获取所有匹配的keys + keys, err := redisClient.Keys(ctx, keyPrefix+"*").Result() + if err != nil { + return 0, fmt.Errorf("failed to get keys: %w", err) + } + + if len(keys) == 0 { + return 0, nil + } + + // 检查每个key的TTL + var staleTasks []string + for _, key := range keys { + ttl, err := redisClient.TTL(ctx, key).Result() + if err != nil { + continue // 跳过错误的key + } + + // 如果TTL小于0(永不过期)或者剩余时间太长(可能是残留的),标记为stale + if ttl < 0 || ttl > maxAge { + staleTasks = append(staleTasks, key) + } + } + + if len(staleTasks) == 0 { + return 0, nil + } + + // 删除stale keys + deleted, err := redisClient.Del(ctx, staleTasks...).Result() + if err != nil { + return 0, fmt.Errorf("failed to delete stale keys: %w", err) + } + + return int(deleted), nil +} + +// CheckRunningTaskStatus 检查指定running task的状态 +func CheckRunningTaskStatus(ctx context.Context, redisClient *redis.Client, runningKey, progressKey string) (map[string]interface{}, error) { + result := make(map[string]interface{}) + + // 检查running key + runningTaskID, err := redisClient.Get(ctx, runningKey).Result() + if err != nil { + if err == redis.Nil { + result["running_task_exists"] = false + } else { + return nil, fmt.Errorf("failed to get running task: %w", err) + } + } else { + result["running_task_exists"] = true + result["running_task_id"] = runningTaskID + + // 获取running key的TTL + ttl, _ := redisClient.TTL(ctx, runningKey).Result() + result["running_task_ttl"] = ttl.String() + } + + // 检查progress key + progressData, err := redisClient.Get(ctx, progressKey).Result() + if err != nil { + if err == redis.Nil { + result["progress_exists"] = false + } else { + return nil, fmt.Errorf("failed to get progress: %w", err) + } + } else { + result["progress_exists"] = true + result["progress_data"] = progressData + + // 获取progress key的TTL + ttl, _ := redisClient.TTL(ctx, progressKey).Result() + result["progress_ttl"] = ttl.String() + } + + return result, nil +} \ No newline at end of file diff --git a/internal/utils/taskid.go b/internal/utils/taskid.go new file mode 100644 index 00000000..afa6209a --- /dev/null +++ b/internal/utils/taskid.go @@ -0,0 +1,115 @@ +package utils + +import ( + "fmt" + "strconv" + "strings" + "time" + + "github.com/google/uuid" +) + +// GenerateTaskID generates a unique task ID with multiple collision-resistant elements. +// The format is: ____ +// +// Parameters: +// - taskType: Type of task (e.g., "faq_import", "kb_clone") +// - tenantID: Tenant ID for multi-tenancy isolation +// - businessID: Optional business-specific ID (e.g., knowledge base ID) +// +// Returns a task ID like: "faq_import_12345_1704628851692_a1b2c3d4_kb789" +func GenerateTaskID(taskType string, tenantID uint64, businessID ...string) string { + // Use current timestamp in milliseconds for temporal uniqueness + timestamp := time.Now().UnixMilli() + + // Generate a short UUID (first 8 characters for brevity) + shortUUID := strings.ReplaceAll(uuid.New().String()[:8], "-", "") + + // Build the task ID components + components := []string{ + sanitizeTaskType(taskType), + strconv.FormatUint(tenantID, 10), + strconv.FormatInt(timestamp, 10), + shortUUID, + } + + // Add business ID if provided + if len(businessID) > 0 && businessID[0] != "" { + components = append(components, sanitizeBusinessID(businessID[0])) + } + + return strings.Join(components, "_") +} + +// GenerateTaskIDWithPrefix generates a task ID with a custom prefix. +// This is useful when you want more control over the task ID format. +func GenerateTaskIDWithPrefix(prefix string, tenantID uint64, businessID ...string) string { + timestamp := time.Now().UnixMilli() + shortUUID := strings.ReplaceAll(uuid.New().String()[:8], "-", "") + + components := []string{ + sanitizeTaskType(prefix), + strconv.FormatUint(tenantID, 10), + strconv.FormatInt(timestamp, 10), + shortUUID, + } + + if len(businessID) > 0 && businessID[0] != "" { + components = append(components, sanitizeBusinessID(businessID[0])) + } + + return strings.Join(components, "_") +} + +// ParseTaskID parses a task ID generated by GenerateTaskID and returns its components. +// Returns taskType, tenantID, timestamp, uuid, businessID, and error. +func ParseTaskID(taskID string) (taskType string, tenantID uint64, timestamp int64, uuidPart string, businessID string, err error) { + parts := strings.Split(taskID, "_") + if len(parts) < 4 { + err = fmt.Errorf("invalid task ID format: %s", taskID) + return + } + + taskType = parts[0] + + tenantID, err = strconv.ParseUint(parts[1], 10, 64) + if err != nil { + err = fmt.Errorf("invalid tenant ID in task ID: %s", parts[1]) + return + } + + timestamp, err = strconv.ParseInt(parts[2], 10, 64) + if err != nil { + err = fmt.Errorf("invalid timestamp in task ID: %s", parts[2]) + return + } + + uuidPart = parts[3] + + if len(parts) > 4 { + businessID = parts[4] + } + + return +} + +// sanitizeTaskType ensures task type is safe for use in task ID +func sanitizeTaskType(taskType string) string { + // Replace colons and other special characters with underscores + taskType = strings.ReplaceAll(taskType, ":", "_") + taskType = strings.ReplaceAll(taskType, "-", "_") + taskType = strings.ReplaceAll(taskType, " ", "_") + return strings.ToLower(taskType) +} + +// sanitizeBusinessID ensures business ID is safe for use in task ID +func sanitizeBusinessID(businessID string) string { + // Take first 12 characters and replace special characters + if len(businessID) > 12 { + businessID = businessID[:12] + } + businessID = strings.ReplaceAll(businessID, "-", "") + businessID = strings.ReplaceAll(businessID, "_", "") + businessID = strings.ReplaceAll(businessID, ":", "") + return businessID +} \ No newline at end of file