feat: 使用增强的任务ID生成器替换UUID

This commit is contained in:
wizardchen
2026-01-07 15:18:18 +08:00
parent 953e0c1a55
commit 042d860329
5 changed files with 228 additions and 8 deletions
+2 -2
View File
@@ -12,7 +12,7 @@ import (
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/google/uuid"
"github.com/Tencent/WeKnora/internal/utils"
"golang.org/x/sync/errgroup"
)
@@ -253,7 +253,7 @@ func (e *EvaluationService) Evaluation(ctx context.Context,
// Create evaluation task with unique ID
logger.Info(ctx, "Creating evaluation task")
taskID := uuid.New().String()
taskID := utils.GenerateTaskID("evaluation", tenantID, datasetID)
logger.Infof(ctx, "Generated task ID: %s", taskID)
// Prepare evaluation detail with all parameters
+17 -3
View File
@@ -2736,10 +2736,10 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context,
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
// 使用传入的TaskID,如果没传则生成UUID
// 使用传入的TaskID,如果没传则生成增强的TaskID
taskID := payload.TaskID
if taskID == "" {
taskID = uuid.New().String()
taskID = secutils.GenerateTaskID("faq_import", tenantID, kbID)
}
var knowledgeID string
@@ -2810,7 +2810,7 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context,
return "", fmt.Errorf("failed to marshal task payload: %w", err)
}
task := asynq.NewTask(types.TypeFAQImport, payloadBytes, asynq.Queue("default"), asynq.MaxRetry(5))
task := asynq.NewTask(types.TypeFAQImport, payloadBytes, asynq.TaskID(taskID), asynq.Queue("default"), asynq.MaxRetry(5))
info, err := s.task.Enqueue(task)
if err != nil {
logger.Errorf(ctx, "Failed to enqueue FAQ import task: %v", err)
@@ -2818,6 +2818,12 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context,
}
logger.Infof(ctx, "Enqueued FAQ import task: id=%s queue=%s task_id=%s dry_run=%v", info.ID, info.Queue, taskID, payload.DryRun)
// 设置 running key 标记任务正在进行
if err := s.setRunningFAQImportTaskID(ctx, kbID, taskID); err != nil {
logger.Errorf(ctx, "Failed to set running FAQ import task ID: %v", err)
// 这个错误不影响任务执行,只是会影响重复提交检查
}
return taskID, nil
}
@@ -5961,10 +5967,18 @@ func (s *knowledgeService) finalizeFAQValidation(ctx context.Context, payload *t
}
}
// 使用 updateFAQImportProgressStatus 来确保正确清理 running key
// 但是需要先保存其他字段,因为 updateFAQImportProgressStatus 不会保存所有字段
if err := s.saveFAQImportProgress(ctx, progress); err != nil {
logger.Warnf(ctx, "Failed to save final FAQ import progress: %v", err)
}
// 然后调用状态更新来清理 running key
if err := s.updateFAQImportProgressStatus(ctx, payload.TaskID, types.FAQImportStatusCompleted,
100, originalTotalEntries, originalTotalEntries, progress.Message, ""); err != nil {
logger.Warnf(ctx, "Failed to update final FAQ import status: %v", err)
}
logger.Infof(ctx, "FAQ task completed: %s, dry_run=%v, success: %d, failed: %d",
payload.TaskID, payload.DryRun, progress.SuccessCount, progress.FailedCount)
+4 -3
View File
@@ -10,9 +10,9 @@ import (
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/Tencent/WeKnora/internal/utils"
secutils "github.com/Tencent/WeKnora/internal/utils"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"github.com/hibiken/asynq"
)
@@ -375,7 +375,7 @@ func (h *KnowledgeBaseHandler) CopyKnowledgeBase(c *gin.Context) {
// Generate task ID if not provided
taskID := req.TaskID
if taskID == "" {
taskID = uuid.New().String()
taskID = utils.GenerateTaskID("kb_clone", tenantID.(uint64), req.SourceID)
}
// Create KB clone payload
@@ -394,7 +394,8 @@ func (h *KnowledgeBaseHandler) CopyKnowledgeBase(c *gin.Context) {
}
// Enqueue KB clone task to Asynq
task := asynq.NewTask(types.TypeKBClone, payloadBytes, asynq.Queue("default"), asynq.MaxRetry(3))
task := asynq.NewTask(types.TypeKBClone, payloadBytes,
asynq.TaskID(taskID), asynq.Queue("default"), asynq.MaxRetry(3))
info, err := h.asynqClient.Enqueue(task)
if err != nil {
logger.Errorf(ctx, "Failed to enqueue KB clone task: %v", err)
+90
View File
@@ -0,0 +1,90 @@
package utils
import (
"context"
"fmt"
"time"
"github.com/redis/go-redis/v9"
)
// CleanupStaleRunningTasks 清理可能残留的running task keys
// 这是一个调试和维护工具,可以用来清理因异常情况导致的残留running keys
func CleanupStaleRunningTasks(ctx context.Context, redisClient *redis.Client, keyPrefix string, maxAge time.Duration) (int, error) {
// 获取所有匹配的keys
keys, err := redisClient.Keys(ctx, keyPrefix+"*").Result()
if err != nil {
return 0, fmt.Errorf("failed to get keys: %w", err)
}
if len(keys) == 0 {
return 0, nil
}
// 检查每个key的TTL
var staleTasks []string
for _, key := range keys {
ttl, err := redisClient.TTL(ctx, key).Result()
if err != nil {
continue // 跳过错误的key
}
// 如果TTL小于0(永不过期)或者剩余时间太长(可能是残留的),标记为stale
if ttl < 0 || ttl > maxAge {
staleTasks = append(staleTasks, key)
}
}
if len(staleTasks) == 0 {
return 0, nil
}
// 删除stale keys
deleted, err := redisClient.Del(ctx, staleTasks...).Result()
if err != nil {
return 0, fmt.Errorf("failed to delete stale keys: %w", err)
}
return int(deleted), nil
}
// CheckRunningTaskStatus 检查指定running task的状态
func CheckRunningTaskStatus(ctx context.Context, redisClient *redis.Client, runningKey, progressKey string) (map[string]interface{}, error) {
result := make(map[string]interface{})
// 检查running key
runningTaskID, err := redisClient.Get(ctx, runningKey).Result()
if err != nil {
if err == redis.Nil {
result["running_task_exists"] = false
} else {
return nil, fmt.Errorf("failed to get running task: %w", err)
}
} else {
result["running_task_exists"] = true
result["running_task_id"] = runningTaskID
// 获取running key的TTL
ttl, _ := redisClient.TTL(ctx, runningKey).Result()
result["running_task_ttl"] = ttl.String()
}
// 检查progress key
progressData, err := redisClient.Get(ctx, progressKey).Result()
if err != nil {
if err == redis.Nil {
result["progress_exists"] = false
} else {
return nil, fmt.Errorf("failed to get progress: %w", err)
}
} else {
result["progress_exists"] = true
result["progress_data"] = progressData
// 获取progress key的TTL
ttl, _ := redisClient.TTL(ctx, progressKey).Result()
result["progress_ttl"] = ttl.String()
}
return result, nil
}
+115
View File
@@ -0,0 +1,115 @@
package utils
import (
"fmt"
"strconv"
"strings"
"time"
"github.com/google/uuid"
)
// GenerateTaskID generates a unique task ID with multiple collision-resistant elements.
// The format is: <taskType>_<tenantID>_<timestamp>_<uuid>_<businessID>
//
// Parameters:
// - taskType: Type of task (e.g., "faq_import", "kb_clone")
// - tenantID: Tenant ID for multi-tenancy isolation
// - businessID: Optional business-specific ID (e.g., knowledge base ID)
//
// Returns a task ID like: "faq_import_12345_1704628851692_a1b2c3d4_kb789"
func GenerateTaskID(taskType string, tenantID uint64, businessID ...string) string {
// Use current timestamp in milliseconds for temporal uniqueness
timestamp := time.Now().UnixMilli()
// Generate a short UUID (first 8 characters for brevity)
shortUUID := strings.ReplaceAll(uuid.New().String()[:8], "-", "")
// Build the task ID components
components := []string{
sanitizeTaskType(taskType),
strconv.FormatUint(tenantID, 10),
strconv.FormatInt(timestamp, 10),
shortUUID,
}
// Add business ID if provided
if len(businessID) > 0 && businessID[0] != "" {
components = append(components, sanitizeBusinessID(businessID[0]))
}
return strings.Join(components, "_")
}
// GenerateTaskIDWithPrefix generates a task ID with a custom prefix.
// This is useful when you want more control over the task ID format.
func GenerateTaskIDWithPrefix(prefix string, tenantID uint64, businessID ...string) string {
timestamp := time.Now().UnixMilli()
shortUUID := strings.ReplaceAll(uuid.New().String()[:8], "-", "")
components := []string{
sanitizeTaskType(prefix),
strconv.FormatUint(tenantID, 10),
strconv.FormatInt(timestamp, 10),
shortUUID,
}
if len(businessID) > 0 && businessID[0] != "" {
components = append(components, sanitizeBusinessID(businessID[0]))
}
return strings.Join(components, "_")
}
// ParseTaskID parses a task ID generated by GenerateTaskID and returns its components.
// Returns taskType, tenantID, timestamp, uuid, businessID, and error.
func ParseTaskID(taskID string) (taskType string, tenantID uint64, timestamp int64, uuidPart string, businessID string, err error) {
parts := strings.Split(taskID, "_")
if len(parts) < 4 {
err = fmt.Errorf("invalid task ID format: %s", taskID)
return
}
taskType = parts[0]
tenantID, err = strconv.ParseUint(parts[1], 10, 64)
if err != nil {
err = fmt.Errorf("invalid tenant ID in task ID: %s", parts[1])
return
}
timestamp, err = strconv.ParseInt(parts[2], 10, 64)
if err != nil {
err = fmt.Errorf("invalid timestamp in task ID: %s", parts[2])
return
}
uuidPart = parts[3]
if len(parts) > 4 {
businessID = parts[4]
}
return
}
// sanitizeTaskType ensures task type is safe for use in task ID
func sanitizeTaskType(taskType string) string {
// Replace colons and other special characters with underscores
taskType = strings.ReplaceAll(taskType, ":", "_")
taskType = strings.ReplaceAll(taskType, "-", "_")
taskType = strings.ReplaceAll(taskType, " ", "_")
return strings.ToLower(taskType)
}
// sanitizeBusinessID ensures business ID is safe for use in task ID
func sanitizeBusinessID(businessID string) string {
// Take first 12 characters and replace special characters
if len(businessID) > 12 {
businessID = businessID[:12]
}
businessID = strings.ReplaceAll(businessID, "-", "")
businessID = strings.ReplaceAll(businessID, "_", "")
businessID = strings.ReplaceAll(businessID, ":", "")
return businessID
}