feat: 使用增强的任务ID生成器替换UUID
This commit is contained in:
@@ -12,7 +12,7 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/google/uuid"
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
@@ -253,7 +253,7 @@ func (e *EvaluationService) Evaluation(ctx context.Context,
|
||||
|
||||
// Create evaluation task with unique ID
|
||||
logger.Info(ctx, "Creating evaluation task")
|
||||
taskID := uuid.New().String()
|
||||
taskID := utils.GenerateTaskID("evaluation", tenantID, datasetID)
|
||||
logger.Infof(ctx, "Generated task ID: %s", taskID)
|
||||
|
||||
// Prepare evaluation detail with all parameters
|
||||
|
||||
@@ -2736,10 +2736,10 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context,
|
||||
|
||||
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
|
||||
|
||||
// 使用传入的TaskID,如果没传则生成UUID
|
||||
// 使用传入的TaskID,如果没传则生成增强的TaskID
|
||||
taskID := payload.TaskID
|
||||
if taskID == "" {
|
||||
taskID = uuid.New().String()
|
||||
taskID = secutils.GenerateTaskID("faq_import", tenantID, kbID)
|
||||
}
|
||||
|
||||
var knowledgeID string
|
||||
@@ -2810,7 +2810,7 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context,
|
||||
return "", fmt.Errorf("failed to marshal task payload: %w", err)
|
||||
}
|
||||
|
||||
task := asynq.NewTask(types.TypeFAQImport, payloadBytes, asynq.Queue("default"), asynq.MaxRetry(5))
|
||||
task := asynq.NewTask(types.TypeFAQImport, payloadBytes, asynq.TaskID(taskID), asynq.Queue("default"), asynq.MaxRetry(5))
|
||||
info, err := s.task.Enqueue(task)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to enqueue FAQ import task: %v", err)
|
||||
@@ -2818,6 +2818,12 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context,
|
||||
}
|
||||
logger.Infof(ctx, "Enqueued FAQ import task: id=%s queue=%s task_id=%s dry_run=%v", info.ID, info.Queue, taskID, payload.DryRun)
|
||||
|
||||
// 设置 running key 标记任务正在进行
|
||||
if err := s.setRunningFAQImportTaskID(ctx, kbID, taskID); err != nil {
|
||||
logger.Errorf(ctx, "Failed to set running FAQ import task ID: %v", err)
|
||||
// 这个错误不影响任务执行,只是会影响重复提交检查
|
||||
}
|
||||
|
||||
return taskID, nil
|
||||
}
|
||||
|
||||
@@ -5961,10 +5967,18 @@ func (s *knowledgeService) finalizeFAQValidation(ctx context.Context, payload *t
|
||||
}
|
||||
}
|
||||
|
||||
// 使用 updateFAQImportProgressStatus 来确保正确清理 running key
|
||||
// 但是需要先保存其他字段,因为 updateFAQImportProgressStatus 不会保存所有字段
|
||||
if err := s.saveFAQImportProgress(ctx, progress); err != nil {
|
||||
logger.Warnf(ctx, "Failed to save final FAQ import progress: %v", err)
|
||||
}
|
||||
|
||||
// 然后调用状态更新来清理 running key
|
||||
if err := s.updateFAQImportProgressStatus(ctx, payload.TaskID, types.FAQImportStatusCompleted,
|
||||
100, originalTotalEntries, originalTotalEntries, progress.Message, ""); err != nil {
|
||||
logger.Warnf(ctx, "Failed to update final FAQ import status: %v", err)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "FAQ task completed: %s, dry_run=%v, success: %d, failed: %d",
|
||||
payload.TaskID, payload.DryRun, progress.SuccessCount, progress.FailedCount)
|
||||
|
||||
|
||||
@@ -10,9 +10,9 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/google/uuid"
|
||||
"github.com/hibiken/asynq"
|
||||
)
|
||||
|
||||
@@ -375,7 +375,7 @@ func (h *KnowledgeBaseHandler) CopyKnowledgeBase(c *gin.Context) {
|
||||
// Generate task ID if not provided
|
||||
taskID := req.TaskID
|
||||
if taskID == "" {
|
||||
taskID = uuid.New().String()
|
||||
taskID = utils.GenerateTaskID("kb_clone", tenantID.(uint64), req.SourceID)
|
||||
}
|
||||
|
||||
// Create KB clone payload
|
||||
@@ -394,7 +394,8 @@ func (h *KnowledgeBaseHandler) CopyKnowledgeBase(c *gin.Context) {
|
||||
}
|
||||
|
||||
// Enqueue KB clone task to Asynq
|
||||
task := asynq.NewTask(types.TypeKBClone, payloadBytes, asynq.Queue("default"), asynq.MaxRetry(3))
|
||||
task := asynq.NewTask(types.TypeKBClone, payloadBytes,
|
||||
asynq.TaskID(taskID), asynq.Queue("default"), asynq.MaxRetry(3))
|
||||
info, err := h.asynqClient.Enqueue(task)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to enqueue KB clone task: %v", err)
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
// CleanupStaleRunningTasks 清理可能残留的running task keys
|
||||
// 这是一个调试和维护工具,可以用来清理因异常情况导致的残留running keys
|
||||
func CleanupStaleRunningTasks(ctx context.Context, redisClient *redis.Client, keyPrefix string, maxAge time.Duration) (int, error) {
|
||||
// 获取所有匹配的keys
|
||||
keys, err := redisClient.Keys(ctx, keyPrefix+"*").Result()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to get keys: %w", err)
|
||||
}
|
||||
|
||||
if len(keys) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// 检查每个key的TTL
|
||||
var staleTasks []string
|
||||
for _, key := range keys {
|
||||
ttl, err := redisClient.TTL(ctx, key).Result()
|
||||
if err != nil {
|
||||
continue // 跳过错误的key
|
||||
}
|
||||
|
||||
// 如果TTL小于0(永不过期)或者剩余时间太长(可能是残留的),标记为stale
|
||||
if ttl < 0 || ttl > maxAge {
|
||||
staleTasks = append(staleTasks, key)
|
||||
}
|
||||
}
|
||||
|
||||
if len(staleTasks) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// 删除stale keys
|
||||
deleted, err := redisClient.Del(ctx, staleTasks...).Result()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to delete stale keys: %w", err)
|
||||
}
|
||||
|
||||
return int(deleted), nil
|
||||
}
|
||||
|
||||
// CheckRunningTaskStatus 检查指定running task的状态
|
||||
func CheckRunningTaskStatus(ctx context.Context, redisClient *redis.Client, runningKey, progressKey string) (map[string]interface{}, error) {
|
||||
result := make(map[string]interface{})
|
||||
|
||||
// 检查running key
|
||||
runningTaskID, err := redisClient.Get(ctx, runningKey).Result()
|
||||
if err != nil {
|
||||
if err == redis.Nil {
|
||||
result["running_task_exists"] = false
|
||||
} else {
|
||||
return nil, fmt.Errorf("failed to get running task: %w", err)
|
||||
}
|
||||
} else {
|
||||
result["running_task_exists"] = true
|
||||
result["running_task_id"] = runningTaskID
|
||||
|
||||
// 获取running key的TTL
|
||||
ttl, _ := redisClient.TTL(ctx, runningKey).Result()
|
||||
result["running_task_ttl"] = ttl.String()
|
||||
}
|
||||
|
||||
// 检查progress key
|
||||
progressData, err := redisClient.Get(ctx, progressKey).Result()
|
||||
if err != nil {
|
||||
if err == redis.Nil {
|
||||
result["progress_exists"] = false
|
||||
} else {
|
||||
return nil, fmt.Errorf("failed to get progress: %w", err)
|
||||
}
|
||||
} else {
|
||||
result["progress_exists"] = true
|
||||
result["progress_data"] = progressData
|
||||
|
||||
// 获取progress key的TTL
|
||||
ttl, _ := redisClient.TTL(ctx, progressKey).Result()
|
||||
result["progress_ttl"] = ttl.String()
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// GenerateTaskID generates a unique task ID with multiple collision-resistant elements.
|
||||
// The format is: <taskType>_<tenantID>_<timestamp>_<uuid>_<businessID>
|
||||
//
|
||||
// Parameters:
|
||||
// - taskType: Type of task (e.g., "faq_import", "kb_clone")
|
||||
// - tenantID: Tenant ID for multi-tenancy isolation
|
||||
// - businessID: Optional business-specific ID (e.g., knowledge base ID)
|
||||
//
|
||||
// Returns a task ID like: "faq_import_12345_1704628851692_a1b2c3d4_kb789"
|
||||
func GenerateTaskID(taskType string, tenantID uint64, businessID ...string) string {
|
||||
// Use current timestamp in milliseconds for temporal uniqueness
|
||||
timestamp := time.Now().UnixMilli()
|
||||
|
||||
// Generate a short UUID (first 8 characters for brevity)
|
||||
shortUUID := strings.ReplaceAll(uuid.New().String()[:8], "-", "")
|
||||
|
||||
// Build the task ID components
|
||||
components := []string{
|
||||
sanitizeTaskType(taskType),
|
||||
strconv.FormatUint(tenantID, 10),
|
||||
strconv.FormatInt(timestamp, 10),
|
||||
shortUUID,
|
||||
}
|
||||
|
||||
// Add business ID if provided
|
||||
if len(businessID) > 0 && businessID[0] != "" {
|
||||
components = append(components, sanitizeBusinessID(businessID[0]))
|
||||
}
|
||||
|
||||
return strings.Join(components, "_")
|
||||
}
|
||||
|
||||
// GenerateTaskIDWithPrefix generates a task ID with a custom prefix.
|
||||
// This is useful when you want more control over the task ID format.
|
||||
func GenerateTaskIDWithPrefix(prefix string, tenantID uint64, businessID ...string) string {
|
||||
timestamp := time.Now().UnixMilli()
|
||||
shortUUID := strings.ReplaceAll(uuid.New().String()[:8], "-", "")
|
||||
|
||||
components := []string{
|
||||
sanitizeTaskType(prefix),
|
||||
strconv.FormatUint(tenantID, 10),
|
||||
strconv.FormatInt(timestamp, 10),
|
||||
shortUUID,
|
||||
}
|
||||
|
||||
if len(businessID) > 0 && businessID[0] != "" {
|
||||
components = append(components, sanitizeBusinessID(businessID[0]))
|
||||
}
|
||||
|
||||
return strings.Join(components, "_")
|
||||
}
|
||||
|
||||
// ParseTaskID parses a task ID generated by GenerateTaskID and returns its components.
|
||||
// Returns taskType, tenantID, timestamp, uuid, businessID, and error.
|
||||
func ParseTaskID(taskID string) (taskType string, tenantID uint64, timestamp int64, uuidPart string, businessID string, err error) {
|
||||
parts := strings.Split(taskID, "_")
|
||||
if len(parts) < 4 {
|
||||
err = fmt.Errorf("invalid task ID format: %s", taskID)
|
||||
return
|
||||
}
|
||||
|
||||
taskType = parts[0]
|
||||
|
||||
tenantID, err = strconv.ParseUint(parts[1], 10, 64)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("invalid tenant ID in task ID: %s", parts[1])
|
||||
return
|
||||
}
|
||||
|
||||
timestamp, err = strconv.ParseInt(parts[2], 10, 64)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("invalid timestamp in task ID: %s", parts[2])
|
||||
return
|
||||
}
|
||||
|
||||
uuidPart = parts[3]
|
||||
|
||||
if len(parts) > 4 {
|
||||
businessID = parts[4]
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// sanitizeTaskType ensures task type is safe for use in task ID
|
||||
func sanitizeTaskType(taskType string) string {
|
||||
// Replace colons and other special characters with underscores
|
||||
taskType = strings.ReplaceAll(taskType, ":", "_")
|
||||
taskType = strings.ReplaceAll(taskType, "-", "_")
|
||||
taskType = strings.ReplaceAll(taskType, " ", "_")
|
||||
return strings.ToLower(taskType)
|
||||
}
|
||||
|
||||
// sanitizeBusinessID ensures business ID is safe for use in task ID
|
||||
func sanitizeBusinessID(businessID string) string {
|
||||
// Take first 12 characters and replace special characters
|
||||
if len(businessID) > 12 {
|
||||
businessID = businessID[:12]
|
||||
}
|
||||
businessID = strings.ReplaceAll(businessID, "-", "")
|
||||
businessID = strings.ReplaceAll(businessID, "_", "")
|
||||
businessID = strings.ReplaceAll(businessID, ":", "")
|
||||
return businessID
|
||||
}
|
||||
Reference in New Issue
Block a user