feat: offload large FAQ imports to object storage
This commit is contained in:
@@ -2785,6 +2785,9 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context,
|
||||
}
|
||||
knowledgeID = faqKnowledge.ID
|
||||
|
||||
// 记录任务入队时间
|
||||
enqueuedAt := time.Now().Unix()
|
||||
|
||||
// 设置 KB 的运行中任务 ID
|
||||
if err := s.setRunningFAQImportTaskID(ctx, kbID, taskID); err != nil {
|
||||
logger.Errorf(ctx, "Failed to set running FAQ import task ID: %v", err)
|
||||
@@ -2818,14 +2821,49 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context,
|
||||
|
||||
// Enqueue FAQ import task to Asynq
|
||||
logger.Info(ctx, "Enqueuing FAQ import task to Asynq")
|
||||
|
||||
// 构建任务 payload
|
||||
taskPayload := types.FAQImportPayload{
|
||||
TenantID: tenantID,
|
||||
TaskID: taskID,
|
||||
KBID: kbID,
|
||||
KnowledgeID: knowledgeID,
|
||||
Entries: payload.Entries,
|
||||
Mode: payload.Mode,
|
||||
DryRun: payload.DryRun,
|
||||
EnqueuedAt: enqueuedAt,
|
||||
}
|
||||
|
||||
// 阈值:超过 200 条或序列化后超过 50KB 时使用对象存储
|
||||
const (
|
||||
entryCountThreshold = 200
|
||||
payloadSizeThreshold = 50 * 1024 // 50KB
|
||||
)
|
||||
|
||||
entryCount := len(payload.Entries)
|
||||
if entryCount > entryCountThreshold {
|
||||
// 数据量较大,上传到对象存储
|
||||
entriesData, err := json.Marshal(payload.Entries)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to marshal FAQ entries: %v", err)
|
||||
return "", fmt.Errorf("failed to marshal entries: %w", err)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "FAQ entries size: %d bytes, uploading to object storage", len(entriesData))
|
||||
|
||||
// 上传到私有桶(主桶),任务处理完成后清理
|
||||
fileName := fmt.Sprintf("faq_import_entries_%s_%d.json", taskID, enqueuedAt)
|
||||
entriesURL, err := s.fileSvc.SaveBytes(ctx, entriesData, tenantID, fileName, false)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to upload FAQ entries to object storage: %v", err)
|
||||
return "", fmt.Errorf("failed to upload entries: %w", err)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "FAQ entries uploaded to: %s", entriesURL)
|
||||
taskPayload.EntriesURL = entriesURL
|
||||
taskPayload.EntryCount = entryCount
|
||||
} else {
|
||||
// 数据量较小,直接存储在 payload 中
|
||||
taskPayload.Entries = payload.Entries
|
||||
}
|
||||
|
||||
payloadBytes, err := json.Marshal(taskPayload)
|
||||
@@ -2834,7 +2872,43 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context,
|
||||
return "", fmt.Errorf("failed to marshal task payload: %w", err)
|
||||
}
|
||||
|
||||
task := asynq.NewTask(types.TypeFAQImport, payloadBytes, asynq.TaskID(taskID), asynq.Queue("default"), asynq.MaxRetry(5))
|
||||
// 再次检查 payload 大小
|
||||
if len(payloadBytes) > payloadSizeThreshold && taskPayload.EntriesURL == "" {
|
||||
// payload 太大但还没上传,现在上传
|
||||
entriesData, _ := json.Marshal(payload.Entries)
|
||||
fileName := fmt.Sprintf("faq_import_entries_%s_%d.json", taskID, enqueuedAt)
|
||||
entriesURL, err := s.fileSvc.SaveBytes(ctx, entriesData, tenantID, fileName, false)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to upload FAQ entries to object storage: %v", err)
|
||||
return "", fmt.Errorf("failed to upload entries: %w", err)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "FAQ entries uploaded to (size exceeded): %s", entriesURL)
|
||||
taskPayload.Entries = nil
|
||||
taskPayload.EntriesURL = entriesURL
|
||||
taskPayload.EntryCount = entryCount
|
||||
|
||||
payloadBytes, _ = json.Marshal(taskPayload)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "FAQ import task payload size: %d bytes", len(payloadBytes))
|
||||
|
||||
maxRetry := 5
|
||||
if payload.DryRun {
|
||||
maxRetry = 3 // dry run 重试次数少一些
|
||||
}
|
||||
|
||||
// 使用 taskID:enqueuedAt 作为 asynq 的唯一任务标识
|
||||
// 这样同一个用户 TaskID 的不同次提交不会冲突
|
||||
asynqTaskID := fmt.Sprintf("%s:%d", taskID, enqueuedAt)
|
||||
|
||||
task := asynq.NewTask(
|
||||
types.TypeFAQImport,
|
||||
payloadBytes,
|
||||
asynq.TaskID(asynqTaskID),
|
||||
asynq.Queue("default"),
|
||||
asynq.MaxRetry(maxRetry),
|
||||
)
|
||||
info, err := s.task.Enqueue(task)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to enqueue FAQ import task: %v", err)
|
||||
@@ -5320,18 +5394,57 @@ func (s *knowledgeService) updateFAQImportProgressStatus(
|
||||
return s.saveFAQImportProgress(ctx, existingProgress)
|
||||
}
|
||||
|
||||
// getRunningFAQImportTaskID checks if there's a running FAQ import task for the given KB
|
||||
// Returns the task ID if found, empty string otherwise
|
||||
func (s *knowledgeService) getRunningFAQImportTaskID(ctx context.Context, kbID string) (string, error) {
|
||||
// cleanupFAQEntriesFileOnFinalFailure 在任务最终失败时清理对象存储中的 entries 文件
|
||||
// 只有当 retryCount >= maxRetry 时才执行清理,否则重试时还需要使用这个文件
|
||||
func (s *knowledgeService) cleanupFAQEntriesFileOnFinalFailure(ctx context.Context, entriesURL string, retryCount, maxRetry int) {
|
||||
if entriesURL == "" || retryCount < maxRetry {
|
||||
return
|
||||
}
|
||||
if err := s.fileSvc.DeleteFile(ctx, entriesURL); err != nil {
|
||||
logger.Warnf(ctx, "Failed to delete FAQ entries file from object storage on final failure: %v", err)
|
||||
} else {
|
||||
logger.Infof(ctx, "Deleted FAQ entries file from object storage on final failure: %s", entriesURL)
|
||||
}
|
||||
}
|
||||
|
||||
// runningFAQImportInfo stores the task ID and enqueued timestamp for uniquely identifying a task instance
|
||||
type runningFAQImportInfo struct {
|
||||
TaskID string `json:"task_id"`
|
||||
EnqueuedAt int64 `json:"enqueued_at"`
|
||||
}
|
||||
|
||||
// getRunningFAQImportInfo checks if there's a running FAQ import task for the given KB
|
||||
// Returns the task info if found, nil otherwise
|
||||
func (s *knowledgeService) getRunningFAQImportInfo(ctx context.Context, kbID string) (*runningFAQImportInfo, error) {
|
||||
key := getFAQImportRunningKey(kbID)
|
||||
taskID, err := s.redisClient.Get(ctx, key).Result()
|
||||
data, err := s.redisClient.Get(ctx, key).Result()
|
||||
if err != nil {
|
||||
if errors.Is(err, redis.Nil) {
|
||||
return "", nil
|
||||
return nil, nil
|
||||
}
|
||||
return "", fmt.Errorf("failed to get running FAQ import task: %w", err)
|
||||
return nil, fmt.Errorf("failed to get running FAQ import task: %w", err)
|
||||
}
|
||||
return taskID, nil
|
||||
|
||||
// Try to parse as JSON first (new format)
|
||||
var info runningFAQImportInfo
|
||||
if err := json.Unmarshal([]byte(data), &info); err != nil {
|
||||
// Fallback: old format was just taskID string
|
||||
return &runningFAQImportInfo{TaskID: data, EnqueuedAt: 0}, nil
|
||||
}
|
||||
return &info, nil
|
||||
}
|
||||
|
||||
// getRunningFAQImportTaskID checks if there's a running FAQ import task for the given KB
|
||||
// Returns the task ID if found, empty string otherwise (for backward compatibility)
|
||||
func (s *knowledgeService) getRunningFAQImportTaskID(ctx context.Context, kbID string) (string, error) {
|
||||
info, err := s.getRunningFAQImportInfo(ctx, kbID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if info == nil {
|
||||
return "", nil
|
||||
}
|
||||
return info.TaskID, nil
|
||||
}
|
||||
|
||||
// setRunningFAQImportTaskID sets the running task ID for a KB
|
||||
@@ -6431,6 +6544,32 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
|
||||
}
|
||||
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo)
|
||||
|
||||
// 如果 entries 存储在对象存储中,先下载
|
||||
if payload.EntriesURL != "" && len(payload.Entries) == 0 {
|
||||
logger.Infof(ctx, "Downloading FAQ entries from object storage: %s", payload.EntriesURL)
|
||||
reader, err := s.fileSvc.GetFile(ctx, payload.EntriesURL)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to download FAQ entries from object storage: %v", err)
|
||||
return fmt.Errorf("failed to download entries: %w", err)
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
entriesData, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to read FAQ entries data: %v", err)
|
||||
return fmt.Errorf("failed to read entries data: %w", err)
|
||||
}
|
||||
|
||||
var entries []types.FAQEntryPayload
|
||||
if err := json.Unmarshal(entriesData, &entries); err != nil {
|
||||
logger.Errorf(ctx, "Failed to unmarshal FAQ entries: %v", err)
|
||||
return fmt.Errorf("failed to unmarshal entries: %w", err)
|
||||
}
|
||||
|
||||
payload.Entries = entries
|
||||
logger.Infof(ctx, "Downloaded %d FAQ entries from object storage", len(entries))
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Processing FAQ import task: task_id=%s, kb_id=%s, total_entries=%d, dry_run=%v, retry=%d/%d",
|
||||
payload.TaskID, payload.KBID, len(payload.Entries), payload.DryRun, retryCount, maxRetry)
|
||||
|
||||
@@ -6525,6 +6664,7 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
|
||||
logger.Errorf(ctx, "Failed to update task status to failed: %v", updateErr)
|
||||
}
|
||||
}
|
||||
s.cleanupFAQEntriesFileOnFinalFailure(ctx, payload.EntriesURL, retryCount, maxRetry)
|
||||
return fmt.Errorf("failed to get knowledge base: %w", err)
|
||||
}
|
||||
|
||||
@@ -6553,6 +6693,7 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
|
||||
logger.Errorf(ctx, "Failed to update task status to failed: %v", updateErr)
|
||||
}
|
||||
}
|
||||
s.cleanupFAQEntriesFileOnFinalFailure(ctx, payload.EntriesURL, retryCount, maxRetry)
|
||||
return fmt.Errorf("failed to delete unindexed chunks: %w", err)
|
||||
}
|
||||
if len(chunksDeleted) > 0 {
|
||||
@@ -6601,6 +6742,7 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
|
||||
logger.Errorf(ctx, "Failed to update task status to failed: %v", updateErr)
|
||||
}
|
||||
}
|
||||
s.cleanupFAQEntriesFileOnFinalFailure(ctx, payload.EntriesURL, retryCount, maxRetry)
|
||||
return fmt.Errorf("FAQ import failed: %w", err)
|
||||
}
|
||||
|
||||
@@ -6615,6 +6757,15 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
|
||||
// finalizeFAQValidation 完成 FAQ 验证/导入任务,生成失败条目 CSV(如果有)
|
||||
func (s *knowledgeService) finalizeFAQValidation(ctx context.Context, payload *types.FAQImportPayload,
|
||||
progress *types.FAQImportProgress, originalTotalEntries int) error {
|
||||
// 清理对象存储中的 entries 文件(如果有)
|
||||
if payload.EntriesURL != "" {
|
||||
if err := s.fileSvc.DeleteFile(ctx, payload.EntriesURL); err != nil {
|
||||
logger.Warnf(ctx, "Failed to delete FAQ entries file from object storage: %v", err)
|
||||
} else {
|
||||
logger.Infof(ctx, "Deleted FAQ entries file from object storage: %s", payload.EntriesURL)
|
||||
}
|
||||
}
|
||||
|
||||
// 更新最终状态
|
||||
progress.Status = types.FAQImportStatusCompleted
|
||||
progress.Progress = 100
|
||||
|
||||
@@ -2,15 +2,15 @@ package types
|
||||
|
||||
const (
|
||||
TypeChunkExtract = "chunk:extract"
|
||||
TypeDocumentProcess = "document:process" // 文档处理任务
|
||||
TypeFAQImport = "faq:import" // FAQ导入任务(包含dry run模式)
|
||||
TypeQuestionGeneration = "question:generation" // 问题生成任务
|
||||
TypeSummaryGeneration = "summary:generation" // 摘要生成任务
|
||||
TypeKBClone = "kb:clone" // 知识库复制任务
|
||||
TypeIndexDelete = "index:delete" // 索引删除任务
|
||||
TypeKBDelete = "kb:delete" // 知识库删除任务
|
||||
TypeDocumentProcess = "document:process" // 文档处理任务
|
||||
TypeFAQImport = "faq:import" // FAQ导入任务(包含dry run模式)
|
||||
TypeQuestionGeneration = "question:generation" // 问题生成任务
|
||||
TypeSummaryGeneration = "summary:generation" // 摘要生成任务
|
||||
TypeKBClone = "kb:clone" // 知识库复制任务
|
||||
TypeIndexDelete = "index:delete" // 索引删除任务
|
||||
TypeKBDelete = "kb:delete" // 知识库删除任务
|
||||
TypeKnowledgeListDelete = "knowledge:list_delete" // 批量删除知识任务
|
||||
TypeDataTableSummary = "datatable:summary" // 表格摘要任务
|
||||
TypeDataTableSummary = "datatable:summary" // 表格摘要任务
|
||||
)
|
||||
|
||||
// ExtractChunkPayload represents the extract chunk task payload
|
||||
@@ -42,9 +42,12 @@ type FAQImportPayload struct {
|
||||
TaskID string `json:"task_id"`
|
||||
KBID string `json:"kb_id"`
|
||||
KnowledgeID string `json:"knowledge_id,omitempty"` // 仅非 dry run 模式需要
|
||||
Entries []FAQEntryPayload `json:"entries"`
|
||||
Entries []FAQEntryPayload `json:"entries,omitempty"` // 小数据量时直接存储在 payload 中
|
||||
EntriesURL string `json:"entries_url,omitempty"` // 大数据量时存储到对象存储,这里存储 URL
|
||||
EntryCount int `json:"entry_count,omitempty"` // 条目总数(使用 EntriesURL 时需要)
|
||||
Mode string `json:"mode"`
|
||||
DryRun bool `json:"dry_run"` // dry run 模式只验证不导入
|
||||
DryRun bool `json:"dry_run"` // dry run 模式只验证不导入
|
||||
EnqueuedAt int64 `json:"enqueued_at"` // 任务入队时间戳,用于区分同一 TaskID 的不同次提交
|
||||
}
|
||||
|
||||
// QuestionGenerationPayload represents the question generation task payload
|
||||
|
||||
Reference in New Issue
Block a user