feat: offload large FAQ imports to object storage

This commit is contained in:
wizardchen
2026-01-16 15:00:30 +08:00
committed by lyingbug
parent 8784018d86
commit a9df80b5db
2 changed files with 173 additions and 19 deletions
+160 -9
View File
@@ -2785,6 +2785,9 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context,
}
knowledgeID = faqKnowledge.ID
// 记录任务入队时间
enqueuedAt := time.Now().Unix()
// 设置 KB 的运行中任务 ID
if err := s.setRunningFAQImportTaskID(ctx, kbID, taskID); err != nil {
logger.Errorf(ctx, "Failed to set running FAQ import task ID: %v", err)
@@ -2818,14 +2821,49 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context,
// Enqueue FAQ import task to Asynq
logger.Info(ctx, "Enqueuing FAQ import task to Asynq")
// 构建任务 payload
taskPayload := types.FAQImportPayload{
TenantID: tenantID,
TaskID: taskID,
KBID: kbID,
KnowledgeID: knowledgeID,
Entries: payload.Entries,
Mode: payload.Mode,
DryRun: payload.DryRun,
EnqueuedAt: enqueuedAt,
}
// 阈值:超过 200 条或序列化后超过 50KB 时使用对象存储
const (
entryCountThreshold = 200
payloadSizeThreshold = 50 * 1024 // 50KB
)
entryCount := len(payload.Entries)
if entryCount > entryCountThreshold {
// 数据量较大,上传到对象存储
entriesData, err := json.Marshal(payload.Entries)
if err != nil {
logger.Errorf(ctx, "Failed to marshal FAQ entries: %v", err)
return "", fmt.Errorf("failed to marshal entries: %w", err)
}
logger.Infof(ctx, "FAQ entries size: %d bytes, uploading to object storage", len(entriesData))
// 上传到私有桶(主桶),任务处理完成后清理
fileName := fmt.Sprintf("faq_import_entries_%s_%d.json", taskID, enqueuedAt)
entriesURL, err := s.fileSvc.SaveBytes(ctx, entriesData, tenantID, fileName, false)
if err != nil {
logger.Errorf(ctx, "Failed to upload FAQ entries to object storage: %v", err)
return "", fmt.Errorf("failed to upload entries: %w", err)
}
logger.Infof(ctx, "FAQ entries uploaded to: %s", entriesURL)
taskPayload.EntriesURL = entriesURL
taskPayload.EntryCount = entryCount
} else {
// 数据量较小,直接存储在 payload 中
taskPayload.Entries = payload.Entries
}
payloadBytes, err := json.Marshal(taskPayload)
@@ -2834,7 +2872,43 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context,
return "", fmt.Errorf("failed to marshal task payload: %w", err)
}
task := asynq.NewTask(types.TypeFAQImport, payloadBytes, asynq.TaskID(taskID), asynq.Queue("default"), asynq.MaxRetry(5))
// 再次检查 payload 大小
if len(payloadBytes) > payloadSizeThreshold && taskPayload.EntriesURL == "" {
// payload 太大但还没上传,现在上传
entriesData, _ := json.Marshal(payload.Entries)
fileName := fmt.Sprintf("faq_import_entries_%s_%d.json", taskID, enqueuedAt)
entriesURL, err := s.fileSvc.SaveBytes(ctx, entriesData, tenantID, fileName, false)
if err != nil {
logger.Errorf(ctx, "Failed to upload FAQ entries to object storage: %v", err)
return "", fmt.Errorf("failed to upload entries: %w", err)
}
logger.Infof(ctx, "FAQ entries uploaded to (size exceeded): %s", entriesURL)
taskPayload.Entries = nil
taskPayload.EntriesURL = entriesURL
taskPayload.EntryCount = entryCount
payloadBytes, _ = json.Marshal(taskPayload)
}
logger.Infof(ctx, "FAQ import task payload size: %d bytes", len(payloadBytes))
maxRetry := 5
if payload.DryRun {
maxRetry = 3 // dry run 重试次数少一些
}
// 使用 taskID:enqueuedAt 作为 asynq 的唯一任务标识
// 这样同一个用户 TaskID 的不同次提交不会冲突
asynqTaskID := fmt.Sprintf("%s:%d", taskID, enqueuedAt)
task := asynq.NewTask(
types.TypeFAQImport,
payloadBytes,
asynq.TaskID(asynqTaskID),
asynq.Queue("default"),
asynq.MaxRetry(maxRetry),
)
info, err := s.task.Enqueue(task)
if err != nil {
logger.Errorf(ctx, "Failed to enqueue FAQ import task: %v", err)
@@ -5320,18 +5394,57 @@ func (s *knowledgeService) updateFAQImportProgressStatus(
return s.saveFAQImportProgress(ctx, existingProgress)
}
// getRunningFAQImportTaskID checks if there's a running FAQ import task for the given KB
// Returns the task ID if found, empty string otherwise
func (s *knowledgeService) getRunningFAQImportTaskID(ctx context.Context, kbID string) (string, error) {
// cleanupFAQEntriesFileOnFinalFailure 在任务最终失败时清理对象存储中的 entries 文件
// 只有当 retryCount >= maxRetry 时才执行清理,否则重试时还需要使用这个文件
func (s *knowledgeService) cleanupFAQEntriesFileOnFinalFailure(ctx context.Context, entriesURL string, retryCount, maxRetry int) {
if entriesURL == "" || retryCount < maxRetry {
return
}
if err := s.fileSvc.DeleteFile(ctx, entriesURL); err != nil {
logger.Warnf(ctx, "Failed to delete FAQ entries file from object storage on final failure: %v", err)
} else {
logger.Infof(ctx, "Deleted FAQ entries file from object storage on final failure: %s", entriesURL)
}
}
// runningFAQImportInfo stores the task ID and enqueued timestamp for uniquely identifying a task instance
type runningFAQImportInfo struct {
TaskID string `json:"task_id"`
EnqueuedAt int64 `json:"enqueued_at"`
}
// getRunningFAQImportInfo checks if there's a running FAQ import task for the given KB
// Returns the task info if found, nil otherwise
func (s *knowledgeService) getRunningFAQImportInfo(ctx context.Context, kbID string) (*runningFAQImportInfo, error) {
key := getFAQImportRunningKey(kbID)
taskID, err := s.redisClient.Get(ctx, key).Result()
data, err := s.redisClient.Get(ctx, key).Result()
if err != nil {
if errors.Is(err, redis.Nil) {
return "", nil
return nil, nil
}
return "", fmt.Errorf("failed to get running FAQ import task: %w", err)
return nil, fmt.Errorf("failed to get running FAQ import task: %w", err)
}
return taskID, nil
// Try to parse as JSON first (new format)
var info runningFAQImportInfo
if err := json.Unmarshal([]byte(data), &info); err != nil {
// Fallback: old format was just taskID string
return &runningFAQImportInfo{TaskID: data, EnqueuedAt: 0}, nil
}
return &info, nil
}
// getRunningFAQImportTaskID checks if there's a running FAQ import task for the given KB
// Returns the task ID if found, empty string otherwise (for backward compatibility)
func (s *knowledgeService) getRunningFAQImportTaskID(ctx context.Context, kbID string) (string, error) {
info, err := s.getRunningFAQImportInfo(ctx, kbID)
if err != nil {
return "", err
}
if info == nil {
return "", nil
}
return info.TaskID, nil
}
// setRunningFAQImportTaskID sets the running task ID for a KB
@@ -6431,6 +6544,32 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
}
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo)
// 如果 entries 存储在对象存储中,先下载
if payload.EntriesURL != "" && len(payload.Entries) == 0 {
logger.Infof(ctx, "Downloading FAQ entries from object storage: %s", payload.EntriesURL)
reader, err := s.fileSvc.GetFile(ctx, payload.EntriesURL)
if err != nil {
logger.Errorf(ctx, "Failed to download FAQ entries from object storage: %v", err)
return fmt.Errorf("failed to download entries: %w", err)
}
defer reader.Close()
entriesData, err := io.ReadAll(reader)
if err != nil {
logger.Errorf(ctx, "Failed to read FAQ entries data: %v", err)
return fmt.Errorf("failed to read entries data: %w", err)
}
var entries []types.FAQEntryPayload
if err := json.Unmarshal(entriesData, &entries); err != nil {
logger.Errorf(ctx, "Failed to unmarshal FAQ entries: %v", err)
return fmt.Errorf("failed to unmarshal entries: %w", err)
}
payload.Entries = entries
logger.Infof(ctx, "Downloaded %d FAQ entries from object storage", len(entries))
}
logger.Infof(ctx, "Processing FAQ import task: task_id=%s, kb_id=%s, total_entries=%d, dry_run=%v, retry=%d/%d",
payload.TaskID, payload.KBID, len(payload.Entries), payload.DryRun, retryCount, maxRetry)
@@ -6525,6 +6664,7 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
logger.Errorf(ctx, "Failed to update task status to failed: %v", updateErr)
}
}
s.cleanupFAQEntriesFileOnFinalFailure(ctx, payload.EntriesURL, retryCount, maxRetry)
return fmt.Errorf("failed to get knowledge base: %w", err)
}
@@ -6553,6 +6693,7 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
logger.Errorf(ctx, "Failed to update task status to failed: %v", updateErr)
}
}
s.cleanupFAQEntriesFileOnFinalFailure(ctx, payload.EntriesURL, retryCount, maxRetry)
return fmt.Errorf("failed to delete unindexed chunks: %w", err)
}
if len(chunksDeleted) > 0 {
@@ -6601,6 +6742,7 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
logger.Errorf(ctx, "Failed to update task status to failed: %v", updateErr)
}
}
s.cleanupFAQEntriesFileOnFinalFailure(ctx, payload.EntriesURL, retryCount, maxRetry)
return fmt.Errorf("FAQ import failed: %w", err)
}
@@ -6615,6 +6757,15 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
// finalizeFAQValidation 完成 FAQ 验证/导入任务,生成失败条目 CSV(如果有)
func (s *knowledgeService) finalizeFAQValidation(ctx context.Context, payload *types.FAQImportPayload,
progress *types.FAQImportProgress, originalTotalEntries int) error {
// 清理对象存储中的 entries 文件(如果有)
if payload.EntriesURL != "" {
if err := s.fileSvc.DeleteFile(ctx, payload.EntriesURL); err != nil {
logger.Warnf(ctx, "Failed to delete FAQ entries file from object storage: %v", err)
} else {
logger.Infof(ctx, "Deleted FAQ entries file from object storage: %s", payload.EntriesURL)
}
}
// 更新最终状态
progress.Status = types.FAQImportStatusCompleted
progress.Progress = 100
+13 -10
View File
@@ -2,15 +2,15 @@ package types
const (
TypeChunkExtract = "chunk:extract"
TypeDocumentProcess = "document:process" // 文档处理任务
TypeFAQImport = "faq:import" // FAQ导入任务(包含dry run模式)
TypeQuestionGeneration = "question:generation" // 问题生成任务
TypeSummaryGeneration = "summary:generation" // 摘要生成任务
TypeKBClone = "kb:clone" // 知识库复制任务
TypeIndexDelete = "index:delete" // 索引删除任务
TypeKBDelete = "kb:delete" // 知识库删除任务
TypeDocumentProcess = "document:process" // 文档处理任务
TypeFAQImport = "faq:import" // FAQ导入任务(包含dry run模式)
TypeQuestionGeneration = "question:generation" // 问题生成任务
TypeSummaryGeneration = "summary:generation" // 摘要生成任务
TypeKBClone = "kb:clone" // 知识库复制任务
TypeIndexDelete = "index:delete" // 索引删除任务
TypeKBDelete = "kb:delete" // 知识库删除任务
TypeKnowledgeListDelete = "knowledge:list_delete" // 批量删除知识任务
TypeDataTableSummary = "datatable:summary" // 表格摘要任务
TypeDataTableSummary = "datatable:summary" // 表格摘要任务
)
// ExtractChunkPayload represents the extract chunk task payload
@@ -42,9 +42,12 @@ type FAQImportPayload struct {
TaskID string `json:"task_id"`
KBID string `json:"kb_id"`
KnowledgeID string `json:"knowledge_id,omitempty"` // 仅非 dry run 模式需要
Entries []FAQEntryPayload `json:"entries"`
Entries []FAQEntryPayload `json:"entries,omitempty"` // 小数据量时直接存储在 payload 中
EntriesURL string `json:"entries_url,omitempty"` // 大数据量时存储到对象存储,这里存储 URL
EntryCount int `json:"entry_count,omitempty"` // 条目总数(使用 EntriesURL 时需要)
Mode string `json:"mode"`
DryRun bool `json:"dry_run"` // dry run 模式只验证不导入
DryRun bool `json:"dry_run"` // dry run 模式只验证不导入
EnqueuedAt int64 `json:"enqueued_at"` // 任务入队时间戳,用于区分同一 TaskID 的不同次提交
}
// QuestionGenerationPayload represents the question generation task payload