feat: 重构FAQ导入进度,改用Redis存储任务状态

This commit is contained in:
wizardchen
2025-12-18 19:41:36 +08:00
committed by lyingbug
parent 7576f08ad5
commit ad450e6bce
8 changed files with 272 additions and 110 deletions
+38
View File
@@ -308,3 +308,41 @@ func (c *Client) ExportFAQEntries(ctx context.Context, knowledgeBaseID string) (
return data, nil
}
// FAQImportProgress represents the progress of an async FAQ import task.
type FAQImportProgress struct {
TaskID string `json:"task_id"`
KBID string `json:"kb_id"`
KnowledgeID string `json:"knowledge_id"`
Status string `json:"status"`
Progress int `json:"progress"`
Total int `json:"total"`
Processed int `json:"processed"`
Message string `json:"message"`
Error string `json:"error,omitempty"`
CreatedAt int64 `json:"created_at"`
UpdatedAt int64 `json:"updated_at"`
}
// FAQImportProgressResponse wraps the FAQ import progress response.
type FAQImportProgressResponse struct {
Success bool `json:"success"`
Data *FAQImportProgress `json:"data"`
Message string `json:"message,omitempty"`
Code string `json:"code,omitempty"`
}
// GetFAQImportProgress retrieves the progress of an async FAQ import task.
func (c *Client) GetFAQImportProgress(ctx context.Context, taskID string) (*FAQImportProgress, error) {
path := fmt.Sprintf("/api/v1/faq/import/progress/%s", taskID)
resp, err := c.doRequest(ctx, http.MethodGet, path, nil, nil)
if err != nil {
return nil, err
}
var response FAQImportProgressResponse
if err := parseResponse(resp, &response); err != nil {
return nil, err
}
return response.Data, nil
}
+19
View File
@@ -211,4 +211,23 @@ export function searchFAQEntries(
export async function exportFAQEntries(kbId: string): Promise<Blob> {
const response = await getDown(`/api/v1/knowledge-bases/${kbId}/faq/entries/export`);
return response as unknown as Blob;
}
// FAQ Import Progress API
export interface FAQImportProgress {
task_id: string
kb_id: string
knowledge_id: string
status: 'pending' | 'processing' | 'completed' | 'failed'
progress: number
total: number
processed: number
message: string
error: string
created_at: number
updated_at: number
}
export function getFAQImportProgress(taskId: string) {
return get(`/api/v1/faq/import/progress/${taskId}`);
}
@@ -1123,7 +1123,7 @@ import {
deleteKnowledgeBaseTag,
getKnowledgeBaseById,
listKnowledgeBases,
getKnowledgeDetails,
getFAQImportProgress,
} from '@/api/knowledge-base'
import * as XLSX from 'xlsx'
import Papa from 'papaparse'
@@ -2223,24 +2223,22 @@ const startPolling = (taskId: string) => {
importState.pollingInterval = setInterval(async () => {
try {
const res: any = await getKnowledgeDetails(taskId)
const knowledge = res?.data
if (knowledge) {
// 从Knowledge对象中提取导入任务状态
// ParseStatus: "pending" -> "pending", "processing" -> "running", "completed" -> "success", "failed" -> "failed"
let status = knowledge.parse_status
const res: any = await getFAQImportProgress(taskId)
const progressData = res?.data
if (progressData) {
// 从Redis进度数据中提取状态
// status: "pending" -> "pending", "processing" -> "running", "completed" -> "success", "failed" -> "failed"
let status = progressData.status
if (status === 'processing') {
status = 'running'
} else if (status === 'completed') {
status = 'success'
}
// 从Metadata中提取导入进度信息
const metadata = knowledge.metadata || {}
const progress = metadata.import_progress || 0
const total = metadata.import_total || 0
const processed = metadata.import_processed || 0
const error = knowledge.error_message || ''
const progress = progressData.progress || 0
const total = progressData.total || 0
const processed = progressData.processed || 0
const error = progressData.error || ''
importState.taskStatus = {
status: status,
@@ -2350,25 +2348,23 @@ const restoreImportTask = async () => {
if (!savedTaskId) return
try {
// 查询Knowledge状态
const res: any = await getKnowledgeDetails(savedTaskId)
const knowledge = res?.data
// 查询Redis中的进度状态
const res: any = await getFAQImportProgress(savedTaskId)
const progressData = res?.data
if (knowledge) {
// 从Knowledge对象中提取导入任务状态
let status = knowledge.parse_status
if (progressData) {
// 从Redis进度数据中提取状态
let status = progressData.status
if (status === 'processing') {
status = 'running'
} else if (status === 'completed') {
status = 'success'
}
// 从Metadata中提取导入进度信息
const metadata = knowledge.metadata || {}
const progress = metadata.import_progress || 0
const total = metadata.import_total || 0
const processed = metadata.import_processed || 0
const error = knowledge.error_message || ''
const progress = progressData.progress || 0
const total = progressData.total || 0
const processed = progressData.processed || 0
const error = progressData.error || ''
importState.taskId = savedTaskId
importState.taskStatus = {
+143 -85
View File
@@ -2680,7 +2680,7 @@ func (s *knowledgeService) ListFAQEntries(ctx context.Context,
}
// UpsertFAQEntries imports or appends FAQ entries asynchronously.
// Returns task ID for tracking import progress.
// Returns task ID (UUID) for tracking import progress.
func (s *knowledgeService) UpsertFAQEntries(ctx context.Context,
kbID string, payload *types.FAQBatchUpsertPayload,
) (string, error) {
@@ -2702,14 +2702,14 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context,
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
// 检查是否有正在进行的导入任务
runningKnowledge, err := s.getRunningFAQImportTask(ctx, kbID, tenantID)
// 检查是否有正在进行的导入任务(通过Redis)
runningTaskID, err := s.getRunningFAQImportTaskID(ctx, kbID)
if err != nil {
logger.Errorf(ctx, "Failed to check running import task: %v", err)
// 检查失败不影响导入,继续执行
} else if runningKnowledge != nil {
logger.Warnf(ctx, "Import task already running for KB %s: %s (status: %s)", kbID, runningKnowledge.ID, runningKnowledge.ParseStatus)
return "", werrors.NewBadRequestError(fmt.Sprintf("该知识库已有导入任务正在进行中(任务ID: %s),请等待完成后再试", runningKnowledge.ID))
} else if runningTaskID != "" {
logger.Warnf(ctx, "Import task already running for KB %s: %s", kbID, runningTaskID)
return "", werrors.NewBadRequestError(fmt.Sprintf("该知识库已有导入任务正在进行中(任务ID: %s),请等待完成后再试", runningTaskID))
}
faqKnowledge, err := s.ensureFAQKnowledge(ctx, tenantID, kb)
@@ -2717,15 +2717,34 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context,
return "", fmt.Errorf("failed to ensure FAQ knowledge: %w", err)
}
// 初始化导入任务状态到Knowledge表
taskID := faqKnowledge.ID // 使用Knowledge ID作为taskID
if err := s.updateFAQImportStatusWithRanges(ctx, taskID, types.FAQImportStatusPending,
0, len(payload.Entries), 0, ""); err != nil {
// 生成UUID作为任务ID
taskID := uuid.New().String()
// 初始化导入任务状态到Redis
progress := &types.FAQImportProgress{
TaskID: taskID,
KBID: kbID,
KnowledgeID: faqKnowledge.ID,
Status: types.FAQImportStatusPending,
Progress: 0,
Total: len(payload.Entries),
Processed: 0,
Message: "任务已创建,等待处理",
CreatedAt: time.Now().Unix(),
UpdatedAt: time.Now().Unix(),
}
if err := s.saveFAQImportProgress(ctx, progress); err != nil {
logger.Errorf(ctx, "Failed to initialize FAQ import task status: %v", err)
return "", fmt.Errorf("failed to initialize task: %w", err)
}
logger.Infof(ctx, "FAQ import task initialized: %s, total entries: %d", taskID, len(payload.Entries))
// 设置 KB 的运行中任务 ID
if err := s.setRunningFAQImportTaskID(ctx, kbID, taskID); err != nil {
logger.Errorf(ctx, "Failed to set running FAQ import task ID: %v", err)
// 不影响任务执行,继续
}
logger.Infof(ctx, "FAQ import task initialized: %s, kb_id: %s, total entries: %d", taskID, kbID, len(payload.Entries))
// Enqueue FAQ import task to Asynq
logger.Info(ctx, "Enqueuing FAQ import task to Asynq")
@@ -3158,7 +3177,7 @@ func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string,
actualProcessed += len(batch)
// 更新任务进度
progress := int(float64(actualProcessed) / float64(totalEntries) * 100)
if err := s.updateFAQImportStatus(ctx, taskID, types.FAQImportStatusProcessing, progress, totalEntries, actualProcessed, ""); err != nil {
if err := s.updateFAQImportProgressStatus(ctx, taskID, types.FAQImportStatusProcessing, progress, totalEntries, actualProcessed, fmt.Sprintf("正在处理第 %d/%d 条", actualProcessed, totalEntries), ""); err != nil {
logger.Errorf(ctx, "Failed to update task progress: %v", err)
}
@@ -4110,77 +4129,73 @@ func (s *knowledgeService) ensureFAQKnowledge(
return knowledge, nil
}
func (s *knowledgeService) updateFAQImportStatus(
// updateFAQImportProgressStatus updates the FAQ import progress in Redis
func (s *knowledgeService) updateFAQImportProgressStatus(
ctx context.Context,
knowledgeID string,
taskID string,
status types.FAQImportTaskStatus,
progress, total, processed int,
errorMsg string,
message, errorMsg string,
) error {
return s.updateFAQImportStatusWithRanges(ctx, knowledgeID, status, progress, total, processed, errorMsg)
}
// updateFAQImportStatusWithRanges 更新FAQ Knowledge的导入任务状态,包含NextChunkIndex
func (s *knowledgeService) updateFAQImportStatusWithRanges(
ctx context.Context,
knowledgeID string,
status types.FAQImportTaskStatus,
progress, total, processed int,
errorMsg string,
) error {
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID)
// Get existing progress from Redis
existingProgress, err := s.GetFAQImportProgress(ctx, taskID)
if err != nil {
return err
// If not found, create a new progress entry
existingProgress = &types.FAQImportProgress{
TaskID: taskID,
CreatedAt: time.Now().Unix(),
}
}
// 更新ParseStatus:将FAQImportTaskStatus映射到ParseStatus
knowledge.ParseStatus = string(status)
knowledge.UpdatedAt = time.Now()
meta, err := types.ParseFAQImportMetadata(knowledge)
if err != nil || meta == nil {
meta = &types.FAQImportMetadata{}
// Update progress fields
existingProgress.Status = status
existingProgress.Progress = progress
existingProgress.Total = total
existingProgress.Processed = processed
if message != "" {
existingProgress.Message = message
}
// 更新ErrorMessage
knowledge.ErrorMessage = errorMsg
existingProgress.Error = errorMsg
if status == types.FAQImportStatusCompleted {
knowledge.ErrorMessage = ""
existingProgress.Error = ""
}
// 更新Metadata中的导入进度信息,保留已有的ChunkIndexRanges和NextChunkIndex
meta.ImportProgress = progress
meta.ImportTotal = total
meta.ImportProcessed = processed
metaJSON, err := meta.ToJSON()
if err != nil {
return fmt.Errorf("failed to marshal import metadata: %w", err)
// 任务完成或失败时,清除 running key
if status == types.FAQImportStatusCompleted || status == types.FAQImportStatusFailed {
if existingProgress.KBID != "" {
if clearErr := s.clearRunningFAQImportTaskID(ctx, existingProgress.KBID); clearErr != nil {
logger.Errorf(ctx, "Failed to clear running FAQ import task ID: %v", clearErr)
}
}
}
knowledge.Metadata = metaJSON
return s.repo.UpdateKnowledge(ctx, knowledge)
return s.saveFAQImportProgress(ctx, existingProgress)
}
// getRunningFAQImportTask 获取指定知识库的进行中导入任务
func (s *knowledgeService) getRunningFAQImportTask(
ctx context.Context,
kbID string,
tenantID uint64,
) (*types.Knowledge, error) {
faqKnowledge, err := s.findFAQKnowledge(ctx, tenantID, kbID)
// getRunningFAQImportTaskID checks if there's a running FAQ import task for the given KB
// Returns the task ID if found, empty string otherwise
func (s *knowledgeService) getRunningFAQImportTaskID(ctx context.Context, kbID string) (string, error) {
key := getFAQImportRunningKey(kbID)
taskID, err := s.redisClient.Get(ctx, key).Result()
if err != nil {
return nil, err
if errors.Is(err, redis.Nil) {
return "", nil
}
return "", fmt.Errorf("failed to get running FAQ import task: %w", err)
}
if faqKnowledge == nil {
return nil, errors.New("FAQ knowledge not found")
}
// 检查ParseStatus是否为pending或processing(进行中状态)
if faqKnowledge.ParseStatus == "pending" || faqKnowledge.ParseStatus == "processing" {
return faqKnowledge, nil
}
return nil, nil
return taskID, nil
}
// setRunningFAQImportTaskID sets the running task ID for a KB
func (s *knowledgeService) setRunningFAQImportTaskID(ctx context.Context, kbID, taskID string) error {
key := getFAQImportRunningKey(kbID)
return s.redisClient.Set(ctx, key, taskID, faqImportProgressTTL).Err()
}
// clearRunningFAQImportTaskID clears the running task ID for a KB
func (s *knowledgeService) clearRunningFAQImportTaskID(ctx context.Context, kbID string) error {
key := getFAQImportRunningKey(kbID)
return s.redisClient.Del(ctx, key).Err()
}
func (s *knowledgeService) chunkToFAQEntry(chunk *types.Chunk, kb *types.KnowledgeBase) (*types.FAQEntry, error) {
@@ -5099,24 +5114,22 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
logger.Errorf(ctx, "Failed to get knowledge base: %v", err)
// 如果是最后一次重试,更新状态为失败
if isLastRetry {
if updateErr := s.updateFAQImportStatus(ctx, payload.TaskID, types.FAQImportStatusFailed, 0, len(payload.Entries), 0, err.Error()); updateErr != nil {
if updateErr := s.updateFAQImportProgressStatus(ctx, payload.TaskID, types.FAQImportStatusFailed, 0, len(payload.Entries), 0, "获取知识库失败", err.Error()); updateErr != nil {
logger.Errorf(ctx, "Failed to update task status to failed: %v", updateErr)
}
}
return fmt.Errorf("failed to get knowledge base: %w", err)
}
// 检查任务状态 - 幂等性处理
if knowledge.ParseStatus == "completed" {
logger.Infof(ctx, "FAQ import already completed, skipping: %s", payload.TaskID)
return nil // 幂等:已完成的任务直接返回
}
// 检查已处理进度
importMeta, _ := types.ParseFAQImportMetadata(knowledge)
// 检查任务状态 - 幂等性处理(先检查Redis中的进度)
existingProgress, _ := s.GetFAQImportProgress(ctx, payload.TaskID)
var processedCount int
if importMeta != nil {
processedCount = importMeta.ImportProcessed
if existingProgress != nil {
if existingProgress.Status == types.FAQImportStatusCompleted {
logger.Infof(ctx, "FAQ import already completed, skipping: %s", payload.TaskID)
return nil // 幂等:已完成的任务直接返回
}
processedCount = existingProgress.Processed
logger.Infof(ctx, "Resuming FAQ import from progress: %d/%d", processedCount, len(payload.Entries))
}
@@ -5131,7 +5144,7 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
logger.Errorf(ctx, "Failed to delete unindexed chunks: %v", err)
// 如果是最后一次重试,更新状态为失败
if isLastRetry {
if updateErr := s.updateFAQImportStatus(ctx, payload.TaskID, types.FAQImportStatusFailed, 0, originalTotalEntries, processedCount, err.Error()); updateErr != nil {
if updateErr := s.updateFAQImportProgressStatus(ctx, payload.TaskID, types.FAQImportStatusFailed, 0, originalTotalEntries, processedCount, "清理未索引数据失败", err.Error()); updateErr != nil {
logger.Errorf(ctx, "Failed to update task status to failed: %v", updateErr)
}
}
@@ -5173,8 +5186,8 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
}
// 更新任务状态为运行中
if err := s.updateFAQImportStatusWithRanges(ctx, payload.TaskID, types.FAQImportStatusProcessing, 0,
originalTotalEntries, processedCount, ""); err != nil {
if err := s.updateFAQImportProgressStatus(ctx, payload.TaskID, types.FAQImportStatusProcessing, 0,
originalTotalEntries, processedCount, "开始处理导入任务", ""); err != nil {
logger.Errorf(ctx, "Failed to update task status to running: %v", err)
}
@@ -5190,12 +5203,12 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
// 如果是最后一次重试,更新状态为失败
if isLastRetry {
// 获取当前已处理的进度
currentMeta, _ := types.ParseFAQImportMetadata(knowledge)
currentProgress, _ := s.GetFAQImportProgress(ctx, payload.TaskID)
currentProcessed := 0
if currentMeta != nil {
currentProcessed = currentMeta.ImportProcessed
if currentProgress != nil {
currentProcessed = currentProgress.Processed
}
if updateErr := s.updateFAQImportStatus(ctx, payload.TaskID, types.FAQImportStatusFailed, 0, originalTotalEntries, currentProcessed, err.Error()); updateErr != nil {
if updateErr := s.updateFAQImportProgressStatus(ctx, payload.TaskID, types.FAQImportStatusFailed, 0, originalTotalEntries, currentProcessed, "导入失败", err.Error()); updateErr != nil {
logger.Errorf(ctx, "Failed to update task status to failed: %v", updateErr)
}
}
@@ -5204,7 +5217,7 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
// 任务成功完成
logger.Infof(ctx, "FAQ import task completed: %s", payload.TaskID)
if err := s.updateFAQImportStatus(ctx, payload.TaskID, types.FAQImportStatusCompleted, 100, originalTotalEntries, originalTotalEntries, ""); err != nil {
if err := s.updateFAQImportProgressStatus(ctx, payload.TaskID, types.FAQImportStatusCompleted, 100, originalTotalEntries, originalTotalEntries, "导入完成", ""); err != nil {
logger.Errorf(ctx, "Failed to update task status to success: %v", err)
}
@@ -5221,6 +5234,51 @@ func getKBCloneProgressKey(taskID string) string {
return kbCloneProgressKeyPrefix + taskID
}
const (
faqImportProgressKeyPrefix = "faq_import_progress:"
faqImportRunningKeyPrefix = "faq_import_running:"
faqImportProgressTTL = 3 * time.Hour
)
// getFAQImportProgressKey returns the Redis key for storing FAQ import progress
func getFAQImportProgressKey(taskID string) string {
return faqImportProgressKeyPrefix + taskID
}
// getFAQImportRunningKey returns the Redis key for storing running task ID by KB ID
func getFAQImportRunningKey(kbID string) string {
return faqImportRunningKeyPrefix + kbID
}
// saveFAQImportProgress saves the FAQ import progress to Redis
func (s *knowledgeService) saveFAQImportProgress(ctx context.Context, progress *types.FAQImportProgress) error {
key := getFAQImportProgressKey(progress.TaskID)
progress.UpdatedAt = time.Now().Unix()
data, err := json.Marshal(progress)
if err != nil {
return fmt.Errorf("failed to marshal FAQ import progress: %w", err)
}
return s.redisClient.Set(ctx, key, data, faqImportProgressTTL).Err()
}
// GetFAQImportProgress retrieves the progress of an FAQ import task
func (s *knowledgeService) GetFAQImportProgress(ctx context.Context, taskID string) (*types.FAQImportProgress, error) {
key := getFAQImportProgressKey(taskID)
data, err := s.redisClient.Get(ctx, key).Bytes()
if err != nil {
if errors.Is(err, redis.Nil) {
return nil, werrors.NewNotFoundError("FAQ import task not found")
}
return nil, fmt.Errorf("failed to get FAQ import progress from Redis: %w", err)
}
var progress types.FAQImportProgress
if err := json.Unmarshal(data, &progress); err != nil {
return nil, fmt.Errorf("failed to unmarshal FAQ import progress: %w", err)
}
return &progress, nil
}
// ProcessKBClone handles Asynq knowledge base clone tasks
func (s *knowledgeService) ProcessKBClone(ctx context.Context, t *asynq.Task) error {
var payload types.KBClonePayload
+28
View File
@@ -341,3 +341,31 @@ func (h *FAQHandler) ExportEntries(c *gin.Context) {
bom := []byte{0xEF, 0xBB, 0xBF}
c.Data(http.StatusOK, "text/csv; charset=utf-8", append(bom, csvData...))
}
// GetImportProgress godoc
// @Summary 获取FAQ导入进度
// @Description 获取FAQ导入任务的进度
// @Tags FAQ管理
// @Accept json
// @Produce json
// @Param task_id path string true "任务ID"
// @Success 200 {object} map[string]interface{} "导入进度"
// @Failure 404 {object} errors.AppError "任务不存在"
// @Security Bearer
// @Router /faq/import/progress/{task_id} [get]
func (h *FAQHandler) GetImportProgress(c *gin.Context) {
ctx := c.Request.Context()
taskID := secutils.SanitizeForLog(c.Param("task_id"))
progress, err := h.knowledgeService.GetFAQImportProgress(ctx, taskID)
if err != nil {
logger.ErrorWithFields(ctx, err, nil)
c.Error(err)
return
}
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": progress,
})
}
+5
View File
@@ -187,6 +187,11 @@ func RegisterFAQRoutes(r *gin.RouterGroup, handler *handler.FAQHandler) {
faq.DELETE("/entries", handler.DeleteEntries)
faq.POST("/search", handler.SearchFAQ)
}
// FAQ import progress route (outside of knowledge-base scope)
faqImport := r.Group("/faq/import")
{
faqImport.GET("/progress/:task_id", handler.GetImportProgress)
}
}
// RegisterKnowledgeBaseRoutes 注册知识库相关的路由
+16
View File
@@ -260,7 +260,23 @@ const (
FAQImportStatusFailed FAQImportTaskStatus = "failed"
)
// FAQImportProgress represents the progress of an FAQ import task stored in Redis
type FAQImportProgress struct {
TaskID string `json:"task_id"` // UUID for the import task
KBID string `json:"kb_id"` // Knowledge Base ID
KnowledgeID string `json:"knowledge_id"` // FAQ Knowledge ID
Status FAQImportTaskStatus `json:"status"` // Task status
Progress int `json:"progress"` // 0-100 percentage
Total int `json:"total"` // Total entries to import
Processed int `json:"processed"` // Entries processed so far
Message string `json:"message"` // Status message
Error string `json:"error"` // Error message if failed
CreatedAt int64 `json:"created_at"` // Task creation timestamp
UpdatedAt int64 `json:"updated_at"` // Last update timestamp
}
// FAQImportMetadata 存储在Knowledge.Metadata中的FAQ导入任务信息
// Deprecated: Use FAQImportProgress with Redis storage instead
type FAQImportMetadata struct {
ImportProgress int `json:"import_progress"` // 0-100
ImportTotal int `json:"import_total"`
+2
View File
@@ -117,6 +117,8 @@ type KnowledgeService interface {
GetKBCloneProgress(ctx context.Context, taskID string) (*types.KBCloneProgress, error)
// SaveKBCloneProgress saves the progress of a knowledge base clone task
SaveKBCloneProgress(ctx context.Context, progress *types.KBCloneProgress) error
// GetFAQImportProgress retrieves the progress of an FAQ import task
GetFAQImportProgress(ctx context.Context, taskID string) (*types.FAQImportProgress, error)
}
// KnowledgeRepository defines the interface for knowledge repositories.