diff --git a/client/faq.go b/client/faq.go index 06b5bc32..69a9ffcf 100644 --- a/client/faq.go +++ b/client/faq.go @@ -51,6 +51,8 @@ type FAQBatchUpsertPayload struct { Entries []FAQEntryPayload `json:"entries"` Mode string `json:"mode"` KnowledgeID string `json:"knowledge_id,omitempty"` + TaskID string `json:"task_id,omitempty"` // Optional, if not provided, a UUID will be generated + DryRun bool `json:"dry_run,omitempty"` // If true, only validate without importing } // FAQEntryFieldsUpdate represents the fields that can be updated for a single FAQ entry. @@ -379,3 +381,57 @@ func (c *Client) GetFAQImportProgress(ctx context.Context, taskID string) (*FAQI } return response.Data, nil } + +// FAQDryRunFailedEntry represents a failed entry in dry run validation. +type FAQDryRunFailedEntry struct { + Index int `json:"index"` + Reason string `json:"reason"` + TagName string `json:"tag_name,omitempty"` + StandardQuestion string `json:"standard_question"` + SimilarQuestions []string `json:"similar_questions,omitempty"` + NegativeQuestions []string `json:"negative_questions,omitempty"` + Answers []string `json:"answers,omitempty"` + AnswerAll bool `json:"answer_all,omitempty"` + IsDisabled bool `json:"is_disabled,omitempty"` +} + +// FAQDryRunProgress represents the progress of an async FAQ dry run validation task. +type FAQDryRunProgress struct { + TaskID string `json:"task_id"` + KBID string `json:"kb_id"` + Status string `json:"status"` + Progress int `json:"progress"` + Total int `json:"total"` + Processed int `json:"processed"` + SuccessCount int `json:"success_count"` + FailedCount int `json:"failed_count"` + FailedEntries []FAQDryRunFailedEntry `json:"failed_entries,omitempty"` + FailedEntriesURL string `json:"failed_entries_url,omitempty"` // CSV download URL when too many failures + Message string `json:"message"` + Error string `json:"error,omitempty"` + CreatedAt int64 `json:"created_at"` + UpdatedAt int64 `json:"updated_at"` +} + +// FAQDryRunProgressResponse wraps the FAQ dry run progress response. +type FAQDryRunProgressResponse struct { + Success bool `json:"success"` + Data *FAQDryRunProgress `json:"data"` + Message string `json:"message,omitempty"` + Code string `json:"code,omitempty"` +} + +// GetFAQDryRunProgress retrieves the progress of an async FAQ dry run validation task. +func (c *Client) GetFAQDryRunProgress(ctx context.Context, taskID string) (*FAQDryRunProgress, error) { + path := fmt.Sprintf("/api/v1/faq/dryrun/progress/%s", taskID) + resp, err := c.doRequest(ctx, http.MethodGet, path, nil, nil) + if err != nil { + return nil, err + } + + var response FAQDryRunProgressResponse + if err := parseResponse(resp, &response); err != nil { + return nil, err + } + return response.Data, nil +} diff --git a/internal/application/service/file/cos.go b/internal/application/service/file/cos.go index 8d921b21..99bc4221 100644 --- a/internal/application/service/file/cos.go +++ b/internal/application/service/file/cos.go @@ -1,6 +1,7 @@ package file import ( + "bytes" "context" "fmt" "io" @@ -9,6 +10,7 @@ import ( "net/url" "path/filepath" "strings" + "time" "github.com/Tencent/WeKnora/internal/types/interfaces" "github.com/google/uuid" @@ -20,11 +22,19 @@ type cosFileService struct { client *cos.Client bucketURL string cosPathPrefix string + // 临时桶配置(用于存放自动过期的临时文件) + tempClient *cos.Client + tempBucketURL string } // NewCosFileService creates a new COS file service instance func NewCosFileService(bucketName, region, secretId, secretKey, cosPathPrefix string) (interfaces.FileService, error) { - bucketURL := fmt.Sprintf("https://%s.cos.%s.myqcloud.com/", bucketName, region) + return NewCosFileServiceWithTempBucket(bucketName, region, secretId, secretKey, cosPathPrefix, "", "") +} + +// NewCosFileServiceWithTempBucket creates a new COS file service instance with optional temp bucket +func NewCosFileServiceWithTempBucket(bucketName, region, secretId, secretKey, cosPathPrefix, tempBucketName, tempRegion string) (interfaces.FileService, error) { + bucketURL := fmt.Sprintf("https://%s.cos.%s.tencentcos.cn/", bucketName, region) u, err := url.Parse(bucketURL) if err != nil { return nil, fmt.Errorf("failed to parse bucketURL: %w", err) @@ -36,11 +46,35 @@ func NewCosFileService(bucketName, region, secretId, secretKey, cosPathPrefix st SecretKey: secretKey, }, }) - return &cosFileService{ + + svc := &cosFileService{ client: client, bucketURL: bucketURL, cosPathPrefix: cosPathPrefix, - }, nil + } + + // 如果配置了临时桶,初始化临时桶客户端 + if tempBucketName != "" { + if tempRegion == "" { + tempRegion = region // 默认使用相同的 region + } + tempBucketURL := fmt.Sprintf("https://%s.cos.%s.tencentcos.cn/", tempBucketName, tempRegion) + tempU, err := url.Parse(tempBucketURL) + if err != nil { + return nil, fmt.Errorf("failed to parse temp bucketURL: %w", err) + } + tempB := &cos.BaseURL{BucketURL: tempU} + tempClient := cos.NewClient(tempB, &http.Client{ + Transport: &cos.AuthorizationTransport{ + SecretID: secretId, + SecretKey: secretKey, + }, + }) + svc.tempClient = tempClient + svc.tempBucketURL = tempBucketURL + } + + return svc, nil } // SaveFile saves a file to COS storage @@ -81,3 +115,53 @@ func (s *cosFileService) DeleteFile(ctx context.Context, filePath string) error } return nil } + +// SaveBytes saves bytes data to COS +// If temp is true and temp bucket is configured, saves to temp bucket (with lifecycle auto-expiration) +// Otherwise saves to main bucket +func (s *cosFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) { + ext := filepath.Ext(fileName) + reader := bytes.NewReader(data) + + // 如果请求写入临时桶且临时桶已配置 + if temp && s.tempClient != nil { + objectName := fmt.Sprintf("exports/%d/%s%s", tenantID, uuid.New().String(), ext) + _, err := s.tempClient.Object.Put(ctx, objectName, reader, nil) + if err != nil { + return "", fmt.Errorf("failed to upload bytes to COS temp bucket: %w", err) + } + return fmt.Sprintf("%s%s", s.tempBucketURL, objectName), nil + } + + // 写入主桶 + objectName := fmt.Sprintf("%s/%d/exports/%s%s", s.cosPathPrefix, tenantID, uuid.New().String(), ext) + _, err := s.client.Object.Put(ctx, objectName, reader, nil) + if err != nil { + return "", fmt.Errorf("failed to upload bytes to COS: %w", err) + } + + return fmt.Sprintf("%s%s", s.bucketURL, objectName), nil +} + +// GetFileURL returns a presigned download URL for the file +func (s *cosFileService) GetFileURL(ctx context.Context, filePath string) (string, error) { + // 判断文件属于哪个桶 + if s.tempClient != nil && strings.HasPrefix(filePath, s.tempBucketURL) { + objectName := strings.TrimPrefix(filePath, s.tempBucketURL) + // Generate presigned URL (valid for 24 hours) + presignedURL, err := s.tempClient.Object.GetPresignedURL(ctx, http.MethodGet, objectName, s.tempClient.GetCredential().SecretID, s.tempClient.GetCredential().SecretKey, 24*time.Hour, nil) + if err != nil { + return "", fmt.Errorf("failed to generate presigned URL for temp bucket: %w", err) + } + return presignedURL.String(), nil + } + + objectName := strings.TrimPrefix(filePath, s.bucketURL) + // Generate presigned URL (valid for 24 hours) + presignedURL, err := s.client.Object.GetPresignedURL(ctx, http.MethodGet, objectName, s.client.GetCredential().SecretID, s.client.GetCredential().SecretKey, 24*time.Hour, nil) + if err != nil { + return "", fmt.Errorf("failed to generate presigned URL: %w", err) + } + + return presignedURL.String(), nil +} diff --git a/internal/application/service/file/dummy.go b/internal/application/service/file/dummy.go index c7bee75a..f48bc88f 100644 --- a/internal/application/service/file/dummy.go +++ b/internal/application/service/file/dummy.go @@ -36,3 +36,13 @@ func (s *DummyFileService) GetFile(ctx context.Context, filePath string) (io.Rea func (s *DummyFileService) DeleteFile(ctx context.Context, filePath string) error { return nil } + +// SaveBytes pretends to save bytes but just returns a random UUID +func (s *DummyFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) { + return uuid.New().String(), nil +} + +// GetFileURL returns the file path as URL (dummy implementation) +func (s *DummyFileService) GetFileURL(ctx context.Context, filePath string) (string, error) { + return filePath, nil +} diff --git a/internal/application/service/file/local.go b/internal/application/service/file/local.go index 7174dcef..07e5b781 100644 --- a/internal/application/service/file/local.go +++ b/internal/application/service/file/local.go @@ -109,3 +109,38 @@ func (s *localFileService) DeleteFile(ctx context.Context, filePath string) erro logger.Info(ctx, "File deleted successfully") return nil } + +// SaveBytes saves bytes data to a file and returns the file path +// temp parameter is ignored for local storage (no auto-expiration support) +func (s *localFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) { + logger.Infof(ctx, "Saving bytes data: fileName=%s, size=%d, tenantID=%d, temp=%v", fileName, len(data), tenantID, temp) + + // Create storage directory with tenant ID + dir := filepath.Join(s.baseDir, fmt.Sprintf("%d", tenantID), "exports") + if err := os.MkdirAll(dir, 0o755); err != nil { + logger.Errorf(ctx, "Failed to create directory: %v", err) + return "", fmt.Errorf("failed to create directory: %w", err) + } + + // Generate unique filename using timestamp + ext := filepath.Ext(fileName) + baseName := fileName[:len(fileName)-len(ext)] + uniqueFileName := fmt.Sprintf("%s_%d%s", baseName, time.Now().UnixNano(), ext) + filePath := filepath.Join(dir, uniqueFileName) + + // Write data to file + if err := os.WriteFile(filePath, data, 0o644); err != nil { + logger.Errorf(ctx, "Failed to write file: %v", err) + return "", fmt.Errorf("failed to write file: %w", err) + } + + logger.Infof(ctx, "Bytes data saved successfully: %s", filePath) + return filePath, nil +} + +// GetFileURL returns a download URL for the file +// For local storage, returns the file path itself (no URL support) +func (s *localFileService) GetFileURL(ctx context.Context, filePath string) (string, error) { + // Local storage doesn't support URLs, return the path + return filePath, nil +} diff --git a/internal/application/service/file/minio.go b/internal/application/service/file/minio.go index e1c05c53..0f0753af 100644 --- a/internal/application/service/file/minio.go +++ b/internal/application/service/file/minio.go @@ -1,11 +1,13 @@ package file import ( + "bytes" "context" "fmt" "io" "mime/multipart" "path/filepath" + "time" "github.com/Tencent/WeKnora/internal/types/interfaces" "github.com/google/uuid" @@ -125,3 +127,43 @@ func (s *minioFileService) DeleteFile(ctx context.Context, filePath string) erro return nil } + +// SaveBytes saves bytes data to MinIO and returns the file path +// temp parameter is ignored for MinIO (no auto-expiration support in this implementation) +func (s *minioFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) { + ext := filepath.Ext(fileName) + objectName := fmt.Sprintf("%d/exports/%s%s", tenantID, uuid.New().String(), ext) + + // Upload bytes to MinIO + reader := bytes.NewReader(data) + _, err := s.client.PutObject(ctx, s.bucketName, objectName, reader, int64(len(data)), minio.PutObjectOptions{ + ContentType: "text/csv; charset=utf-8", + }) + if err != nil { + return "", fmt.Errorf("failed to upload bytes to MinIO: %w", err) + } + + return fmt.Sprintf("minio://%s/%s", s.bucketName, objectName), nil +} + +// GetFileURL returns a presigned download URL for the file +func (s *minioFileService) GetFileURL(ctx context.Context, filePath string) (string, error) { + // Parse MinIO path + if len(filePath) < 9 || filePath[:8] != "minio://" { + return "", fmt.Errorf("invalid MinIO file path: %s", filePath) + } + + // Extract object name + objectName := filePath[9+len(s.bucketName):] + if objectName[0] == '/' { + objectName = objectName[1:] + } + + // Generate presigned URL (valid for 24 hours) + presignedURL, err := s.client.PresignedGetObject(ctx, s.bucketName, objectName, 24*time.Hour, nil) + if err != nil { + return "", fmt.Errorf("failed to generate presigned URL: %w", err) + } + + return presignedURL.String(), nil +} diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index 47a2513d..9b1754a3 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -2808,6 +2808,518 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context, return taskID, nil } +// ValidateFAQEntriesDryRun validates FAQ entries asynchronously without actually importing them. +// Returns task ID for tracking validation progress. +func (s *knowledgeService) ValidateFAQEntriesDryRun(ctx context.Context, + kbID string, payload *types.FAQBatchUpsertPayload, +) (string, error) { + if payload == nil || len(payload.Entries) == 0 { + return "", werrors.NewBadRequestError("FAQ 条目不能为空") + } + if payload.Mode == "" { + payload.Mode = types.FAQBatchModeAppend + } + if payload.Mode != types.FAQBatchModeAppend && payload.Mode != types.FAQBatchModeReplace { + return "", werrors.NewBadRequestError("模式仅支持 append 或 replace") + } + + // 验证知识库是否存在且有效 + _, err := s.validateFAQKnowledgeBase(ctx, kbID) + if err != nil { + return "", err + } + + tenantID := ctx.Value(types.TenantIDContextKey).(uint64) + + // 使用传入的 task_id,如果不存在则生成新的 + taskID := payload.TaskID + if taskID == "" { + taskID = uuid.New().String() + } + + // 初始化 dry run 任务状态到 Redis + progress := &types.FAQDryRunProgress{ + TaskID: taskID, + KBID: kbID, + Status: types.FAQImportStatusPending, + Progress: 0, + Total: len(payload.Entries), + Processed: 0, + SuccessCount: 0, + FailedCount: 0, + FailedEntries: make([]types.FAQDryRunFailedEntry, 0), + Message: "任务已创建,等待处理", + CreatedAt: time.Now().Unix(), + UpdatedAt: time.Now().Unix(), + } + if err := s.saveFAQDryRunProgress(ctx, progress); err != nil { + logger.Errorf(ctx, "Failed to initialize FAQ dry run task status: %v", err) + return "", fmt.Errorf("failed to initialize task: %w", err) + } + + logger.Infof(ctx, "FAQ dry run task initialized: %s, kb_id: %s, total entries: %d", taskID, kbID, len(payload.Entries)) + + // Enqueue FAQ dry run task to Asynq + taskPayload := types.FAQDryRunPayload{ + TenantID: tenantID, + TaskID: taskID, + KBID: kbID, + Entries: payload.Entries, + Mode: payload.Mode, + } + + payloadBytes, err := json.Marshal(taskPayload) + if err != nil { + logger.Errorf(ctx, "Failed to marshal FAQ dry run task payload: %v", err) + return "", fmt.Errorf("failed to marshal task payload: %w", err) + } + + task := asynq.NewTask( + types.TypeFAQDryRun, + payloadBytes, + asynq.Queue(getAsynqQueueName("default")), + asynq.MaxRetry(3), + ) + info, err := s.task.Enqueue(task) + if err != nil { + logger.Errorf(ctx, "Failed to enqueue FAQ dry run task: %v", err) + return "", fmt.Errorf("failed to enqueue task: %w", err) + } + logger.Infof(ctx, "Enqueued FAQ dry run task: id=%s queue=%s task_id=%s", info.ID, info.Queue, taskID) + + return taskID, nil +} + +// ProcessFAQDryRun handles Asynq FAQ dry run validation tasks. +func (s *knowledgeService) ProcessFAQDryRun(ctx context.Context, t *asynq.Task) error { + var payload types.FAQDryRunPayload + if err := json.Unmarshal(t.Payload(), &payload); err != nil { + return fmt.Errorf("failed to unmarshal FAQ dry run payload: %w", err) + } + + logger.Infof(ctx, "Processing FAQ dry run task: %s, kb_id: %s, entries: %d", + payload.TaskID, payload.KBID, len(payload.Entries)) + + // 更新状态为处理中 + progress := &types.FAQDryRunProgress{ + TaskID: payload.TaskID, + KBID: payload.KBID, + Status: types.FAQImportStatusProcessing, + Progress: 0, + Total: len(payload.Entries), + Processed: 0, + SuccessCount: 0, + FailedCount: 0, + FailedEntries: make([]types.FAQDryRunFailedEntry, 0), + Message: "正在验证条目...", + CreatedAt: time.Now().Unix(), + UpdatedAt: time.Now().Unix(), + } + if err := s.saveFAQDryRunProgress(ctx, progress); err != nil { + logger.Warnf(ctx, "Failed to update FAQ dry run progress: %v", err) + } + + // 执行验证 + s.executeFAQDryRunValidation(ctx, &payload, progress) + + // 更新最终状态 + progress.Status = types.FAQImportStatusCompleted + progress.Progress = 100 + progress.SuccessCount = progress.Total - progress.FailedCount + progress.Message = fmt.Sprintf("验证完成: 成功 %d 条, 失败 %d 条", progress.SuccessCount, progress.FailedCount) + progress.UpdatedAt = time.Now().Unix() + + // 如果失败条目超过阈值,生成 CSV 文件 + csvURL, err := s.generateFailedEntriesCSV(ctx, payload.TenantID, payload.TaskID, progress.FailedEntries) + if err != nil { + logger.Warnf(ctx, "Failed to generate failed entries CSV: %v", err) + } else { + progress.FailedEntriesURL = csvURL + progress.FailedEntries = nil // 清空内联数据,使用 URL + progress.Message += " (失败记录已导出为CSV)" + } + + if err := s.saveFAQDryRunProgress(ctx, progress); err != nil { + logger.Warnf(ctx, "Failed to save final FAQ dry run progress: %v", err) + } + + logger.Infof(ctx, "FAQ dry run task completed: %s, success: %d, failed: %d", + payload.TaskID, progress.SuccessCount, progress.FailedCount) + + return nil +} + +// generateFailedEntriesCSV 生成失败条目的 CSV 文件并上传 +func (s *knowledgeService) generateFailedEntriesCSV(ctx context.Context, + tenantID uint64, taskID string, failedEntries []types.FAQDryRunFailedEntry, +) (string, error) { + // 生成 CSV 内容 + var buf strings.Builder + + // 写入 BOM 以支持 Excel 正确识别 UTF-8 + buf.WriteString("\xEF\xBB\xBF") + + // 写入表头 + buf.WriteString("错误原因,分类(必填),问题(必填),相似问题(选填-多个用##分隔),反例问题(选填-多个用##分隔),机器人回答(必填-多个用##分隔),是否全部回复(选填-默认FALSE),是否停用(选填-默认FALSE)\n") + + // 写入数据行 + for _, entry := range failedEntries { + // CSV 转义:如果内容包含逗号、引号或换行,需要用引号包裹并转义内部引号 + reason := csvEscape(entry.Reason) + tagName := csvEscape(entry.TagName) + standardQ := csvEscape(entry.StandardQuestion) + similarQs := "" + if len(entry.SimilarQuestions) > 0 { + similarQs = csvEscape(strings.Join(entry.SimilarQuestions, "##")) + } + negativeQs := "" + if len(entry.NegativeQuestions) > 0 { + negativeQs = csvEscape(strings.Join(entry.NegativeQuestions, "##")) + } + answers := "" + if len(entry.Answers) > 0 { + answers = csvEscape(strings.Join(entry.Answers, "##")) + } + answerAll := "false" + if entry.AnswerAll { + answerAll = "true" + } + isDisabled := "false" + if entry.IsDisabled { + isDisabled = "true" + } + + buf.WriteString(fmt.Sprintf("%s,%s,%s,%s,%s,%s,%s,%s\n", + reason, tagName, standardQ, similarQs, negativeQs, answers, answerAll, isDisabled)) + } + + // 上传 CSV 文件到临时存储(会自动过期) + fileName := fmt.Sprintf("faq_dryrun_failed_%s.csv", taskID) + filePath, err := s.fileSvc.SaveBytes(ctx, []byte(buf.String()), tenantID, fileName, true) + if err != nil { + return "", fmt.Errorf("failed to save CSV file: %w", err) + } + + // 获取下载 URL + fileURL, err := s.fileSvc.GetFileURL(ctx, filePath) + if err != nil { + return "", fmt.Errorf("failed to get file URL: %w", err) + } + + logger.Infof(ctx, "Generated failed entries CSV: %s, entries: %d", fileURL, len(failedEntries)) + return fileURL, nil +} + +// csvEscape 转义 CSV 字段 +func csvEscape(s string) string { + if strings.ContainsAny(s, ",\"\n\r") { + // 将内部引号替换为两个引号,并用引号包裹整个字段 + return "\"" + strings.ReplaceAll(s, "\"", "\"\"") + "\"" + } + return s +} + +// buildFAQDryRunFailedEntry 构建 FAQDryRunFailedEntry +func buildFAQDryRunFailedEntry(idx int, reason string, entry *types.FAQEntryPayload) types.FAQDryRunFailedEntry { + answerAll := false + if entry.AnswerStrategy != nil && *entry.AnswerStrategy == types.AnswerStrategyAll { + answerAll = true + } + isDisabled := false + if entry.IsEnabled != nil && !*entry.IsEnabled { + isDisabled = true + } + return types.FAQDryRunFailedEntry{ + Index: idx, + Reason: reason, + TagName: entry.TagName, + StandardQuestion: strings.TrimSpace(entry.StandardQuestion), + SimilarQuestions: entry.SimilarQuestions, + NegativeQuestions: entry.NegativeQuestions, + Answers: entry.Answers, + AnswerAll: answerAll, + IsDisabled: isDisabled, + } +} + +// executeFAQDryRunValidation 执行 FAQ dry run 验证 +func (s *knowledgeService) executeFAQDryRunValidation(ctx context.Context, + payload *types.FAQDryRunPayload, progress *types.FAQDryRunProgress, +) { + entries := payload.Entries + + // 用于记录已通过基本验证和重复检查的条目索引,后续进行安全检查 + validEntryIndices := make([]int, 0, len(entries)) + + // 根据模式选择不同的验证逻辑 + if payload.Mode == types.FAQBatchModeAppend { + validEntryIndices = s.validateEntriesForAppendModeWithProgress(ctx, payload.TenantID, payload.KBID, entries, progress) + } else { + validEntryIndices = s.validateEntriesForReplaceModeWithProgress(ctx, entries, progress) + } + + // 安全检查:对通过基本验证的条目进行安全检查(检查所有条目) + if s.securityClient != nil && len(validEntryIndices) > 0 { + s.validateEntriesSecurityWithProgress(ctx, payload.TenantID, entries, validEntryIndices, progress) + } +} + +// validateEntriesSecurityWithProgress 对条目进行安全检查(检查所有条目) +func (s *knowledgeService) validateEntriesSecurityWithProgress(ctx context.Context, + tenantID uint64, entries []types.FAQEntryPayload, validIndices []int, progress *types.FAQDryRunProgress, +) { + for _, idx := range validIndices { + entry := entries[idx] + blockStatus, err := s.CheckFAQContent(ctx, tenantID, &entry) + if err != nil { + // 安全检查服务出错,记录但不阻止 + logger.Warnf(ctx, "Security check failed for entry %d: %v", idx, err) + continue + } + if blockStatus != 0 { + progress.FailedCount++ + progress.FailedEntries = append(progress.FailedEntries, buildFAQDryRunFailedEntry(idx, "内容安全检查不通过", &entry)) + } + } +} + +// validateEntriesForAppendModeWithProgress 验证 Append 模式下的条目(带进度更新) +func (s *knowledgeService) validateEntriesForAppendModeWithProgress(ctx context.Context, + tenantID uint64, kbID string, entries []types.FAQEntryPayload, progress *types.FAQDryRunProgress, +) []int { + validIndices := make([]int, 0, len(entries)) + + // 查询知识库中已有的所有FAQ chunks的metadata + existingChunks, err := s.chunkRepo.ListAllFAQChunksWithMetadataByKnowledgeBaseID(ctx, tenantID, kbID) + if err != nil { + logger.Warnf(ctx, "Failed to list existing FAQ chunks for dry run: %v", err) + // 无法获取已有数据时,仅做批次内验证 + } + + // 构建已存在的标准问和相似问集合 + existingQuestions := make(map[string]bool) + for _, chunk := range existingChunks { + meta, err := chunk.FAQMetadata() + if err != nil || meta == nil { + continue + } + if meta.StandardQuestion != "" { + existingQuestions[meta.StandardQuestion] = true + } + for _, q := range meta.SimilarQuestions { + if q != "" { + existingQuestions[q] = true + } + } + } + + // 构建当前批次的标准问和相似问集合(用于批次内去重) + batchQuestions := make(map[string]int) // value 为首次出现的索引 + + for i, entry := range entries { + // 验证条目基本格式 + if err := validateFAQEntryPayloadBasic(&entry); err != nil { + progress.FailedCount++ + progress.FailedEntries = append(progress.FailedEntries, buildFAQDryRunFailedEntry(i, err.Error(), &entry)) + progress.Processed++ + continue + } + + standardQ := strings.TrimSpace(entry.StandardQuestion) + + // 检查标准问是否与已有知识库重复 + if existingQuestions[standardQ] { + progress.FailedCount++ + progress.FailedEntries = append(progress.FailedEntries, buildFAQDryRunFailedEntry(i, "标准问与知识库中已有问题重复", &entry)) + progress.Processed++ + continue + } + + // 检查标准问是否与同批次重复 + if firstIdx, exists := batchQuestions[standardQ]; exists { + progress.FailedCount++ + progress.FailedEntries = append(progress.FailedEntries, buildFAQDryRunFailedEntry(i, fmt.Sprintf("标准问与批次内第 %d 条重复", firstIdx+1), &entry)) + progress.Processed++ + continue + } + + // 检查相似问是否有重复 + hasDuplicate := false + for _, q := range entry.SimilarQuestions { + q = strings.TrimSpace(q) + if q == "" { + continue + } + if existingQuestions[q] { + progress.FailedCount++ + progress.FailedEntries = append(progress.FailedEntries, buildFAQDryRunFailedEntry(i, fmt.Sprintf("相似问 \"%s\" 与知识库中已有问题重复", q), &entry)) + hasDuplicate = true + break + } + if firstIdx, exists := batchQuestions[q]; exists { + progress.FailedCount++ + progress.FailedEntries = append(progress.FailedEntries, buildFAQDryRunFailedEntry(i, fmt.Sprintf("相似问 \"%s\" 与批次内第 %d 条重复", q, firstIdx+1), &entry)) + hasDuplicate = true + break + } + } + if hasDuplicate { + progress.Processed++ + continue + } + + // 将当前条目的标准问和相似问加入批次集合 + batchQuestions[standardQ] = i + for _, q := range entry.SimilarQuestions { + q = strings.TrimSpace(q) + if q != "" { + batchQuestions[q] = i + } + } + + // 记录通过验证的条目索引 + validIndices = append(validIndices, i) + progress.Processed++ + + // 定期更新进度 + if progress.Processed%100 == 0 { + progress.Progress = progress.Processed * 80 / progress.Total // 前80%用于基本验证 + progress.UpdatedAt = time.Now().Unix() + if err := s.saveFAQDryRunProgress(ctx, progress); err != nil { + logger.Warnf(ctx, "Failed to update FAQ dry run progress: %v", err) + } + } + } + + return validIndices +} + +// validateEntriesForReplaceModeWithProgress 验证 Replace 模式下的条目(带进度更新) +func (s *knowledgeService) validateEntriesForReplaceModeWithProgress(ctx context.Context, + entries []types.FAQEntryPayload, progress *types.FAQDryRunProgress, +) []int { + validIndices := make([]int, 0, len(entries)) + + // Replace 模式下只检查批次内重复 + batchQuestions := make(map[string]int) // value 为首次出现的索引 + + for i, entry := range entries { + // 验证条目基本格式 + if err := validateFAQEntryPayloadBasic(&entry); err != nil { + progress.FailedCount++ + progress.FailedEntries = append(progress.FailedEntries, buildFAQDryRunFailedEntry(i, err.Error(), &entry)) + progress.Processed++ + continue + } + + standardQ := strings.TrimSpace(entry.StandardQuestion) + + // 检查标准问是否与同批次重复 + if firstIdx, exists := batchQuestions[standardQ]; exists { + progress.FailedCount++ + progress.FailedEntries = append(progress.FailedEntries, buildFAQDryRunFailedEntry(i, fmt.Sprintf("标准问与批次内第 %d 条重复", firstIdx+1), &entry)) + progress.Processed++ + continue + } + + // 检查相似问是否有重复 + hasDuplicate := false + for _, q := range entry.SimilarQuestions { + q = strings.TrimSpace(q) + if q == "" { + continue + } + if firstIdx, exists := batchQuestions[q]; exists { + progress.FailedCount++ + progress.FailedEntries = append(progress.FailedEntries, buildFAQDryRunFailedEntry(i, fmt.Sprintf("相似问 \"%s\" 与批次内第 %d 条重复", q, firstIdx+1), &entry)) + hasDuplicate = true + break + } + } + if hasDuplicate { + progress.Processed++ + continue + } + + // 将当前条目的标准问和相似问加入批次集合 + batchQuestions[standardQ] = i + for _, q := range entry.SimilarQuestions { + q = strings.TrimSpace(q) + if q != "" { + batchQuestions[q] = i + } + } + + // 记录通过验证的条目索引 + validIndices = append(validIndices, i) + progress.Processed++ + + // 定期更新进度 + if progress.Processed%100 == 0 { + progress.Progress = progress.Processed * 80 / progress.Total // 前80%用于基本验证 + progress.UpdatedAt = time.Now().Unix() + if err := s.saveFAQDryRunProgress(ctx, progress); err != nil { + logger.Warnf(ctx, "Failed to update FAQ dry run progress: %v", err) + } + } + } + + return validIndices +} + +// saveFAQDryRunProgress 保存 FAQ dry run 进度到 Redis +func (s *knowledgeService) saveFAQDryRunProgress(ctx context.Context, progress *types.FAQDryRunProgress) error { + key := fmt.Sprintf("faq:dryrun:progress:%s", progress.TaskID) + data, err := json.Marshal(progress) + if err != nil { + return fmt.Errorf("failed to marshal progress: %w", err) + } + // 设置 24 小时过期 + return s.redisClient.Set(ctx, key, data, 24*time.Hour).Err() +} + +// GetFAQDryRunProgress 获取 FAQ dry run 进度 +func (s *knowledgeService) GetFAQDryRunProgress(ctx context.Context, taskID string) (*types.FAQDryRunProgress, error) { + key := fmt.Sprintf("faq:dryrun:progress:%s", taskID) + data, err := s.redisClient.Get(ctx, key).Bytes() + if err != nil { + if err == redis.Nil { + return nil, werrors.NewNotFoundError("任务不存在或已过期") + } + return nil, fmt.Errorf("failed to get progress: %w", err) + } + var progress types.FAQDryRunProgress + if err := json.Unmarshal(data, &progress); err != nil { + return nil, fmt.Errorf("failed to unmarshal progress: %w", err) + } + return &progress, nil +} + +// validateFAQEntryPayloadBasic 验证 FAQ 条目的基本格式 +func validateFAQEntryPayloadBasic(entry *types.FAQEntryPayload) error { + if entry == nil { + return fmt.Errorf("条目不能为空") + } + standardQ := strings.TrimSpace(entry.StandardQuestion) + if standardQ == "" { + return fmt.Errorf("标准问不能为空") + } + if len(entry.Answers) == 0 { + return fmt.Errorf("答案不能为空") + } + hasValidAnswer := false + for _, a := range entry.Answers { + if strings.TrimSpace(a) != "" { + hasValidAnswer = true + break + } + } + if !hasValidAnswer { + return fmt.Errorf("答案不能全为空") + } + return nil +} + // calculateAppendOperations 计算Append模式下需要处理的条目,跳过已存在且内容相同的条目 // 同时过滤掉标准问或相似问与同批次或已有知识库中重复的条目 func (s *knowledgeService) calculateAppendOperations(ctx context.Context, diff --git a/internal/container/container.go b/internal/container/container.go index ade5b7ab..6dc18a8d 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -391,12 +391,14 @@ func initFileService(cfg *config.Config) (interfaces.FileService, error) { os.Getenv("COS_PATH_PREFIX") == "" { return nil, fmt.Errorf("missing COS configuration") } - return file.NewCosFileService( + return file.NewCosFileServiceWithTempBucket( os.Getenv("COS_BUCKET_NAME"), os.Getenv("COS_REGION"), os.Getenv("COS_SECRET_ID"), os.Getenv("COS_SECRET_KEY"), os.Getenv("COS_PATH_PREFIX"), + os.Getenv("COS_TEMP_BUCKET_NAME"), // 可选:临时桶名称(桶需配置生命周期规则自动过期) + os.Getenv("COS_TEMP_REGION"), // 可选:临时桶 region,默认与主桶相同 ) case "local": return file.NewLocalFileService(os.Getenv("LOCAL_STORAGE_BASE_DIR")), nil diff --git a/internal/handler/faq.go b/internal/handler/faq.go index b88f94b1..bba98532 100644 --- a/internal/handler/faq.go +++ b/internal/handler/faq.go @@ -69,7 +69,9 @@ func (h *FAQHandler) ListEntries(c *gin.Context) { // UpsertEntries godoc // @Summary 批量更新/插入FAQ条目 -// @Description 异步批量更新或插入FAQ条目 +// @Description 异步批量更新或插入FAQ条目。支持 dry_run 模式(设置 dry_run=true),异步验证不实际导入。 +// @Description dry_run 模式是异步操作,返回 task_id,通过 /faq/dryrun/progress/{task_id} 查询进度和结果。 +// @Description 验证内容包括:1) 条目基本格式 2) 重复问题(批次内和知识库已有) 3) 内容安全检查。 // @Tags FAQ管理 // @Accept json // @Produce json @@ -89,7 +91,26 @@ func (h *FAQHandler) UpsertEntries(c *gin.Context) { return } - taskID, err := h.knowledgeService.UpsertFAQEntries(ctx, secutils.SanitizeForLog(c.Param("id")), &req) + kbID := secutils.SanitizeForLog(c.Param("id")) + + // dry_run 模式:异步验证,不实际导入 + if req.DryRun { + taskID, err := h.knowledgeService.ValidateFAQEntriesDryRun(ctx, kbID, &req) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(err) + return + } + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": gin.H{ + "task_id": taskID, + }, + }) + return + } + + taskID, err := h.knowledgeService.UpsertFAQEntries(ctx, kbID, &req) if err != nil { logger.ErrorWithFields(ctx, err, nil) c.Error(err) @@ -417,3 +438,32 @@ func (h *FAQHandler) GetImportProgress(c *gin.Context) { "data": progress, }) } + +// GetDryRunProgress godoc +// @Summary 获取FAQ预检进度 +// @Description 获取FAQ导入预检任务的进度和结果 +// @Tags FAQ管理 +// @Accept json +// @Produce json +// @Param task_id path string true "任务ID" +// @Success 200 {object} map[string]interface{} "预检进度和结果" +// @Failure 404 {object} errors.AppError "任务不存在" +// @Security Bearer +// @Security ApiKeyAuth +// @Router /faq/dryrun/progress/{task_id} [get] +func (h *FAQHandler) GetDryRunProgress(c *gin.Context) { + ctx := c.Request.Context() + taskID := secutils.SanitizeForLog(c.Param("task_id")) + + progress, err := h.knowledgeService.GetFAQDryRunProgress(ctx, taskID) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(err) + return + } + + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": progress, + }) +} diff --git a/internal/router/router.go b/internal/router/router.go index d3863aa0..6f834db6 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -200,6 +200,11 @@ func RegisterFAQRoutes(r *gin.RouterGroup, handler *handler.FAQHandler) { { faqImport.GET("/progress/:task_id", handler.GetImportProgress) } + // FAQ dry run progress route (outside of knowledge-base scope) + faqDryRun := r.Group("/faq/dryrun") + { + faqDryRun.GET("/progress/:task_id", handler.GetDryRunProgress) + } } // RegisterKnowledgeBaseRoutes 注册知识库相关的路由 diff --git a/internal/router/task.go b/internal/router/task.go index 4df6f92f..9739f235 100644 --- a/internal/router/task.go +++ b/internal/router/task.go @@ -79,6 +79,9 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux { // Register FAQ import handler mux.HandleFunc(types.TypeFAQImport, params.KnowledgeService.ProcessFAQImport) + // Register FAQ dry run handler + mux.HandleFunc(types.TypeFAQDryRun, params.KnowledgeService.ProcessFAQDryRun) + // Register question generation handler mux.HandleFunc(types.TypeQuestionGeneration, params.KnowledgeService.ProcessQuestionGeneration) diff --git a/internal/types/extract_graph.go b/internal/types/extract_graph.go index aac06184..c6e6f693 100644 --- a/internal/types/extract_graph.go +++ b/internal/types/extract_graph.go @@ -4,6 +4,7 @@ const ( TypeChunkExtract = "chunk:extract" TypeDocumentProcess = "document:process" // 文档处理任务 TypeFAQImport = "faq:import" // FAQ导入任务 + TypeFAQDryRun = "faq:dryrun" // FAQ导入预检任务 TypeQuestionGeneration = "question:generation" // 问题生成任务 TypeSummaryGeneration = "summary:generation" // 摘要生成任务 TypeKBClone = "kb:clone" // 知识库复制任务 @@ -45,6 +46,15 @@ type FAQImportPayload struct { Mode string `json:"mode"` } +// FAQDryRunPayload represents the FAQ dry run validation task payload +type FAQDryRunPayload struct { + TenantID uint64 `json:"tenant_id"` + TaskID string `json:"task_id"` + KBID string `json:"kb_id"` + Entries []FAQEntryPayload `json:"entries"` + Mode string `json:"mode"` +} + // QuestionGenerationPayload represents the question generation task payload type QuestionGenerationPayload struct { TenantID uint64 `json:"tenant_id"` diff --git a/internal/types/faq.go b/internal/types/faq.go index 484ff682..895fe338 100644 --- a/internal/types/faq.go +++ b/internal/types/faq.go @@ -219,6 +219,48 @@ type FAQBatchUpsertPayload struct { Entries []FAQEntryPayload `json:"entries" binding:"required"` Mode string `json:"mode" binding:"oneof=append replace"` KnowledgeID string `json:"knowledge_id"` + TaskID string `json:"task_id"` // 可选,如果不传则自动生成UUID + DryRun bool `json:"dry_run"` // 仅验证,不实际导入 +} + +// FAQDryRunFailedEntry 表示 dry_run 模式下验证失败的条目 +type FAQDryRunFailedEntry struct { + Index int `json:"index"` // 条目在批次中的索引(从0开始) + Reason string `json:"reason"` // 失败原因 + TagName string `json:"tag_name,omitempty"` // 分类 + StandardQuestion string `json:"standard_question"` // 标准问题 + SimilarQuestions []string `json:"similar_questions,omitempty"` // 相似问题 + NegativeQuestions []string `json:"negative_questions,omitempty"` // 反例问题 + Answers []string `json:"answers,omitempty"` // 答案 + AnswerAll bool `json:"answer_all,omitempty"` // 是否全部回复 + IsDisabled bool `json:"is_disabled,omitempty"` // 是否停用 +} + +// FAQDryRunResult 表示 dry_run 模式的验证结果 +type FAQDryRunResult struct { + TaskID string `json:"task_id,omitempty"` // 异步任务ID(异步模式时返回) + Total int `json:"total"` // 总条目数 + SuccessCount int `json:"success_count"` // 验证通过的条目数 + FailedCount int `json:"failed_count"` // 验证失败的条目数 + FailedEntries []FAQDryRunFailedEntry `json:"failed_entries"` // 失败条目详情 +} + +// FAQDryRunProgress 表示 dry_run 异步任务的进度(存储在 Redis) +type FAQDryRunProgress struct { + TaskID string `json:"task_id"` + KBID string `json:"kb_id"` + Status FAQImportTaskStatus `json:"status"` // pending, processing, completed, failed + Progress int `json:"progress"` // 0-100 percentage + Total int `json:"total"` // 总条目数 + Processed int `json:"processed"` // 已处理条目数 + SuccessCount int `json:"success_count"` // 验证通过的条目数 + FailedCount int `json:"failed_count"` // 验证失败的条目数 + FailedEntries []FAQDryRunFailedEntry `json:"failed_entries,omitempty"` // 失败条目详情(少量时直接返回) + FailedEntriesURL string `json:"failed_entries_url,omitempty"` // 失败条目CSV下载URL(大量时返回URL) + Message string `json:"message"` // 状态消息 + Error string `json:"error,omitempty"` // 错误信息 + CreatedAt int64 `json:"created_at"` + UpdatedAt int64 `json:"updated_at"` } // FAQSearchRequest FAQ检索请求参数 diff --git a/internal/types/interfaces/file.go b/internal/types/interfaces/file.go index 448121f2..413bb434 100644 --- a/internal/types/interfaces/file.go +++ b/internal/types/interfaces/file.go @@ -11,8 +11,13 @@ import ( type FileService interface { // SaveFile saves a file. SaveFile(ctx context.Context, file *multipart.FileHeader, tenantID uint64, knowledgeID string) (string, error) + // SaveBytes saves bytes data to a file and returns the file path. + // If temp is true, the file will be saved to a temporary storage that may auto-expire. + SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) // GetFile retrieves a file. GetFile(ctx context.Context, filePath string) (io.ReadCloser, error) + // GetFileURL returns a download URL for the file (if supported by the storage backend). + GetFileURL(ctx context.Context, filePath string) (string, error) // DeleteFile deletes a file. DeleteFile(ctx context.Context, filePath string) error } diff --git a/internal/types/interfaces/knowledge.go b/internal/types/interfaces/knowledge.go index d8d2a7c4..ee307a20 100644 --- a/internal/types/interfaces/knowledge.go +++ b/internal/types/interfaces/knowledge.go @@ -88,6 +88,13 @@ type KnowledgeService interface { // UpsertFAQEntries imports or appends FAQ entries asynchronously. // Returns task ID (Knowledge ID) for tracking import progress. UpsertFAQEntries(ctx context.Context, kbID string, payload *types.FAQBatchUpsertPayload) (string, error) + // ValidateFAQEntriesDryRun validates FAQ entries asynchronously without actually importing them. + // Returns task ID for tracking validation progress. + ValidateFAQEntriesDryRun(ctx context.Context, kbID string, payload *types.FAQBatchUpsertPayload) (string, error) + // GetFAQDryRunProgress retrieves the progress of an FAQ dry run validation task. + GetFAQDryRunProgress(ctx context.Context, taskID string) (*types.FAQDryRunProgress, error) + // ProcessFAQDryRun handles Asynq FAQ dry run validation tasks. + ProcessFAQDryRun(ctx context.Context, t *asynq.Task) error // CreateFAQEntry creates a single FAQ entry synchronously. CreateFAQEntry(ctx context.Context, kbID string, payload *types.FAQEntryPayload) (*types.FAQEntry, error) // GetFAQEntry retrieves a single FAQ entry by ID.