feat: 新增FAQ批量导入dry run功能

This commit is contained in:
wizardchen
2026-01-06 10:51:40 +08:00
parent a35999466c
commit 100484f13d
14 changed files with 869 additions and 6 deletions
+56
View File
@@ -51,6 +51,8 @@ type FAQBatchUpsertPayload struct {
Entries []FAQEntryPayload `json:"entries"`
Mode string `json:"mode"`
KnowledgeID string `json:"knowledge_id,omitempty"`
TaskID string `json:"task_id,omitempty"` // Optional, if not provided, a UUID will be generated
DryRun bool `json:"dry_run,omitempty"` // If true, only validate without importing
}
// FAQEntryFieldsUpdate represents the fields that can be updated for a single FAQ entry.
@@ -379,3 +381,57 @@ func (c *Client) GetFAQImportProgress(ctx context.Context, taskID string) (*FAQI
}
return response.Data, nil
}
// FAQDryRunFailedEntry represents a failed entry in dry run validation.
type FAQDryRunFailedEntry struct {
Index int `json:"index"`
Reason string `json:"reason"`
TagName string `json:"tag_name,omitempty"`
StandardQuestion string `json:"standard_question"`
SimilarQuestions []string `json:"similar_questions,omitempty"`
NegativeQuestions []string `json:"negative_questions,omitempty"`
Answers []string `json:"answers,omitempty"`
AnswerAll bool `json:"answer_all,omitempty"`
IsDisabled bool `json:"is_disabled,omitempty"`
}
// FAQDryRunProgress represents the progress of an async FAQ dry run validation task.
type FAQDryRunProgress struct {
TaskID string `json:"task_id"`
KBID string `json:"kb_id"`
Status string `json:"status"`
Progress int `json:"progress"`
Total int `json:"total"`
Processed int `json:"processed"`
SuccessCount int `json:"success_count"`
FailedCount int `json:"failed_count"`
FailedEntries []FAQDryRunFailedEntry `json:"failed_entries,omitempty"`
FailedEntriesURL string `json:"failed_entries_url,omitempty"` // CSV download URL when too many failures
Message string `json:"message"`
Error string `json:"error,omitempty"`
CreatedAt int64 `json:"created_at"`
UpdatedAt int64 `json:"updated_at"`
}
// FAQDryRunProgressResponse wraps the FAQ dry run progress response.
type FAQDryRunProgressResponse struct {
Success bool `json:"success"`
Data *FAQDryRunProgress `json:"data"`
Message string `json:"message,omitempty"`
Code string `json:"code,omitempty"`
}
// GetFAQDryRunProgress retrieves the progress of an async FAQ dry run validation task.
func (c *Client) GetFAQDryRunProgress(ctx context.Context, taskID string) (*FAQDryRunProgress, error) {
path := fmt.Sprintf("/api/v1/faq/dryrun/progress/%s", taskID)
resp, err := c.doRequest(ctx, http.MethodGet, path, nil, nil)
if err != nil {
return nil, err
}
var response FAQDryRunProgressResponse
if err := parseResponse(resp, &response); err != nil {
return nil, err
}
return response.Data, nil
}
+87 -3
View File
@@ -1,6 +1,7 @@
package file
import (
"bytes"
"context"
"fmt"
"io"
@@ -9,6 +10,7 @@ import (
"net/url"
"path/filepath"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/google/uuid"
@@ -20,11 +22,19 @@ type cosFileService struct {
client *cos.Client
bucketURL string
cosPathPrefix string
// 临时桶配置(用于存放自动过期的临时文件)
tempClient *cos.Client
tempBucketURL string
}
// NewCosFileService creates a new COS file service instance
func NewCosFileService(bucketName, region, secretId, secretKey, cosPathPrefix string) (interfaces.FileService, error) {
bucketURL := fmt.Sprintf("https://%s.cos.%s.myqcloud.com/", bucketName, region)
return NewCosFileServiceWithTempBucket(bucketName, region, secretId, secretKey, cosPathPrefix, "", "")
}
// NewCosFileServiceWithTempBucket creates a new COS file service instance with optional temp bucket
func NewCosFileServiceWithTempBucket(bucketName, region, secretId, secretKey, cosPathPrefix, tempBucketName, tempRegion string) (interfaces.FileService, error) {
bucketURL := fmt.Sprintf("https://%s.cos.%s.tencentcos.cn/", bucketName, region)
u, err := url.Parse(bucketURL)
if err != nil {
return nil, fmt.Errorf("failed to parse bucketURL: %w", err)
@@ -36,11 +46,35 @@ func NewCosFileService(bucketName, region, secretId, secretKey, cosPathPrefix st
SecretKey: secretKey,
},
})
return &cosFileService{
svc := &cosFileService{
client: client,
bucketURL: bucketURL,
cosPathPrefix: cosPathPrefix,
}, nil
}
// 如果配置了临时桶,初始化临时桶客户端
if tempBucketName != "" {
if tempRegion == "" {
tempRegion = region // 默认使用相同的 region
}
tempBucketURL := fmt.Sprintf("https://%s.cos.%s.tencentcos.cn/", tempBucketName, tempRegion)
tempU, err := url.Parse(tempBucketURL)
if err != nil {
return nil, fmt.Errorf("failed to parse temp bucketURL: %w", err)
}
tempB := &cos.BaseURL{BucketURL: tempU}
tempClient := cos.NewClient(tempB, &http.Client{
Transport: &cos.AuthorizationTransport{
SecretID: secretId,
SecretKey: secretKey,
},
})
svc.tempClient = tempClient
svc.tempBucketURL = tempBucketURL
}
return svc, nil
}
// SaveFile saves a file to COS storage
@@ -81,3 +115,53 @@ func (s *cosFileService) DeleteFile(ctx context.Context, filePath string) error
}
return nil
}
// SaveBytes saves bytes data to COS
// If temp is true and temp bucket is configured, saves to temp bucket (with lifecycle auto-expiration)
// Otherwise saves to main bucket
func (s *cosFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) {
ext := filepath.Ext(fileName)
reader := bytes.NewReader(data)
// 如果请求写入临时桶且临时桶已配置
if temp && s.tempClient != nil {
objectName := fmt.Sprintf("exports/%d/%s%s", tenantID, uuid.New().String(), ext)
_, err := s.tempClient.Object.Put(ctx, objectName, reader, nil)
if err != nil {
return "", fmt.Errorf("failed to upload bytes to COS temp bucket: %w", err)
}
return fmt.Sprintf("%s%s", s.tempBucketURL, objectName), nil
}
// 写入主桶
objectName := fmt.Sprintf("%s/%d/exports/%s%s", s.cosPathPrefix, tenantID, uuid.New().String(), ext)
_, err := s.client.Object.Put(ctx, objectName, reader, nil)
if err != nil {
return "", fmt.Errorf("failed to upload bytes to COS: %w", err)
}
return fmt.Sprintf("%s%s", s.bucketURL, objectName), nil
}
// GetFileURL returns a presigned download URL for the file
func (s *cosFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
// 判断文件属于哪个桶
if s.tempClient != nil && strings.HasPrefix(filePath, s.tempBucketURL) {
objectName := strings.TrimPrefix(filePath, s.tempBucketURL)
// Generate presigned URL (valid for 24 hours)
presignedURL, err := s.tempClient.Object.GetPresignedURL(ctx, http.MethodGet, objectName, s.tempClient.GetCredential().SecretID, s.tempClient.GetCredential().SecretKey, 24*time.Hour, nil)
if err != nil {
return "", fmt.Errorf("failed to generate presigned URL for temp bucket: %w", err)
}
return presignedURL.String(), nil
}
objectName := strings.TrimPrefix(filePath, s.bucketURL)
// Generate presigned URL (valid for 24 hours)
presignedURL, err := s.client.Object.GetPresignedURL(ctx, http.MethodGet, objectName, s.client.GetCredential().SecretID, s.client.GetCredential().SecretKey, 24*time.Hour, nil)
if err != nil {
return "", fmt.Errorf("failed to generate presigned URL: %w", err)
}
return presignedURL.String(), nil
}
@@ -36,3 +36,13 @@ func (s *DummyFileService) GetFile(ctx context.Context, filePath string) (io.Rea
func (s *DummyFileService) DeleteFile(ctx context.Context, filePath string) error {
return nil
}
// SaveBytes pretends to save bytes but just returns a random UUID
func (s *DummyFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) {
return uuid.New().String(), nil
}
// GetFileURL returns the file path as URL (dummy implementation)
func (s *DummyFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
return filePath, nil
}
@@ -109,3 +109,38 @@ func (s *localFileService) DeleteFile(ctx context.Context, filePath string) erro
logger.Info(ctx, "File deleted successfully")
return nil
}
// SaveBytes saves bytes data to a file and returns the file path
// temp parameter is ignored for local storage (no auto-expiration support)
func (s *localFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) {
logger.Infof(ctx, "Saving bytes data: fileName=%s, size=%d, tenantID=%d, temp=%v", fileName, len(data), tenantID, temp)
// Create storage directory with tenant ID
dir := filepath.Join(s.baseDir, fmt.Sprintf("%d", tenantID), "exports")
if err := os.MkdirAll(dir, 0o755); err != nil {
logger.Errorf(ctx, "Failed to create directory: %v", err)
return "", fmt.Errorf("failed to create directory: %w", err)
}
// Generate unique filename using timestamp
ext := filepath.Ext(fileName)
baseName := fileName[:len(fileName)-len(ext)]
uniqueFileName := fmt.Sprintf("%s_%d%s", baseName, time.Now().UnixNano(), ext)
filePath := filepath.Join(dir, uniqueFileName)
// Write data to file
if err := os.WriteFile(filePath, data, 0o644); err != nil {
logger.Errorf(ctx, "Failed to write file: %v", err)
return "", fmt.Errorf("failed to write file: %w", err)
}
logger.Infof(ctx, "Bytes data saved successfully: %s", filePath)
return filePath, nil
}
// GetFileURL returns a download URL for the file
// For local storage, returns the file path itself (no URL support)
func (s *localFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
// Local storage doesn't support URLs, return the path
return filePath, nil
}
@@ -1,11 +1,13 @@
package file
import (
"bytes"
"context"
"fmt"
"io"
"mime/multipart"
"path/filepath"
"time"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/google/uuid"
@@ -125,3 +127,43 @@ func (s *minioFileService) DeleteFile(ctx context.Context, filePath string) erro
return nil
}
// SaveBytes saves bytes data to MinIO and returns the file path
// temp parameter is ignored for MinIO (no auto-expiration support in this implementation)
func (s *minioFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) {
ext := filepath.Ext(fileName)
objectName := fmt.Sprintf("%d/exports/%s%s", tenantID, uuid.New().String(), ext)
// Upload bytes to MinIO
reader := bytes.NewReader(data)
_, err := s.client.PutObject(ctx, s.bucketName, objectName, reader, int64(len(data)), minio.PutObjectOptions{
ContentType: "text/csv; charset=utf-8",
})
if err != nil {
return "", fmt.Errorf("failed to upload bytes to MinIO: %w", err)
}
return fmt.Sprintf("minio://%s/%s", s.bucketName, objectName), nil
}
// GetFileURL returns a presigned download URL for the file
func (s *minioFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
// Parse MinIO path
if len(filePath) < 9 || filePath[:8] != "minio://" {
return "", fmt.Errorf("invalid MinIO file path: %s", filePath)
}
// Extract object name
objectName := filePath[9+len(s.bucketName):]
if objectName[0] == '/' {
objectName = objectName[1:]
}
// Generate presigned URL (valid for 24 hours)
presignedURL, err := s.client.PresignedGetObject(ctx, s.bucketName, objectName, 24*time.Hour, nil)
if err != nil {
return "", fmt.Errorf("failed to generate presigned URL: %w", err)
}
return presignedURL.String(), nil
}
+512
View File
@@ -2808,6 +2808,518 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context,
return taskID, nil
}
// ValidateFAQEntriesDryRun validates FAQ entries asynchronously without actually importing them.
// Returns task ID for tracking validation progress.
func (s *knowledgeService) ValidateFAQEntriesDryRun(ctx context.Context,
kbID string, payload *types.FAQBatchUpsertPayload,
) (string, error) {
if payload == nil || len(payload.Entries) == 0 {
return "", werrors.NewBadRequestError("FAQ 条目不能为空")
}
if payload.Mode == "" {
payload.Mode = types.FAQBatchModeAppend
}
if payload.Mode != types.FAQBatchModeAppend && payload.Mode != types.FAQBatchModeReplace {
return "", werrors.NewBadRequestError("模式仅支持 append 或 replace")
}
// 验证知识库是否存在且有效
_, err := s.validateFAQKnowledgeBase(ctx, kbID)
if err != nil {
return "", err
}
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
// 使用传入的 task_id,如果不存在则生成新的
taskID := payload.TaskID
if taskID == "" {
taskID = uuid.New().String()
}
// 初始化 dry run 任务状态到 Redis
progress := &types.FAQDryRunProgress{
TaskID: taskID,
KBID: kbID,
Status: types.FAQImportStatusPending,
Progress: 0,
Total: len(payload.Entries),
Processed: 0,
SuccessCount: 0,
FailedCount: 0,
FailedEntries: make([]types.FAQDryRunFailedEntry, 0),
Message: "任务已创建,等待处理",
CreatedAt: time.Now().Unix(),
UpdatedAt: time.Now().Unix(),
}
if err := s.saveFAQDryRunProgress(ctx, progress); err != nil {
logger.Errorf(ctx, "Failed to initialize FAQ dry run task status: %v", err)
return "", fmt.Errorf("failed to initialize task: %w", err)
}
logger.Infof(ctx, "FAQ dry run task initialized: %s, kb_id: %s, total entries: %d", taskID, kbID, len(payload.Entries))
// Enqueue FAQ dry run task to Asynq
taskPayload := types.FAQDryRunPayload{
TenantID: tenantID,
TaskID: taskID,
KBID: kbID,
Entries: payload.Entries,
Mode: payload.Mode,
}
payloadBytes, err := json.Marshal(taskPayload)
if err != nil {
logger.Errorf(ctx, "Failed to marshal FAQ dry run task payload: %v", err)
return "", fmt.Errorf("failed to marshal task payload: %w", err)
}
task := asynq.NewTask(
types.TypeFAQDryRun,
payloadBytes,
asynq.Queue(getAsynqQueueName("default")),
asynq.MaxRetry(3),
)
info, err := s.task.Enqueue(task)
if err != nil {
logger.Errorf(ctx, "Failed to enqueue FAQ dry run task: %v", err)
return "", fmt.Errorf("failed to enqueue task: %w", err)
}
logger.Infof(ctx, "Enqueued FAQ dry run task: id=%s queue=%s task_id=%s", info.ID, info.Queue, taskID)
return taskID, nil
}
// ProcessFAQDryRun handles Asynq FAQ dry run validation tasks.
func (s *knowledgeService) ProcessFAQDryRun(ctx context.Context, t *asynq.Task) error {
var payload types.FAQDryRunPayload
if err := json.Unmarshal(t.Payload(), &payload); err != nil {
return fmt.Errorf("failed to unmarshal FAQ dry run payload: %w", err)
}
logger.Infof(ctx, "Processing FAQ dry run task: %s, kb_id: %s, entries: %d",
payload.TaskID, payload.KBID, len(payload.Entries))
// 更新状态为处理中
progress := &types.FAQDryRunProgress{
TaskID: payload.TaskID,
KBID: payload.KBID,
Status: types.FAQImportStatusProcessing,
Progress: 0,
Total: len(payload.Entries),
Processed: 0,
SuccessCount: 0,
FailedCount: 0,
FailedEntries: make([]types.FAQDryRunFailedEntry, 0),
Message: "正在验证条目...",
CreatedAt: time.Now().Unix(),
UpdatedAt: time.Now().Unix(),
}
if err := s.saveFAQDryRunProgress(ctx, progress); err != nil {
logger.Warnf(ctx, "Failed to update FAQ dry run progress: %v", err)
}
// 执行验证
s.executeFAQDryRunValidation(ctx, &payload, progress)
// 更新最终状态
progress.Status = types.FAQImportStatusCompleted
progress.Progress = 100
progress.SuccessCount = progress.Total - progress.FailedCount
progress.Message = fmt.Sprintf("验证完成: 成功 %d 条, 失败 %d 条", progress.SuccessCount, progress.FailedCount)
progress.UpdatedAt = time.Now().Unix()
// 如果失败条目超过阈值,生成 CSV 文件
csvURL, err := s.generateFailedEntriesCSV(ctx, payload.TenantID, payload.TaskID, progress.FailedEntries)
if err != nil {
logger.Warnf(ctx, "Failed to generate failed entries CSV: %v", err)
} else {
progress.FailedEntriesURL = csvURL
progress.FailedEntries = nil // 清空内联数据,使用 URL
progress.Message += " (失败记录已导出为CSV)"
}
if err := s.saveFAQDryRunProgress(ctx, progress); err != nil {
logger.Warnf(ctx, "Failed to save final FAQ dry run progress: %v", err)
}
logger.Infof(ctx, "FAQ dry run task completed: %s, success: %d, failed: %d",
payload.TaskID, progress.SuccessCount, progress.FailedCount)
return nil
}
// generateFailedEntriesCSV 生成失败条目的 CSV 文件并上传
func (s *knowledgeService) generateFailedEntriesCSV(ctx context.Context,
tenantID uint64, taskID string, failedEntries []types.FAQDryRunFailedEntry,
) (string, error) {
// 生成 CSV 内容
var buf strings.Builder
// 写入 BOM 以支持 Excel 正确识别 UTF-8
buf.WriteString("\xEF\xBB\xBF")
// 写入表头
buf.WriteString("错误原因,分类(必填),问题(必填),相似问题(选填-多个用##分隔),反例问题(选填-多个用##分隔),机器人回答(必填-多个用##分隔),是否全部回复(选填-默认FALSE),是否停用(选填-默认FALSE)\n")
// 写入数据行
for _, entry := range failedEntries {
// CSV 转义:如果内容包含逗号、引号或换行,需要用引号包裹并转义内部引号
reason := csvEscape(entry.Reason)
tagName := csvEscape(entry.TagName)
standardQ := csvEscape(entry.StandardQuestion)
similarQs := ""
if len(entry.SimilarQuestions) > 0 {
similarQs = csvEscape(strings.Join(entry.SimilarQuestions, "##"))
}
negativeQs := ""
if len(entry.NegativeQuestions) > 0 {
negativeQs = csvEscape(strings.Join(entry.NegativeQuestions, "##"))
}
answers := ""
if len(entry.Answers) > 0 {
answers = csvEscape(strings.Join(entry.Answers, "##"))
}
answerAll := "false"
if entry.AnswerAll {
answerAll = "true"
}
isDisabled := "false"
if entry.IsDisabled {
isDisabled = "true"
}
buf.WriteString(fmt.Sprintf("%s,%s,%s,%s,%s,%s,%s,%s\n",
reason, tagName, standardQ, similarQs, negativeQs, answers, answerAll, isDisabled))
}
// 上传 CSV 文件到临时存储(会自动过期)
fileName := fmt.Sprintf("faq_dryrun_failed_%s.csv", taskID)
filePath, err := s.fileSvc.SaveBytes(ctx, []byte(buf.String()), tenantID, fileName, true)
if err != nil {
return "", fmt.Errorf("failed to save CSV file: %w", err)
}
// 获取下载 URL
fileURL, err := s.fileSvc.GetFileURL(ctx, filePath)
if err != nil {
return "", fmt.Errorf("failed to get file URL: %w", err)
}
logger.Infof(ctx, "Generated failed entries CSV: %s, entries: %d", fileURL, len(failedEntries))
return fileURL, nil
}
// csvEscape 转义 CSV 字段
func csvEscape(s string) string {
if strings.ContainsAny(s, ",\"\n\r") {
// 将内部引号替换为两个引号,并用引号包裹整个字段
return "\"" + strings.ReplaceAll(s, "\"", "\"\"") + "\""
}
return s
}
// buildFAQDryRunFailedEntry 构建 FAQDryRunFailedEntry
func buildFAQDryRunFailedEntry(idx int, reason string, entry *types.FAQEntryPayload) types.FAQDryRunFailedEntry {
answerAll := false
if entry.AnswerStrategy != nil && *entry.AnswerStrategy == types.AnswerStrategyAll {
answerAll = true
}
isDisabled := false
if entry.IsEnabled != nil && !*entry.IsEnabled {
isDisabled = true
}
return types.FAQDryRunFailedEntry{
Index: idx,
Reason: reason,
TagName: entry.TagName,
StandardQuestion: strings.TrimSpace(entry.StandardQuestion),
SimilarQuestions: entry.SimilarQuestions,
NegativeQuestions: entry.NegativeQuestions,
Answers: entry.Answers,
AnswerAll: answerAll,
IsDisabled: isDisabled,
}
}
// executeFAQDryRunValidation 执行 FAQ dry run 验证
func (s *knowledgeService) executeFAQDryRunValidation(ctx context.Context,
payload *types.FAQDryRunPayload, progress *types.FAQDryRunProgress,
) {
entries := payload.Entries
// 用于记录已通过基本验证和重复检查的条目索引,后续进行安全检查
validEntryIndices := make([]int, 0, len(entries))
// 根据模式选择不同的验证逻辑
if payload.Mode == types.FAQBatchModeAppend {
validEntryIndices = s.validateEntriesForAppendModeWithProgress(ctx, payload.TenantID, payload.KBID, entries, progress)
} else {
validEntryIndices = s.validateEntriesForReplaceModeWithProgress(ctx, entries, progress)
}
// 安全检查:对通过基本验证的条目进行安全检查(检查所有条目)
if s.securityClient != nil && len(validEntryIndices) > 0 {
s.validateEntriesSecurityWithProgress(ctx, payload.TenantID, entries, validEntryIndices, progress)
}
}
// validateEntriesSecurityWithProgress 对条目进行安全检查(检查所有条目)
func (s *knowledgeService) validateEntriesSecurityWithProgress(ctx context.Context,
tenantID uint64, entries []types.FAQEntryPayload, validIndices []int, progress *types.FAQDryRunProgress,
) {
for _, idx := range validIndices {
entry := entries[idx]
blockStatus, err := s.CheckFAQContent(ctx, tenantID, &entry)
if err != nil {
// 安全检查服务出错,记录但不阻止
logger.Warnf(ctx, "Security check failed for entry %d: %v", idx, err)
continue
}
if blockStatus != 0 {
progress.FailedCount++
progress.FailedEntries = append(progress.FailedEntries, buildFAQDryRunFailedEntry(idx, "内容安全检查不通过", &entry))
}
}
}
// validateEntriesForAppendModeWithProgress 验证 Append 模式下的条目(带进度更新)
func (s *knowledgeService) validateEntriesForAppendModeWithProgress(ctx context.Context,
tenantID uint64, kbID string, entries []types.FAQEntryPayload, progress *types.FAQDryRunProgress,
) []int {
validIndices := make([]int, 0, len(entries))
// 查询知识库中已有的所有FAQ chunks的metadata
existingChunks, err := s.chunkRepo.ListAllFAQChunksWithMetadataByKnowledgeBaseID(ctx, tenantID, kbID)
if err != nil {
logger.Warnf(ctx, "Failed to list existing FAQ chunks for dry run: %v", err)
// 无法获取已有数据时,仅做批次内验证
}
// 构建已存在的标准问和相似问集合
existingQuestions := make(map[string]bool)
for _, chunk := range existingChunks {
meta, err := chunk.FAQMetadata()
if err != nil || meta == nil {
continue
}
if meta.StandardQuestion != "" {
existingQuestions[meta.StandardQuestion] = true
}
for _, q := range meta.SimilarQuestions {
if q != "" {
existingQuestions[q] = true
}
}
}
// 构建当前批次的标准问和相似问集合(用于批次内去重)
batchQuestions := make(map[string]int) // value 为首次出现的索引
for i, entry := range entries {
// 验证条目基本格式
if err := validateFAQEntryPayloadBasic(&entry); err != nil {
progress.FailedCount++
progress.FailedEntries = append(progress.FailedEntries, buildFAQDryRunFailedEntry(i, err.Error(), &entry))
progress.Processed++
continue
}
standardQ := strings.TrimSpace(entry.StandardQuestion)
// 检查标准问是否与已有知识库重复
if existingQuestions[standardQ] {
progress.FailedCount++
progress.FailedEntries = append(progress.FailedEntries, buildFAQDryRunFailedEntry(i, "标准问与知识库中已有问题重复", &entry))
progress.Processed++
continue
}
// 检查标准问是否与同批次重复
if firstIdx, exists := batchQuestions[standardQ]; exists {
progress.FailedCount++
progress.FailedEntries = append(progress.FailedEntries, buildFAQDryRunFailedEntry(i, fmt.Sprintf("标准问与批次内第 %d 条重复", firstIdx+1), &entry))
progress.Processed++
continue
}
// 检查相似问是否有重复
hasDuplicate := false
for _, q := range entry.SimilarQuestions {
q = strings.TrimSpace(q)
if q == "" {
continue
}
if existingQuestions[q] {
progress.FailedCount++
progress.FailedEntries = append(progress.FailedEntries, buildFAQDryRunFailedEntry(i, fmt.Sprintf("相似问 \"%s\" 与知识库中已有问题重复", q), &entry))
hasDuplicate = true
break
}
if firstIdx, exists := batchQuestions[q]; exists {
progress.FailedCount++
progress.FailedEntries = append(progress.FailedEntries, buildFAQDryRunFailedEntry(i, fmt.Sprintf("相似问 \"%s\" 与批次内第 %d 条重复", q, firstIdx+1), &entry))
hasDuplicate = true
break
}
}
if hasDuplicate {
progress.Processed++
continue
}
// 将当前条目的标准问和相似问加入批次集合
batchQuestions[standardQ] = i
for _, q := range entry.SimilarQuestions {
q = strings.TrimSpace(q)
if q != "" {
batchQuestions[q] = i
}
}
// 记录通过验证的条目索引
validIndices = append(validIndices, i)
progress.Processed++
// 定期更新进度
if progress.Processed%100 == 0 {
progress.Progress = progress.Processed * 80 / progress.Total // 前80%用于基本验证
progress.UpdatedAt = time.Now().Unix()
if err := s.saveFAQDryRunProgress(ctx, progress); err != nil {
logger.Warnf(ctx, "Failed to update FAQ dry run progress: %v", err)
}
}
}
return validIndices
}
// validateEntriesForReplaceModeWithProgress 验证 Replace 模式下的条目(带进度更新)
func (s *knowledgeService) validateEntriesForReplaceModeWithProgress(ctx context.Context,
entries []types.FAQEntryPayload, progress *types.FAQDryRunProgress,
) []int {
validIndices := make([]int, 0, len(entries))
// Replace 模式下只检查批次内重复
batchQuestions := make(map[string]int) // value 为首次出现的索引
for i, entry := range entries {
// 验证条目基本格式
if err := validateFAQEntryPayloadBasic(&entry); err != nil {
progress.FailedCount++
progress.FailedEntries = append(progress.FailedEntries, buildFAQDryRunFailedEntry(i, err.Error(), &entry))
progress.Processed++
continue
}
standardQ := strings.TrimSpace(entry.StandardQuestion)
// 检查标准问是否与同批次重复
if firstIdx, exists := batchQuestions[standardQ]; exists {
progress.FailedCount++
progress.FailedEntries = append(progress.FailedEntries, buildFAQDryRunFailedEntry(i, fmt.Sprintf("标准问与批次内第 %d 条重复", firstIdx+1), &entry))
progress.Processed++
continue
}
// 检查相似问是否有重复
hasDuplicate := false
for _, q := range entry.SimilarQuestions {
q = strings.TrimSpace(q)
if q == "" {
continue
}
if firstIdx, exists := batchQuestions[q]; exists {
progress.FailedCount++
progress.FailedEntries = append(progress.FailedEntries, buildFAQDryRunFailedEntry(i, fmt.Sprintf("相似问 \"%s\" 与批次内第 %d 条重复", q, firstIdx+1), &entry))
hasDuplicate = true
break
}
}
if hasDuplicate {
progress.Processed++
continue
}
// 将当前条目的标准问和相似问加入批次集合
batchQuestions[standardQ] = i
for _, q := range entry.SimilarQuestions {
q = strings.TrimSpace(q)
if q != "" {
batchQuestions[q] = i
}
}
// 记录通过验证的条目索引
validIndices = append(validIndices, i)
progress.Processed++
// 定期更新进度
if progress.Processed%100 == 0 {
progress.Progress = progress.Processed * 80 / progress.Total // 前80%用于基本验证
progress.UpdatedAt = time.Now().Unix()
if err := s.saveFAQDryRunProgress(ctx, progress); err != nil {
logger.Warnf(ctx, "Failed to update FAQ dry run progress: %v", err)
}
}
}
return validIndices
}
// saveFAQDryRunProgress 保存 FAQ dry run 进度到 Redis
func (s *knowledgeService) saveFAQDryRunProgress(ctx context.Context, progress *types.FAQDryRunProgress) error {
key := fmt.Sprintf("faq:dryrun:progress:%s", progress.TaskID)
data, err := json.Marshal(progress)
if err != nil {
return fmt.Errorf("failed to marshal progress: %w", err)
}
// 设置 24 小时过期
return s.redisClient.Set(ctx, key, data, 24*time.Hour).Err()
}
// GetFAQDryRunProgress 获取 FAQ dry run 进度
func (s *knowledgeService) GetFAQDryRunProgress(ctx context.Context, taskID string) (*types.FAQDryRunProgress, error) {
key := fmt.Sprintf("faq:dryrun:progress:%s", taskID)
data, err := s.redisClient.Get(ctx, key).Bytes()
if err != nil {
if err == redis.Nil {
return nil, werrors.NewNotFoundError("任务不存在或已过期")
}
return nil, fmt.Errorf("failed to get progress: %w", err)
}
var progress types.FAQDryRunProgress
if err := json.Unmarshal(data, &progress); err != nil {
return nil, fmt.Errorf("failed to unmarshal progress: %w", err)
}
return &progress, nil
}
// validateFAQEntryPayloadBasic 验证 FAQ 条目的基本格式
func validateFAQEntryPayloadBasic(entry *types.FAQEntryPayload) error {
if entry == nil {
return fmt.Errorf("条目不能为空")
}
standardQ := strings.TrimSpace(entry.StandardQuestion)
if standardQ == "" {
return fmt.Errorf("标准问不能为空")
}
if len(entry.Answers) == 0 {
return fmt.Errorf("答案不能为空")
}
hasValidAnswer := false
for _, a := range entry.Answers {
if strings.TrimSpace(a) != "" {
hasValidAnswer = true
break
}
}
if !hasValidAnswer {
return fmt.Errorf("答案不能全为空")
}
return nil
}
// calculateAppendOperations 计算Append模式下需要处理的条目,跳过已存在且内容相同的条目
// 同时过滤掉标准问或相似问与同批次或已有知识库中重复的条目
func (s *knowledgeService) calculateAppendOperations(ctx context.Context,
+3 -1
View File
@@ -391,12 +391,14 @@ func initFileService(cfg *config.Config) (interfaces.FileService, error) {
os.Getenv("COS_PATH_PREFIX") == "" {
return nil, fmt.Errorf("missing COS configuration")
}
return file.NewCosFileService(
return file.NewCosFileServiceWithTempBucket(
os.Getenv("COS_BUCKET_NAME"),
os.Getenv("COS_REGION"),
os.Getenv("COS_SECRET_ID"),
os.Getenv("COS_SECRET_KEY"),
os.Getenv("COS_PATH_PREFIX"),
os.Getenv("COS_TEMP_BUCKET_NAME"), // 可选:临时桶名称(桶需配置生命周期规则自动过期)
os.Getenv("COS_TEMP_REGION"), // 可选:临时桶 region,默认与主桶相同
)
case "local":
return file.NewLocalFileService(os.Getenv("LOCAL_STORAGE_BASE_DIR")), nil
+52 -2
View File
@@ -69,7 +69,9 @@ func (h *FAQHandler) ListEntries(c *gin.Context) {
// UpsertEntries godoc
// @Summary 批量更新/插入FAQ条目
// @Description 异步批量更新或插入FAQ条目
// @Description 异步批量更新或插入FAQ条目。支持 dry_run 模式(设置 dry_run=true),异步验证不实际导入。
// @Description dry_run 模式是异步操作,返回 task_id,通过 /faq/dryrun/progress/{task_id} 查询进度和结果。
// @Description 验证内容包括:1) 条目基本格式 2) 重复问题(批次内和知识库已有) 3) 内容安全检查。
// @Tags FAQ管理
// @Accept json
// @Produce json
@@ -89,7 +91,26 @@ func (h *FAQHandler) UpsertEntries(c *gin.Context) {
return
}
taskID, err := h.knowledgeService.UpsertFAQEntries(ctx, secutils.SanitizeForLog(c.Param("id")), &req)
kbID := secutils.SanitizeForLog(c.Param("id"))
// dry_run 模式:异步验证,不实际导入
if req.DryRun {
taskID, err := h.knowledgeService.ValidateFAQEntriesDryRun(ctx, kbID, &req)
if err != nil {
logger.ErrorWithFields(ctx, err, nil)
c.Error(err)
return
}
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": gin.H{
"task_id": taskID,
},
})
return
}
taskID, err := h.knowledgeService.UpsertFAQEntries(ctx, kbID, &req)
if err != nil {
logger.ErrorWithFields(ctx, err, nil)
c.Error(err)
@@ -417,3 +438,32 @@ func (h *FAQHandler) GetImportProgress(c *gin.Context) {
"data": progress,
})
}
// GetDryRunProgress godoc
// @Summary 获取FAQ预检进度
// @Description 获取FAQ导入预检任务的进度和结果
// @Tags FAQ管理
// @Accept json
// @Produce json
// @Param task_id path string true "任务ID"
// @Success 200 {object} map[string]interface{} "预检进度和结果"
// @Failure 404 {object} errors.AppError "任务不存在"
// @Security Bearer
// @Security ApiKeyAuth
// @Router /faq/dryrun/progress/{task_id} [get]
func (h *FAQHandler) GetDryRunProgress(c *gin.Context) {
ctx := c.Request.Context()
taskID := secutils.SanitizeForLog(c.Param("task_id"))
progress, err := h.knowledgeService.GetFAQDryRunProgress(ctx, taskID)
if err != nil {
logger.ErrorWithFields(ctx, err, nil)
c.Error(err)
return
}
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": progress,
})
}
+5
View File
@@ -200,6 +200,11 @@ func RegisterFAQRoutes(r *gin.RouterGroup, handler *handler.FAQHandler) {
{
faqImport.GET("/progress/:task_id", handler.GetImportProgress)
}
// FAQ dry run progress route (outside of knowledge-base scope)
faqDryRun := r.Group("/faq/dryrun")
{
faqDryRun.GET("/progress/:task_id", handler.GetDryRunProgress)
}
}
// RegisterKnowledgeBaseRoutes 注册知识库相关的路由
+3
View File
@@ -79,6 +79,9 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux {
// Register FAQ import handler
mux.HandleFunc(types.TypeFAQImport, params.KnowledgeService.ProcessFAQImport)
// Register FAQ dry run handler
mux.HandleFunc(types.TypeFAQDryRun, params.KnowledgeService.ProcessFAQDryRun)
// Register question generation handler
mux.HandleFunc(types.TypeQuestionGeneration, params.KnowledgeService.ProcessQuestionGeneration)
+10
View File
@@ -4,6 +4,7 @@ const (
TypeChunkExtract = "chunk:extract"
TypeDocumentProcess = "document:process" // 文档处理任务
TypeFAQImport = "faq:import" // FAQ导入任务
TypeFAQDryRun = "faq:dryrun" // FAQ导入预检任务
TypeQuestionGeneration = "question:generation" // 问题生成任务
TypeSummaryGeneration = "summary:generation" // 摘要生成任务
TypeKBClone = "kb:clone" // 知识库复制任务
@@ -45,6 +46,15 @@ type FAQImportPayload struct {
Mode string `json:"mode"`
}
// FAQDryRunPayload represents the FAQ dry run validation task payload
type FAQDryRunPayload struct {
TenantID uint64 `json:"tenant_id"`
TaskID string `json:"task_id"`
KBID string `json:"kb_id"`
Entries []FAQEntryPayload `json:"entries"`
Mode string `json:"mode"`
}
// QuestionGenerationPayload represents the question generation task payload
type QuestionGenerationPayload struct {
TenantID uint64 `json:"tenant_id"`
+42
View File
@@ -219,6 +219,48 @@ type FAQBatchUpsertPayload struct {
Entries []FAQEntryPayload `json:"entries" binding:"required"`
Mode string `json:"mode" binding:"oneof=append replace"`
KnowledgeID string `json:"knowledge_id"`
TaskID string `json:"task_id"` // 可选,如果不传则自动生成UUID
DryRun bool `json:"dry_run"` // 仅验证,不实际导入
}
// FAQDryRunFailedEntry 表示 dry_run 模式下验证失败的条目
type FAQDryRunFailedEntry struct {
Index int `json:"index"` // 条目在批次中的索引(从0开始)
Reason string `json:"reason"` // 失败原因
TagName string `json:"tag_name,omitempty"` // 分类
StandardQuestion string `json:"standard_question"` // 标准问题
SimilarQuestions []string `json:"similar_questions,omitempty"` // 相似问题
NegativeQuestions []string `json:"negative_questions,omitempty"` // 反例问题
Answers []string `json:"answers,omitempty"` // 答案
AnswerAll bool `json:"answer_all,omitempty"` // 是否全部回复
IsDisabled bool `json:"is_disabled,omitempty"` // 是否停用
}
// FAQDryRunResult 表示 dry_run 模式的验证结果
type FAQDryRunResult struct {
TaskID string `json:"task_id,omitempty"` // 异步任务ID(异步模式时返回)
Total int `json:"total"` // 总条目数
SuccessCount int `json:"success_count"` // 验证通过的条目数
FailedCount int `json:"failed_count"` // 验证失败的条目数
FailedEntries []FAQDryRunFailedEntry `json:"failed_entries"` // 失败条目详情
}
// FAQDryRunProgress 表示 dry_run 异步任务的进度(存储在 Redis)
type FAQDryRunProgress struct {
TaskID string `json:"task_id"`
KBID string `json:"kb_id"`
Status FAQImportTaskStatus `json:"status"` // pending, processing, completed, failed
Progress int `json:"progress"` // 0-100 percentage
Total int `json:"total"` // 总条目数
Processed int `json:"processed"` // 已处理条目数
SuccessCount int `json:"success_count"` // 验证通过的条目数
FailedCount int `json:"failed_count"` // 验证失败的条目数
FailedEntries []FAQDryRunFailedEntry `json:"failed_entries,omitempty"` // 失败条目详情(少量时直接返回)
FailedEntriesURL string `json:"failed_entries_url,omitempty"` // 失败条目CSV下载URL(大量时返回URL)
Message string `json:"message"` // 状态消息
Error string `json:"error,omitempty"` // 错误信息
CreatedAt int64 `json:"created_at"`
UpdatedAt int64 `json:"updated_at"`
}
// FAQSearchRequest FAQ检索请求参数
+5
View File
@@ -11,8 +11,13 @@ import (
type FileService interface {
// SaveFile saves a file.
SaveFile(ctx context.Context, file *multipart.FileHeader, tenantID uint64, knowledgeID string) (string, error)
// SaveBytes saves bytes data to a file and returns the file path.
// If temp is true, the file will be saved to a temporary storage that may auto-expire.
SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error)
// GetFile retrieves a file.
GetFile(ctx context.Context, filePath string) (io.ReadCloser, error)
// GetFileURL returns a download URL for the file (if supported by the storage backend).
GetFileURL(ctx context.Context, filePath string) (string, error)
// DeleteFile deletes a file.
DeleteFile(ctx context.Context, filePath string) error
}
+7
View File
@@ -88,6 +88,13 @@ type KnowledgeService interface {
// UpsertFAQEntries imports or appends FAQ entries asynchronously.
// Returns task ID (Knowledge ID) for tracking import progress.
UpsertFAQEntries(ctx context.Context, kbID string, payload *types.FAQBatchUpsertPayload) (string, error)
// ValidateFAQEntriesDryRun validates FAQ entries asynchronously without actually importing them.
// Returns task ID for tracking validation progress.
ValidateFAQEntriesDryRun(ctx context.Context, kbID string, payload *types.FAQBatchUpsertPayload) (string, error)
// GetFAQDryRunProgress retrieves the progress of an FAQ dry run validation task.
GetFAQDryRunProgress(ctx context.Context, taskID string) (*types.FAQDryRunProgress, error)
// ProcessFAQDryRun handles Asynq FAQ dry run validation tasks.
ProcessFAQDryRun(ctx context.Context, t *asynq.Task) error
// CreateFAQEntry creates a single FAQ entry synchronously.
CreateFAQEntry(ctx context.Context, kbID string, payload *types.FAQEntryPayload) (*types.FAQEntry, error)
// GetFAQEntry retrieves a single FAQ entry by ID.