feat: 新增FAQ批量导入dry run功能
This commit is contained in:
@@ -51,6 +51,8 @@ type FAQBatchUpsertPayload struct {
|
||||
Entries []FAQEntryPayload `json:"entries"`
|
||||
Mode string `json:"mode"`
|
||||
KnowledgeID string `json:"knowledge_id,omitempty"`
|
||||
TaskID string `json:"task_id,omitempty"` // Optional, if not provided, a UUID will be generated
|
||||
DryRun bool `json:"dry_run,omitempty"` // If true, only validate without importing
|
||||
}
|
||||
|
||||
// FAQEntryFieldsUpdate represents the fields that can be updated for a single FAQ entry.
|
||||
@@ -379,3 +381,57 @@ func (c *Client) GetFAQImportProgress(ctx context.Context, taskID string) (*FAQI
|
||||
}
|
||||
return response.Data, nil
|
||||
}
|
||||
|
||||
// FAQDryRunFailedEntry represents a failed entry in dry run validation.
|
||||
type FAQDryRunFailedEntry struct {
|
||||
Index int `json:"index"`
|
||||
Reason string `json:"reason"`
|
||||
TagName string `json:"tag_name,omitempty"`
|
||||
StandardQuestion string `json:"standard_question"`
|
||||
SimilarQuestions []string `json:"similar_questions,omitempty"`
|
||||
NegativeQuestions []string `json:"negative_questions,omitempty"`
|
||||
Answers []string `json:"answers,omitempty"`
|
||||
AnswerAll bool `json:"answer_all,omitempty"`
|
||||
IsDisabled bool `json:"is_disabled,omitempty"`
|
||||
}
|
||||
|
||||
// FAQDryRunProgress represents the progress of an async FAQ dry run validation task.
|
||||
type FAQDryRunProgress struct {
|
||||
TaskID string `json:"task_id"`
|
||||
KBID string `json:"kb_id"`
|
||||
Status string `json:"status"`
|
||||
Progress int `json:"progress"`
|
||||
Total int `json:"total"`
|
||||
Processed int `json:"processed"`
|
||||
SuccessCount int `json:"success_count"`
|
||||
FailedCount int `json:"failed_count"`
|
||||
FailedEntries []FAQDryRunFailedEntry `json:"failed_entries,omitempty"`
|
||||
FailedEntriesURL string `json:"failed_entries_url,omitempty"` // CSV download URL when too many failures
|
||||
Message string `json:"message"`
|
||||
Error string `json:"error,omitempty"`
|
||||
CreatedAt int64 `json:"created_at"`
|
||||
UpdatedAt int64 `json:"updated_at"`
|
||||
}
|
||||
|
||||
// FAQDryRunProgressResponse wraps the FAQ dry run progress response.
|
||||
type FAQDryRunProgressResponse struct {
|
||||
Success bool `json:"success"`
|
||||
Data *FAQDryRunProgress `json:"data"`
|
||||
Message string `json:"message,omitempty"`
|
||||
Code string `json:"code,omitempty"`
|
||||
}
|
||||
|
||||
// GetFAQDryRunProgress retrieves the progress of an async FAQ dry run validation task.
|
||||
func (c *Client) GetFAQDryRunProgress(ctx context.Context, taskID string) (*FAQDryRunProgress, error) {
|
||||
path := fmt.Sprintf("/api/v1/faq/dryrun/progress/%s", taskID)
|
||||
resp, err := c.doRequest(ctx, http.MethodGet, path, nil, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var response FAQDryRunProgressResponse
|
||||
if err := parseResponse(resp, &response); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return response.Data, nil
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -9,6 +10,7 @@ import (
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/google/uuid"
|
||||
@@ -20,11 +22,19 @@ type cosFileService struct {
|
||||
client *cos.Client
|
||||
bucketURL string
|
||||
cosPathPrefix string
|
||||
// 临时桶配置(用于存放自动过期的临时文件)
|
||||
tempClient *cos.Client
|
||||
tempBucketURL string
|
||||
}
|
||||
|
||||
// NewCosFileService creates a new COS file service instance
|
||||
func NewCosFileService(bucketName, region, secretId, secretKey, cosPathPrefix string) (interfaces.FileService, error) {
|
||||
bucketURL := fmt.Sprintf("https://%s.cos.%s.myqcloud.com/", bucketName, region)
|
||||
return NewCosFileServiceWithTempBucket(bucketName, region, secretId, secretKey, cosPathPrefix, "", "")
|
||||
}
|
||||
|
||||
// NewCosFileServiceWithTempBucket creates a new COS file service instance with optional temp bucket
|
||||
func NewCosFileServiceWithTempBucket(bucketName, region, secretId, secretKey, cosPathPrefix, tempBucketName, tempRegion string) (interfaces.FileService, error) {
|
||||
bucketURL := fmt.Sprintf("https://%s.cos.%s.tencentcos.cn/", bucketName, region)
|
||||
u, err := url.Parse(bucketURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse bucketURL: %w", err)
|
||||
@@ -36,11 +46,35 @@ func NewCosFileService(bucketName, region, secretId, secretKey, cosPathPrefix st
|
||||
SecretKey: secretKey,
|
||||
},
|
||||
})
|
||||
return &cosFileService{
|
||||
|
||||
svc := &cosFileService{
|
||||
client: client,
|
||||
bucketURL: bucketURL,
|
||||
cosPathPrefix: cosPathPrefix,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// 如果配置了临时桶,初始化临时桶客户端
|
||||
if tempBucketName != "" {
|
||||
if tempRegion == "" {
|
||||
tempRegion = region // 默认使用相同的 region
|
||||
}
|
||||
tempBucketURL := fmt.Sprintf("https://%s.cos.%s.tencentcos.cn/", tempBucketName, tempRegion)
|
||||
tempU, err := url.Parse(tempBucketURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse temp bucketURL: %w", err)
|
||||
}
|
||||
tempB := &cos.BaseURL{BucketURL: tempU}
|
||||
tempClient := cos.NewClient(tempB, &http.Client{
|
||||
Transport: &cos.AuthorizationTransport{
|
||||
SecretID: secretId,
|
||||
SecretKey: secretKey,
|
||||
},
|
||||
})
|
||||
svc.tempClient = tempClient
|
||||
svc.tempBucketURL = tempBucketURL
|
||||
}
|
||||
|
||||
return svc, nil
|
||||
}
|
||||
|
||||
// SaveFile saves a file to COS storage
|
||||
@@ -81,3 +115,53 @@ func (s *cosFileService) DeleteFile(ctx context.Context, filePath string) error
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SaveBytes saves bytes data to COS
|
||||
// If temp is true and temp bucket is configured, saves to temp bucket (with lifecycle auto-expiration)
|
||||
// Otherwise saves to main bucket
|
||||
func (s *cosFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) {
|
||||
ext := filepath.Ext(fileName)
|
||||
reader := bytes.NewReader(data)
|
||||
|
||||
// 如果请求写入临时桶且临时桶已配置
|
||||
if temp && s.tempClient != nil {
|
||||
objectName := fmt.Sprintf("exports/%d/%s%s", tenantID, uuid.New().String(), ext)
|
||||
_, err := s.tempClient.Object.Put(ctx, objectName, reader, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to upload bytes to COS temp bucket: %w", err)
|
||||
}
|
||||
return fmt.Sprintf("%s%s", s.tempBucketURL, objectName), nil
|
||||
}
|
||||
|
||||
// 写入主桶
|
||||
objectName := fmt.Sprintf("%s/%d/exports/%s%s", s.cosPathPrefix, tenantID, uuid.New().String(), ext)
|
||||
_, err := s.client.Object.Put(ctx, objectName, reader, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to upload bytes to COS: %w", err)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s%s", s.bucketURL, objectName), nil
|
||||
}
|
||||
|
||||
// GetFileURL returns a presigned download URL for the file
|
||||
func (s *cosFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
|
||||
// 判断文件属于哪个桶
|
||||
if s.tempClient != nil && strings.HasPrefix(filePath, s.tempBucketURL) {
|
||||
objectName := strings.TrimPrefix(filePath, s.tempBucketURL)
|
||||
// Generate presigned URL (valid for 24 hours)
|
||||
presignedURL, err := s.tempClient.Object.GetPresignedURL(ctx, http.MethodGet, objectName, s.tempClient.GetCredential().SecretID, s.tempClient.GetCredential().SecretKey, 24*time.Hour, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to generate presigned URL for temp bucket: %w", err)
|
||||
}
|
||||
return presignedURL.String(), nil
|
||||
}
|
||||
|
||||
objectName := strings.TrimPrefix(filePath, s.bucketURL)
|
||||
// Generate presigned URL (valid for 24 hours)
|
||||
presignedURL, err := s.client.Object.GetPresignedURL(ctx, http.MethodGet, objectName, s.client.GetCredential().SecretID, s.client.GetCredential().SecretKey, 24*time.Hour, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to generate presigned URL: %w", err)
|
||||
}
|
||||
|
||||
return presignedURL.String(), nil
|
||||
}
|
||||
|
||||
@@ -36,3 +36,13 @@ func (s *DummyFileService) GetFile(ctx context.Context, filePath string) (io.Rea
|
||||
func (s *DummyFileService) DeleteFile(ctx context.Context, filePath string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// SaveBytes pretends to save bytes but just returns a random UUID
|
||||
func (s *DummyFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) {
|
||||
return uuid.New().String(), nil
|
||||
}
|
||||
|
||||
// GetFileURL returns the file path as URL (dummy implementation)
|
||||
func (s *DummyFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
|
||||
return filePath, nil
|
||||
}
|
||||
|
||||
@@ -109,3 +109,38 @@ func (s *localFileService) DeleteFile(ctx context.Context, filePath string) erro
|
||||
logger.Info(ctx, "File deleted successfully")
|
||||
return nil
|
||||
}
|
||||
|
||||
// SaveBytes saves bytes data to a file and returns the file path
|
||||
// temp parameter is ignored for local storage (no auto-expiration support)
|
||||
func (s *localFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) {
|
||||
logger.Infof(ctx, "Saving bytes data: fileName=%s, size=%d, tenantID=%d, temp=%v", fileName, len(data), tenantID, temp)
|
||||
|
||||
// Create storage directory with tenant ID
|
||||
dir := filepath.Join(s.baseDir, fmt.Sprintf("%d", tenantID), "exports")
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
logger.Errorf(ctx, "Failed to create directory: %v", err)
|
||||
return "", fmt.Errorf("failed to create directory: %w", err)
|
||||
}
|
||||
|
||||
// Generate unique filename using timestamp
|
||||
ext := filepath.Ext(fileName)
|
||||
baseName := fileName[:len(fileName)-len(ext)]
|
||||
uniqueFileName := fmt.Sprintf("%s_%d%s", baseName, time.Now().UnixNano(), ext)
|
||||
filePath := filepath.Join(dir, uniqueFileName)
|
||||
|
||||
// Write data to file
|
||||
if err := os.WriteFile(filePath, data, 0o644); err != nil {
|
||||
logger.Errorf(ctx, "Failed to write file: %v", err)
|
||||
return "", fmt.Errorf("failed to write file: %w", err)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Bytes data saved successfully: %s", filePath)
|
||||
return filePath, nil
|
||||
}
|
||||
|
||||
// GetFileURL returns a download URL for the file
|
||||
// For local storage, returns the file path itself (no URL support)
|
||||
func (s *localFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
|
||||
// Local storage doesn't support URLs, return the path
|
||||
return filePath, nil
|
||||
}
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/google/uuid"
|
||||
@@ -125,3 +127,43 @@ func (s *minioFileService) DeleteFile(ctx context.Context, filePath string) erro
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SaveBytes saves bytes data to MinIO and returns the file path
|
||||
// temp parameter is ignored for MinIO (no auto-expiration support in this implementation)
|
||||
func (s *minioFileService) SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error) {
|
||||
ext := filepath.Ext(fileName)
|
||||
objectName := fmt.Sprintf("%d/exports/%s%s", tenantID, uuid.New().String(), ext)
|
||||
|
||||
// Upload bytes to MinIO
|
||||
reader := bytes.NewReader(data)
|
||||
_, err := s.client.PutObject(ctx, s.bucketName, objectName, reader, int64(len(data)), minio.PutObjectOptions{
|
||||
ContentType: "text/csv; charset=utf-8",
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to upload bytes to MinIO: %w", err)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("minio://%s/%s", s.bucketName, objectName), nil
|
||||
}
|
||||
|
||||
// GetFileURL returns a presigned download URL for the file
|
||||
func (s *minioFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
|
||||
// Parse MinIO path
|
||||
if len(filePath) < 9 || filePath[:8] != "minio://" {
|
||||
return "", fmt.Errorf("invalid MinIO file path: %s", filePath)
|
||||
}
|
||||
|
||||
// Extract object name
|
||||
objectName := filePath[9+len(s.bucketName):]
|
||||
if objectName[0] == '/' {
|
||||
objectName = objectName[1:]
|
||||
}
|
||||
|
||||
// Generate presigned URL (valid for 24 hours)
|
||||
presignedURL, err := s.client.PresignedGetObject(ctx, s.bucketName, objectName, 24*time.Hour, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to generate presigned URL: %w", err)
|
||||
}
|
||||
|
||||
return presignedURL.String(), nil
|
||||
}
|
||||
|
||||
@@ -2808,6 +2808,518 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context,
|
||||
return taskID, nil
|
||||
}
|
||||
|
||||
// ValidateFAQEntriesDryRun validates FAQ entries asynchronously without actually importing them.
|
||||
// Returns task ID for tracking validation progress.
|
||||
func (s *knowledgeService) ValidateFAQEntriesDryRun(ctx context.Context,
|
||||
kbID string, payload *types.FAQBatchUpsertPayload,
|
||||
) (string, error) {
|
||||
if payload == nil || len(payload.Entries) == 0 {
|
||||
return "", werrors.NewBadRequestError("FAQ 条目不能为空")
|
||||
}
|
||||
if payload.Mode == "" {
|
||||
payload.Mode = types.FAQBatchModeAppend
|
||||
}
|
||||
if payload.Mode != types.FAQBatchModeAppend && payload.Mode != types.FAQBatchModeReplace {
|
||||
return "", werrors.NewBadRequestError("模式仅支持 append 或 replace")
|
||||
}
|
||||
|
||||
// 验证知识库是否存在且有效
|
||||
_, err := s.validateFAQKnowledgeBase(ctx, kbID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
|
||||
|
||||
// 使用传入的 task_id,如果不存在则生成新的
|
||||
taskID := payload.TaskID
|
||||
if taskID == "" {
|
||||
taskID = uuid.New().String()
|
||||
}
|
||||
|
||||
// 初始化 dry run 任务状态到 Redis
|
||||
progress := &types.FAQDryRunProgress{
|
||||
TaskID: taskID,
|
||||
KBID: kbID,
|
||||
Status: types.FAQImportStatusPending,
|
||||
Progress: 0,
|
||||
Total: len(payload.Entries),
|
||||
Processed: 0,
|
||||
SuccessCount: 0,
|
||||
FailedCount: 0,
|
||||
FailedEntries: make([]types.FAQDryRunFailedEntry, 0),
|
||||
Message: "任务已创建,等待处理",
|
||||
CreatedAt: time.Now().Unix(),
|
||||
UpdatedAt: time.Now().Unix(),
|
||||
}
|
||||
if err := s.saveFAQDryRunProgress(ctx, progress); err != nil {
|
||||
logger.Errorf(ctx, "Failed to initialize FAQ dry run task status: %v", err)
|
||||
return "", fmt.Errorf("failed to initialize task: %w", err)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "FAQ dry run task initialized: %s, kb_id: %s, total entries: %d", taskID, kbID, len(payload.Entries))
|
||||
|
||||
// Enqueue FAQ dry run task to Asynq
|
||||
taskPayload := types.FAQDryRunPayload{
|
||||
TenantID: tenantID,
|
||||
TaskID: taskID,
|
||||
KBID: kbID,
|
||||
Entries: payload.Entries,
|
||||
Mode: payload.Mode,
|
||||
}
|
||||
|
||||
payloadBytes, err := json.Marshal(taskPayload)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to marshal FAQ dry run task payload: %v", err)
|
||||
return "", fmt.Errorf("failed to marshal task payload: %w", err)
|
||||
}
|
||||
|
||||
task := asynq.NewTask(
|
||||
types.TypeFAQDryRun,
|
||||
payloadBytes,
|
||||
asynq.Queue(getAsynqQueueName("default")),
|
||||
asynq.MaxRetry(3),
|
||||
)
|
||||
info, err := s.task.Enqueue(task)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to enqueue FAQ dry run task: %v", err)
|
||||
return "", fmt.Errorf("failed to enqueue task: %w", err)
|
||||
}
|
||||
logger.Infof(ctx, "Enqueued FAQ dry run task: id=%s queue=%s task_id=%s", info.ID, info.Queue, taskID)
|
||||
|
||||
return taskID, nil
|
||||
}
|
||||
|
||||
// ProcessFAQDryRun handles Asynq FAQ dry run validation tasks.
|
||||
func (s *knowledgeService) ProcessFAQDryRun(ctx context.Context, t *asynq.Task) error {
|
||||
var payload types.FAQDryRunPayload
|
||||
if err := json.Unmarshal(t.Payload(), &payload); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal FAQ dry run payload: %w", err)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Processing FAQ dry run task: %s, kb_id: %s, entries: %d",
|
||||
payload.TaskID, payload.KBID, len(payload.Entries))
|
||||
|
||||
// 更新状态为处理中
|
||||
progress := &types.FAQDryRunProgress{
|
||||
TaskID: payload.TaskID,
|
||||
KBID: payload.KBID,
|
||||
Status: types.FAQImportStatusProcessing,
|
||||
Progress: 0,
|
||||
Total: len(payload.Entries),
|
||||
Processed: 0,
|
||||
SuccessCount: 0,
|
||||
FailedCount: 0,
|
||||
FailedEntries: make([]types.FAQDryRunFailedEntry, 0),
|
||||
Message: "正在验证条目...",
|
||||
CreatedAt: time.Now().Unix(),
|
||||
UpdatedAt: time.Now().Unix(),
|
||||
}
|
||||
if err := s.saveFAQDryRunProgress(ctx, progress); err != nil {
|
||||
logger.Warnf(ctx, "Failed to update FAQ dry run progress: %v", err)
|
||||
}
|
||||
|
||||
// 执行验证
|
||||
s.executeFAQDryRunValidation(ctx, &payload, progress)
|
||||
|
||||
// 更新最终状态
|
||||
progress.Status = types.FAQImportStatusCompleted
|
||||
progress.Progress = 100
|
||||
progress.SuccessCount = progress.Total - progress.FailedCount
|
||||
progress.Message = fmt.Sprintf("验证完成: 成功 %d 条, 失败 %d 条", progress.SuccessCount, progress.FailedCount)
|
||||
progress.UpdatedAt = time.Now().Unix()
|
||||
|
||||
// 如果失败条目超过阈值,生成 CSV 文件
|
||||
csvURL, err := s.generateFailedEntriesCSV(ctx, payload.TenantID, payload.TaskID, progress.FailedEntries)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "Failed to generate failed entries CSV: %v", err)
|
||||
} else {
|
||||
progress.FailedEntriesURL = csvURL
|
||||
progress.FailedEntries = nil // 清空内联数据,使用 URL
|
||||
progress.Message += " (失败记录已导出为CSV)"
|
||||
}
|
||||
|
||||
if err := s.saveFAQDryRunProgress(ctx, progress); err != nil {
|
||||
logger.Warnf(ctx, "Failed to save final FAQ dry run progress: %v", err)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "FAQ dry run task completed: %s, success: %d, failed: %d",
|
||||
payload.TaskID, progress.SuccessCount, progress.FailedCount)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// generateFailedEntriesCSV 生成失败条目的 CSV 文件并上传
|
||||
func (s *knowledgeService) generateFailedEntriesCSV(ctx context.Context,
|
||||
tenantID uint64, taskID string, failedEntries []types.FAQDryRunFailedEntry,
|
||||
) (string, error) {
|
||||
// 生成 CSV 内容
|
||||
var buf strings.Builder
|
||||
|
||||
// 写入 BOM 以支持 Excel 正确识别 UTF-8
|
||||
buf.WriteString("\xEF\xBB\xBF")
|
||||
|
||||
// 写入表头
|
||||
buf.WriteString("错误原因,分类(必填),问题(必填),相似问题(选填-多个用##分隔),反例问题(选填-多个用##分隔),机器人回答(必填-多个用##分隔),是否全部回复(选填-默认FALSE),是否停用(选填-默认FALSE)\n")
|
||||
|
||||
// 写入数据行
|
||||
for _, entry := range failedEntries {
|
||||
// CSV 转义:如果内容包含逗号、引号或换行,需要用引号包裹并转义内部引号
|
||||
reason := csvEscape(entry.Reason)
|
||||
tagName := csvEscape(entry.TagName)
|
||||
standardQ := csvEscape(entry.StandardQuestion)
|
||||
similarQs := ""
|
||||
if len(entry.SimilarQuestions) > 0 {
|
||||
similarQs = csvEscape(strings.Join(entry.SimilarQuestions, "##"))
|
||||
}
|
||||
negativeQs := ""
|
||||
if len(entry.NegativeQuestions) > 0 {
|
||||
negativeQs = csvEscape(strings.Join(entry.NegativeQuestions, "##"))
|
||||
}
|
||||
answers := ""
|
||||
if len(entry.Answers) > 0 {
|
||||
answers = csvEscape(strings.Join(entry.Answers, "##"))
|
||||
}
|
||||
answerAll := "false"
|
||||
if entry.AnswerAll {
|
||||
answerAll = "true"
|
||||
}
|
||||
isDisabled := "false"
|
||||
if entry.IsDisabled {
|
||||
isDisabled = "true"
|
||||
}
|
||||
|
||||
buf.WriteString(fmt.Sprintf("%s,%s,%s,%s,%s,%s,%s,%s\n",
|
||||
reason, tagName, standardQ, similarQs, negativeQs, answers, answerAll, isDisabled))
|
||||
}
|
||||
|
||||
// 上传 CSV 文件到临时存储(会自动过期)
|
||||
fileName := fmt.Sprintf("faq_dryrun_failed_%s.csv", taskID)
|
||||
filePath, err := s.fileSvc.SaveBytes(ctx, []byte(buf.String()), tenantID, fileName, true)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to save CSV file: %w", err)
|
||||
}
|
||||
|
||||
// 获取下载 URL
|
||||
fileURL, err := s.fileSvc.GetFileURL(ctx, filePath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to get file URL: %w", err)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Generated failed entries CSV: %s, entries: %d", fileURL, len(failedEntries))
|
||||
return fileURL, nil
|
||||
}
|
||||
|
||||
// csvEscape 转义 CSV 字段
|
||||
func csvEscape(s string) string {
|
||||
if strings.ContainsAny(s, ",\"\n\r") {
|
||||
// 将内部引号替换为两个引号,并用引号包裹整个字段
|
||||
return "\"" + strings.ReplaceAll(s, "\"", "\"\"") + "\""
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// buildFAQDryRunFailedEntry 构建 FAQDryRunFailedEntry
|
||||
func buildFAQDryRunFailedEntry(idx int, reason string, entry *types.FAQEntryPayload) types.FAQDryRunFailedEntry {
|
||||
answerAll := false
|
||||
if entry.AnswerStrategy != nil && *entry.AnswerStrategy == types.AnswerStrategyAll {
|
||||
answerAll = true
|
||||
}
|
||||
isDisabled := false
|
||||
if entry.IsEnabled != nil && !*entry.IsEnabled {
|
||||
isDisabled = true
|
||||
}
|
||||
return types.FAQDryRunFailedEntry{
|
||||
Index: idx,
|
||||
Reason: reason,
|
||||
TagName: entry.TagName,
|
||||
StandardQuestion: strings.TrimSpace(entry.StandardQuestion),
|
||||
SimilarQuestions: entry.SimilarQuestions,
|
||||
NegativeQuestions: entry.NegativeQuestions,
|
||||
Answers: entry.Answers,
|
||||
AnswerAll: answerAll,
|
||||
IsDisabled: isDisabled,
|
||||
}
|
||||
}
|
||||
|
||||
// executeFAQDryRunValidation 执行 FAQ dry run 验证
|
||||
func (s *knowledgeService) executeFAQDryRunValidation(ctx context.Context,
|
||||
payload *types.FAQDryRunPayload, progress *types.FAQDryRunProgress,
|
||||
) {
|
||||
entries := payload.Entries
|
||||
|
||||
// 用于记录已通过基本验证和重复检查的条目索引,后续进行安全检查
|
||||
validEntryIndices := make([]int, 0, len(entries))
|
||||
|
||||
// 根据模式选择不同的验证逻辑
|
||||
if payload.Mode == types.FAQBatchModeAppend {
|
||||
validEntryIndices = s.validateEntriesForAppendModeWithProgress(ctx, payload.TenantID, payload.KBID, entries, progress)
|
||||
} else {
|
||||
validEntryIndices = s.validateEntriesForReplaceModeWithProgress(ctx, entries, progress)
|
||||
}
|
||||
|
||||
// 安全检查:对通过基本验证的条目进行安全检查(检查所有条目)
|
||||
if s.securityClient != nil && len(validEntryIndices) > 0 {
|
||||
s.validateEntriesSecurityWithProgress(ctx, payload.TenantID, entries, validEntryIndices, progress)
|
||||
}
|
||||
}
|
||||
|
||||
// validateEntriesSecurityWithProgress 对条目进行安全检查(检查所有条目)
|
||||
func (s *knowledgeService) validateEntriesSecurityWithProgress(ctx context.Context,
|
||||
tenantID uint64, entries []types.FAQEntryPayload, validIndices []int, progress *types.FAQDryRunProgress,
|
||||
) {
|
||||
for _, idx := range validIndices {
|
||||
entry := entries[idx]
|
||||
blockStatus, err := s.CheckFAQContent(ctx, tenantID, &entry)
|
||||
if err != nil {
|
||||
// 安全检查服务出错,记录但不阻止
|
||||
logger.Warnf(ctx, "Security check failed for entry %d: %v", idx, err)
|
||||
continue
|
||||
}
|
||||
if blockStatus != 0 {
|
||||
progress.FailedCount++
|
||||
progress.FailedEntries = append(progress.FailedEntries, buildFAQDryRunFailedEntry(idx, "内容安全检查不通过", &entry))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// validateEntriesForAppendModeWithProgress 验证 Append 模式下的条目(带进度更新)
|
||||
func (s *knowledgeService) validateEntriesForAppendModeWithProgress(ctx context.Context,
|
||||
tenantID uint64, kbID string, entries []types.FAQEntryPayload, progress *types.FAQDryRunProgress,
|
||||
) []int {
|
||||
validIndices := make([]int, 0, len(entries))
|
||||
|
||||
// 查询知识库中已有的所有FAQ chunks的metadata
|
||||
existingChunks, err := s.chunkRepo.ListAllFAQChunksWithMetadataByKnowledgeBaseID(ctx, tenantID, kbID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "Failed to list existing FAQ chunks for dry run: %v", err)
|
||||
// 无法获取已有数据时,仅做批次内验证
|
||||
}
|
||||
|
||||
// 构建已存在的标准问和相似问集合
|
||||
existingQuestions := make(map[string]bool)
|
||||
for _, chunk := range existingChunks {
|
||||
meta, err := chunk.FAQMetadata()
|
||||
if err != nil || meta == nil {
|
||||
continue
|
||||
}
|
||||
if meta.StandardQuestion != "" {
|
||||
existingQuestions[meta.StandardQuestion] = true
|
||||
}
|
||||
for _, q := range meta.SimilarQuestions {
|
||||
if q != "" {
|
||||
existingQuestions[q] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 构建当前批次的标准问和相似问集合(用于批次内去重)
|
||||
batchQuestions := make(map[string]int) // value 为首次出现的索引
|
||||
|
||||
for i, entry := range entries {
|
||||
// 验证条目基本格式
|
||||
if err := validateFAQEntryPayloadBasic(&entry); err != nil {
|
||||
progress.FailedCount++
|
||||
progress.FailedEntries = append(progress.FailedEntries, buildFAQDryRunFailedEntry(i, err.Error(), &entry))
|
||||
progress.Processed++
|
||||
continue
|
||||
}
|
||||
|
||||
standardQ := strings.TrimSpace(entry.StandardQuestion)
|
||||
|
||||
// 检查标准问是否与已有知识库重复
|
||||
if existingQuestions[standardQ] {
|
||||
progress.FailedCount++
|
||||
progress.FailedEntries = append(progress.FailedEntries, buildFAQDryRunFailedEntry(i, "标准问与知识库中已有问题重复", &entry))
|
||||
progress.Processed++
|
||||
continue
|
||||
}
|
||||
|
||||
// 检查标准问是否与同批次重复
|
||||
if firstIdx, exists := batchQuestions[standardQ]; exists {
|
||||
progress.FailedCount++
|
||||
progress.FailedEntries = append(progress.FailedEntries, buildFAQDryRunFailedEntry(i, fmt.Sprintf("标准问与批次内第 %d 条重复", firstIdx+1), &entry))
|
||||
progress.Processed++
|
||||
continue
|
||||
}
|
||||
|
||||
// 检查相似问是否有重复
|
||||
hasDuplicate := false
|
||||
for _, q := range entry.SimilarQuestions {
|
||||
q = strings.TrimSpace(q)
|
||||
if q == "" {
|
||||
continue
|
||||
}
|
||||
if existingQuestions[q] {
|
||||
progress.FailedCount++
|
||||
progress.FailedEntries = append(progress.FailedEntries, buildFAQDryRunFailedEntry(i, fmt.Sprintf("相似问 \"%s\" 与知识库中已有问题重复", q), &entry))
|
||||
hasDuplicate = true
|
||||
break
|
||||
}
|
||||
if firstIdx, exists := batchQuestions[q]; exists {
|
||||
progress.FailedCount++
|
||||
progress.FailedEntries = append(progress.FailedEntries, buildFAQDryRunFailedEntry(i, fmt.Sprintf("相似问 \"%s\" 与批次内第 %d 条重复", q, firstIdx+1), &entry))
|
||||
hasDuplicate = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if hasDuplicate {
|
||||
progress.Processed++
|
||||
continue
|
||||
}
|
||||
|
||||
// 将当前条目的标准问和相似问加入批次集合
|
||||
batchQuestions[standardQ] = i
|
||||
for _, q := range entry.SimilarQuestions {
|
||||
q = strings.TrimSpace(q)
|
||||
if q != "" {
|
||||
batchQuestions[q] = i
|
||||
}
|
||||
}
|
||||
|
||||
// 记录通过验证的条目索引
|
||||
validIndices = append(validIndices, i)
|
||||
progress.Processed++
|
||||
|
||||
// 定期更新进度
|
||||
if progress.Processed%100 == 0 {
|
||||
progress.Progress = progress.Processed * 80 / progress.Total // 前80%用于基本验证
|
||||
progress.UpdatedAt = time.Now().Unix()
|
||||
if err := s.saveFAQDryRunProgress(ctx, progress); err != nil {
|
||||
logger.Warnf(ctx, "Failed to update FAQ dry run progress: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return validIndices
|
||||
}
|
||||
|
||||
// validateEntriesForReplaceModeWithProgress 验证 Replace 模式下的条目(带进度更新)
|
||||
func (s *knowledgeService) validateEntriesForReplaceModeWithProgress(ctx context.Context,
|
||||
entries []types.FAQEntryPayload, progress *types.FAQDryRunProgress,
|
||||
) []int {
|
||||
validIndices := make([]int, 0, len(entries))
|
||||
|
||||
// Replace 模式下只检查批次内重复
|
||||
batchQuestions := make(map[string]int) // value 为首次出现的索引
|
||||
|
||||
for i, entry := range entries {
|
||||
// 验证条目基本格式
|
||||
if err := validateFAQEntryPayloadBasic(&entry); err != nil {
|
||||
progress.FailedCount++
|
||||
progress.FailedEntries = append(progress.FailedEntries, buildFAQDryRunFailedEntry(i, err.Error(), &entry))
|
||||
progress.Processed++
|
||||
continue
|
||||
}
|
||||
|
||||
standardQ := strings.TrimSpace(entry.StandardQuestion)
|
||||
|
||||
// 检查标准问是否与同批次重复
|
||||
if firstIdx, exists := batchQuestions[standardQ]; exists {
|
||||
progress.FailedCount++
|
||||
progress.FailedEntries = append(progress.FailedEntries, buildFAQDryRunFailedEntry(i, fmt.Sprintf("标准问与批次内第 %d 条重复", firstIdx+1), &entry))
|
||||
progress.Processed++
|
||||
continue
|
||||
}
|
||||
|
||||
// 检查相似问是否有重复
|
||||
hasDuplicate := false
|
||||
for _, q := range entry.SimilarQuestions {
|
||||
q = strings.TrimSpace(q)
|
||||
if q == "" {
|
||||
continue
|
||||
}
|
||||
if firstIdx, exists := batchQuestions[q]; exists {
|
||||
progress.FailedCount++
|
||||
progress.FailedEntries = append(progress.FailedEntries, buildFAQDryRunFailedEntry(i, fmt.Sprintf("相似问 \"%s\" 与批次内第 %d 条重复", q, firstIdx+1), &entry))
|
||||
hasDuplicate = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if hasDuplicate {
|
||||
progress.Processed++
|
||||
continue
|
||||
}
|
||||
|
||||
// 将当前条目的标准问和相似问加入批次集合
|
||||
batchQuestions[standardQ] = i
|
||||
for _, q := range entry.SimilarQuestions {
|
||||
q = strings.TrimSpace(q)
|
||||
if q != "" {
|
||||
batchQuestions[q] = i
|
||||
}
|
||||
}
|
||||
|
||||
// 记录通过验证的条目索引
|
||||
validIndices = append(validIndices, i)
|
||||
progress.Processed++
|
||||
|
||||
// 定期更新进度
|
||||
if progress.Processed%100 == 0 {
|
||||
progress.Progress = progress.Processed * 80 / progress.Total // 前80%用于基本验证
|
||||
progress.UpdatedAt = time.Now().Unix()
|
||||
if err := s.saveFAQDryRunProgress(ctx, progress); err != nil {
|
||||
logger.Warnf(ctx, "Failed to update FAQ dry run progress: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return validIndices
|
||||
}
|
||||
|
||||
// saveFAQDryRunProgress 保存 FAQ dry run 进度到 Redis
|
||||
func (s *knowledgeService) saveFAQDryRunProgress(ctx context.Context, progress *types.FAQDryRunProgress) error {
|
||||
key := fmt.Sprintf("faq:dryrun:progress:%s", progress.TaskID)
|
||||
data, err := json.Marshal(progress)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal progress: %w", err)
|
||||
}
|
||||
// 设置 24 小时过期
|
||||
return s.redisClient.Set(ctx, key, data, 24*time.Hour).Err()
|
||||
}
|
||||
|
||||
// GetFAQDryRunProgress 获取 FAQ dry run 进度
|
||||
func (s *knowledgeService) GetFAQDryRunProgress(ctx context.Context, taskID string) (*types.FAQDryRunProgress, error) {
|
||||
key := fmt.Sprintf("faq:dryrun:progress:%s", taskID)
|
||||
data, err := s.redisClient.Get(ctx, key).Bytes()
|
||||
if err != nil {
|
||||
if err == redis.Nil {
|
||||
return nil, werrors.NewNotFoundError("任务不存在或已过期")
|
||||
}
|
||||
return nil, fmt.Errorf("failed to get progress: %w", err)
|
||||
}
|
||||
var progress types.FAQDryRunProgress
|
||||
if err := json.Unmarshal(data, &progress); err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal progress: %w", err)
|
||||
}
|
||||
return &progress, nil
|
||||
}
|
||||
|
||||
// validateFAQEntryPayloadBasic 验证 FAQ 条目的基本格式
|
||||
func validateFAQEntryPayloadBasic(entry *types.FAQEntryPayload) error {
|
||||
if entry == nil {
|
||||
return fmt.Errorf("条目不能为空")
|
||||
}
|
||||
standardQ := strings.TrimSpace(entry.StandardQuestion)
|
||||
if standardQ == "" {
|
||||
return fmt.Errorf("标准问不能为空")
|
||||
}
|
||||
if len(entry.Answers) == 0 {
|
||||
return fmt.Errorf("答案不能为空")
|
||||
}
|
||||
hasValidAnswer := false
|
||||
for _, a := range entry.Answers {
|
||||
if strings.TrimSpace(a) != "" {
|
||||
hasValidAnswer = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !hasValidAnswer {
|
||||
return fmt.Errorf("答案不能全为空")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// calculateAppendOperations 计算Append模式下需要处理的条目,跳过已存在且内容相同的条目
|
||||
// 同时过滤掉标准问或相似问与同批次或已有知识库中重复的条目
|
||||
func (s *knowledgeService) calculateAppendOperations(ctx context.Context,
|
||||
|
||||
@@ -391,12 +391,14 @@ func initFileService(cfg *config.Config) (interfaces.FileService, error) {
|
||||
os.Getenv("COS_PATH_PREFIX") == "" {
|
||||
return nil, fmt.Errorf("missing COS configuration")
|
||||
}
|
||||
return file.NewCosFileService(
|
||||
return file.NewCosFileServiceWithTempBucket(
|
||||
os.Getenv("COS_BUCKET_NAME"),
|
||||
os.Getenv("COS_REGION"),
|
||||
os.Getenv("COS_SECRET_ID"),
|
||||
os.Getenv("COS_SECRET_KEY"),
|
||||
os.Getenv("COS_PATH_PREFIX"),
|
||||
os.Getenv("COS_TEMP_BUCKET_NAME"), // 可选:临时桶名称(桶需配置生命周期规则自动过期)
|
||||
os.Getenv("COS_TEMP_REGION"), // 可选:临时桶 region,默认与主桶相同
|
||||
)
|
||||
case "local":
|
||||
return file.NewLocalFileService(os.Getenv("LOCAL_STORAGE_BASE_DIR")), nil
|
||||
|
||||
+52
-2
@@ -69,7 +69,9 @@ func (h *FAQHandler) ListEntries(c *gin.Context) {
|
||||
|
||||
// UpsertEntries godoc
|
||||
// @Summary 批量更新/插入FAQ条目
|
||||
// @Description 异步批量更新或插入FAQ条目
|
||||
// @Description 异步批量更新或插入FAQ条目。支持 dry_run 模式(设置 dry_run=true),异步验证不实际导入。
|
||||
// @Description dry_run 模式是异步操作,返回 task_id,通过 /faq/dryrun/progress/{task_id} 查询进度和结果。
|
||||
// @Description 验证内容包括:1) 条目基本格式 2) 重复问题(批次内和知识库已有) 3) 内容安全检查。
|
||||
// @Tags FAQ管理
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
@@ -89,7 +91,26 @@ func (h *FAQHandler) UpsertEntries(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
taskID, err := h.knowledgeService.UpsertFAQEntries(ctx, secutils.SanitizeForLog(c.Param("id")), &req)
|
||||
kbID := secutils.SanitizeForLog(c.Param("id"))
|
||||
|
||||
// dry_run 模式:异步验证,不实际导入
|
||||
if req.DryRun {
|
||||
taskID, err := h.knowledgeService.ValidateFAQEntriesDryRun(ctx, kbID, &req)
|
||||
if err != nil {
|
||||
logger.ErrorWithFields(ctx, err, nil)
|
||||
c.Error(err)
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"success": true,
|
||||
"data": gin.H{
|
||||
"task_id": taskID,
|
||||
},
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
taskID, err := h.knowledgeService.UpsertFAQEntries(ctx, kbID, &req)
|
||||
if err != nil {
|
||||
logger.ErrorWithFields(ctx, err, nil)
|
||||
c.Error(err)
|
||||
@@ -417,3 +438,32 @@ func (h *FAQHandler) GetImportProgress(c *gin.Context) {
|
||||
"data": progress,
|
||||
})
|
||||
}
|
||||
|
||||
// GetDryRunProgress godoc
|
||||
// @Summary 获取FAQ预检进度
|
||||
// @Description 获取FAQ导入预检任务的进度和结果
|
||||
// @Tags FAQ管理
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param task_id path string true "任务ID"
|
||||
// @Success 200 {object} map[string]interface{} "预检进度和结果"
|
||||
// @Failure 404 {object} errors.AppError "任务不存在"
|
||||
// @Security Bearer
|
||||
// @Security ApiKeyAuth
|
||||
// @Router /faq/dryrun/progress/{task_id} [get]
|
||||
func (h *FAQHandler) GetDryRunProgress(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
taskID := secutils.SanitizeForLog(c.Param("task_id"))
|
||||
|
||||
progress, err := h.knowledgeService.GetFAQDryRunProgress(ctx, taskID)
|
||||
if err != nil {
|
||||
logger.ErrorWithFields(ctx, err, nil)
|
||||
c.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"success": true,
|
||||
"data": progress,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -200,6 +200,11 @@ func RegisterFAQRoutes(r *gin.RouterGroup, handler *handler.FAQHandler) {
|
||||
{
|
||||
faqImport.GET("/progress/:task_id", handler.GetImportProgress)
|
||||
}
|
||||
// FAQ dry run progress route (outside of knowledge-base scope)
|
||||
faqDryRun := r.Group("/faq/dryrun")
|
||||
{
|
||||
faqDryRun.GET("/progress/:task_id", handler.GetDryRunProgress)
|
||||
}
|
||||
}
|
||||
|
||||
// RegisterKnowledgeBaseRoutes 注册知识库相关的路由
|
||||
|
||||
@@ -79,6 +79,9 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux {
|
||||
// Register FAQ import handler
|
||||
mux.HandleFunc(types.TypeFAQImport, params.KnowledgeService.ProcessFAQImport)
|
||||
|
||||
// Register FAQ dry run handler
|
||||
mux.HandleFunc(types.TypeFAQDryRun, params.KnowledgeService.ProcessFAQDryRun)
|
||||
|
||||
// Register question generation handler
|
||||
mux.HandleFunc(types.TypeQuestionGeneration, params.KnowledgeService.ProcessQuestionGeneration)
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ const (
|
||||
TypeChunkExtract = "chunk:extract"
|
||||
TypeDocumentProcess = "document:process" // 文档处理任务
|
||||
TypeFAQImport = "faq:import" // FAQ导入任务
|
||||
TypeFAQDryRun = "faq:dryrun" // FAQ导入预检任务
|
||||
TypeQuestionGeneration = "question:generation" // 问题生成任务
|
||||
TypeSummaryGeneration = "summary:generation" // 摘要生成任务
|
||||
TypeKBClone = "kb:clone" // 知识库复制任务
|
||||
@@ -45,6 +46,15 @@ type FAQImportPayload struct {
|
||||
Mode string `json:"mode"`
|
||||
}
|
||||
|
||||
// FAQDryRunPayload represents the FAQ dry run validation task payload
|
||||
type FAQDryRunPayload struct {
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
TaskID string `json:"task_id"`
|
||||
KBID string `json:"kb_id"`
|
||||
Entries []FAQEntryPayload `json:"entries"`
|
||||
Mode string `json:"mode"`
|
||||
}
|
||||
|
||||
// QuestionGenerationPayload represents the question generation task payload
|
||||
type QuestionGenerationPayload struct {
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
|
||||
@@ -219,6 +219,48 @@ type FAQBatchUpsertPayload struct {
|
||||
Entries []FAQEntryPayload `json:"entries" binding:"required"`
|
||||
Mode string `json:"mode" binding:"oneof=append replace"`
|
||||
KnowledgeID string `json:"knowledge_id"`
|
||||
TaskID string `json:"task_id"` // 可选,如果不传则自动生成UUID
|
||||
DryRun bool `json:"dry_run"` // 仅验证,不实际导入
|
||||
}
|
||||
|
||||
// FAQDryRunFailedEntry 表示 dry_run 模式下验证失败的条目
|
||||
type FAQDryRunFailedEntry struct {
|
||||
Index int `json:"index"` // 条目在批次中的索引(从0开始)
|
||||
Reason string `json:"reason"` // 失败原因
|
||||
TagName string `json:"tag_name,omitempty"` // 分类
|
||||
StandardQuestion string `json:"standard_question"` // 标准问题
|
||||
SimilarQuestions []string `json:"similar_questions,omitempty"` // 相似问题
|
||||
NegativeQuestions []string `json:"negative_questions,omitempty"` // 反例问题
|
||||
Answers []string `json:"answers,omitempty"` // 答案
|
||||
AnswerAll bool `json:"answer_all,omitempty"` // 是否全部回复
|
||||
IsDisabled bool `json:"is_disabled,omitempty"` // 是否停用
|
||||
}
|
||||
|
||||
// FAQDryRunResult 表示 dry_run 模式的验证结果
|
||||
type FAQDryRunResult struct {
|
||||
TaskID string `json:"task_id,omitempty"` // 异步任务ID(异步模式时返回)
|
||||
Total int `json:"total"` // 总条目数
|
||||
SuccessCount int `json:"success_count"` // 验证通过的条目数
|
||||
FailedCount int `json:"failed_count"` // 验证失败的条目数
|
||||
FailedEntries []FAQDryRunFailedEntry `json:"failed_entries"` // 失败条目详情
|
||||
}
|
||||
|
||||
// FAQDryRunProgress 表示 dry_run 异步任务的进度(存储在 Redis)
|
||||
type FAQDryRunProgress struct {
|
||||
TaskID string `json:"task_id"`
|
||||
KBID string `json:"kb_id"`
|
||||
Status FAQImportTaskStatus `json:"status"` // pending, processing, completed, failed
|
||||
Progress int `json:"progress"` // 0-100 percentage
|
||||
Total int `json:"total"` // 总条目数
|
||||
Processed int `json:"processed"` // 已处理条目数
|
||||
SuccessCount int `json:"success_count"` // 验证通过的条目数
|
||||
FailedCount int `json:"failed_count"` // 验证失败的条目数
|
||||
FailedEntries []FAQDryRunFailedEntry `json:"failed_entries,omitempty"` // 失败条目详情(少量时直接返回)
|
||||
FailedEntriesURL string `json:"failed_entries_url,omitempty"` // 失败条目CSV下载URL(大量时返回URL)
|
||||
Message string `json:"message"` // 状态消息
|
||||
Error string `json:"error,omitempty"` // 错误信息
|
||||
CreatedAt int64 `json:"created_at"`
|
||||
UpdatedAt int64 `json:"updated_at"`
|
||||
}
|
||||
|
||||
// FAQSearchRequest FAQ检索请求参数
|
||||
|
||||
@@ -11,8 +11,13 @@ import (
|
||||
type FileService interface {
|
||||
// SaveFile saves a file.
|
||||
SaveFile(ctx context.Context, file *multipart.FileHeader, tenantID uint64, knowledgeID string) (string, error)
|
||||
// SaveBytes saves bytes data to a file and returns the file path.
|
||||
// If temp is true, the file will be saved to a temporary storage that may auto-expire.
|
||||
SaveBytes(ctx context.Context, data []byte, tenantID uint64, fileName string, temp bool) (string, error)
|
||||
// GetFile retrieves a file.
|
||||
GetFile(ctx context.Context, filePath string) (io.ReadCloser, error)
|
||||
// GetFileURL returns a download URL for the file (if supported by the storage backend).
|
||||
GetFileURL(ctx context.Context, filePath string) (string, error)
|
||||
// DeleteFile deletes a file.
|
||||
DeleteFile(ctx context.Context, filePath string) error
|
||||
}
|
||||
|
||||
@@ -88,6 +88,13 @@ type KnowledgeService interface {
|
||||
// UpsertFAQEntries imports or appends FAQ entries asynchronously.
|
||||
// Returns task ID (Knowledge ID) for tracking import progress.
|
||||
UpsertFAQEntries(ctx context.Context, kbID string, payload *types.FAQBatchUpsertPayload) (string, error)
|
||||
// ValidateFAQEntriesDryRun validates FAQ entries asynchronously without actually importing them.
|
||||
// Returns task ID for tracking validation progress.
|
||||
ValidateFAQEntriesDryRun(ctx context.Context, kbID string, payload *types.FAQBatchUpsertPayload) (string, error)
|
||||
// GetFAQDryRunProgress retrieves the progress of an FAQ dry run validation task.
|
||||
GetFAQDryRunProgress(ctx context.Context, taskID string) (*types.FAQDryRunProgress, error)
|
||||
// ProcessFAQDryRun handles Asynq FAQ dry run validation tasks.
|
||||
ProcessFAQDryRun(ctx context.Context, t *asynq.Task) error
|
||||
// CreateFAQEntry creates a single FAQ entry synchronously.
|
||||
CreateFAQEntry(ctx context.Context, kbID string, payload *types.FAQEntryPayload) (*types.FAQEntry, error)
|
||||
// GetFAQEntry retrieves a single FAQ entry by ID.
|
||||
|
||||
Reference in New Issue
Block a user