feat: Add synchronous creation of single FAQ entry and duplicate question detection, update API documentation and frontend handling

This commit is contained in:
wizardchen
2025-12-02 21:14:24 +08:00
parent 845d990a48
commit f26cbe02e1
11 changed files with 447 additions and 11 deletions
+42
View File
@@ -251,3 +251,45 @@ func (r *chunkRepository) ListAllFAQChunksByKnowledgeID(
return allChunks, nil
}
// ListAllFAQChunksWithMetadataByKnowledgeBaseID lists all FAQ chunks for a knowledge base ID
// Returns ID and Metadata fields for duplicate question checking
// Uses batch query to handle large datasets
func (r *chunkRepository) ListAllFAQChunksWithMetadataByKnowledgeBaseID(
ctx context.Context,
tenantID uint64,
kbID string,
) ([]*types.Chunk, error) {
const batchSize = 1000 // 每批查询1000条
var allChunks []*types.Chunk
offset := 0
for {
var batchChunks []*types.Chunk
if err := r.db.WithContext(ctx).
Select("id, metadata").
Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ? AND status = ?",
tenantID, kbID, types.ChunkTypeFAQ, types.ChunkStatusIndexed).
Offset(offset).
Limit(batchSize).
Find(&batchChunks).Error; err != nil {
return nil, err
}
// 如果没有查询到数据,说明已经查询完毕
if len(batchChunks) == 0 {
break
}
allChunks = append(allChunks, batchChunks...)
// 如果返回的数据少于批次大小,说明已经是最后一批
if len(batchChunks) < batchSize {
break
}
offset += batchSize
}
return allChunks, nil
}
+271 -6
View File
@@ -2065,16 +2065,88 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context,
}
// calculateAppendOperations 计算Append模式下需要处理的条目,跳过已存在且内容相同的条目
// 同时过滤掉标准问或相似问与同批次或已有知识库中重复的条目
func (s *knowledgeService) calculateAppendOperations(ctx context.Context,
tenantID uint64, knowledgeID string, entries []types.FAQEntryPayload,
tenantID uint64, kbID string, entries []types.FAQEntryPayload,
) ([]types.FAQEntryPayload, int, error) {
if len(entries) == 0 {
return []types.FAQEntryPayload{}, 0, nil
}
return entries, 0, nil
// 1. 查询知识库中已有的所有FAQ chunks的metadata
existingChunks, err := s.chunkRepo.ListAllFAQChunksWithMetadataByKnowledgeBaseID(ctx, tenantID, kbID)
if err != nil {
return nil, 0, fmt.Errorf("failed to list existing FAQ chunks: %w", err)
}
// 2. 构建已存在的标准问和相似问集合
existingQuestions := make(map[string]bool)
for _, chunk := range existingChunks {
meta, err := chunk.FAQMetadata()
if err != nil || meta == nil {
continue
}
// 添加标准问
if meta.StandardQuestion != "" {
existingQuestions[meta.StandardQuestion] = true
}
// 添加相似问
for _, q := range meta.SimilarQuestions {
if q != "" {
existingQuestions[q] = true
}
}
}
// 3. 构建当前批次的标准问和相似问集合(用于批次内去重)
batchQuestions := make(map[string]bool)
entriesToProcess := make([]types.FAQEntryPayload, 0, len(entries))
skippedCount := 0
for _, entry := range entries {
meta, err := sanitizeFAQEntryPayload(&entry)
if err != nil {
// 跳过无效条目
skippedCount++
logger.Warnf(ctx, "Skipping invalid FAQ entry: %v", err)
continue
}
// 检查标准问是否重复(与已有或同批次)
if existingQuestions[meta.StandardQuestion] || batchQuestions[meta.StandardQuestion] {
skippedCount++
logger.Infof(ctx, "Skipping FAQ entry with duplicate standard question: %s", meta.StandardQuestion)
continue
}
// 检查相似问是否有重复(与已有或同批次)
hasDuplicateSimilar := false
for _, q := range meta.SimilarQuestions {
if existingQuestions[q] || batchQuestions[q] {
hasDuplicateSimilar = true
logger.Infof(ctx, "Skipping FAQ entry with duplicate similar question: %s (standard: %s)", q, meta.StandardQuestion)
break
}
}
if hasDuplicateSimilar {
skippedCount++
continue
}
// 将当前条目的标准问和相似问加入批次集合
batchQuestions[meta.StandardQuestion] = true
for _, q := range meta.SimilarQuestions {
batchQuestions[q] = true
}
entriesToProcess = append(entriesToProcess, entry)
}
return entriesToProcess, skippedCount, nil
}
// calculateReplaceOperations 计算Replace模式下需要删除、创建、更新的条目
// 同时过滤掉同批次内标准问或相似问重复的条目
func (s *knowledgeService) calculateReplaceOperations(ctx context.Context,
tenantID uint64, knowledgeID string, newEntries []types.FAQEntryPayload,
) ([]types.FAQEntryPayload, []*types.Chunk, int, error) {
@@ -2082,18 +2154,52 @@ func (s *knowledgeService) calculateReplaceOperations(ctx context.Context,
type entryWithHash struct {
entry types.FAQEntryPayload
hash string
meta *types.FAQChunkMetadata
}
entriesWithHash := make([]entryWithHash, 0, len(newEntries))
newHashSet := make(map[string]bool)
// 用于批次内标准问和相似问去重
batchQuestions := make(map[string]bool)
batchSkippedCount := 0
for _, entry := range newEntries {
meta, err := sanitizeFAQEntryPayload(&entry)
if err != nil {
return nil, nil, 0, fmt.Errorf("failed to sanitize entry: %w", err)
batchSkippedCount++
logger.Warnf(ctx, "Skipping invalid FAQ entry in replace mode: %v", err)
continue
}
// 检查标准问是否在同批次中重复
if batchQuestions[meta.StandardQuestion] {
batchSkippedCount++
logger.Infof(ctx, "Skipping FAQ entry with duplicate standard question in batch: %s", meta.StandardQuestion)
continue
}
// 检查相似问是否在同批次中重复
hasDuplicateSimilar := false
for _, q := range meta.SimilarQuestions {
if batchQuestions[q] {
hasDuplicateSimilar = true
logger.Infof(ctx, "Skipping FAQ entry with duplicate similar question in batch: %s (standard: %s)", q, meta.StandardQuestion)
break
}
}
if hasDuplicateSimilar {
batchSkippedCount++
continue
}
// 将当前条目的标准问和相似问加入批次集合
batchQuestions[meta.StandardQuestion] = true
for _, q := range meta.SimilarQuestions {
batchQuestions[q] = true
}
hash := types.CalculateFAQContentHash(meta)
if hash != "" {
entriesWithHash = append(entriesWithHash, entryWithHash{entry: entry, hash: hash})
entriesWithHash = append(entriesWithHash, entryWithHash{entry: entry, hash: hash, meta: meta})
newHashSet[hash] = true
}
}
@@ -2126,7 +2232,7 @@ func (s *knowledgeService) calculateReplaceOperations(ctx context.Context,
// 计算需要创建的条目(利用已经计算好的hash,避免重复计算)
entriesToProcess := make([]types.FAQEntryPayload, 0, len(entriesWithHash))
skippedCount := 0
skippedCount := batchSkippedCount
for _, ewh := range entriesWithHash {
if existingHashMap[ewh.hash] != nil {
@@ -2220,7 +2326,7 @@ func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string,
}
} else {
// Append模式:查询已存在的条目,跳过未变化的
entriesToProcess, skippedCount, err = s.calculateAppendOperations(ctx, tenantID, faqKnowledge.ID, payload.Entries)
entriesToProcess, skippedCount, err = s.calculateAppendOperations(ctx, tenantID, kb.ID, payload.Entries)
if err != nil {
return fmt.Errorf("failed to calculate append operations: %w", err)
}
@@ -2385,6 +2491,100 @@ func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string,
return nil
}
// CreateFAQEntry creates a single FAQ entry synchronously.
func (s *knowledgeService) CreateFAQEntry(ctx context.Context,
kbID string, payload *types.FAQEntryPayload,
) (*types.FAQEntry, error) {
if payload == nil {
return nil, werrors.NewBadRequestError("请求体不能为空")
}
kb, err := s.validateFAQKnowledgeBase(ctx, kbID)
if err != nil {
return nil, err
}
kb.EnsureDefaults()
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
// 验证并清理输入
meta, err := sanitizeFAQEntryPayload(payload)
if err != nil {
return nil, err
}
// 检查标准问和相似问是否与其他条目重复
if err := s.checkFAQQuestionDuplicate(ctx, tenantID, kb.ID, "", meta); err != nil {
return nil, err
}
// 确保FAQ Knowledge存在
faqKnowledge, err := s.ensureFAQKnowledge(ctx, tenantID, kb)
if err != nil {
return nil, fmt.Errorf("failed to ensure FAQ knowledge: %w", err)
}
// 获取索引模式
indexMode := types.FAQIndexModeQuestionOnly
if kb.FAQConfig != nil && kb.FAQConfig.IndexMode != "" {
indexMode = kb.FAQConfig.IndexMode
}
// 获取embedding模型
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID)
if err != nil {
return nil, fmt.Errorf("failed to get embedding model: %w", err)
}
// 创建chunk
isEnabled := true
if payload.IsEnabled != nil {
isEnabled = *payload.IsEnabled
}
chunk := &types.Chunk{
ID: uuid.New().String(),
TenantID: tenantID,
KnowledgeID: faqKnowledge.ID,
KnowledgeBaseID: kb.ID,
Content: buildFAQChunkContent(meta, indexMode),
IsEnabled: isEnabled,
ChunkType: types.ChunkTypeFAQ,
TagID: payload.TagID,
Status: int(types.ChunkStatusStored),
}
if err := chunk.SetFAQMetadata(meta); err != nil {
return nil, fmt.Errorf("failed to set FAQ metadata: %w", err)
}
// 保存chunk
if err := s.chunkService.CreateChunks(ctx, []*types.Chunk{chunk}); err != nil {
return nil, fmt.Errorf("failed to create chunk: %w", err)
}
// 索引chunk
if err := s.indexFAQChunks(ctx, kb, faqKnowledge, []*types.Chunk{chunk}, embeddingModel, true, false); err != nil {
// 如果索引失败,删除已创建的chunk
_ = s.chunkService.DeleteChunk(ctx, chunk.ID)
return nil, fmt.Errorf("failed to index chunk: %w", err)
}
// 更新chunk状态为已索引
chunk.Status = int(types.ChunkStatusIndexed)
if err := s.chunkService.UpdateChunk(ctx, chunk); err != nil {
return nil, fmt.Errorf("failed to update chunk status: %w", err)
}
// 转换为FAQEntry返回
entry, err := s.chunkToFAQEntry(chunk, kb)
if err != nil {
return nil, err
}
return entry, nil
}
// UpdateFAQEntry updates a single FAQ entry.
func (s *knowledgeService) UpdateFAQEntry(ctx context.Context,
kbID string, entryID string, payload *types.FAQEntryPayload,
@@ -2412,6 +2612,12 @@ func (s *knowledgeService) UpdateFAQEntry(ctx context.Context,
if err != nil {
return err
}
// 检查标准问和相似问是否与其他条目重复
if err := s.checkFAQQuestionDuplicate(ctx, tenantID, kb.ID, entryID, meta); err != nil {
return err
}
if existing, err := chunk.FAQMetadata(); err == nil && existing != nil {
meta.Version = existing.Version + 1
}
@@ -3095,6 +3301,65 @@ func buildFAQChunkContent(meta *types.FAQChunkMetadata, mode types.FAQIndexMode)
return builder.String()
}
// checkFAQQuestionDuplicate 检查标准问和相似问是否与知识库中其他条目重复
// excludeChunkID 用于排除当前正在编辑的条目(更新时使用)
func (s *knowledgeService) checkFAQQuestionDuplicate(
ctx context.Context,
tenantID uint64,
kbID string,
excludeChunkID string,
meta *types.FAQChunkMetadata,
) error {
// 查询知识库中已有的所有FAQ chunks的metadata
existingChunks, err := s.chunkRepo.ListAllFAQChunksWithMetadataByKnowledgeBaseID(ctx, tenantID, kbID)
if err != nil {
return fmt.Errorf("failed to list existing FAQ chunks: %w", err)
}
// 构建已存在的标准问和相似问集合
for _, chunk := range existingChunks {
// 排除当前正在编辑的条目
if chunk.ID == excludeChunkID {
continue
}
existingMeta, err := chunk.FAQMetadata()
if err != nil || existingMeta == nil {
continue
}
// 检查标准问是否重复
if existingMeta.StandardQuestion == meta.StandardQuestion {
return werrors.NewBadRequestError(fmt.Sprintf("标准问「%s」已存在", meta.StandardQuestion))
}
// 检查当前标准问是否与已有相似问重复
for _, q := range existingMeta.SimilarQuestions {
if q == meta.StandardQuestion {
return werrors.NewBadRequestError(fmt.Sprintf("标准问「%s」与已有相似问重复", meta.StandardQuestion))
}
}
// 检查当前相似问是否与已有标准问重复
for _, q := range meta.SimilarQuestions {
if q == existingMeta.StandardQuestion {
return werrors.NewBadRequestError(fmt.Sprintf("相似问「%s」与已有标准问重复", q))
}
}
// 检查当前相似问是否与已有相似问重复
for _, q := range meta.SimilarQuestions {
for _, existingQ := range existingMeta.SimilarQuestions {
if q == existingQ {
return werrors.NewBadRequestError(fmt.Sprintf("相似问「%s」已存在", q))
}
}
}
}
return nil
}
func sanitizeFAQEntryPayload(payload *types.FAQEntryPayload) (*types.FAQChunkMetadata, error) {
meta := &types.FAQChunkMetadata{
StandardQuestion: strings.TrimSpace(payload.StandardQuestion),