feat(chunk): add FindFAQChunkWithDuplicateQuestion method for enhanced duplicate question checking

- Implemented a new method in the chunk repository to find FAQ chunks with overlapping standard or similar questions.
- Updated the knowledge service to utilize this method for checking duplicates during FAQ question creation.
- Improved error handling for duplicate questions, providing clearer feedback on conflicts.
This commit is contained in:
wizardchen
2026-03-20 16:56:18 +08:00
committed by lyingbug
parent 3a8bd36d8a
commit 9650188a35
3 changed files with 95 additions and 37 deletions
+61
View File
@@ -2,6 +2,7 @@ package repository
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
@@ -602,6 +603,66 @@ func (r *chunkRepository) ListAllFAQChunksWithMetadataByKnowledgeBaseID(
return allChunks, nil
}
// FindFAQChunkWithDuplicateQuestion finds a single FAQ chunk whose standard_question or
// similar_questions overlap with the given question list.
// Uses dialect-specific JSON queries (MySQL / PostgreSQL / SQLite).
func (r *chunkRepository) FindFAQChunkWithDuplicateQuestion(
ctx context.Context,
tenantID uint64,
kbID string,
excludeChunkID string,
questions []string,
) (*types.Chunk, error) {
if len(questions) == 0 {
return nil, nil
}
db := r.db.WithContext(ctx).
Select("id, metadata").
Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ? AND status = ? AND id != ?",
tenantID, kbID, types.ChunkTypeFAQ, types.ChunkStatusIndexed, excludeChunkID)
switch r.db.Name() {
case "mysql":
// MySQL 5.7+: JSON_EXTRACT for standard_question, JSON_CONTAINS for similar_questions
parts := []string{
"JSON_UNQUOTE(JSON_EXTRACT(metadata, '$.standard_question')) IN ?",
}
args := []interface{}{questions}
for _, q := range questions {
parts = append(parts,
"JSON_CONTAINS(metadata, ?, '$.similar_questions')")
jsonVal, _ := json.Marshal(q)
args = append(args, string(jsonVal))
}
db = db.Where(strings.Join(parts, " OR "), args...)
case "postgres":
db = db.Where(
"(metadata->>'standard_question' IN ? OR EXISTS ("+
"SELECT 1 FROM jsonb_array_elements_text("+
"COALESCE(metadata->'similar_questions', '[]'::jsonb)) elem "+
"WHERE elem.value IN ?))",
questions, questions)
default: // sqlite
db = db.Where(
"(json_extract(metadata, '$.standard_question') IN ? OR EXISTS ("+
"SELECT 1 FROM json_each("+
"CASE WHEN json_extract(metadata, '$.similar_questions') IS NOT NULL "+
"THEN json_extract(metadata, '$.similar_questions') ELSE '[]' END) "+
"WHERE value IN ?))",
questions, questions)
}
var chunk types.Chunk
if err := db.Limit(1).Find(&chunk).Error; err != nil {
return nil, err
}
if chunk.ID == "" {
return nil, nil
}
return &chunk, nil
}
// ListAllFAQChunksForExport lists all FAQ chunks for export with full metadata, tag_id, is_enabled, and flags.
// Uses batch query to handle large datasets.
func (r *chunkRepository) ListAllFAQChunksForExport(
+31 -37
View File
@@ -6238,54 +6238,48 @@ func (s *knowledgeService) checkFAQQuestionDuplicate(
seen[q] = struct{}{}
}
// 查询知识库中已有的所有FAQ chunks的metadata
existingChunks, err := s.chunkRepo.ListAllFAQChunksWithMetadataByKnowledgeBaseID(ctx, tenantID, kbID)
// 将标准问和所有相似问合并,用一条 DB 查询检查是否与其他条目冲突
allQuestions := make([]string, 0, 1+len(meta.SimilarQuestions))
allQuestions = append(allQuestions, meta.StandardQuestion)
allQuestions = append(allQuestions, meta.SimilarQuestions...)
dupChunk, err := s.chunkRepo.FindFAQChunkWithDuplicateQuestion(ctx, tenantID, kbID, excludeChunkID, allQuestions)
if err != nil {
return fmt.Errorf("failed to list existing FAQ chunks: %w", err)
return fmt.Errorf("failed to check FAQ question duplicate: %w", err)
}
if dupChunk == nil {
return nil
}
// 构建已存在的标准问和相似问集合
for _, chunk := range existingChunks {
// 排除当前正在编辑的条目
if chunk.ID == excludeChunkID {
continue
}
// 找到冲突条目,解析其 metadata 以生成精确的错误信息
existingMeta, err := dupChunk.FAQMetadata()
if err != nil || existingMeta == nil {
return werrors.NewBadRequestError("标准问或相似问与已有条目重复")
}
existingMeta, err := chunk.FAQMetadata()
if err != nil || existingMeta == nil {
continue
}
if existingMeta.StandardQuestion == meta.StandardQuestion {
return werrors.NewBadRequestError(fmt.Sprintf("标准问「%s」已存在", meta.StandardQuestion))
}
// 检查标准问是否重复
if existingMeta.StandardQuestion == meta.StandardQuestion {
return werrors.NewBadRequestError(fmt.Sprintf("标准问「%s」已存在", meta.StandardQuestion))
}
existingSimilarSet := make(map[string]struct{}, len(existingMeta.SimilarQuestions))
for _, q := range existingMeta.SimilarQuestions {
existingSimilarSet[q] = struct{}{}
}
// 检查当前标准问是否与已有相似问重复
for _, q := range existingMeta.SimilarQuestions {
if q == meta.StandardQuestion {
return werrors.NewBadRequestError(fmt.Sprintf("标准问「%s」与已有相似问重复", meta.StandardQuestion))
}
}
if _, ok := existingSimilarSet[meta.StandardQuestion]; ok {
return werrors.NewBadRequestError(fmt.Sprintf("标准问「%s」与已有相似问重复", meta.StandardQuestion))
}
// 检查当前相似问是否与已有标准问重复
for _, q := range meta.SimilarQuestions {
if q == existingMeta.StandardQuestion {
return werrors.NewBadRequestError(fmt.Sprintf("相似问「%s」与已有标准问重复", q))
}
for _, q := range meta.SimilarQuestions {
if q == existingMeta.StandardQuestion {
return werrors.NewBadRequestError(fmt.Sprintf("相似问「%s」与已有标准问重复", q))
}
// 检查当前相似问是否与已有相似问重复
for _, q := range meta.SimilarQuestions {
for _, existingQ := range existingMeta.SimilarQuestions {
if q == existingQ {
return werrors.NewBadRequestError(fmt.Sprintf("相似问「%s」已存在", q))
}
}
if _, ok := existingSimilarSet[q]; ok {
return werrors.NewBadRequestError(fmt.Sprintf("相似问「%s」已存在", q))
}
}
return nil
return werrors.NewBadRequestError("标准问或相似问与已有条目重复")
}
// resolveTagID resolves tag ID (UUID) from payload, prioritizing tag_id (seq_id) over tag_name
+3
View File
@@ -73,6 +73,9 @@ type ChunkRepository interface {
// ListAllFAQChunksWithMetadataByKnowledgeBaseID lists all FAQ chunks for a knowledge base ID
// returns ID and Metadata fields for duplicate question checking
ListAllFAQChunksWithMetadataByKnowledgeBaseID(ctx context.Context, tenantID uint64, kbID string) ([]*types.Chunk, error)
// FindFAQChunkWithDuplicateQuestion finds a single FAQ chunk whose standard_question or
// similar_questions overlap with the given question list. Returns nil if no duplicate found.
FindFAQChunkWithDuplicateQuestion(ctx context.Context, tenantID uint64, kbID string, excludeChunkID string, questions []string) (*types.Chunk, error)
// ListAllFAQChunksForExport lists all FAQ chunks for export with full metadata, tag_id, is_enabled, and flags
ListAllFAQChunksForExport(ctx context.Context, tenantID uint64, knowledgeID string) ([]*types.Chunk, error)
// UpdateChunkFlagsBatch updates flags for multiple chunks in batch using a single SQL statement.