feat(chunk): add FindFAQChunkWithDuplicateQuestion method for enhanced duplicate question checking
- Implemented a new method in the chunk repository to find FAQ chunks with overlapping standard or similar questions. - Updated the knowledge service to utilize this method for checking duplicates during FAQ question creation. - Improved error handling for duplicate questions, providing clearer feedback on conflicts.
This commit is contained in:
@@ -2,6 +2,7 @@ package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
@@ -602,6 +603,66 @@ func (r *chunkRepository) ListAllFAQChunksWithMetadataByKnowledgeBaseID(
|
||||
return allChunks, nil
|
||||
}
|
||||
|
||||
// FindFAQChunkWithDuplicateQuestion finds a single FAQ chunk whose standard_question or
|
||||
// similar_questions overlap with the given question list.
|
||||
// Uses dialect-specific JSON queries (MySQL / PostgreSQL / SQLite).
|
||||
func (r *chunkRepository) FindFAQChunkWithDuplicateQuestion(
|
||||
ctx context.Context,
|
||||
tenantID uint64,
|
||||
kbID string,
|
||||
excludeChunkID string,
|
||||
questions []string,
|
||||
) (*types.Chunk, error) {
|
||||
if len(questions) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
db := r.db.WithContext(ctx).
|
||||
Select("id, metadata").
|
||||
Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ? AND status = ? AND id != ?",
|
||||
tenantID, kbID, types.ChunkTypeFAQ, types.ChunkStatusIndexed, excludeChunkID)
|
||||
|
||||
switch r.db.Name() {
|
||||
case "mysql":
|
||||
// MySQL 5.7+: JSON_EXTRACT for standard_question, JSON_CONTAINS for similar_questions
|
||||
parts := []string{
|
||||
"JSON_UNQUOTE(JSON_EXTRACT(metadata, '$.standard_question')) IN ?",
|
||||
}
|
||||
args := []interface{}{questions}
|
||||
for _, q := range questions {
|
||||
parts = append(parts,
|
||||
"JSON_CONTAINS(metadata, ?, '$.similar_questions')")
|
||||
jsonVal, _ := json.Marshal(q)
|
||||
args = append(args, string(jsonVal))
|
||||
}
|
||||
db = db.Where(strings.Join(parts, " OR "), args...)
|
||||
case "postgres":
|
||||
db = db.Where(
|
||||
"(metadata->>'standard_question' IN ? OR EXISTS ("+
|
||||
"SELECT 1 FROM jsonb_array_elements_text("+
|
||||
"COALESCE(metadata->'similar_questions', '[]'::jsonb)) elem "+
|
||||
"WHERE elem.value IN ?))",
|
||||
questions, questions)
|
||||
default: // sqlite
|
||||
db = db.Where(
|
||||
"(json_extract(metadata, '$.standard_question') IN ? OR EXISTS ("+
|
||||
"SELECT 1 FROM json_each("+
|
||||
"CASE WHEN json_extract(metadata, '$.similar_questions') IS NOT NULL "+
|
||||
"THEN json_extract(metadata, '$.similar_questions') ELSE '[]' END) "+
|
||||
"WHERE value IN ?))",
|
||||
questions, questions)
|
||||
}
|
||||
|
||||
var chunk types.Chunk
|
||||
if err := db.Limit(1).Find(&chunk).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if chunk.ID == "" {
|
||||
return nil, nil
|
||||
}
|
||||
return &chunk, nil
|
||||
}
|
||||
|
||||
// ListAllFAQChunksForExport lists all FAQ chunks for export with full metadata, tag_id, is_enabled, and flags.
|
||||
// Uses batch query to handle large datasets.
|
||||
func (r *chunkRepository) ListAllFAQChunksForExport(
|
||||
|
||||
@@ -6238,54 +6238,48 @@ func (s *knowledgeService) checkFAQQuestionDuplicate(
|
||||
seen[q] = struct{}{}
|
||||
}
|
||||
|
||||
// 查询知识库中已有的所有FAQ chunks的metadata
|
||||
existingChunks, err := s.chunkRepo.ListAllFAQChunksWithMetadataByKnowledgeBaseID(ctx, tenantID, kbID)
|
||||
// 将标准问和所有相似问合并,用一条 DB 查询检查是否与其他条目冲突
|
||||
allQuestions := make([]string, 0, 1+len(meta.SimilarQuestions))
|
||||
allQuestions = append(allQuestions, meta.StandardQuestion)
|
||||
allQuestions = append(allQuestions, meta.SimilarQuestions...)
|
||||
|
||||
dupChunk, err := s.chunkRepo.FindFAQChunkWithDuplicateQuestion(ctx, tenantID, kbID, excludeChunkID, allQuestions)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to list existing FAQ chunks: %w", err)
|
||||
return fmt.Errorf("failed to check FAQ question duplicate: %w", err)
|
||||
}
|
||||
if dupChunk == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 构建已存在的标准问和相似问集合
|
||||
for _, chunk := range existingChunks {
|
||||
// 排除当前正在编辑的条目
|
||||
if chunk.ID == excludeChunkID {
|
||||
continue
|
||||
}
|
||||
// 找到冲突条目,解析其 metadata 以生成精确的错误信息
|
||||
existingMeta, err := dupChunk.FAQMetadata()
|
||||
if err != nil || existingMeta == nil {
|
||||
return werrors.NewBadRequestError("标准问或相似问与已有条目重复")
|
||||
}
|
||||
|
||||
existingMeta, err := chunk.FAQMetadata()
|
||||
if err != nil || existingMeta == nil {
|
||||
continue
|
||||
}
|
||||
if existingMeta.StandardQuestion == meta.StandardQuestion {
|
||||
return werrors.NewBadRequestError(fmt.Sprintf("标准问「%s」已存在", meta.StandardQuestion))
|
||||
}
|
||||
|
||||
// 检查标准问是否重复
|
||||
if existingMeta.StandardQuestion == meta.StandardQuestion {
|
||||
return werrors.NewBadRequestError(fmt.Sprintf("标准问「%s」已存在", meta.StandardQuestion))
|
||||
}
|
||||
existingSimilarSet := make(map[string]struct{}, len(existingMeta.SimilarQuestions))
|
||||
for _, q := range existingMeta.SimilarQuestions {
|
||||
existingSimilarSet[q] = struct{}{}
|
||||
}
|
||||
|
||||
// 检查当前标准问是否与已有相似问重复
|
||||
for _, q := range existingMeta.SimilarQuestions {
|
||||
if q == meta.StandardQuestion {
|
||||
return werrors.NewBadRequestError(fmt.Sprintf("标准问「%s」与已有相似问重复", meta.StandardQuestion))
|
||||
}
|
||||
}
|
||||
if _, ok := existingSimilarSet[meta.StandardQuestion]; ok {
|
||||
return werrors.NewBadRequestError(fmt.Sprintf("标准问「%s」与已有相似问重复", meta.StandardQuestion))
|
||||
}
|
||||
|
||||
// 检查当前相似问是否与已有标准问重复
|
||||
for _, q := range meta.SimilarQuestions {
|
||||
if q == existingMeta.StandardQuestion {
|
||||
return werrors.NewBadRequestError(fmt.Sprintf("相似问「%s」与已有标准问重复", q))
|
||||
}
|
||||
for _, q := range meta.SimilarQuestions {
|
||||
if q == existingMeta.StandardQuestion {
|
||||
return werrors.NewBadRequestError(fmt.Sprintf("相似问「%s」与已有标准问重复", q))
|
||||
}
|
||||
|
||||
// 检查当前相似问是否与已有相似问重复
|
||||
for _, q := range meta.SimilarQuestions {
|
||||
for _, existingQ := range existingMeta.SimilarQuestions {
|
||||
if q == existingQ {
|
||||
return werrors.NewBadRequestError(fmt.Sprintf("相似问「%s」已存在", q))
|
||||
}
|
||||
}
|
||||
if _, ok := existingSimilarSet[q]; ok {
|
||||
return werrors.NewBadRequestError(fmt.Sprintf("相似问「%s」已存在", q))
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
return werrors.NewBadRequestError("标准问或相似问与已有条目重复")
|
||||
}
|
||||
|
||||
// resolveTagID resolves tag ID (UUID) from payload, prioritizing tag_id (seq_id) over tag_name
|
||||
|
||||
@@ -73,6 +73,9 @@ type ChunkRepository interface {
|
||||
// ListAllFAQChunksWithMetadataByKnowledgeBaseID lists all FAQ chunks for a knowledge base ID
|
||||
// returns ID and Metadata fields for duplicate question checking
|
||||
ListAllFAQChunksWithMetadataByKnowledgeBaseID(ctx context.Context, tenantID uint64, kbID string) ([]*types.Chunk, error)
|
||||
// FindFAQChunkWithDuplicateQuestion finds a single FAQ chunk whose standard_question or
|
||||
// similar_questions overlap with the given question list. Returns nil if no duplicate found.
|
||||
FindFAQChunkWithDuplicateQuestion(ctx context.Context, tenantID uint64, kbID string, excludeChunkID string, questions []string) (*types.Chunk, error)
|
||||
// ListAllFAQChunksForExport lists all FAQ chunks for export with full metadata, tag_id, is_enabled, and flags
|
||||
ListAllFAQChunksForExport(ctx context.Context, tenantID uint64, knowledgeID string) ([]*types.Chunk, error)
|
||||
// UpdateChunkFlagsBatch updates flags for multiple chunks in batch using a single SQL statement.
|
||||
|
||||
Reference in New Issue
Block a user