From 9650188a35af7e84fb6d755e2f5e2a07f9e6996d Mon Sep 17 00:00:00 2001 From: wizardchen Date: Fri, 20 Mar 2026 16:56:18 +0800 Subject: [PATCH] feat(chunk): add FindFAQChunkWithDuplicateQuestion method for enhanced duplicate question checking - Implemented a new method in the chunk repository to find FAQ chunks with overlapping standard or similar questions. - Updated the knowledge service to utilize this method for checking duplicates during FAQ question creation. - Improved error handling for duplicate questions, providing clearer feedback on conflicts. --- internal/application/repository/chunk.go | 61 ++++++++++++++++++++ internal/application/service/knowledge.go | 68 +++++++++++------------ internal/types/interfaces/chunk.go | 3 + 3 files changed, 95 insertions(+), 37 deletions(-) diff --git a/internal/application/repository/chunk.go b/internal/application/repository/chunk.go index 61789a48..cb375e16 100644 --- a/internal/application/repository/chunk.go +++ b/internal/application/repository/chunk.go @@ -2,6 +2,7 @@ package repository import ( "context" + "encoding/json" "errors" "fmt" "strings" @@ -602,6 +603,66 @@ func (r *chunkRepository) ListAllFAQChunksWithMetadataByKnowledgeBaseID( return allChunks, nil } +// FindFAQChunkWithDuplicateQuestion finds a single FAQ chunk whose standard_question or +// similar_questions overlap with the given question list. +// Uses dialect-specific JSON queries (MySQL / PostgreSQL / SQLite). +func (r *chunkRepository) FindFAQChunkWithDuplicateQuestion( + ctx context.Context, + tenantID uint64, + kbID string, + excludeChunkID string, + questions []string, +) (*types.Chunk, error) { + if len(questions) == 0 { + return nil, nil + } + + db := r.db.WithContext(ctx). + Select("id, metadata"). + Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ? AND status = ? AND id != ?", + tenantID, kbID, types.ChunkTypeFAQ, types.ChunkStatusIndexed, excludeChunkID) + + switch r.db.Name() { + case "mysql": + // MySQL 5.7+: JSON_EXTRACT for standard_question, JSON_CONTAINS for similar_questions + parts := []string{ + "JSON_UNQUOTE(JSON_EXTRACT(metadata, '$.standard_question')) IN ?", + } + args := []interface{}{questions} + for _, q := range questions { + parts = append(parts, + "JSON_CONTAINS(metadata, ?, '$.similar_questions')") + jsonVal, _ := json.Marshal(q) + args = append(args, string(jsonVal)) + } + db = db.Where(strings.Join(parts, " OR "), args...) + case "postgres": + db = db.Where( + "(metadata->>'standard_question' IN ? OR EXISTS ("+ + "SELECT 1 FROM jsonb_array_elements_text("+ + "COALESCE(metadata->'similar_questions', '[]'::jsonb)) elem "+ + "WHERE elem.value IN ?))", + questions, questions) + default: // sqlite + db = db.Where( + "(json_extract(metadata, '$.standard_question') IN ? OR EXISTS ("+ + "SELECT 1 FROM json_each("+ + "CASE WHEN json_extract(metadata, '$.similar_questions') IS NOT NULL "+ + "THEN json_extract(metadata, '$.similar_questions') ELSE '[]' END) "+ + "WHERE value IN ?))", + questions, questions) + } + + var chunk types.Chunk + if err := db.Limit(1).Find(&chunk).Error; err != nil { + return nil, err + } + if chunk.ID == "" { + return nil, nil + } + return &chunk, nil +} + // ListAllFAQChunksForExport lists all FAQ chunks for export with full metadata, tag_id, is_enabled, and flags. // Uses batch query to handle large datasets. func (r *chunkRepository) ListAllFAQChunksForExport( diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index 5e9fb730..6dfb9b39 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -6238,54 +6238,48 @@ func (s *knowledgeService) checkFAQQuestionDuplicate( seen[q] = struct{}{} } - // 查询知识库中已有的所有FAQ chunks的metadata - existingChunks, err := s.chunkRepo.ListAllFAQChunksWithMetadataByKnowledgeBaseID(ctx, tenantID, kbID) + // 将标准问和所有相似问合并,用一条 DB 查询检查是否与其他条目冲突 + allQuestions := make([]string, 0, 1+len(meta.SimilarQuestions)) + allQuestions = append(allQuestions, meta.StandardQuestion) + allQuestions = append(allQuestions, meta.SimilarQuestions...) + + dupChunk, err := s.chunkRepo.FindFAQChunkWithDuplicateQuestion(ctx, tenantID, kbID, excludeChunkID, allQuestions) if err != nil { - return fmt.Errorf("failed to list existing FAQ chunks: %w", err) + return fmt.Errorf("failed to check FAQ question duplicate: %w", err) + } + if dupChunk == nil { + return nil } - // 构建已存在的标准问和相似问集合 - for _, chunk := range existingChunks { - // 排除当前正在编辑的条目 - if chunk.ID == excludeChunkID { - continue - } + // 找到冲突条目,解析其 metadata 以生成精确的错误信息 + existingMeta, err := dupChunk.FAQMetadata() + if err != nil || existingMeta == nil { + return werrors.NewBadRequestError("标准问或相似问与已有条目重复") + } - existingMeta, err := chunk.FAQMetadata() - if err != nil || existingMeta == nil { - continue - } + if existingMeta.StandardQuestion == meta.StandardQuestion { + return werrors.NewBadRequestError(fmt.Sprintf("标准问「%s」已存在", meta.StandardQuestion)) + } - // 检查标准问是否重复 - if existingMeta.StandardQuestion == meta.StandardQuestion { - return werrors.NewBadRequestError(fmt.Sprintf("标准问「%s」已存在", meta.StandardQuestion)) - } + existingSimilarSet := make(map[string]struct{}, len(existingMeta.SimilarQuestions)) + for _, q := range existingMeta.SimilarQuestions { + existingSimilarSet[q] = struct{}{} + } - // 检查当前标准问是否与已有相似问重复 - for _, q := range existingMeta.SimilarQuestions { - if q == meta.StandardQuestion { - return werrors.NewBadRequestError(fmt.Sprintf("标准问「%s」与已有相似问重复", meta.StandardQuestion)) - } - } + if _, ok := existingSimilarSet[meta.StandardQuestion]; ok { + return werrors.NewBadRequestError(fmt.Sprintf("标准问「%s」与已有相似问重复", meta.StandardQuestion)) + } - // 检查当前相似问是否与已有标准问重复 - for _, q := range meta.SimilarQuestions { - if q == existingMeta.StandardQuestion { - return werrors.NewBadRequestError(fmt.Sprintf("相似问「%s」与已有标准问重复", q)) - } + for _, q := range meta.SimilarQuestions { + if q == existingMeta.StandardQuestion { + return werrors.NewBadRequestError(fmt.Sprintf("相似问「%s」与已有标准问重复", q)) } - - // 检查当前相似问是否与已有相似问重复 - for _, q := range meta.SimilarQuestions { - for _, existingQ := range existingMeta.SimilarQuestions { - if q == existingQ { - return werrors.NewBadRequestError(fmt.Sprintf("相似问「%s」已存在", q)) - } - } + if _, ok := existingSimilarSet[q]; ok { + return werrors.NewBadRequestError(fmt.Sprintf("相似问「%s」已存在", q)) } } - return nil + return werrors.NewBadRequestError("标准问或相似问与已有条目重复") } // resolveTagID resolves tag ID (UUID) from payload, prioritizing tag_id (seq_id) over tag_name diff --git a/internal/types/interfaces/chunk.go b/internal/types/interfaces/chunk.go index 74940d2e..7dce2f6e 100644 --- a/internal/types/interfaces/chunk.go +++ b/internal/types/interfaces/chunk.go @@ -73,6 +73,9 @@ type ChunkRepository interface { // ListAllFAQChunksWithMetadataByKnowledgeBaseID lists all FAQ chunks for a knowledge base ID // returns ID and Metadata fields for duplicate question checking ListAllFAQChunksWithMetadataByKnowledgeBaseID(ctx context.Context, tenantID uint64, kbID string) ([]*types.Chunk, error) + // FindFAQChunkWithDuplicateQuestion finds a single FAQ chunk whose standard_question or + // similar_questions overlap with the given question list. Returns nil if no duplicate found. + FindFAQChunkWithDuplicateQuestion(ctx context.Context, tenantID uint64, kbID string, excludeChunkID string, questions []string) (*types.Chunk, error) // ListAllFAQChunksForExport lists all FAQ chunks for export with full metadata, tag_id, is_enabled, and flags ListAllFAQChunksForExport(ctx context.Context, tenantID uint64, knowledgeID string) ([]*types.Chunk, error) // UpdateChunkFlagsBatch updates flags for multiple chunks in batch using a single SQL statement.