diff --git a/frontend/src/api/knowledge-base/index.ts b/frontend/src/api/knowledge-base/index.ts index 37cf86ea..fdb10583 100644 --- a/frontend/src/api/knowledge-base/index.ts +++ b/frontend/src/api/knowledge-base/index.ts @@ -445,10 +445,11 @@ export function searchFAQEntries( return post(`/api/v1/knowledge-bases/${kbId}/faq/search`, data); } -// Export FAQ entries as CSV file -export async function exportFAQEntries(kbId: string): Promise { - const response = await getDown(`/api/v1/knowledge-bases/${kbId}/faq/entries/export`); - return response as unknown as Blob; +// Export FAQ entries as CSV or JSON file +export async function exportFAQEntries(kbId: string, format: 'csv' | 'json' = 'csv'): Promise { + const suffix = format === 'json' ? '?format=json' : '' + const response = await getDown(`/api/v1/knowledge-bases/${kbId}/faq/entries/export${suffix}`) + return response as unknown as Blob } // FAQ Import Progress API diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index 208f5d87..fb744a52 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -2992,7 +2992,9 @@ export default { downloadExampleExcel: 'Download Excel Example', }, faqExport: { - exportButton: 'Export CSV', + exportButton: 'Export', + exportCSV: 'Export CSV', + exportJSON: 'Export JSON', exportSuccess: 'Export successful', exportFailed: 'Export failed', }, diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index e5b93041..486b67b1 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -3689,7 +3689,9 @@ export default { downloadExampleExcel: 'Скачать пример Excel' }, faqExport: { - exportButton: 'Экспорт CSV', + exportButton: 'Экспорт', + exportCSV: 'Экспорт CSV', + exportJSON: 'Экспорт JSON', exportSuccess: 'Экспорт успешен', exportFailed: 'Ошибка экспорта' }, diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index 15e52496..e47bd964 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -4281,7 +4281,9 @@ export default { downloadExampleExcel: "下载 Excel 示例", }, faqExport: { - exportButton: "导出 CSV", + exportButton: "导出", + exportCSV: "导出 CSV", + exportJSON: "导出 JSON", exportSuccess: "导出成功", exportFailed: "导出失败", }, diff --git a/frontend/src/views/knowledge/components/FAQEntryManager.vue b/frontend/src/views/knowledge/components/FAQEntryManager.vue index 475ad3cd..a49f0b36 100644 --- a/frontend/src/views/knowledge/components/FAQEntryManager.vue +++ b/frontend/src/views/knowledge/components/FAQEntryManager.vue @@ -238,12 +238,15 @@ - - - - - + + + + + + + { return orgStore.canManageKB(props.kbId, false) }) +const faqExportOptions = computed(() => [ + { content: t('knowledgeEditor.faqExport.exportCSV'), value: 'export_csv' }, + { content: t('knowledgeEditor.faqExport.exportJSON'), value: 'export_json' }, +]) + // FAQ 操作:新建组(新建条目 + 导入) const faqCreateOptions = computed(() => { if (!canEdit.value) return [] @@ -1019,6 +1027,12 @@ const handleFaqAction = (data: { value: string }) => { case 'search': searchDrawerVisible.value = true break + case 'export_csv': + handleExportCSV() + break + case 'export_json': + handleExportJSON() + break case 'export': handleExportCSV() break @@ -2647,9 +2661,19 @@ const downloadExcelExample = () => { XLSX.writeFile(workbook, 'faq_example.xlsx') } -// 导出 FAQ 数据为 CSV +// 导出 FAQ 数据 const exportLoading = ref(false) -const handleExportCSV = async () => { +const downloadExportBlob = (blob: Blob, ext: 'csv' | 'json') => { + const url = URL.createObjectURL(blob) + const link = document.createElement('a') + link.href = url + link.download = `faq_export_${new Date().toISOString().slice(0, 10)}.${ext}` + document.body.appendChild(link) + link.click() + document.body.removeChild(link) + URL.revokeObjectURL(url) +} +const handleExportFAQ = async (format: 'csv' | 'json') => { if (!props.kbId) { MessagePlugin.warning(t('knowledgeBase.selectKnowledgeBase')) return @@ -2657,15 +2681,8 @@ const handleExportCSV = async () => { exportLoading.value = true try { - const blob = await exportFAQEntries(props.kbId) - const url = URL.createObjectURL(blob) - const link = document.createElement('a') - link.href = url - link.download = `faq_export_${new Date().toISOString().slice(0, 10)}.csv` - document.body.appendChild(link) - link.click() - document.body.removeChild(link) - URL.revokeObjectURL(url) + const blob = await exportFAQEntries(props.kbId, format) + downloadExportBlob(blob, format) MessagePlugin.success(t('knowledgeEditor.faqExport.exportSuccess')) } catch (error: any) { console.error('Export failed:', error) @@ -2674,6 +2691,8 @@ const handleExportCSV = async () => { exportLoading.value = false } } +const handleExportCSV = () => handleExportFAQ('csv') +const handleExportJSON = () => handleExportFAQ('json') watch( () => props.kbId, diff --git a/internal/application/repository/chunk.go b/internal/application/repository/chunk.go index 8c8af637..1fd7ada7 100644 --- a/internal/application/repository/chunk.go +++ b/internal/application/repository/chunk.go @@ -931,44 +931,170 @@ func (r *chunkRepository) UpdateChunkFieldsByTagID( return affectedIDs, nil } +type chunkIDHash struct { + ID string `gorm:"column:id"` + ContentHash string `gorm:"column:content_hash"` +} + +const faqChunkDiffBatchSize = 5000 + +// listFAQChunkIDHashesByKB loads id/content_hash pairs for all FAQ chunks in a KB. +func (r *chunkRepository) listFAQChunkIDHashesByKB( + ctx context.Context, + tenantID uint64, + kbID string, +) ([]chunkIDHash, error) { + var all []chunkIDHash + var lastID string + + for { + var batch []chunkIDHash + query := r.db.WithContext(ctx).Model(&types.Chunk{}). + Select("id, content_hash"). + Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?", + tenantID, kbID, types.ChunkTypeFAQ). + Order("id ASC"). + Limit(faqChunkDiffBatchSize) + if lastID != "" { + query = query.Where("id > ?", lastID) + } + if err := query.Find(&batch).Error; err != nil { + return nil, err + } + if len(batch) == 0 { + break + } + all = append(all, batch...) + lastID = batch[len(batch)-1].ID + if len(batch) < faqChunkDiffBatchSize { + break + } + } + return all, nil +} + +func diffFAQChunkIDsByContentHash(src, dst []chunkIDHash) ( + chunksToAdd, chunksToDelete []string, + matched []types.FAQChunkSyncPair, +) { + dstHashes := make(map[string]struct{}, len(dst)) + dstIDByHash := make(map[string]string, len(dst)) + dstIDsByHash := make(map[string][]string, len(dst)) + for _, pair := range dst { + dstHashes[pair.ContentHash] = struct{}{} + if _, ok := dstIDByHash[pair.ContentHash]; !ok { + dstIDByHash[pair.ContentHash] = pair.ID + } + dstIDsByHash[pair.ContentHash] = append(dstIDsByHash[pair.ContentHash], pair.ID) + } + srcHashes := make(map[string]struct{}, len(src)) + for _, pair := range src { + srcHashes[pair.ContentHash] = struct{}{} + } + + for _, pair := range src { + if _, exists := dstHashes[pair.ContentHash]; !exists { + chunksToAdd = append(chunksToAdd, pair.ID) + continue + } + if dstID, ok := dstIDByHash[pair.ContentHash]; ok { + matched = append(matched, types.FAQChunkSyncPair{ + SrcChunkID: pair.ID, + DstChunkID: dstID, + }) + } + } + for _, pair := range dst { + if _, exists := srcHashes[pair.ContentHash]; !exists { + chunksToDelete = append(chunksToDelete, pair.ID) + } + } + for hash, ids := range dstIDsByHash { + if hash == "" || len(ids) <= 1 { + continue + } + if _, inSrc := srcHashes[hash]; !inSrc { + continue + } + canonical := dstIDByHash[hash] + for _, id := range ids { + if id != canonical { + chunksToDelete = append(chunksToDelete, id) + } + } + } + return chunksToAdd, chunksToDelete, matched +} + // FAQChunkDiff compares FAQ chunks between two knowledge bases and returns the differences. // Returns: chunksToAdd (IDs of chunks in src whose content_hash is not in dst), // -// chunksToDelete (IDs of chunks in dst whose content_hash is not in src) +// chunksToDelete (IDs of chunks in dst whose content_hash is not in src, plus +// duplicate dst chunks that share a content_hash with another dst chunk when +// that hash still exists in src) func (r *chunkRepository) FAQChunkDiff( ctx context.Context, srcTenantID uint64, srcKBID string, dstTenantID uint64, dstKBID string, -) (chunksToAdd []string, chunksToDelete []string, err error) { - // Get content_hash set from destination KB - dstHashSubQuery := r.db.Model(&types.Chunk{}). - Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?", dstTenantID, dstKBID, types.ChunkTypeFAQ). - Select("content_hash") - - // Find chunks in source that don't exist in destination (by content_hash) - err = r.db.WithContext(ctx).Model(&types.Chunk{}). - Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?", srcTenantID, srcKBID, types.ChunkTypeFAQ). - Where("content_hash NOT IN (?)", dstHashSubQuery). - Pluck("id", &chunksToAdd).Error - if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { - return nil, nil, fmt.Errorf("failed to get chunks to add: %w", err) +) (*types.FAQChunkDiffResult, error) { + srcPairs, err := r.listFAQChunkIDHashesByKB(ctx, srcTenantID, srcKBID) + if err != nil { + return nil, fmt.Errorf("failed to list source FAQ chunks: %w", err) + } + dstPairs, err := r.listFAQChunkIDHashesByKB(ctx, dstTenantID, dstKBID) + if err != nil { + return nil, fmt.Errorf("failed to list destination FAQ chunks: %w", err) } - // Get content_hash set from source KB - srcHashSubQuery := r.db.Model(&types.Chunk{}). - Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?", srcTenantID, srcKBID, types.ChunkTypeFAQ). - Select("content_hash") + add, del, matched := diffFAQChunkIDsByContentHash(srcPairs, dstPairs) + return &types.FAQChunkDiffResult{ + ChunksToAdd: add, + ChunksToDelete: del, + MatchedPairs: matched, + }, nil +} - // Find chunks in destination that don't exist in source (by content_hash) - err = r.db.WithContext(ctx).Model(&types.Chunk{}). - Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?", dstTenantID, dstKBID, types.ChunkTypeFAQ). - Where("content_hash NOT IN (?)", srcHashSubQuery). - Pluck("id", &chunksToDelete).Error - if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { - return nil, nil, fmt.Errorf("failed to get chunks to delete: %w", err) +// ListFAQChunkStatusByIDs loads status fields for FAQ clone sync. +func (r *chunkRepository) ListFAQChunkStatusByIDs( + ctx context.Context, + tenantID uint64, + ids []string, +) (map[string]*types.FAQChunkStatus, error) { + if len(ids) == 0 { + return map[string]*types.FAQChunkStatus{}, nil } - - return chunksToAdd, chunksToDelete, nil + const batchSize = 5000 + var chunks []*types.Chunk + for i := 0; i < len(ids); i += batchSize { + end := i + batchSize + if end > len(ids) { + end = len(ids) + } + var batch []*types.Chunk + if err := r.db.WithContext(ctx). + Select("id, tag_id, is_enabled, flags, metadata"). + Where("tenant_id = ? AND id IN ?", tenantID, ids[i:end]). + Find(&batch).Error; err != nil { + return nil, err + } + chunks = append(chunks, batch...) + } + out := make(map[string]*types.FAQChunkStatus, len(chunks)) + for _, chunk := range chunks { + strategy := types.AnswerStrategyAll + if meta, err := chunk.FAQMetadata(); err == nil && meta != nil { + strategy = meta.AnswerStrategy + } + out[chunk.ID] = &types.FAQChunkStatus{ + ID: chunk.ID, + TagID: chunk.TagID, + IsEnabled: chunk.IsEnabled, + Flags: chunk.Flags, + AnswerStrategy: strategy, + Metadata: chunk.Metadata, + } + } + return out, nil } // ListRecommendedFAQChunks lists FAQ chunks with the recommended flag set. diff --git a/internal/application/repository/chunk_faq_diff_test.go b/internal/application/repository/chunk_faq_diff_test.go new file mode 100644 index 00000000..ba578772 --- /dev/null +++ b/internal/application/repository/chunk_faq_diff_test.go @@ -0,0 +1,117 @@ +package repository + +import ( + "context" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDiffFAQChunkIDsByContentHash(t *testing.T) { + toAdd, toDelete, matched := diffFAQChunkIDsByContentHash( + []chunkIDHash{ + {ID: "src-1", ContentHash: "hash-a"}, + {ID: "src-2", ContentHash: "hash-b"}, + {ID: "src-3", ContentHash: "hash-only-src"}, + }, + []chunkIDHash{ + {ID: "dst-1", ContentHash: "hash-a"}, + {ID: "dst-2", ContentHash: "hash-only-dst"}, + }, + ) + + assert.ElementsMatch(t, []string{"src-2", "src-3"}, toAdd) + assert.ElementsMatch(t, []string{"dst-2"}, toDelete) + require.Len(t, matched, 1) + assert.Equal(t, "src-1", matched[0].SrcChunkID) + assert.Equal(t, "dst-1", matched[0].DstChunkID) +} + +func TestDiffFAQChunkIDsByContentHash_dstDuplicateHash(t *testing.T) { + // Two destination chunks share hash-a (same FAQ content, e.g. stale copy after + // answer_strategy-only change). Source has one chunk for hash-a → keep dst-1, + // delete dst-dup, match for status sync. + toAdd, toDelete, matched := diffFAQChunkIDsByContentHash( + []chunkIDHash{{ID: "src-1", ContentHash: "hash-a"}}, + []chunkIDHash{ + {ID: "dst-1", ContentHash: "hash-a"}, + {ID: "dst-dup", ContentHash: "hash-a"}, + }, + ) + + assert.Empty(t, toAdd) + assert.ElementsMatch(t, []string{"dst-dup"}, toDelete) + require.Len(t, matched, 1) + assert.Equal(t, "src-1", matched[0].SrcChunkID) + assert.Equal(t, "dst-1", matched[0].DstChunkID) +} + +func TestDiffFAQChunkIDsByContentHash_dstDuplicateHash_notInSrc(t *testing.T) { + // Duplicates whose hash is absent from source are all removed (orphan content). + toAdd, toDelete, matched := diffFAQChunkIDsByContentHash( + nil, + []chunkIDHash{ + {ID: "dst-orphan-1", ContentHash: "hash-orphan"}, + {ID: "dst-orphan-2", ContentHash: "hash-orphan"}, + }, + ) + + assert.Empty(t, toAdd) + assert.ElementsMatch(t, []string{"dst-orphan-1", "dst-orphan-2"}, toDelete) + assert.Empty(t, matched) +} + +func TestDiffFAQChunkIDsByContentHash_emptySide(t *testing.T) { + toAdd, toDelete, matched := diffFAQChunkIDsByContentHash( + []chunkIDHash{{ID: "src-1", ContentHash: "hash-a"}}, + nil, + ) + assert.Equal(t, []string{"src-1"}, toAdd) + assert.Empty(t, toDelete) + assert.Empty(t, matched) + + toAdd, toDelete, matched = diffFAQChunkIDsByContentHash( + nil, + []chunkIDHash{{ID: "dst-1", ContentHash: "hash-a"}}, + ) + assert.Empty(t, toAdd) + assert.Equal(t, []string{"dst-1"}, toDelete) + assert.Empty(t, matched) +} + +func TestFAQChunkDiff_SQLite(t *testing.T) { + db := setupChunkTestDB(t) + repo := NewChunkRepository(db) + ctx := context.Background() + + srcKBID := uuid.New().String() + dstKBID := uuid.New().String() + srcKnowledgeID := uuid.New().String() + dstKnowledgeID := uuid.New().String() + + require.NoError(t, db.Create(&types.Chunk{ + ID: uuid.New().String(), TenantID: 1, KnowledgeBaseID: srcKBID, KnowledgeID: srcKnowledgeID, + ChunkType: types.ChunkTypeFAQ, ContentHash: "shared", Status: int(types.ChunkStatusIndexed), + }).Error) + require.NoError(t, db.Create(&types.Chunk{ + ID: uuid.New().String(), TenantID: 1, KnowledgeBaseID: srcKBID, KnowledgeID: srcKnowledgeID, + ChunkType: types.ChunkTypeFAQ, ContentHash: "src-only", Status: int(types.ChunkStatusIndexed), + }).Error) + require.NoError(t, db.Create(&types.Chunk{ + ID: uuid.New().String(), TenantID: 1, KnowledgeBaseID: dstKBID, KnowledgeID: dstKnowledgeID, + ChunkType: types.ChunkTypeFAQ, ContentHash: "shared", Status: int(types.ChunkStatusIndexed), + }).Error) + require.NoError(t, db.Create(&types.Chunk{ + ID: uuid.New().String(), TenantID: 1, KnowledgeBaseID: dstKBID, KnowledgeID: dstKnowledgeID, + ChunkType: types.ChunkTypeFAQ, ContentHash: "dst-only", Status: int(types.ChunkStatusIndexed), + }).Error) + + diff, err := repo.FAQChunkDiff(ctx, 1, srcKBID, 1, dstKBID) + require.NoError(t, err) + assert.Len(t, diff.ChunksToAdd, 1) + assert.Len(t, diff.ChunksToDelete, 1) + assert.Len(t, diff.MatchedPairs, 1) +} diff --git a/internal/application/service/faq_clone_sync.go b/internal/application/service/faq_clone_sync.go new file mode 100644 index 00000000..c40368eb --- /dev/null +++ b/internal/application/service/faq_clone_sync.go @@ -0,0 +1,165 @@ +package service + +import ( + "context" + "fmt" + "time" + + "github.com/Tencent/WeKnora/internal/application/service/retriever" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" +) + +type faqStatusSyncPlan struct { + Pairs []types.FAQChunkSyncPair + SrcByID map[string]*types.FAQChunkStatus + DstByID map[string]*types.FAQChunkStatus +} + +func (s *knowledgeService) buildFAQStatusSyncPlan( + ctx context.Context, + srcTenantID, dstTenantID uint64, + matched []types.FAQChunkSyncPair, + resolveTag func(srcTagID string) string, +) (*faqStatusSyncPlan, error) { + if len(matched) == 0 { + return &faqStatusSyncPlan{}, nil + } + srcIDs := make([]string, 0, len(matched)) + dstIDs := make([]string, 0, len(matched)) + for _, p := range matched { + srcIDs = append(srcIDs, p.SrcChunkID) + dstIDs = append(dstIDs, p.DstChunkID) + } + srcByID, err := s.chunkRepo.ListFAQChunkStatusByIDs(ctx, srcTenantID, srcIDs) + if err != nil { + return nil, err + } + dstByID, err := s.chunkRepo.ListFAQChunkStatusByIDs(ctx, dstTenantID, dstIDs) + if err != nil { + return nil, err + } + pairs := make([]types.FAQChunkSyncPair, 0) + for _, p := range matched { + src, dst := srcByID[p.SrcChunkID], dstByID[p.DstChunkID] + if src == nil || dst == nil { + continue + } + mappedTag := "" + if src.TagID != "" { + mappedTag = resolveTag(src.TagID) + } + if types.FAQChunkNeedsStatusSync(src, dst, mappedTag) { + pairs = append(pairs, p) + } + } + return &faqStatusSyncPlan{Pairs: pairs, SrcByID: srcByID, DstByID: dstByID}, nil +} + +func (s *knowledgeService) syncFAQChunkStatusBatch( + ctx context.Context, + dstKB *types.KnowledgeBase, + pairs []types.FAQChunkSyncPair, + srcByID, dstByID map[string]*types.FAQChunkStatus, + resolveTag func(srcTagID string) string, +) error { + if len(pairs) == 0 { + return nil + } + tenantID := types.MustTenantIDFromContext(ctx) + enabledUpdates := make(map[string]bool) + recommendedUpdates := make(map[string]bool) + tagUpdates := make(map[string]string) + rows := make([]*types.Chunk, 0, len(pairs)) + metadataIDs := make([]string, 0) + + for _, p := range pairs { + src, dst := srcByID[p.SrcChunkID], dstByID[p.DstChunkID] + if src == nil || dst == nil { + continue + } + mappedTag := "" + if src.TagID != "" { + mappedTag = resolveTag(src.TagID) + } + if !types.FAQChunkNeedsStatusSync(src, dst, mappedTag) { + continue + } + rows = append(rows, &types.Chunk{ + ID: dst.ID, IsEnabled: src.IsEnabled, Flags: src.Flags, + TagID: mappedTag, UpdatedAt: time.Now(), + }) + if dst.IsEnabled != src.IsEnabled { + enabledUpdates[dst.ID] = src.IsEnabled + } + srcRec := src.Flags.HasFlag(types.ChunkFlagRecommended) + dstRec := dst.Flags.HasFlag(types.ChunkFlagRecommended) + if srcRec != dstRec { + recommendedUpdates[dst.ID] = srcRec + } + if mappedTag != dst.TagID { + tagUpdates[dst.ID] = mappedTag + } + if types.NormalizeAnswerStrategy(src.AnswerStrategy) != types.NormalizeAnswerStrategy(dst.AnswerStrategy) { + metadataIDs = append(metadataIDs, dst.ID) + } + } + if len(rows) > 0 { + if err := s.chunkRepo.UpdateChunks(ctx, rows); err != nil { + return err + } + } + for _, dstID := range metadataIDs { + src := srcForDst(pairs, srcByID, dstID) + dst := dstByID[dstID] + if src == nil || dst == nil { + continue + } + patched, err := types.PatchFAQAnswerStrategy(dst.Metadata, src.Metadata) + if err != nil { + return fmt.Errorf("patch answer_strategy for %s: %w", dstID, err) + } + full, err := s.chunkRepo.GetChunkByID(ctx, tenantID, dstID) + if err != nil { + return err + } + full.Metadata = patched + full.UpdatedAt = time.Now() + if err := s.chunkRepo.UpdateChunk(ctx, full); err != nil { + return err + } + } + if len(enabledUpdates) == 0 && len(tagUpdates) == 0 && len(recommendedUpdates) == 0 { + return nil + } + engine, err := retriever.CreateRetrieveEngineForKB( + ctx, s.retrieveEngine, s.ownership, tenantID, dstKB.VectorStoreID) + if err != nil { + return err + } + if len(enabledUpdates) > 0 { + if err := engine.BatchUpdateChunkEnabledStatus(ctx, enabledUpdates); err != nil { + return err + } + } + if len(tagUpdates) > 0 { + if err := engine.BatchUpdateChunkTagID(ctx, tagUpdates); err != nil { + return err + } + } + if len(recommendedUpdates) > 0 { + // Vector-store recommended flag sync is not yet available on all backends in + // this branch; DB flags were already updated via UpdateChunks above. + logger.Warnf(ctx, "FAQ clone sync: skipped vector recommended update for %d chunks (DB flags updated)", len(recommendedUpdates)) + } + return nil +} + +func srcForDst(pairs []types.FAQChunkSyncPair, srcByID map[string]*types.FAQChunkStatus, dstID string) *types.FAQChunkStatus { + for _, p := range pairs { + if p.DstChunkID == dstID { + return srcByID[p.SrcChunkID] + } + } + return nil +} diff --git a/internal/application/service/knowledge_clone_move.go b/internal/application/service/knowledge_clone_move.go index 45ce46f0..89895ccb 100644 --- a/internal/application/service/knowledge_clone_move.go +++ b/internal/application/service/knowledge_clone_move.go @@ -675,23 +675,47 @@ func (s *knowledgeService) cloneFAQKnowledgeBase( } srcKnowledge := srcKnowledgeList[0] - // Get chunk-level differences based on content_hash - chunksToAdd, chunksToDelete, err := s.chunkRepo.FAQChunkDiff(ctx, srcKB.TenantID, srcKB.ID, dstKB.TenantID, dstKB.ID) + // Get chunk-level differences based on content_hash. + diff, err := s.chunkRepo.FAQChunkDiff(ctx, srcKB.TenantID, srcKB.ID, dstKB.TenantID, dstKB.ID) if err != nil { logger.Errorf(ctx, "Failed to calculate FAQ chunk difference: %v", err) handleError(progress, err, "Failed to calculate FAQ chunk difference") return err } + chunksToAdd := diff.ChunksToAdd + chunksToDelete := diff.ChunksToDelete - totalOperations := len(chunksToAdd) + len(chunksToDelete) + tagIDMapping := map[string]string{} + resolveFAQTag := func(srcTagID string) string { + if srcTagID == "" { + return "" + } + if id, ok := tagIDMapping[srcTagID]; ok { + return id + } + return s.getOrCreateTagInTarget(ctx, srcKB.TenantID, dstKB.TenantID, dstKB.ID, srcTagID, tagIDMapping) + } + + syncPlan, err := s.buildFAQStatusSyncPlan(ctx, srcKB.TenantID, dstKB.TenantID, diff.MatchedPairs, resolveFAQTag) + if err != nil { + logger.Errorf(ctx, "Failed to build FAQ status sync plan: %v", err) + handleError(progress, err, "Failed to build FAQ status sync plan") + return err + } + chunksToUpdate := syncPlan.Pairs + + totalOperations := len(chunksToAdd) + len(chunksToDelete) + len(chunksToUpdate) progress.Total = totalOperations - progress.Message = fmt.Sprintf("Found %d FAQ entries to add, %d to delete", len(chunksToAdd), len(chunksToDelete)) + progress.Message = fmt.Sprintf( + "Found %d FAQ entries to add, %d to delete, %d to update status", + len(chunksToAdd), len(chunksToDelete), len(chunksToUpdate), + ) progress.UpdatedAt = time.Now().Unix() _ = s.saveKBCloneProgress(ctx, progress) - logger.Infof(ctx, "FAQ chunks to add: %d, delete: %d", len(chunksToAdd), len(chunksToDelete)) + logger.Infof(ctx, "FAQ chunks to add: %d, delete: %d, update status: %d", + len(chunksToAdd), len(chunksToDelete), len(chunksToUpdate)) - // If nothing to do, mark as completed if totalOperations == 0 { progress.Status = types.KBCloneStatusCompleted progress.Progress = 100 @@ -760,7 +784,6 @@ func (s *knowledgeService) cloneFAQKnowledgeBase( // Clone FAQ chunks from source to destination batch := 50 - tagIDMapping := map[string]string{} // srcTagID -> dstTagID for i := 0; i < len(chunksToAdd); i += batch { end := i + batch if end > len(chunksToAdd) { @@ -782,19 +805,11 @@ func (s *knowledgeService) cloneFAQKnowledgeBase( // Map TagID to target knowledge base targetTagID := "" if srcChunk.TagID != "" { - if mappedTagID, ok := tagIDMapping[srcChunk.TagID]; ok { - targetTagID = mappedTagID - } else { - // Try to find or create the tag in target knowledge base - targetTagID = s.getOrCreateTagInTarget(ctx, srcKB.TenantID, dstKB.TenantID, dstKB.ID, srcChunk.TagID, tagIDMapping) - } + targetTagID = resolveFAQTag(srcChunk.TagID) } // Deep-copy extracted images into objects owned by the destination - // FAQ knowledge so deleting the source never breaks this clone. A FAQ - // chunk is self-contained (its own Content + image_info), so its - // ![](url) references can be rewritten immediately using the mapping - // just accumulated in imageURLCache. + // FAQ knowledge so deleting the source never breaks this clone. newImageInfo, copied, copyErr := cloneChunkImageInfo( ctx, dstSvc, srcChunk.ImageInfo, dstKB.TenantID, dstKnowledge.ID, imageURLCache) if copyErr != nil { @@ -860,6 +875,27 @@ func (s *knowledgeService) cloneFAQKnowledgeBase( _ = s.saveKBCloneProgress(ctx, progress) } + for i := 0; i < len(chunksToUpdate); i += batch { + end := i + batch + if end > len(chunksToUpdate) { + end = len(chunksToUpdate) + } + if err := s.syncFAQChunkStatusBatch( + ctx, dstKB, chunksToUpdate[i:end], syncPlan.SrcByID, syncPlan.DstByID, resolveFAQTag); err != nil { + logger.Errorf(ctx, "Failed to sync FAQ status fields: %v", err) + handleError(progress, err, "Failed to sync FAQ status fields") + return err + } + processedCount += end - i + if totalOperations > 0 { + progress.Progress = processedCount * 100 / totalOperations + } + progress.Processed = processedCount + progress.Message = fmt.Sprintf("Updated %d/%d FAQ status fields", processedCount-len(chunksToDelete)-len(chunksToAdd), len(chunksToUpdate)) + progress.UpdatedAt = time.Now().Unix() + _ = s.saveKBCloneProgress(ctx, progress) + } + // Mark as completed progress.Status = types.KBCloneStatusCompleted progress.Progress = 100 diff --git a/internal/application/service/knowledge_faq.go b/internal/application/service/knowledge_faq.go index 5894845b..9171fab5 100644 --- a/internal/application/service/knowledge_faq.go +++ b/internal/application/service/knowledge_faq.go @@ -2,6 +2,9 @@ package service import ( "context" + "crypto/md5" + "encoding/hex" + "encoding/json" "fmt" "slices" "strings" @@ -1445,6 +1448,66 @@ func (s *knowledgeService) ExportFAQEntries(ctx context.Context, kbID string) ([ return s.buildFAQCSV(chunks, tagMap), nil } +// ExportFAQEntriesJSON 以 JSON 数组形式导出 FAQ 知识库下的全部条目, +// 字段与 FAQEntryPayload 兼容,便于"导出 → 编辑 → 重新 append 导入"循环。 +func (s *knowledgeService) ExportFAQEntriesJSON(ctx context.Context, kbID string) ([]byte, error) { + kb, err := s.validateFAQKnowledgeBase(ctx, kbID) + if err != nil { + return nil, err + } + + tenantID := ctx.Value(types.TenantIDContextKey).(uint64) + faqKnowledge, err := s.findFAQKnowledge(ctx, tenantID, kb.ID) + if err != nil { + return nil, err + } + if faqKnowledge == nil { + return json.Marshal([]types.FAQExportEntry{}) + } + + chunks, err := s.chunkRepo.ListAllFAQChunksForExport(ctx, tenantID, faqKnowledge.ID) + if err != nil { + return nil, fmt.Errorf("failed to list FAQ chunks: %w", err) + } + + tagMap, err := s.buildTagMap(ctx, tenantID, kbID) + if err != nil { + return nil, fmt.Errorf("failed to build tag map: %w", err) + } + + return s.buildFAQJSON(chunks, tagMap) +} + +func (s *knowledgeService) buildFAQJSON(chunks []*types.Chunk, tagMap map[string]string) ([]byte, error) { + entries := make([]types.FAQExportEntry, 0, len(chunks)) + for _, chunk := range chunks { + meta, err := chunk.FAQMetadata() + if err != nil || meta == nil { + continue + } + + tagName := "" + if chunk.TagID != "" && tagMap != nil { + if name, ok := tagMap[chunk.TagID]; ok { + tagName = name + } + } + + entries = append(entries, types.FAQExportEntry{ + ID: chunk.SeqID, + TagName: tagName, + StandardQuestion: meta.StandardQuestion, + SimilarQuestions: meta.SimilarQuestions, + NegativeQuestions: meta.NegativeQuestions, + Answers: meta.Answers, + AnswerStrategy: meta.AnswerStrategy, + IsEnabled: chunk.IsEnabled, + IsRecommended: chunk.Flags.HasFlag(types.ChunkFlagRecommended), + }) + } + return json.Marshal(entries) +} + // buildTagMap builds a map from tag_id to tag_name for the given knowledge base. func (s *knowledgeService) buildTagMap(ctx context.Context, tenantID uint64, kbID string) (map[string]string, error) { const pageSize = 1000 @@ -1787,6 +1850,74 @@ func (s *knowledgeService) checkFAQQuestionDuplicate( return werrors.NewBadRequestError("标准问或相似问与已有条目重复") } +// faqTagResolver resolves a payload's TagID/TagName to the tag's internal UUID. +type faqTagResolver func(*types.FAQEntryPayload) (string, error) + +// buildFAQTagResolver 预加载 entries 引用的 tag UUID 映射,避免逐条 resolveTagID。 +func (s *knowledgeService) buildFAQTagResolver( + ctx context.Context, kbID string, entries []types.FAQEntryPayload, +) faqTagResolver { + tenantID := ctx.Value(types.TenantIDContextKey).(uint64) + + var seqIDs []int64 + var names []string + seqSeen := make(map[int64]struct{}) + nameSeen := make(map[string]struct{}) + for i := range entries { + e := &entries[i] + switch { + case e.TagID != 0: + if _, ok := seqSeen[e.TagID]; !ok { + seqSeen[e.TagID] = struct{}{} + seqIDs = append(seqIDs, e.TagID) + } + case e.TagName != "": + if _, ok := nameSeen[e.TagName]; !ok { + nameSeen[e.TagName] = struct{}{} + names = append(names, e.TagName) + } + } + } + + seqIDToUUID := make(map[int64]string) + if len(seqIDs) > 0 { + if tags, err := s.tagRepo.GetBySeqIDs(ctx, tenantID, seqIDs); err == nil { + for _, t := range tags { + seqIDToUUID[t.SeqID] = t.ID + } + } else { + logger.Warnf(ctx, "buildFAQTagResolver: batch GetBySeqIDs failed (%d ids), fallback per-entry: %v", + len(seqIDs), err) + } + } + + nameToUUID := make(map[string]string) + for _, name := range names { + if tag, err := s.tagRepo.GetByName(ctx, tenantID, kbID, name); err == nil && tag != nil { + nameToUUID[name] = tag.ID + } + } + + return func(e *types.FAQEntryPayload) (string, error) { + if e.TagID != 0 { + if uuid, ok := seqIDToUUID[e.TagID]; ok { + return uuid, nil + } + } else if e.TagName != "" { + if uuid, ok := nameToUUID[e.TagName]; ok { + return uuid, nil + } + } + return s.resolveTagID(ctx, kbID, e) + } +} + +// hashQuestion 计算问题内容的哈希值,用于生成稳定的 sourceID。 +func hashQuestion(question string) string { + h := md5.Sum([]byte(question)) + return hex.EncodeToString(h[:4]) +} + // resolveTagID resolves tag ID (UUID) from payload, prioritizing tag_id (seq_id) over tag_name // If no tag is specified, creates or finds the "未分类" tag // Returns the internal UUID of the tag diff --git a/internal/application/service/knowledge_faq_import.go b/internal/application/service/knowledge_faq_import.go index 7a70b06b..c253d011 100644 --- a/internal/application/service/knowledge_faq_import.go +++ b/internal/application/service/knowledge_faq_import.go @@ -73,15 +73,29 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context, // 记录任务入队时间 enqueuedAt := time.Now().Unix() + instanceID := uuid.NewString() + runningInfoSet := false + enqueueSucceeded := false // 设置 KB 的运行中任务信息 if err := s.setRunningFAQImportInfo(ctx, kbID, &runningFAQImportInfo{ TaskID: taskID, EnqueuedAt: enqueuedAt, + InstanceID: instanceID, }); err != nil { logger.Errorf(ctx, "Failed to set running FAQ import task info: %v", err) // 不影响任务执行,继续 + } else { + runningInfoSet = true } + defer func() { + if !runningInfoSet || enqueueSucceeded { + return + } + if clearErr := s.clearRunningFAQImportInfoIfMatches(ctx, kbID, taskID, instanceID, enqueuedAt); clearErr != nil { + logger.Warnf(ctx, "Failed to clear FAQ import running info after setup failure: %v", clearErr) + } + }() // 初始化导入任务状态到Redis progress := &types.FAQImportProgress{ @@ -120,6 +134,7 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context, Mode: payload.Mode, DryRun: payload.DryRun, EnqueuedAt: enqueuedAt, + InstanceID: instanceID, Initiator: types.TaskInitiatorFromContext(ctx), } @@ -189,9 +204,8 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context, maxRetry = 3 // dry run 重试次数少一些 } - // 使用 taskID:enqueuedAt 作为 asynq 的唯一任务标识 - // 这样同一个用户 TaskID 的不同次提交不会冲突 - asynqTaskID := fmt.Sprintf("%s:%d", taskID, enqueuedAt) + // 使用 taskID:instanceID 作为 asynq 的唯一任务标识 + asynqTaskID := fmt.Sprintf("%s:%s", taskID, instanceID) task := asynq.NewTask( types.TypeFAQImport, @@ -207,6 +221,7 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context, return "", fmt.Errorf("failed to enqueue task: %w", err) } logger.Infof(ctx, "Enqueued FAQ import task: id=%s queue=%s task_id=%s dry_run=%v", info.ID, info.Queue, taskID, payload.DryRun) + enqueueSucceeded = true if !payload.DryRun { recordKBActivity(ctx, s.audit, tenantID, kbID, types.AuditActionFAQImportStarted, @@ -1215,34 +1230,118 @@ func (s *knowledgeService) calculateReplaceOperations(ctx context.Context, } } + // 批量预加载 tag 信息,避免循环内逐条查询数据库 + // 收集所有需要查询的 tag_id 和 tag_name + tagSeqIDSet := make(map[int64]bool) + tagNameSet := make(map[string]bool) + for _, ewh := range entriesWithHash { + if ewh.entry.TagID != 0 { + tagSeqIDSet[ewh.entry.TagID] = true + } else if ewh.entry.TagName != "" { + tagNameSet[ewh.entry.TagName] = true + } else { + tagNameSet[types.UntaggedTagName] = true + } + } + + // 批量查询 tag by seq_id + tagSeqIDToUUID := make(map[int64]string) + if len(tagSeqIDSet) > 0 { + seqIDs := make([]int64, 0, len(tagSeqIDSet)) + for seqID := range tagSeqIDSet { + seqIDs = append(seqIDs, seqID) + } + tags, err := s.tagRepo.GetBySeqIDs(ctx, tenantID, seqIDs) + if err != nil { + logger.Warnf(ctx, "Failed to batch load tags by seq_ids: %v", err) + } else { + for _, tag := range tags { + tagSeqIDToUUID[tag.SeqID] = tag.ID + } + } + } + + // 批量查询 tag by name + tagNameToUUID := make(map[string]string) + if len(tagNameSet) > 0 && kbID != "" { + for name := range tagNameSet { + if tag, err := s.tagRepo.GetByName(ctx, tenantID, kbID, name); err == nil && tag != nil { + tagNameToUUID[name] = tag.ID + } + } + } + + logger.Infof(ctx, "Preloaded %d tags by seq_id, %d tags by name for %d entries", + len(tagSeqIDToUUID), len(tagNameToUUID), len(entriesWithHash)) + + // resolveTagIDFromCache 从缓存中解析 tag ID,缓存未命中时回退到数据库查询 + resolveTagIDFromCache := func(entry *types.FAQEntryPayload) (string, error) { + if entry.TagID != 0 { + if uuid, ok := tagSeqIDToUUID[entry.TagID]; ok { + return uuid, nil + } + // 缓存未命中,回退到数据库查询(可能需要创建) + return s.resolveTagID(ctx, kbID, entry) + } + tagName := entry.TagName + if tagName == "" { + tagName = types.UntaggedTagName + } + if uuid, ok := tagNameToUUID[tagName]; ok { + return uuid, nil + } + // 缓存未命中,回退到数据库查询(可能需要创建) + return s.resolveTagID(ctx, kbID, entry) + } + // 计算需要创建的条目(利用已经计算好的hash,避免重复计算) entriesToProcess := make([]types.FAQEntryPayload, 0, len(entriesWithHash)) skippedCount := batchSkippedCount - for _, ewh := range entriesWithHash { + for idx, ewh := range entriesWithHash { + // 每处理 1000 条打印一次进度日志 + if idx > 0 && idx%1000 == 0 { + logger.Infof(ctx, "calculateReplaceOperations progress: %d/%d entries processed", idx, len(entriesWithHash)) + } + existingChunk := existingHashMap[ewh.hash] if existingChunk != nil { // hash 匹配,检查 tag 是否变化 - newTagID, err := s.resolveTagID(ctx, kbID, &ewh.entry) + newTagID, err := resolveTagIDFromCache(&ewh.entry) if err != nil { logger.Warnf(ctx, "Failed to resolve tag for entry, treating as new: %v", err) entriesToProcess = append(entriesToProcess, ewh.entry) continue } - if existingChunk.TagID != newTagID { - // tag 变化了,需要删除旧的并创建新的 - logger.Infof(ctx, "FAQ entry tag changed from %s to %s, will update", existingChunk.TagID, newTagID) + enabledChanged := ewh.entry.IsEnabled != nil && *ewh.entry.IsEnabled != existingChunk.IsEnabled + recommendedChanged := ewh.entry.IsRecommended != nil && + *ewh.entry.IsRecommended != existingChunk.Flags.HasFlag(types.ChunkFlagRecommended) + answerStrategyChanged := false + if existingMeta, metaErr := existingChunk.FAQMetadata(); metaErr == nil && existingMeta != nil { + answerStrategyChanged = ewh.meta.AnswerStrategy != existingMeta.AnswerStrategy + } + + if existingChunk.TagID != newTagID || enabledChanged || recommendedChanged || answerStrategyChanged { + if existingChunk.TagID != newTagID { + logger.Infof(ctx, "FAQ entry tag changed from %s to %s, will update", existingChunk.TagID, newTagID) + } + if enabledChanged || recommendedChanged || answerStrategyChanged { + logger.Infof(ctx, "FAQ entry operational fields changed (enabled: %v->%v, recommended: %v->%v, answerStrategy changed: %v), will update", + existingChunk.IsEnabled, ewh.entry.IsEnabled, + existingChunk.Flags.HasFlag(types.ChunkFlagRecommended), ewh.entry.IsRecommended, + answerStrategyChanged) + } chunksToDelete = append(chunksToDelete, existingChunk) entriesToProcess = append(entriesToProcess, ewh.entry) } else { - // hash 和 tag 都相同,跳过 + // hash、tag、运营状态都相同,跳过 skippedCount++ } continue } - // hash不匹配或不存在,需要创建 + // hash 不匹配或不存在,需要创建 entriesToProcess = append(entriesToProcess, ewh.entry) } @@ -1512,7 +1611,7 @@ func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string, actualProcessed += len(batch) // 更新任务进度 progress := int(float64(actualProcessed) / float64(totalEntries) * 100) - if err := s.updateFAQImportProgressStatus(ctx, taskID, types.FAQImportStatusProcessing, progress, totalEntries, actualProcessed, fmt.Sprintf("正在处理第 %d/%d 条", actualProcessed, totalEntries), ""); err != nil { + if err := s.updateFAQImportProgressStatus(ctx, taskID, "", 0, types.FAQImportStatusProcessing, progress, totalEntries, actualProcessed, fmt.Sprintf("正在处理第 %d/%d 条", actualProcessed, totalEntries), ""); err != nil { logger.Errorf(ctx, "Failed to update task progress: %v", err) } @@ -1555,6 +1654,8 @@ func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string, func (s *knowledgeService) updateFAQImportProgressStatus( ctx context.Context, taskID string, + instanceID string, + enqueuedAt int64, status types.FAQImportTaskStatus, progress, total, processed int, message, errorMsg string, @@ -1585,7 +1686,7 @@ func (s *knowledgeService) updateFAQImportProgressStatus( // 任务完成或失败时,清除 running key if status == types.FAQImportStatusCompleted || status == types.FAQImportStatusFailed { if existingProgress.KBID != "" { - if clearErr := s.clearRunningFAQImportTaskID(ctx, existingProgress.KBID); clearErr != nil { + if clearErr := s.clearRunningFAQImportInfoIfMatches(ctx, existingProgress.KBID, taskID, instanceID, enqueuedAt); clearErr != nil { logger.Errorf(ctx, "Failed to clear running FAQ import task ID: %v", clearErr) } } @@ -1611,6 +1712,7 @@ func (s *knowledgeService) cleanupFAQEntriesFileOnFinalFailure(ctx context.Conte type runningFAQImportInfo struct { TaskID string `json:"task_id"` EnqueuedAt int64 `json:"enqueued_at"` + InstanceID string `json:"instance_id,omitempty"` } // getRunningFAQImportInfo checks if there's a running FAQ import task for the given KB @@ -1677,6 +1779,39 @@ func (s *knowledgeService) clearRunningFAQImportTaskID(ctx context.Context, kbID return s.redisClient.Del(ctx, key).Err() } +func (s *knowledgeService) clearRunningFAQImportInfoIfMatches(ctx context.Context, kbID, taskID, instanceID string, enqueuedAt int64) error { + if s.redisClient == nil { + if v, ok := s.memFAQRunningImport.Load(kbID); ok { + info, _ := v.(*runningFAQImportInfo) + if runningFAQImportInfoMatches(info, taskID, instanceID, enqueuedAt) { + s.memFAQRunningImport.Delete(kbID) + } + } + return nil + } + + info, err := s.getRunningFAQImportInfo(ctx, kbID) + if err != nil { + return err + } + if !runningFAQImportInfoMatches(info, taskID, instanceID, enqueuedAt) { + return nil + } + + key := getFAQImportRunningKey(kbID) + return s.redisClient.Del(ctx, key).Err() +} + +func runningFAQImportInfoMatches(info *runningFAQImportInfo, taskID, instanceID string, enqueuedAt int64) bool { + if info == nil || info.TaskID != taskID { + return false + } + if info.InstanceID != "" && instanceID != "" { + return info.InstanceID == instanceID + } + return enqueuedAt == 0 || info.EnqueuedAt == 0 || info.EnqueuedAt == enqueuedAt +} + // incrementalIndexFAQEntry 增量更新FAQ条目的索引 // 只对内容变化的部分进行embedding计算和索引更新,跳过未变化的部分 func (s *knowledgeService) incrementalIndexFAQEntry( @@ -1691,6 +1826,8 @@ func (s *knowledgeService) incrementalIndexFAQEntry( newMeta *types.FAQChunkMetadata, ) error { indexStartTime := time.Now() + logger.Debugf(ctx, "incrementalIndexFAQEntry: starting for chunk=%s, oldSimilarQuestions=%d, newSimilarQuestions=%d", + chunk.ID, len(oldSimilarQuestions), len(newMeta.SimilarQuestions)) retrieveEngine, err := retriever.CreateRetrieveEngineForKB( ctx, s.retrieveEngine, s.ownership, types.MustTenantIDFromContext(ctx), kb.VectorStoreID) @@ -1703,12 +1840,26 @@ func (s *knowledgeService) incrementalIndexFAQEntry( indexMode = kb.FAQConfig.IndexMode } - // 构建旧的内容(用于比较) - buildOldContent := func(question string) string { - if indexMode == types.FAQIndexModeQuestionAnswer && len(oldAnswers) > 0 { + // 对新旧数据进行归一化处理,确保与 buildFAQIndexInfoList 的行为一致 + // 旧数据归一化 + oldStandardQuestion = types.NormalizeQuestion(oldStandardQuestion) + normalizedOldSimilarQuestions := make([]string, 0, len(oldSimilarQuestions)) + for _, q := range oldSimilarQuestions { + if nq := types.NormalizeQuestion(q); nq != "" { + normalizedOldSimilarQuestions = append(normalizedOldSimilarQuestions, nq) + } + } + oldSimilarQuestions = normalizedOldSimilarQuestions + oldAnswers = types.SanitizeStrings(oldAnswers) + // 新数据归一化 + normalizedNewMeta := newMeta.Normalize() + + // 构建索引内容 + buildContent := func(question string, answers []string) string { + if indexMode == types.FAQIndexModeQuestionAnswer && len(answers) > 0 { var builder strings.Builder builder.WriteString(question) - for _, ans := range oldAnswers { + for _, ans := range answers { builder.WriteString("\n") builder.WriteString(ans) } @@ -1717,30 +1868,20 @@ func (s *knowledgeService) incrementalIndexFAQEntry( return question } - // 构建新的内容 - buildNewContent := func(question string) string { - if indexMode == types.FAQIndexModeQuestionAnswer && len(newMeta.Answers) > 0 { - var builder strings.Builder - builder.WriteString(question) - for _, ans := range newMeta.Answers { - builder.WriteString("\n") - builder.WriteString(ans) - } - return builder.String() - } - return question - } - - // 检查答案是否变化 - answersChanged := !slices.Equal(oldAnswers, newMeta.Answers) + // 检查答案是否变化(仅在 QuestionAnswer 模式下才影响索引) + answersChanged := indexMode == types.FAQIndexModeQuestionAnswer && !slices.Equal(oldAnswers, normalizedNewMeta.Answers) + logger.Debugf(ctx, "incrementalIndexFAQEntry: answersChanged=%v (indexMode=%s), oldAnswers=%d, newAnswers=%d", + answersChanged, indexMode, len(oldAnswers), len(normalizedNewMeta.Answers)) // 收集需要更新的索引项 var indexInfoToUpdate []*types.IndexInfo // 1. 检查标准问是否需要更新 - oldStdContent := buildOldContent(oldStandardQuestion) - newStdContent := buildNewContent(newMeta.StandardQuestion) - if oldStdContent != newStdContent { + oldStdContent := buildContent(oldStandardQuestion, oldAnswers) + newStdContent := buildContent(normalizedNewMeta.StandardQuestion, normalizedNewMeta.Answers) + stdQuestionChanged := oldStdContent != newStdContent + if stdQuestionChanged { + logger.Debugf(ctx, "incrementalIndexFAQEntry: standard question changed, sourceID=%s", chunk.ID) indexInfoToUpdate = append(indexInfoToUpdate, &types.IndexInfo{ Content: newStdContent, SourceID: chunk.ID, @@ -1755,27 +1896,41 @@ func (s *knowledgeService) incrementalIndexFAQEntry( }) } - // 2. 检查每个相似问是否需要更新 - oldCount := len(oldSimilarQuestions) - newCount := len(newMeta.SimilarQuestions) + // 2. 基于内容哈希处理相似问的增删改 + // 构建旧问题集合 (问题 -> 是否存在) + oldQuestionsSet := make(map[string]struct{}, len(oldSimilarQuestions)) + for _, q := range oldSimilarQuestions { + oldQuestionsSet[q] = struct{}{} + } - for i, newQ := range newMeta.SimilarQuestions { - needUpdate := false - if i >= oldCount { - // 新增的相似问 - needUpdate = true - } else { - // 已存在的相似问,检查内容是否变化 - oldQ := oldSimilarQuestions[i] - if oldQ != newQ || answersChanged { - needUpdate = true - } + // 构建新问题集合 + newQuestionsSet := make(map[string]struct{}, len(normalizedNewMeta.SimilarQuestions)) + for _, q := range normalizedNewMeta.SimilarQuestions { + newQuestionsSet[q] = struct{}{} + } + + // 找出需要删除的问题(在旧集合中但不在新集合中) + var sourceIDsToDelete []string + var deletedQuestions []string + for oldQ := range oldQuestionsSet { + if _, exists := newQuestionsSet[oldQ]; !exists { + sourceID := fmt.Sprintf("%s-%s", chunk.ID, hashQuestion(oldQ)) + sourceIDsToDelete = append(sourceIDsToDelete, sourceID) + deletedQuestions = append(deletedQuestions, oldQ) } + } - if needUpdate { - sourceID := fmt.Sprintf("%s-%d", chunk.ID, i) + // 找出需要新增或更新的问题 + var addedQuestions, updatedQuestions []string + for newQ := range newQuestionsSet { + _, existedBefore := oldQuestionsSet[newQ] + // 需要更新的条件: + // 1. 新问题(之前不存在) + // 2. 答案变化(需要重新embedding) + if !existedBefore || answersChanged { + sourceID := fmt.Sprintf("%s-%s", chunk.ID, hashQuestion(newQ)) indexInfoToUpdate = append(indexInfoToUpdate, &types.IndexInfo{ - Content: buildNewContent(newQ), + Content: buildContent(newQ, normalizedNewMeta.Answers), SourceID: sourceID, SourceType: types.ChunkSourceType, ChunkID: chunk.ID, @@ -1786,22 +1941,35 @@ func (s *knowledgeService) incrementalIndexFAQEntry( IsEnabled: chunk.IsEnabled, IsRecommended: chunk.Flags.HasFlag(types.ChunkFlagRecommended), }) + if !existedBefore { + addedQuestions = append(addedQuestions, newQ) + } else { + updatedQuestions = append(updatedQuestions, newQ) + } } } - // 3. 删除多余的旧相似问索引 - if oldCount > newCount { - sourceIDsToDelete := make([]string, 0, oldCount-newCount) - for i := newCount; i < oldCount; i++ { - sourceIDsToDelete = append(sourceIDsToDelete, fmt.Sprintf("%s-%d", chunk.ID, i)) - } - logger.Debugf(ctx, "incrementalIndexFAQEntry: deleting %d obsolete source IDs", len(sourceIDsToDelete)) + // 输出详细的变化日志 + if len(deletedQuestions) > 0 { + logger.Debugf(ctx, "incrementalIndexFAQEntry: deleted similar questions: %v", deletedQuestions) + } + if len(addedQuestions) > 0 { + logger.Debugf(ctx, "incrementalIndexFAQEntry: added similar questions: %v", addedQuestions) + } + if len(updatedQuestions) > 0 { + logger.Debugf(ctx, "incrementalIndexFAQEntry: updated similar questions (answers changed): %v", updatedQuestions) + } + + // 3. 删除不再存在的相似问索引 + if len(sourceIDsToDelete) > 0 { + logger.Debugf(ctx, "incrementalIndexFAQEntry: deleting %d obsolete sourceIDs: %v", len(sourceIDsToDelete), sourceIDsToDelete) if delErr := retrieveEngine.DeleteBySourceIDList(ctx, sourceIDsToDelete, embeddingModel.GetDimensions(), types.KnowledgeTypeFAQ); delErr != nil { logger.Warnf(ctx, "incrementalIndexFAQEntry: failed to delete obsolete source IDs: %v", delErr) } } // 4. 批量索引需要更新的内容 + newCount := len(normalizedNewMeta.SimilarQuestions) if len(indexInfoToUpdate) > 0 { logger.Debugf(ctx, "incrementalIndexFAQEntry: updating %d index entries (skipped %d unchanged)", len(indexInfoToUpdate), 1+newCount-len(indexInfoToUpdate)) @@ -2193,7 +2361,7 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task) logger.Errorf(ctx, "Failed to get knowledge base: %v", err) // 如果是最后一次重试,更新状态为失败 if isLastRetry { - if updateErr := s.updateFAQImportProgressStatus(ctx, payload.TaskID, types.FAQImportStatusFailed, 0, originalTotalEntries, 0, "获取知识库失败", err.Error()); updateErr != nil { + if updateErr := s.updateFAQImportProgressStatus(ctx, payload.TaskID, payload.InstanceID, payload.EnqueuedAt, types.FAQImportStatusFailed, 0, originalTotalEntries, 0, "获取知识库失败", err.Error()); updateErr != nil { logger.Errorf(ctx, "Failed to update task status to failed: %v", updateErr) } s.recordFAQImportKBActivity(ctx, &payload, progress, originalTotalEntries, types.AuditActionFAQImportFailed, types.AuditOutcomeFailed) @@ -2207,6 +2375,9 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task) if existingProgress != nil { if existingProgress.Status == types.FAQImportStatusCompleted { logger.Infof(ctx, "FAQ import already completed, skipping: %s", payload.TaskID) + if clearErr := s.clearRunningFAQImportInfoIfMatches(ctx, payload.KBID, payload.TaskID, payload.InstanceID, payload.EnqueuedAt); clearErr != nil { + logger.Warnf(ctx, "Failed to clear running FAQ import info for completed task: %v", clearErr) + } return nil // 幂等:已完成的任务直接返回 } // 获取已处理的数量(注意:这是相对于 validEntries 的索引) @@ -2223,7 +2394,7 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task) logger.Errorf(ctx, "Failed to delete unindexed chunks: %v", err) // 如果是最后一次重试,更新状态为失败 if isLastRetry { - if updateErr := s.updateFAQImportProgressStatus(ctx, payload.TaskID, types.FAQImportStatusFailed, 0, originalTotalEntries, 0, "清理未索引数据失败", err.Error()); updateErr != nil { + if updateErr := s.updateFAQImportProgressStatus(ctx, payload.TaskID, payload.InstanceID, payload.EnqueuedAt, types.FAQImportStatusFailed, 0, originalTotalEntries, 0, "清理未索引数据失败", err.Error()); updateErr != nil { logger.Errorf(ctx, "Failed to update task status to failed: %v", updateErr) } s.recordFAQImportKBActivity(ctx, &payload, progress, originalTotalEntries, types.AuditActionFAQImportFailed, types.AuditOutcomeFailed) @@ -2280,7 +2451,7 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task) logger.Errorf(ctx, "FAQ import task failed: %s, error: %v", payload.TaskID, err) // 如果是最后一次重试,更新状态为失败 if isLastRetry { - if updateErr := s.updateFAQImportProgressStatus(ctx, payload.TaskID, types.FAQImportStatusFailed, 0, originalTotalEntries, len(validEntries), "导入失败", err.Error()); updateErr != nil { + if updateErr := s.updateFAQImportProgressStatus(ctx, payload.TaskID, payload.InstanceID, payload.EnqueuedAt, types.FAQImportStatusFailed, 0, originalTotalEntries, len(validEntries), "导入失败", err.Error()); updateErr != nil { logger.Errorf(ctx, "Failed to update task status to failed: %v", updateErr) } s.recordFAQImportKBActivity(ctx, &payload, progress, originalTotalEntries, types.AuditActionFAQImportFailed, types.AuditOutcomeFailed) @@ -2385,7 +2556,7 @@ func (s *knowledgeService) finalizeFAQValidation(ctx context.Context, payload *t } // 然后调用状态更新来清理 running key - if err := s.updateFAQImportProgressStatus(ctx, payload.TaskID, types.FAQImportStatusCompleted, + if err := s.updateFAQImportProgressStatus(ctx, payload.TaskID, payload.InstanceID, payload.EnqueuedAt, types.FAQImportStatusCompleted, 100, originalTotalEntries, originalTotalEntries, progress.Message, ""); err != nil { logger.Warnf(ctx, "Failed to update final FAQ import status: %v", err) } diff --git a/internal/handler/faq.go b/internal/handler/faq.go index 8f4fb3ef..8cd8aa9a 100644 --- a/internal/handler/faq.go +++ b/internal/handler/faq.go @@ -3,6 +3,7 @@ package handler import ( "net/http" "strconv" + "strings" "github.com/gin-gonic/gin" @@ -381,19 +382,35 @@ func (h *FAQHandler) SearchFAQ(c *gin.Context) { // ExportEntries godoc // @Summary 导出FAQ条目 -// @Description 将所有FAQ条目导出为CSV文件 +// @Description 将所有FAQ条目导出为 CSV(默认)或 JSON。?format=json 返回与 FAQEntryPayload 结构兼容的数组。 // @Tags FAQ管理 // @Accept json // @Produce text/csv -// @Param id path string true "知识库ID" -// @Success 200 {file} file "CSV文件" -// @Failure 400 {object} errors.AppError "请求参数错误" +// @Produce application/json +// @Param id path string true "知识库ID" +// @Param format query string false "导出格式:csv(默认)或 json" +// @Success 200 {file} file "导出文件" +// @Failure 400 {object} errors.AppError "请求参数错误" // @Security Bearer // @Security ApiKeyAuth // @Router /knowledge-bases/{id}/faq/entries/export [get] func (h *FAQHandler) ExportEntries(c *gin.Context) { ctx := c.Request.Context() kbID := secutils.SanitizeForLog(c.Param("id")) + format := strings.ToLower(strings.TrimSpace(c.Query("format"))) + + if format == "json" { + jsonData, err := h.knowledgeService.ExportFAQEntriesJSON(ctx, kbID) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(err) + return + } + c.Header("Content-Type", "application/json; charset=utf-8") + c.Header("Content-Disposition", "attachment; filename=faq_export.json") + c.Data(http.StatusOK, "application/json; charset=utf-8", jsonData) + return + } csvData, err := h.knowledgeService.ExportFAQEntries(ctx, kbID) if err != nil { diff --git a/internal/types/faq.go b/internal/types/faq.go index 90046609..cd8021a5 100644 --- a/internal/types/faq.go +++ b/internal/types/faq.go @@ -251,6 +251,21 @@ type FAQEntry struct { MatchedQuestion string `json:"matched_question,omitempty"` } +// FAQExportEntry 表示 JSON 导出的 FAQ 条目,与 FAQEntryPayload 的导入格式兼容, +// 便于"导出 → 编辑 → 重新导入"循环。新增字段时务必保留 omitempty,避免破坏 +// 历史导出文件的兼容性。 +type FAQExportEntry struct { + ID int64 `json:"id"` + TagName string `json:"tag_name,omitempty"` + StandardQuestion string `json:"standard_question"` + SimilarQuestions []string `json:"similar_questions,omitempty"` + NegativeQuestions []string `json:"negative_questions,omitempty"` + Answers []string `json:"answers,omitempty"` + AnswerStrategy AnswerStrategy `json:"answer_strategy,omitempty"` + IsEnabled bool `json:"is_enabled"` + IsRecommended bool `json:"is_recommended"` +} + // FAQEntryPayload 用于创建/更新 FAQ 条目的 payload type FAQEntryPayload struct { // ID 可选,用于数据迁移时指定 seq_id(必须小于自增起始值 100000000) diff --git a/internal/types/faq_sync.go b/internal/types/faq_sync.go new file mode 100644 index 00000000..d45b980a --- /dev/null +++ b/internal/types/faq_sync.go @@ -0,0 +1,77 @@ +package types + +import "encoding/json" + +// FAQChunkSyncPair links source/destination FAQ chunks sharing content_hash. +type FAQChunkSyncPair struct { + SrcChunkID string + DstChunkID string +} + +// FAQChunkDiffResult is the output of FAQChunkDiff. +type FAQChunkDiffResult struct { + ChunksToAdd []string + ChunksToDelete []string + MatchedPairs []FAQChunkSyncPair +} + +// FAQChunkStatus holds lightweight fields for clone status sync. +type FAQChunkStatus struct { + ID string + TagID string + IsEnabled bool + Flags ChunkFlags + AnswerStrategy AnswerStrategy + Metadata JSON +} + +// FAQChunkStatusFieldsEqual compares status fields (tag excluded). +func FAQChunkStatusFieldsEqual(a, b *FAQChunkStatus) bool { + if a == nil || b == nil { + return a == b + } + if a.IsEnabled != b.IsEnabled || a.Flags != b.Flags { + return false + } + return NormalizeAnswerStrategy(a.AnswerStrategy) == NormalizeAnswerStrategy(b.AnswerStrategy) +} + +// FAQChunkNeedsStatusSync reports whether a matched pair needs status sync. +func FAQChunkNeedsStatusSync(src, dst *FAQChunkStatus, mappedSrcTagID string) bool { + if src == nil || dst == nil { + return false + } + if mappedSrcTagID != dst.TagID { + return true + } + return !FAQChunkStatusFieldsEqual(src, dst) +} + +// NormalizeAnswerStrategy treats empty as "all". +func NormalizeAnswerStrategy(s AnswerStrategy) AnswerStrategy { + if s == "" { + return AnswerStrategyAll + } + return s +} + +// PatchFAQAnswerStrategy copies answer_strategy from src metadata onto dst metadata. +func PatchFAQAnswerStrategy(dstMeta, srcMeta JSON) (JSON, error) { + var dstM, srcM FAQChunkMetadata + if len(dstMeta) > 0 { + if err := json.Unmarshal(dstMeta, &dstM); err != nil { + return nil, err + } + } + if len(srcMeta) > 0 { + if err := json.Unmarshal(srcMeta, &srcM); err != nil { + return nil, err + } + } + dstM.AnswerStrategy = srcM.AnswerStrategy + bytes, err := json.Marshal(&dstM) + if err != nil { + return nil, err + } + return JSON(bytes), nil +} diff --git a/internal/types/faq_sync_test.go b/internal/types/faq_sync_test.go new file mode 100644 index 00000000..f6be2a77 --- /dev/null +++ b/internal/types/faq_sync_test.go @@ -0,0 +1,19 @@ +package types + +import "testing" + +func TestFAQChunkNeedsStatusSync(t *testing.T) { + src := &FAQChunkStatus{TagID: "a", IsEnabled: true, Flags: ChunkFlagRecommended, AnswerStrategy: AnswerStrategyAll} + dst := &FAQChunkStatus{TagID: "old", IsEnabled: true, Flags: ChunkFlagRecommended, AnswerStrategy: AnswerStrategyAll} + if !FAQChunkNeedsStatusSync(src, dst, "mapped") { + t.Fatal("tag mismatch should need sync") + } + dst.TagID = "mapped" + if FAQChunkNeedsStatusSync(src, dst, "mapped") { + t.Fatal("in sync after tag mapped") + } + dst.IsEnabled = false + if !FAQChunkNeedsStatusSync(src, dst, "mapped") { + t.Fatal("is_enabled mismatch should need sync") + } +} diff --git a/internal/types/interfaces/chunk.go b/internal/types/interfaces/chunk.go index 1e62f5e3..11b0fba7 100644 --- a/internal/types/interfaces/chunk.go +++ b/internal/types/interfaces/chunk.go @@ -97,8 +97,9 @@ type ChunkRepository interface { // newTagID: if not nil, updates tag_id to this value (empty string means uncategorized) UpdateChunkFieldsByTagID(ctx context.Context, tenantID uint64, kbID string, tagID string, isEnabled *bool, setFlags types.ChunkFlags, clearFlags types.ChunkFlags, newTagID *string, excludeIDs []string) ([]string, error) // FAQChunkDiff compares FAQ chunks between two knowledge bases and returns the differences. - // Returns: chunksToAdd (content_hash in src but not in dst), chunksToDelete (content_hash in dst but not in src) - FAQChunkDiff(ctx context.Context, srcTenantID uint64, srcKBID string, dstTenantID uint64, dstKBID string) (chunksToAdd []string, chunksToDelete []string, err error) + FAQChunkDiff(ctx context.Context, srcTenantID uint64, srcKBID string, dstTenantID uint64, dstKBID string) (*types.FAQChunkDiffResult, error) + // ListFAQChunkStatusByIDs loads status fields for FAQ clone sync. + ListFAQChunkStatusByIDs(ctx context.Context, tenantID uint64, ids []string) (map[string]*types.FAQChunkStatus, error) // ListRecommendedFAQChunks lists FAQ chunks with the recommended flag set. // Filter by explicitly selected kbIDs, knowledgeIDs, and/or FAQ tagIDs. diff --git a/internal/types/interfaces/knowledge.go b/internal/types/interfaces/knowledge.go index ae176da3..aea4499f 100644 --- a/internal/types/interfaces/knowledge.go +++ b/internal/types/interfaces/knowledge.go @@ -152,6 +152,8 @@ type KnowledgeService interface { SearchFAQEntries(ctx context.Context, kbID string, req *types.FAQSearchRequest) ([]*types.FAQEntry, error) // ExportFAQEntries exports all FAQ entries for a knowledge base as CSV data. ExportFAQEntries(ctx context.Context, kbID string) ([]byte, error) + // ExportFAQEntriesJSON exports all FAQ entries as a JSON array compatible with FAQEntryPayload. + ExportFAQEntriesJSON(ctx context.Context, kbID string) ([]byte, error) // UpdateKnowledgeTagBatch updates tag for document knowledge items in batch. // authorizedKBID restricts all updates to knowledge items belonging to this KB; // pass empty string to skip (caller must ensure authorization by other means). diff --git a/internal/types/task.go b/internal/types/task.go index 1461b7b2..fdcd916b 100644 --- a/internal/types/task.go +++ b/internal/types/task.go @@ -305,6 +305,7 @@ type FAQImportPayload struct { Mode string `json:"mode"` DryRun bool `json:"dry_run"` // dry run 模式只验证不导入 EnqueuedAt int64 `json:"enqueued_at"` // 任务入队时间戳,用于区分同一 TaskID 的不同次提交 + InstanceID string `json:"instance_id,omitempty"` Initiator TaskInitiator `json:"initiator,omitempty"` }