From b6a3b347dae816d7c24cc9887354fd5e2799dcc7 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Wed, 22 Jul 2026 20:57:31 +0800 Subject: [PATCH] feat(knowledge): enhance FAQ export functionality - Updated the exportFAQEntries function to support exporting FAQ entries in both CSV and JSON formats. - Modified the frontend to include options for selecting the export format. - Enhanced localization files to reflect the new export options in English, Russian, and Chinese. - Implemented new tests to validate the export functionality for both formats. This commit improves the flexibility of FAQ exports, allowing users to choose their preferred format for better integration with other tools. --- frontend/src/api/knowledge-base/index.ts | 9 +- frontend/src/i18n/locales/en-US.ts | 4 +- frontend/src/i18n/locales/ru-RU.ts | 4 +- frontend/src/i18n/locales/zh-CN.ts | 4 +- .../knowledge/components/FAQEntryManager.vue | 53 +++- internal/application/repository/chunk.go | 180 +++++++++-- .../repository/chunk_faq_diff_test.go | 117 +++++++ .../application/service/faq_clone_sync.go | 165 ++++++++++ .../service/knowledge_clone_move.go | 70 ++++- internal/application/service/knowledge_faq.go | 131 ++++++++ .../service/knowledge_faq_import.go | 297 ++++++++++++++---- internal/handler/faq.go | 25 +- internal/types/faq.go | 15 + internal/types/faq_sync.go | 77 +++++ internal/types/faq_sync_test.go | 19 ++ internal/types/interfaces/chunk.go | 5 +- internal/types/interfaces/knowledge.go | 2 + internal/types/task.go | 1 + 18 files changed, 1041 insertions(+), 137 deletions(-) create mode 100644 internal/application/repository/chunk_faq_diff_test.go create mode 100644 internal/application/service/faq_clone_sync.go create mode 100644 internal/types/faq_sync.go create mode 100644 internal/types/faq_sync_test.go diff --git a/frontend/src/api/knowledge-base/index.ts b/frontend/src/api/knowledge-base/index.ts index 37cf86ea..fdb10583 100644 --- a/frontend/src/api/knowledge-base/index.ts +++ b/frontend/src/api/knowledge-base/index.ts @@ -445,10 +445,11 @@ export function searchFAQEntries( return post(`/api/v1/knowledge-bases/${kbId}/faq/search`, data); } -// Export FAQ entries as CSV file -export async function exportFAQEntries(kbId: string): Promise { - const response = await getDown(`/api/v1/knowledge-bases/${kbId}/faq/entries/export`); - return response as unknown as Blob; +// Export FAQ entries as CSV or JSON file +export async function exportFAQEntries(kbId: string, format: 'csv' | 'json' = 'csv'): Promise { + const suffix = format === 'json' ? '?format=json' : '' + const response = await getDown(`/api/v1/knowledge-bases/${kbId}/faq/entries/export${suffix}`) + return response as unknown as Blob } // FAQ Import Progress API diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index 208f5d87..fb744a52 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -2992,7 +2992,9 @@ export default { downloadExampleExcel: 'Download Excel Example', }, faqExport: { - exportButton: 'Export CSV', + exportButton: 'Export', + exportCSV: 'Export CSV', + exportJSON: 'Export JSON', exportSuccess: 'Export successful', exportFailed: 'Export failed', }, diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index e5b93041..486b67b1 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -3689,7 +3689,9 @@ export default { downloadExampleExcel: 'Скачать пример Excel' }, faqExport: { - exportButton: 'Экспорт CSV', + exportButton: 'Экспорт', + exportCSV: 'Экспорт CSV', + exportJSON: 'Экспорт JSON', exportSuccess: 'Экспорт успешен', exportFailed: 'Ошибка экспорта' }, diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index 15e52496..e47bd964 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -4281,7 +4281,9 @@ export default { downloadExampleExcel: "下载 Excel 示例", }, faqExport: { - exportButton: "导出 CSV", + exportButton: "导出", + exportCSV: "导出 CSV", + exportJSON: "导出 JSON", exportSuccess: "导出成功", exportFailed: "导出失败", }, diff --git a/frontend/src/views/knowledge/components/FAQEntryManager.vue b/frontend/src/views/knowledge/components/FAQEntryManager.vue index 475ad3cd..a49f0b36 100644 --- a/frontend/src/views/knowledge/components/FAQEntryManager.vue +++ b/frontend/src/views/knowledge/components/FAQEntryManager.vue @@ -238,12 +238,15 @@ - - - - - + + + + + + + { return orgStore.canManageKB(props.kbId, false) }) +const faqExportOptions = computed(() => [ + { content: t('knowledgeEditor.faqExport.exportCSV'), value: 'export_csv' }, + { content: t('knowledgeEditor.faqExport.exportJSON'), value: 'export_json' }, +]) + // FAQ 操作:新建组(新建条目 + 导入) const faqCreateOptions = computed(() => { if (!canEdit.value) return [] @@ -1019,6 +1027,12 @@ const handleFaqAction = (data: { value: string }) => { case 'search': searchDrawerVisible.value = true break + case 'export_csv': + handleExportCSV() + break + case 'export_json': + handleExportJSON() + break case 'export': handleExportCSV() break @@ -2647,9 +2661,19 @@ const downloadExcelExample = () => { XLSX.writeFile(workbook, 'faq_example.xlsx') } -// 导出 FAQ 数据为 CSV +// 导出 FAQ 数据 const exportLoading = ref(false) -const handleExportCSV = async () => { +const downloadExportBlob = (blob: Blob, ext: 'csv' | 'json') => { + const url = URL.createObjectURL(blob) + const link = document.createElement('a') + link.href = url + link.download = `faq_export_${new Date().toISOString().slice(0, 10)}.${ext}` + document.body.appendChild(link) + link.click() + document.body.removeChild(link) + URL.revokeObjectURL(url) +} +const handleExportFAQ = async (format: 'csv' | 'json') => { if (!props.kbId) { MessagePlugin.warning(t('knowledgeBase.selectKnowledgeBase')) return @@ -2657,15 +2681,8 @@ const handleExportCSV = async () => { exportLoading.value = true try { - const blob = await exportFAQEntries(props.kbId) - const url = URL.createObjectURL(blob) - const link = document.createElement('a') - link.href = url - link.download = `faq_export_${new Date().toISOString().slice(0, 10)}.csv` - document.body.appendChild(link) - link.click() - document.body.removeChild(link) - URL.revokeObjectURL(url) + const blob = await exportFAQEntries(props.kbId, format) + downloadExportBlob(blob, format) MessagePlugin.success(t('knowledgeEditor.faqExport.exportSuccess')) } catch (error: any) { console.error('Export failed:', error) @@ -2674,6 +2691,8 @@ const handleExportCSV = async () => { exportLoading.value = false } } +const handleExportCSV = () => handleExportFAQ('csv') +const handleExportJSON = () => handleExportFAQ('json') watch( () => props.kbId, diff --git a/internal/application/repository/chunk.go b/internal/application/repository/chunk.go index 8c8af637..1fd7ada7 100644 --- a/internal/application/repository/chunk.go +++ b/internal/application/repository/chunk.go @@ -931,44 +931,170 @@ func (r *chunkRepository) UpdateChunkFieldsByTagID( return affectedIDs, nil } +type chunkIDHash struct { + ID string `gorm:"column:id"` + ContentHash string `gorm:"column:content_hash"` +} + +const faqChunkDiffBatchSize = 5000 + +// listFAQChunkIDHashesByKB loads id/content_hash pairs for all FAQ chunks in a KB. +func (r *chunkRepository) listFAQChunkIDHashesByKB( + ctx context.Context, + tenantID uint64, + kbID string, +) ([]chunkIDHash, error) { + var all []chunkIDHash + var lastID string + + for { + var batch []chunkIDHash + query := r.db.WithContext(ctx).Model(&types.Chunk{}). + Select("id, content_hash"). + Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?", + tenantID, kbID, types.ChunkTypeFAQ). + Order("id ASC"). + Limit(faqChunkDiffBatchSize) + if lastID != "" { + query = query.Where("id > ?", lastID) + } + if err := query.Find(&batch).Error; err != nil { + return nil, err + } + if len(batch) == 0 { + break + } + all = append(all, batch...) + lastID = batch[len(batch)-1].ID + if len(batch) < faqChunkDiffBatchSize { + break + } + } + return all, nil +} + +func diffFAQChunkIDsByContentHash(src, dst []chunkIDHash) ( + chunksToAdd, chunksToDelete []string, + matched []types.FAQChunkSyncPair, +) { + dstHashes := make(map[string]struct{}, len(dst)) + dstIDByHash := make(map[string]string, len(dst)) + dstIDsByHash := make(map[string][]string, len(dst)) + for _, pair := range dst { + dstHashes[pair.ContentHash] = struct{}{} + if _, ok := dstIDByHash[pair.ContentHash]; !ok { + dstIDByHash[pair.ContentHash] = pair.ID + } + dstIDsByHash[pair.ContentHash] = append(dstIDsByHash[pair.ContentHash], pair.ID) + } + srcHashes := make(map[string]struct{}, len(src)) + for _, pair := range src { + srcHashes[pair.ContentHash] = struct{}{} + } + + for _, pair := range src { + if _, exists := dstHashes[pair.ContentHash]; !exists { + chunksToAdd = append(chunksToAdd, pair.ID) + continue + } + if dstID, ok := dstIDByHash[pair.ContentHash]; ok { + matched = append(matched, types.FAQChunkSyncPair{ + SrcChunkID: pair.ID, + DstChunkID: dstID, + }) + } + } + for _, pair := range dst { + if _, exists := srcHashes[pair.ContentHash]; !exists { + chunksToDelete = append(chunksToDelete, pair.ID) + } + } + for hash, ids := range dstIDsByHash { + if hash == "" || len(ids) <= 1 { + continue + } + if _, inSrc := srcHashes[hash]; !inSrc { + continue + } + canonical := dstIDByHash[hash] + for _, id := range ids { + if id != canonical { + chunksToDelete = append(chunksToDelete, id) + } + } + } + return chunksToAdd, chunksToDelete, matched +} + // FAQChunkDiff compares FAQ chunks between two knowledge bases and returns the differences. // Returns: chunksToAdd (IDs of chunks in src whose content_hash is not in dst), // -// chunksToDelete (IDs of chunks in dst whose content_hash is not in src) +// chunksToDelete (IDs of chunks in dst whose content_hash is not in src, plus +// duplicate dst chunks that share a content_hash with another dst chunk when +// that hash still exists in src) func (r *chunkRepository) FAQChunkDiff( ctx context.Context, srcTenantID uint64, srcKBID string, dstTenantID uint64, dstKBID string, -) (chunksToAdd []string, chunksToDelete []string, err error) { - // Get content_hash set from destination KB - dstHashSubQuery := r.db.Model(&types.Chunk{}). - Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?", dstTenantID, dstKBID, types.ChunkTypeFAQ). - Select("content_hash") - - // Find chunks in source that don't exist in destination (by content_hash) - err = r.db.WithContext(ctx).Model(&types.Chunk{}). - Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?", srcTenantID, srcKBID, types.ChunkTypeFAQ). - Where("content_hash NOT IN (?)", dstHashSubQuery). - Pluck("id", &chunksToAdd).Error - if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { - return nil, nil, fmt.Errorf("failed to get chunks to add: %w", err) +) (*types.FAQChunkDiffResult, error) { + srcPairs, err := r.listFAQChunkIDHashesByKB(ctx, srcTenantID, srcKBID) + if err != nil { + return nil, fmt.Errorf("failed to list source FAQ chunks: %w", err) + } + dstPairs, err := r.listFAQChunkIDHashesByKB(ctx, dstTenantID, dstKBID) + if err != nil { + return nil, fmt.Errorf("failed to list destination FAQ chunks: %w", err) } - // Get content_hash set from source KB - srcHashSubQuery := r.db.Model(&types.Chunk{}). - Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?", srcTenantID, srcKBID, types.ChunkTypeFAQ). - Select("content_hash") + add, del, matched := diffFAQChunkIDsByContentHash(srcPairs, dstPairs) + return &types.FAQChunkDiffResult{ + ChunksToAdd: add, + ChunksToDelete: del, + MatchedPairs: matched, + }, nil +} - // Find chunks in destination that don't exist in source (by content_hash) - err = r.db.WithContext(ctx).Model(&types.Chunk{}). - Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?", dstTenantID, dstKBID, types.ChunkTypeFAQ). - Where("content_hash NOT IN (?)", srcHashSubQuery). - Pluck("id", &chunksToDelete).Error - if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { - return nil, nil, fmt.Errorf("failed to get chunks to delete: %w", err) +// ListFAQChunkStatusByIDs loads status fields for FAQ clone sync. +func (r *chunkRepository) ListFAQChunkStatusByIDs( + ctx context.Context, + tenantID uint64, + ids []string, +) (map[string]*types.FAQChunkStatus, error) { + if len(ids) == 0 { + return map[string]*types.FAQChunkStatus{}, nil } - - return chunksToAdd, chunksToDelete, nil + const batchSize = 5000 + var chunks []*types.Chunk + for i := 0; i < len(ids); i += batchSize { + end := i + batchSize + if end > len(ids) { + end = len(ids) + } + var batch []*types.Chunk + if err := r.db.WithContext(ctx). + Select("id, tag_id, is_enabled, flags, metadata"). + Where("tenant_id = ? AND id IN ?", tenantID, ids[i:end]). + Find(&batch).Error; err != nil { + return nil, err + } + chunks = append(chunks, batch...) + } + out := make(map[string]*types.FAQChunkStatus, len(chunks)) + for _, chunk := range chunks { + strategy := types.AnswerStrategyAll + if meta, err := chunk.FAQMetadata(); err == nil && meta != nil { + strategy = meta.AnswerStrategy + } + out[chunk.ID] = &types.FAQChunkStatus{ + ID: chunk.ID, + TagID: chunk.TagID, + IsEnabled: chunk.IsEnabled, + Flags: chunk.Flags, + AnswerStrategy: strategy, + Metadata: chunk.Metadata, + } + } + return out, nil } // ListRecommendedFAQChunks lists FAQ chunks with the recommended flag set. diff --git a/internal/application/repository/chunk_faq_diff_test.go b/internal/application/repository/chunk_faq_diff_test.go new file mode 100644 index 00000000..ba578772 --- /dev/null +++ b/internal/application/repository/chunk_faq_diff_test.go @@ -0,0 +1,117 @@ +package repository + +import ( + "context" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDiffFAQChunkIDsByContentHash(t *testing.T) { + toAdd, toDelete, matched := diffFAQChunkIDsByContentHash( + []chunkIDHash{ + {ID: "src-1", ContentHash: "hash-a"}, + {ID: "src-2", ContentHash: "hash-b"}, + {ID: "src-3", ContentHash: "hash-only-src"}, + }, + []chunkIDHash{ + {ID: "dst-1", ContentHash: "hash-a"}, + {ID: "dst-2", ContentHash: "hash-only-dst"}, + }, + ) + + assert.ElementsMatch(t, []string{"src-2", "src-3"}, toAdd) + assert.ElementsMatch(t, []string{"dst-2"}, toDelete) + require.Len(t, matched, 1) + assert.Equal(t, "src-1", matched[0].SrcChunkID) + assert.Equal(t, "dst-1", matched[0].DstChunkID) +} + +func TestDiffFAQChunkIDsByContentHash_dstDuplicateHash(t *testing.T) { + // Two destination chunks share hash-a (same FAQ content, e.g. stale copy after + // answer_strategy-only change). Source has one chunk for hash-a → keep dst-1, + // delete dst-dup, match for status sync. + toAdd, toDelete, matched := diffFAQChunkIDsByContentHash( + []chunkIDHash{{ID: "src-1", ContentHash: "hash-a"}}, + []chunkIDHash{ + {ID: "dst-1", ContentHash: "hash-a"}, + {ID: "dst-dup", ContentHash: "hash-a"}, + }, + ) + + assert.Empty(t, toAdd) + assert.ElementsMatch(t, []string{"dst-dup"}, toDelete) + require.Len(t, matched, 1) + assert.Equal(t, "src-1", matched[0].SrcChunkID) + assert.Equal(t, "dst-1", matched[0].DstChunkID) +} + +func TestDiffFAQChunkIDsByContentHash_dstDuplicateHash_notInSrc(t *testing.T) { + // Duplicates whose hash is absent from source are all removed (orphan content). + toAdd, toDelete, matched := diffFAQChunkIDsByContentHash( + nil, + []chunkIDHash{ + {ID: "dst-orphan-1", ContentHash: "hash-orphan"}, + {ID: "dst-orphan-2", ContentHash: "hash-orphan"}, + }, + ) + + assert.Empty(t, toAdd) + assert.ElementsMatch(t, []string{"dst-orphan-1", "dst-orphan-2"}, toDelete) + assert.Empty(t, matched) +} + +func TestDiffFAQChunkIDsByContentHash_emptySide(t *testing.T) { + toAdd, toDelete, matched := diffFAQChunkIDsByContentHash( + []chunkIDHash{{ID: "src-1", ContentHash: "hash-a"}}, + nil, + ) + assert.Equal(t, []string{"src-1"}, toAdd) + assert.Empty(t, toDelete) + assert.Empty(t, matched) + + toAdd, toDelete, matched = diffFAQChunkIDsByContentHash( + nil, + []chunkIDHash{{ID: "dst-1", ContentHash: "hash-a"}}, + ) + assert.Empty(t, toAdd) + assert.Equal(t, []string{"dst-1"}, toDelete) + assert.Empty(t, matched) +} + +func TestFAQChunkDiff_SQLite(t *testing.T) { + db := setupChunkTestDB(t) + repo := NewChunkRepository(db) + ctx := context.Background() + + srcKBID := uuid.New().String() + dstKBID := uuid.New().String() + srcKnowledgeID := uuid.New().String() + dstKnowledgeID := uuid.New().String() + + require.NoError(t, db.Create(&types.Chunk{ + ID: uuid.New().String(), TenantID: 1, KnowledgeBaseID: srcKBID, KnowledgeID: srcKnowledgeID, + ChunkType: types.ChunkTypeFAQ, ContentHash: "shared", Status: int(types.ChunkStatusIndexed), + }).Error) + require.NoError(t, db.Create(&types.Chunk{ + ID: uuid.New().String(), TenantID: 1, KnowledgeBaseID: srcKBID, KnowledgeID: srcKnowledgeID, + ChunkType: types.ChunkTypeFAQ, ContentHash: "src-only", Status: int(types.ChunkStatusIndexed), + }).Error) + require.NoError(t, db.Create(&types.Chunk{ + ID: uuid.New().String(), TenantID: 1, KnowledgeBaseID: dstKBID, KnowledgeID: dstKnowledgeID, + ChunkType: types.ChunkTypeFAQ, ContentHash: "shared", Status: int(types.ChunkStatusIndexed), + }).Error) + require.NoError(t, db.Create(&types.Chunk{ + ID: uuid.New().String(), TenantID: 1, KnowledgeBaseID: dstKBID, KnowledgeID: dstKnowledgeID, + ChunkType: types.ChunkTypeFAQ, ContentHash: "dst-only", Status: int(types.ChunkStatusIndexed), + }).Error) + + diff, err := repo.FAQChunkDiff(ctx, 1, srcKBID, 1, dstKBID) + require.NoError(t, err) + assert.Len(t, diff.ChunksToAdd, 1) + assert.Len(t, diff.ChunksToDelete, 1) + assert.Len(t, diff.MatchedPairs, 1) +} diff --git a/internal/application/service/faq_clone_sync.go b/internal/application/service/faq_clone_sync.go new file mode 100644 index 00000000..c40368eb --- /dev/null +++ b/internal/application/service/faq_clone_sync.go @@ -0,0 +1,165 @@ +package service + +import ( + "context" + "fmt" + "time" + + "github.com/Tencent/WeKnora/internal/application/service/retriever" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" +) + +type faqStatusSyncPlan struct { + Pairs []types.FAQChunkSyncPair + SrcByID map[string]*types.FAQChunkStatus + DstByID map[string]*types.FAQChunkStatus +} + +func (s *knowledgeService) buildFAQStatusSyncPlan( + ctx context.Context, + srcTenantID, dstTenantID uint64, + matched []types.FAQChunkSyncPair, + resolveTag func(srcTagID string) string, +) (*faqStatusSyncPlan, error) { + if len(matched) == 0 { + return &faqStatusSyncPlan{}, nil + } + srcIDs := make([]string, 0, len(matched)) + dstIDs := make([]string, 0, len(matched)) + for _, p := range matched { + srcIDs = append(srcIDs, p.SrcChunkID) + dstIDs = append(dstIDs, p.DstChunkID) + } + srcByID, err := s.chunkRepo.ListFAQChunkStatusByIDs(ctx, srcTenantID, srcIDs) + if err != nil { + return nil, err + } + dstByID, err := s.chunkRepo.ListFAQChunkStatusByIDs(ctx, dstTenantID, dstIDs) + if err != nil { + return nil, err + } + pairs := make([]types.FAQChunkSyncPair, 0) + for _, p := range matched { + src, dst := srcByID[p.SrcChunkID], dstByID[p.DstChunkID] + if src == nil || dst == nil { + continue + } + mappedTag := "" + if src.TagID != "" { + mappedTag = resolveTag(src.TagID) + } + if types.FAQChunkNeedsStatusSync(src, dst, mappedTag) { + pairs = append(pairs, p) + } + } + return &faqStatusSyncPlan{Pairs: pairs, SrcByID: srcByID, DstByID: dstByID}, nil +} + +func (s *knowledgeService) syncFAQChunkStatusBatch( + ctx context.Context, + dstKB *types.KnowledgeBase, + pairs []types.FAQChunkSyncPair, + srcByID, dstByID map[string]*types.FAQChunkStatus, + resolveTag func(srcTagID string) string, +) error { + if len(pairs) == 0 { + return nil + } + tenantID := types.MustTenantIDFromContext(ctx) + enabledUpdates := make(map[string]bool) + recommendedUpdates := make(map[string]bool) + tagUpdates := make(map[string]string) + rows := make([]*types.Chunk, 0, len(pairs)) + metadataIDs := make([]string, 0) + + for _, p := range pairs { + src, dst := srcByID[p.SrcChunkID], dstByID[p.DstChunkID] + if src == nil || dst == nil { + continue + } + mappedTag := "" + if src.TagID != "" { + mappedTag = resolveTag(src.TagID) + } + if !types.FAQChunkNeedsStatusSync(src, dst, mappedTag) { + continue + } + rows = append(rows, &types.Chunk{ + ID: dst.ID, IsEnabled: src.IsEnabled, Flags: src.Flags, + TagID: mappedTag, UpdatedAt: time.Now(), + }) + if dst.IsEnabled != src.IsEnabled { + enabledUpdates[dst.ID] = src.IsEnabled + } + srcRec := src.Flags.HasFlag(types.ChunkFlagRecommended) + dstRec := dst.Flags.HasFlag(types.ChunkFlagRecommended) + if srcRec != dstRec { + recommendedUpdates[dst.ID] = srcRec + } + if mappedTag != dst.TagID { + tagUpdates[dst.ID] = mappedTag + } + if types.NormalizeAnswerStrategy(src.AnswerStrategy) != types.NormalizeAnswerStrategy(dst.AnswerStrategy) { + metadataIDs = append(metadataIDs, dst.ID) + } + } + if len(rows) > 0 { + if err := s.chunkRepo.UpdateChunks(ctx, rows); err != nil { + return err + } + } + for _, dstID := range metadataIDs { + src := srcForDst(pairs, srcByID, dstID) + dst := dstByID[dstID] + if src == nil || dst == nil { + continue + } + patched, err := types.PatchFAQAnswerStrategy(dst.Metadata, src.Metadata) + if err != nil { + return fmt.Errorf("patch answer_strategy for %s: %w", dstID, err) + } + full, err := s.chunkRepo.GetChunkByID(ctx, tenantID, dstID) + if err != nil { + return err + } + full.Metadata = patched + full.UpdatedAt = time.Now() + if err := s.chunkRepo.UpdateChunk(ctx, full); err != nil { + return err + } + } + if len(enabledUpdates) == 0 && len(tagUpdates) == 0 && len(recommendedUpdates) == 0 { + return nil + } + engine, err := retriever.CreateRetrieveEngineForKB( + ctx, s.retrieveEngine, s.ownership, tenantID, dstKB.VectorStoreID) + if err != nil { + return err + } + if len(enabledUpdates) > 0 { + if err := engine.BatchUpdateChunkEnabledStatus(ctx, enabledUpdates); err != nil { + return err + } + } + if len(tagUpdates) > 0 { + if err := engine.BatchUpdateChunkTagID(ctx, tagUpdates); err != nil { + return err + } + } + if len(recommendedUpdates) > 0 { + // Vector-store recommended flag sync is not yet available on all backends in + // this branch; DB flags were already updated via UpdateChunks above. + logger.Warnf(ctx, "FAQ clone sync: skipped vector recommended update for %d chunks (DB flags updated)", len(recommendedUpdates)) + } + return nil +} + +func srcForDst(pairs []types.FAQChunkSyncPair, srcByID map[string]*types.FAQChunkStatus, dstID string) *types.FAQChunkStatus { + for _, p := range pairs { + if p.DstChunkID == dstID { + return srcByID[p.SrcChunkID] + } + } + return nil +} diff --git a/internal/application/service/knowledge_clone_move.go b/internal/application/service/knowledge_clone_move.go index 45ce46f0..89895ccb 100644 --- a/internal/application/service/knowledge_clone_move.go +++ b/internal/application/service/knowledge_clone_move.go @@ -675,23 +675,47 @@ func (s *knowledgeService) cloneFAQKnowledgeBase( } srcKnowledge := srcKnowledgeList[0] - // Get chunk-level differences based on content_hash - chunksToAdd, chunksToDelete, err := s.chunkRepo.FAQChunkDiff(ctx, srcKB.TenantID, srcKB.ID, dstKB.TenantID, dstKB.ID) + // Get chunk-level differences based on content_hash. + diff, err := s.chunkRepo.FAQChunkDiff(ctx, srcKB.TenantID, srcKB.ID, dstKB.TenantID, dstKB.ID) if err != nil { logger.Errorf(ctx, "Failed to calculate FAQ chunk difference: %v", err) handleError(progress, err, "Failed to calculate FAQ chunk difference") return err } + chunksToAdd := diff.ChunksToAdd + chunksToDelete := diff.ChunksToDelete - totalOperations := len(chunksToAdd) + len(chunksToDelete) + tagIDMapping := map[string]string{} + resolveFAQTag := func(srcTagID string) string { + if srcTagID == "" { + return "" + } + if id, ok := tagIDMapping[srcTagID]; ok { + return id + } + return s.getOrCreateTagInTarget(ctx, srcKB.TenantID, dstKB.TenantID, dstKB.ID, srcTagID, tagIDMapping) + } + + syncPlan, err := s.buildFAQStatusSyncPlan(ctx, srcKB.TenantID, dstKB.TenantID, diff.MatchedPairs, resolveFAQTag) + if err != nil { + logger.Errorf(ctx, "Failed to build FAQ status sync plan: %v", err) + handleError(progress, err, "Failed to build FAQ status sync plan") + return err + } + chunksToUpdate := syncPlan.Pairs + + totalOperations := len(chunksToAdd) + len(chunksToDelete) + len(chunksToUpdate) progress.Total = totalOperations - progress.Message = fmt.Sprintf("Found %d FAQ entries to add, %d to delete", len(chunksToAdd), len(chunksToDelete)) + progress.Message = fmt.Sprintf( + "Found %d FAQ entries to add, %d to delete, %d to update status", + len(chunksToAdd), len(chunksToDelete), len(chunksToUpdate), + ) progress.UpdatedAt = time.Now().Unix() _ = s.saveKBCloneProgress(ctx, progress) - logger.Infof(ctx, "FAQ chunks to add: %d, delete: %d", len(chunksToAdd), len(chunksToDelete)) + logger.Infof(ctx, "FAQ chunks to add: %d, delete: %d, update status: %d", + len(chunksToAdd), len(chunksToDelete), len(chunksToUpdate)) - // If nothing to do, mark as completed if totalOperations == 0 { progress.Status = types.KBCloneStatusCompleted progress.Progress = 100 @@ -760,7 +784,6 @@ func (s *knowledgeService) cloneFAQKnowledgeBase( // Clone FAQ chunks from source to destination batch := 50 - tagIDMapping := map[string]string{} // srcTagID -> dstTagID for i := 0; i < len(chunksToAdd); i += batch { end := i + batch if end > len(chunksToAdd) { @@ -782,19 +805,11 @@ func (s *knowledgeService) cloneFAQKnowledgeBase( // Map TagID to target knowledge base targetTagID := "" if srcChunk.TagID != "" { - if mappedTagID, ok := tagIDMapping[srcChunk.TagID]; ok { - targetTagID = mappedTagID - } else { - // Try to find or create the tag in target knowledge base - targetTagID = s.getOrCreateTagInTarget(ctx, srcKB.TenantID, dstKB.TenantID, dstKB.ID, srcChunk.TagID, tagIDMapping) - } + targetTagID = resolveFAQTag(srcChunk.TagID) } // Deep-copy extracted images into objects owned by the destination - // FAQ knowledge so deleting the source never breaks this clone. A FAQ - // chunk is self-contained (its own Content + image_info), so its - // ![](url) references can be rewritten immediately using the mapping - // just accumulated in imageURLCache. + // FAQ knowledge so deleting the source never breaks this clone. newImageInfo, copied, copyErr := cloneChunkImageInfo( ctx, dstSvc, srcChunk.ImageInfo, dstKB.TenantID, dstKnowledge.ID, imageURLCache) if copyErr != nil { @@ -860,6 +875,27 @@ func (s *knowledgeService) cloneFAQKnowledgeBase( _ = s.saveKBCloneProgress(ctx, progress) } + for i := 0; i < len(chunksToUpdate); i += batch { + end := i + batch + if end > len(chunksToUpdate) { + end = len(chunksToUpdate) + } + if err := s.syncFAQChunkStatusBatch( + ctx, dstKB, chunksToUpdate[i:end], syncPlan.SrcByID, syncPlan.DstByID, resolveFAQTag); err != nil { + logger.Errorf(ctx, "Failed to sync FAQ status fields: %v", err) + handleError(progress, err, "Failed to sync FAQ status fields") + return err + } + processedCount += end - i + if totalOperations > 0 { + progress.Progress = processedCount * 100 / totalOperations + } + progress.Processed = processedCount + progress.Message = fmt.Sprintf("Updated %d/%d FAQ status fields", processedCount-len(chunksToDelete)-len(chunksToAdd), len(chunksToUpdate)) + progress.UpdatedAt = time.Now().Unix() + _ = s.saveKBCloneProgress(ctx, progress) + } + // Mark as completed progress.Status = types.KBCloneStatusCompleted progress.Progress = 100 diff --git a/internal/application/service/knowledge_faq.go b/internal/application/service/knowledge_faq.go index 5894845b..9171fab5 100644 --- a/internal/application/service/knowledge_faq.go +++ b/internal/application/service/knowledge_faq.go @@ -2,6 +2,9 @@ package service import ( "context" + "crypto/md5" + "encoding/hex" + "encoding/json" "fmt" "slices" "strings" @@ -1445,6 +1448,66 @@ func (s *knowledgeService) ExportFAQEntries(ctx context.Context, kbID string) ([ return s.buildFAQCSV(chunks, tagMap), nil } +// ExportFAQEntriesJSON 以 JSON 数组形式导出 FAQ 知识库下的全部条目, +// 字段与 FAQEntryPayload 兼容,便于"导出 → 编辑 → 重新 append 导入"循环。 +func (s *knowledgeService) ExportFAQEntriesJSON(ctx context.Context, kbID string) ([]byte, error) { + kb, err := s.validateFAQKnowledgeBase(ctx, kbID) + if err != nil { + return nil, err + } + + tenantID := ctx.Value(types.TenantIDContextKey).(uint64) + faqKnowledge, err := s.findFAQKnowledge(ctx, tenantID, kb.ID) + if err != nil { + return nil, err + } + if faqKnowledge == nil { + return json.Marshal([]types.FAQExportEntry{}) + } + + chunks, err := s.chunkRepo.ListAllFAQChunksForExport(ctx, tenantID, faqKnowledge.ID) + if err != nil { + return nil, fmt.Errorf("failed to list FAQ chunks: %w", err) + } + + tagMap, err := s.buildTagMap(ctx, tenantID, kbID) + if err != nil { + return nil, fmt.Errorf("failed to build tag map: %w", err) + } + + return s.buildFAQJSON(chunks, tagMap) +} + +func (s *knowledgeService) buildFAQJSON(chunks []*types.Chunk, tagMap map[string]string) ([]byte, error) { + entries := make([]types.FAQExportEntry, 0, len(chunks)) + for _, chunk := range chunks { + meta, err := chunk.FAQMetadata() + if err != nil || meta == nil { + continue + } + + tagName := "" + if chunk.TagID != "" && tagMap != nil { + if name, ok := tagMap[chunk.TagID]; ok { + tagName = name + } + } + + entries = append(entries, types.FAQExportEntry{ + ID: chunk.SeqID, + TagName: tagName, + StandardQuestion: meta.StandardQuestion, + SimilarQuestions: meta.SimilarQuestions, + NegativeQuestions: meta.NegativeQuestions, + Answers: meta.Answers, + AnswerStrategy: meta.AnswerStrategy, + IsEnabled: chunk.IsEnabled, + IsRecommended: chunk.Flags.HasFlag(types.ChunkFlagRecommended), + }) + } + return json.Marshal(entries) +} + // buildTagMap builds a map from tag_id to tag_name for the given knowledge base. func (s *knowledgeService) buildTagMap(ctx context.Context, tenantID uint64, kbID string) (map[string]string, error) { const pageSize = 1000 @@ -1787,6 +1850,74 @@ func (s *knowledgeService) checkFAQQuestionDuplicate( return werrors.NewBadRequestError("标准问或相似问与已有条目重复") } +// faqTagResolver resolves a payload's TagID/TagName to the tag's internal UUID. +type faqTagResolver func(*types.FAQEntryPayload) (string, error) + +// buildFAQTagResolver 预加载 entries 引用的 tag UUID 映射,避免逐条 resolveTagID。 +func (s *knowledgeService) buildFAQTagResolver( + ctx context.Context, kbID string, entries []types.FAQEntryPayload, +) faqTagResolver { + tenantID := ctx.Value(types.TenantIDContextKey).(uint64) + + var seqIDs []int64 + var names []string + seqSeen := make(map[int64]struct{}) + nameSeen := make(map[string]struct{}) + for i := range entries { + e := &entries[i] + switch { + case e.TagID != 0: + if _, ok := seqSeen[e.TagID]; !ok { + seqSeen[e.TagID] = struct{}{} + seqIDs = append(seqIDs, e.TagID) + } + case e.TagName != "": + if _, ok := nameSeen[e.TagName]; !ok { + nameSeen[e.TagName] = struct{}{} + names = append(names, e.TagName) + } + } + } + + seqIDToUUID := make(map[int64]string) + if len(seqIDs) > 0 { + if tags, err := s.tagRepo.GetBySeqIDs(ctx, tenantID, seqIDs); err == nil { + for _, t := range tags { + seqIDToUUID[t.SeqID] = t.ID + } + } else { + logger.Warnf(ctx, "buildFAQTagResolver: batch GetBySeqIDs failed (%d ids), fallback per-entry: %v", + len(seqIDs), err) + } + } + + nameToUUID := make(map[string]string) + for _, name := range names { + if tag, err := s.tagRepo.GetByName(ctx, tenantID, kbID, name); err == nil && tag != nil { + nameToUUID[name] = tag.ID + } + } + + return func(e *types.FAQEntryPayload) (string, error) { + if e.TagID != 0 { + if uuid, ok := seqIDToUUID[e.TagID]; ok { + return uuid, nil + } + } else if e.TagName != "" { + if uuid, ok := nameToUUID[e.TagName]; ok { + return uuid, nil + } + } + return s.resolveTagID(ctx, kbID, e) + } +} + +// hashQuestion 计算问题内容的哈希值,用于生成稳定的 sourceID。 +func hashQuestion(question string) string { + h := md5.Sum([]byte(question)) + return hex.EncodeToString(h[:4]) +} + // resolveTagID resolves tag ID (UUID) from payload, prioritizing tag_id (seq_id) over tag_name // If no tag is specified, creates or finds the "未分类" tag // Returns the internal UUID of the tag diff --git a/internal/application/service/knowledge_faq_import.go b/internal/application/service/knowledge_faq_import.go index 7a70b06b..c253d011 100644 --- a/internal/application/service/knowledge_faq_import.go +++ b/internal/application/service/knowledge_faq_import.go @@ -73,15 +73,29 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context, // 记录任务入队时间 enqueuedAt := time.Now().Unix() + instanceID := uuid.NewString() + runningInfoSet := false + enqueueSucceeded := false // 设置 KB 的运行中任务信息 if err := s.setRunningFAQImportInfo(ctx, kbID, &runningFAQImportInfo{ TaskID: taskID, EnqueuedAt: enqueuedAt, + InstanceID: instanceID, }); err != nil { logger.Errorf(ctx, "Failed to set running FAQ import task info: %v", err) // 不影响任务执行,继续 + } else { + runningInfoSet = true } + defer func() { + if !runningInfoSet || enqueueSucceeded { + return + } + if clearErr := s.clearRunningFAQImportInfoIfMatches(ctx, kbID, taskID, instanceID, enqueuedAt); clearErr != nil { + logger.Warnf(ctx, "Failed to clear FAQ import running info after setup failure: %v", clearErr) + } + }() // 初始化导入任务状态到Redis progress := &types.FAQImportProgress{ @@ -120,6 +134,7 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context, Mode: payload.Mode, DryRun: payload.DryRun, EnqueuedAt: enqueuedAt, + InstanceID: instanceID, Initiator: types.TaskInitiatorFromContext(ctx), } @@ -189,9 +204,8 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context, maxRetry = 3 // dry run 重试次数少一些 } - // 使用 taskID:enqueuedAt 作为 asynq 的唯一任务标识 - // 这样同一个用户 TaskID 的不同次提交不会冲突 - asynqTaskID := fmt.Sprintf("%s:%d", taskID, enqueuedAt) + // 使用 taskID:instanceID 作为 asynq 的唯一任务标识 + asynqTaskID := fmt.Sprintf("%s:%s", taskID, instanceID) task := asynq.NewTask( types.TypeFAQImport, @@ -207,6 +221,7 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context, return "", fmt.Errorf("failed to enqueue task: %w", err) } logger.Infof(ctx, "Enqueued FAQ import task: id=%s queue=%s task_id=%s dry_run=%v", info.ID, info.Queue, taskID, payload.DryRun) + enqueueSucceeded = true if !payload.DryRun { recordKBActivity(ctx, s.audit, tenantID, kbID, types.AuditActionFAQImportStarted, @@ -1215,34 +1230,118 @@ func (s *knowledgeService) calculateReplaceOperations(ctx context.Context, } } + // 批量预加载 tag 信息,避免循环内逐条查询数据库 + // 收集所有需要查询的 tag_id 和 tag_name + tagSeqIDSet := make(map[int64]bool) + tagNameSet := make(map[string]bool) + for _, ewh := range entriesWithHash { + if ewh.entry.TagID != 0 { + tagSeqIDSet[ewh.entry.TagID] = true + } else if ewh.entry.TagName != "" { + tagNameSet[ewh.entry.TagName] = true + } else { + tagNameSet[types.UntaggedTagName] = true + } + } + + // 批量查询 tag by seq_id + tagSeqIDToUUID := make(map[int64]string) + if len(tagSeqIDSet) > 0 { + seqIDs := make([]int64, 0, len(tagSeqIDSet)) + for seqID := range tagSeqIDSet { + seqIDs = append(seqIDs, seqID) + } + tags, err := s.tagRepo.GetBySeqIDs(ctx, tenantID, seqIDs) + if err != nil { + logger.Warnf(ctx, "Failed to batch load tags by seq_ids: %v", err) + } else { + for _, tag := range tags { + tagSeqIDToUUID[tag.SeqID] = tag.ID + } + } + } + + // 批量查询 tag by name + tagNameToUUID := make(map[string]string) + if len(tagNameSet) > 0 && kbID != "" { + for name := range tagNameSet { + if tag, err := s.tagRepo.GetByName(ctx, tenantID, kbID, name); err == nil && tag != nil { + tagNameToUUID[name] = tag.ID + } + } + } + + logger.Infof(ctx, "Preloaded %d tags by seq_id, %d tags by name for %d entries", + len(tagSeqIDToUUID), len(tagNameToUUID), len(entriesWithHash)) + + // resolveTagIDFromCache 从缓存中解析 tag ID,缓存未命中时回退到数据库查询 + resolveTagIDFromCache := func(entry *types.FAQEntryPayload) (string, error) { + if entry.TagID != 0 { + if uuid, ok := tagSeqIDToUUID[entry.TagID]; ok { + return uuid, nil + } + // 缓存未命中,回退到数据库查询(可能需要创建) + return s.resolveTagID(ctx, kbID, entry) + } + tagName := entry.TagName + if tagName == "" { + tagName = types.UntaggedTagName + } + if uuid, ok := tagNameToUUID[tagName]; ok { + return uuid, nil + } + // 缓存未命中,回退到数据库查询(可能需要创建) + return s.resolveTagID(ctx, kbID, entry) + } + // 计算需要创建的条目(利用已经计算好的hash,避免重复计算) entriesToProcess := make([]types.FAQEntryPayload, 0, len(entriesWithHash)) skippedCount := batchSkippedCount - for _, ewh := range entriesWithHash { + for idx, ewh := range entriesWithHash { + // 每处理 1000 条打印一次进度日志 + if idx > 0 && idx%1000 == 0 { + logger.Infof(ctx, "calculateReplaceOperations progress: %d/%d entries processed", idx, len(entriesWithHash)) + } + existingChunk := existingHashMap[ewh.hash] if existingChunk != nil { // hash 匹配,检查 tag 是否变化 - newTagID, err := s.resolveTagID(ctx, kbID, &ewh.entry) + newTagID, err := resolveTagIDFromCache(&ewh.entry) if err != nil { logger.Warnf(ctx, "Failed to resolve tag for entry, treating as new: %v", err) entriesToProcess = append(entriesToProcess, ewh.entry) continue } - if existingChunk.TagID != newTagID { - // tag 变化了,需要删除旧的并创建新的 - logger.Infof(ctx, "FAQ entry tag changed from %s to %s, will update", existingChunk.TagID, newTagID) + enabledChanged := ewh.entry.IsEnabled != nil && *ewh.entry.IsEnabled != existingChunk.IsEnabled + recommendedChanged := ewh.entry.IsRecommended != nil && + *ewh.entry.IsRecommended != existingChunk.Flags.HasFlag(types.ChunkFlagRecommended) + answerStrategyChanged := false + if existingMeta, metaErr := existingChunk.FAQMetadata(); metaErr == nil && existingMeta != nil { + answerStrategyChanged = ewh.meta.AnswerStrategy != existingMeta.AnswerStrategy + } + + if existingChunk.TagID != newTagID || enabledChanged || recommendedChanged || answerStrategyChanged { + if existingChunk.TagID != newTagID { + logger.Infof(ctx, "FAQ entry tag changed from %s to %s, will update", existingChunk.TagID, newTagID) + } + if enabledChanged || recommendedChanged || answerStrategyChanged { + logger.Infof(ctx, "FAQ entry operational fields changed (enabled: %v->%v, recommended: %v->%v, answerStrategy changed: %v), will update", + existingChunk.IsEnabled, ewh.entry.IsEnabled, + existingChunk.Flags.HasFlag(types.ChunkFlagRecommended), ewh.entry.IsRecommended, + answerStrategyChanged) + } chunksToDelete = append(chunksToDelete, existingChunk) entriesToProcess = append(entriesToProcess, ewh.entry) } else { - // hash 和 tag 都相同,跳过 + // hash、tag、运营状态都相同,跳过 skippedCount++ } continue } - // hash不匹配或不存在,需要创建 + // hash 不匹配或不存在,需要创建 entriesToProcess = append(entriesToProcess, ewh.entry) } @@ -1512,7 +1611,7 @@ func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string, actualProcessed += len(batch) // 更新任务进度 progress := int(float64(actualProcessed) / float64(totalEntries) * 100) - if err := s.updateFAQImportProgressStatus(ctx, taskID, types.FAQImportStatusProcessing, progress, totalEntries, actualProcessed, fmt.Sprintf("正在处理第 %d/%d 条", actualProcessed, totalEntries), ""); err != nil { + if err := s.updateFAQImportProgressStatus(ctx, taskID, "", 0, types.FAQImportStatusProcessing, progress, totalEntries, actualProcessed, fmt.Sprintf("正在处理第 %d/%d 条", actualProcessed, totalEntries), ""); err != nil { logger.Errorf(ctx, "Failed to update task progress: %v", err) } @@ -1555,6 +1654,8 @@ func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string, func (s *knowledgeService) updateFAQImportProgressStatus( ctx context.Context, taskID string, + instanceID string, + enqueuedAt int64, status types.FAQImportTaskStatus, progress, total, processed int, message, errorMsg string, @@ -1585,7 +1686,7 @@ func (s *knowledgeService) updateFAQImportProgressStatus( // 任务完成或失败时,清除 running key if status == types.FAQImportStatusCompleted || status == types.FAQImportStatusFailed { if existingProgress.KBID != "" { - if clearErr := s.clearRunningFAQImportTaskID(ctx, existingProgress.KBID); clearErr != nil { + if clearErr := s.clearRunningFAQImportInfoIfMatches(ctx, existingProgress.KBID, taskID, instanceID, enqueuedAt); clearErr != nil { logger.Errorf(ctx, "Failed to clear running FAQ import task ID: %v", clearErr) } } @@ -1611,6 +1712,7 @@ func (s *knowledgeService) cleanupFAQEntriesFileOnFinalFailure(ctx context.Conte type runningFAQImportInfo struct { TaskID string `json:"task_id"` EnqueuedAt int64 `json:"enqueued_at"` + InstanceID string `json:"instance_id,omitempty"` } // getRunningFAQImportInfo checks if there's a running FAQ import task for the given KB @@ -1677,6 +1779,39 @@ func (s *knowledgeService) clearRunningFAQImportTaskID(ctx context.Context, kbID return s.redisClient.Del(ctx, key).Err() } +func (s *knowledgeService) clearRunningFAQImportInfoIfMatches(ctx context.Context, kbID, taskID, instanceID string, enqueuedAt int64) error { + if s.redisClient == nil { + if v, ok := s.memFAQRunningImport.Load(kbID); ok { + info, _ := v.(*runningFAQImportInfo) + if runningFAQImportInfoMatches(info, taskID, instanceID, enqueuedAt) { + s.memFAQRunningImport.Delete(kbID) + } + } + return nil + } + + info, err := s.getRunningFAQImportInfo(ctx, kbID) + if err != nil { + return err + } + if !runningFAQImportInfoMatches(info, taskID, instanceID, enqueuedAt) { + return nil + } + + key := getFAQImportRunningKey(kbID) + return s.redisClient.Del(ctx, key).Err() +} + +func runningFAQImportInfoMatches(info *runningFAQImportInfo, taskID, instanceID string, enqueuedAt int64) bool { + if info == nil || info.TaskID != taskID { + return false + } + if info.InstanceID != "" && instanceID != "" { + return info.InstanceID == instanceID + } + return enqueuedAt == 0 || info.EnqueuedAt == 0 || info.EnqueuedAt == enqueuedAt +} + // incrementalIndexFAQEntry 增量更新FAQ条目的索引 // 只对内容变化的部分进行embedding计算和索引更新,跳过未变化的部分 func (s *knowledgeService) incrementalIndexFAQEntry( @@ -1691,6 +1826,8 @@ func (s *knowledgeService) incrementalIndexFAQEntry( newMeta *types.FAQChunkMetadata, ) error { indexStartTime := time.Now() + logger.Debugf(ctx, "incrementalIndexFAQEntry: starting for chunk=%s, oldSimilarQuestions=%d, newSimilarQuestions=%d", + chunk.ID, len(oldSimilarQuestions), len(newMeta.SimilarQuestions)) retrieveEngine, err := retriever.CreateRetrieveEngineForKB( ctx, s.retrieveEngine, s.ownership, types.MustTenantIDFromContext(ctx), kb.VectorStoreID) @@ -1703,12 +1840,26 @@ func (s *knowledgeService) incrementalIndexFAQEntry( indexMode = kb.FAQConfig.IndexMode } - // 构建旧的内容(用于比较) - buildOldContent := func(question string) string { - if indexMode == types.FAQIndexModeQuestionAnswer && len(oldAnswers) > 0 { + // 对新旧数据进行归一化处理,确保与 buildFAQIndexInfoList 的行为一致 + // 旧数据归一化 + oldStandardQuestion = types.NormalizeQuestion(oldStandardQuestion) + normalizedOldSimilarQuestions := make([]string, 0, len(oldSimilarQuestions)) + for _, q := range oldSimilarQuestions { + if nq := types.NormalizeQuestion(q); nq != "" { + normalizedOldSimilarQuestions = append(normalizedOldSimilarQuestions, nq) + } + } + oldSimilarQuestions = normalizedOldSimilarQuestions + oldAnswers = types.SanitizeStrings(oldAnswers) + // 新数据归一化 + normalizedNewMeta := newMeta.Normalize() + + // 构建索引内容 + buildContent := func(question string, answers []string) string { + if indexMode == types.FAQIndexModeQuestionAnswer && len(answers) > 0 { var builder strings.Builder builder.WriteString(question) - for _, ans := range oldAnswers { + for _, ans := range answers { builder.WriteString("\n") builder.WriteString(ans) } @@ -1717,30 +1868,20 @@ func (s *knowledgeService) incrementalIndexFAQEntry( return question } - // 构建新的内容 - buildNewContent := func(question string) string { - if indexMode == types.FAQIndexModeQuestionAnswer && len(newMeta.Answers) > 0 { - var builder strings.Builder - builder.WriteString(question) - for _, ans := range newMeta.Answers { - builder.WriteString("\n") - builder.WriteString(ans) - } - return builder.String() - } - return question - } - - // 检查答案是否变化 - answersChanged := !slices.Equal(oldAnswers, newMeta.Answers) + // 检查答案是否变化(仅在 QuestionAnswer 模式下才影响索引) + answersChanged := indexMode == types.FAQIndexModeQuestionAnswer && !slices.Equal(oldAnswers, normalizedNewMeta.Answers) + logger.Debugf(ctx, "incrementalIndexFAQEntry: answersChanged=%v (indexMode=%s), oldAnswers=%d, newAnswers=%d", + answersChanged, indexMode, len(oldAnswers), len(normalizedNewMeta.Answers)) // 收集需要更新的索引项 var indexInfoToUpdate []*types.IndexInfo // 1. 检查标准问是否需要更新 - oldStdContent := buildOldContent(oldStandardQuestion) - newStdContent := buildNewContent(newMeta.StandardQuestion) - if oldStdContent != newStdContent { + oldStdContent := buildContent(oldStandardQuestion, oldAnswers) + newStdContent := buildContent(normalizedNewMeta.StandardQuestion, normalizedNewMeta.Answers) + stdQuestionChanged := oldStdContent != newStdContent + if stdQuestionChanged { + logger.Debugf(ctx, "incrementalIndexFAQEntry: standard question changed, sourceID=%s", chunk.ID) indexInfoToUpdate = append(indexInfoToUpdate, &types.IndexInfo{ Content: newStdContent, SourceID: chunk.ID, @@ -1755,27 +1896,41 @@ func (s *knowledgeService) incrementalIndexFAQEntry( }) } - // 2. 检查每个相似问是否需要更新 - oldCount := len(oldSimilarQuestions) - newCount := len(newMeta.SimilarQuestions) + // 2. 基于内容哈希处理相似问的增删改 + // 构建旧问题集合 (问题 -> 是否存在) + oldQuestionsSet := make(map[string]struct{}, len(oldSimilarQuestions)) + for _, q := range oldSimilarQuestions { + oldQuestionsSet[q] = struct{}{} + } - for i, newQ := range newMeta.SimilarQuestions { - needUpdate := false - if i >= oldCount { - // 新增的相似问 - needUpdate = true - } else { - // 已存在的相似问,检查内容是否变化 - oldQ := oldSimilarQuestions[i] - if oldQ != newQ || answersChanged { - needUpdate = true - } + // 构建新问题集合 + newQuestionsSet := make(map[string]struct{}, len(normalizedNewMeta.SimilarQuestions)) + for _, q := range normalizedNewMeta.SimilarQuestions { + newQuestionsSet[q] = struct{}{} + } + + // 找出需要删除的问题(在旧集合中但不在新集合中) + var sourceIDsToDelete []string + var deletedQuestions []string + for oldQ := range oldQuestionsSet { + if _, exists := newQuestionsSet[oldQ]; !exists { + sourceID := fmt.Sprintf("%s-%s", chunk.ID, hashQuestion(oldQ)) + sourceIDsToDelete = append(sourceIDsToDelete, sourceID) + deletedQuestions = append(deletedQuestions, oldQ) } + } - if needUpdate { - sourceID := fmt.Sprintf("%s-%d", chunk.ID, i) + // 找出需要新增或更新的问题 + var addedQuestions, updatedQuestions []string + for newQ := range newQuestionsSet { + _, existedBefore := oldQuestionsSet[newQ] + // 需要更新的条件: + // 1. 新问题(之前不存在) + // 2. 答案变化(需要重新embedding) + if !existedBefore || answersChanged { + sourceID := fmt.Sprintf("%s-%s", chunk.ID, hashQuestion(newQ)) indexInfoToUpdate = append(indexInfoToUpdate, &types.IndexInfo{ - Content: buildNewContent(newQ), + Content: buildContent(newQ, normalizedNewMeta.Answers), SourceID: sourceID, SourceType: types.ChunkSourceType, ChunkID: chunk.ID, @@ -1786,22 +1941,35 @@ func (s *knowledgeService) incrementalIndexFAQEntry( IsEnabled: chunk.IsEnabled, IsRecommended: chunk.Flags.HasFlag(types.ChunkFlagRecommended), }) + if !existedBefore { + addedQuestions = append(addedQuestions, newQ) + } else { + updatedQuestions = append(updatedQuestions, newQ) + } } } - // 3. 删除多余的旧相似问索引 - if oldCount > newCount { - sourceIDsToDelete := make([]string, 0, oldCount-newCount) - for i := newCount; i < oldCount; i++ { - sourceIDsToDelete = append(sourceIDsToDelete, fmt.Sprintf("%s-%d", chunk.ID, i)) - } - logger.Debugf(ctx, "incrementalIndexFAQEntry: deleting %d obsolete source IDs", len(sourceIDsToDelete)) + // 输出详细的变化日志 + if len(deletedQuestions) > 0 { + logger.Debugf(ctx, "incrementalIndexFAQEntry: deleted similar questions: %v", deletedQuestions) + } + if len(addedQuestions) > 0 { + logger.Debugf(ctx, "incrementalIndexFAQEntry: added similar questions: %v", addedQuestions) + } + if len(updatedQuestions) > 0 { + logger.Debugf(ctx, "incrementalIndexFAQEntry: updated similar questions (answers changed): %v", updatedQuestions) + } + + // 3. 删除不再存在的相似问索引 + if len(sourceIDsToDelete) > 0 { + logger.Debugf(ctx, "incrementalIndexFAQEntry: deleting %d obsolete sourceIDs: %v", len(sourceIDsToDelete), sourceIDsToDelete) if delErr := retrieveEngine.DeleteBySourceIDList(ctx, sourceIDsToDelete, embeddingModel.GetDimensions(), types.KnowledgeTypeFAQ); delErr != nil { logger.Warnf(ctx, "incrementalIndexFAQEntry: failed to delete obsolete source IDs: %v", delErr) } } // 4. 批量索引需要更新的内容 + newCount := len(normalizedNewMeta.SimilarQuestions) if len(indexInfoToUpdate) > 0 { logger.Debugf(ctx, "incrementalIndexFAQEntry: updating %d index entries (skipped %d unchanged)", len(indexInfoToUpdate), 1+newCount-len(indexInfoToUpdate)) @@ -2193,7 +2361,7 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task) logger.Errorf(ctx, "Failed to get knowledge base: %v", err) // 如果是最后一次重试,更新状态为失败 if isLastRetry { - if updateErr := s.updateFAQImportProgressStatus(ctx, payload.TaskID, types.FAQImportStatusFailed, 0, originalTotalEntries, 0, "获取知识库失败", err.Error()); updateErr != nil { + if updateErr := s.updateFAQImportProgressStatus(ctx, payload.TaskID, payload.InstanceID, payload.EnqueuedAt, types.FAQImportStatusFailed, 0, originalTotalEntries, 0, "获取知识库失败", err.Error()); updateErr != nil { logger.Errorf(ctx, "Failed to update task status to failed: %v", updateErr) } s.recordFAQImportKBActivity(ctx, &payload, progress, originalTotalEntries, types.AuditActionFAQImportFailed, types.AuditOutcomeFailed) @@ -2207,6 +2375,9 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task) if existingProgress != nil { if existingProgress.Status == types.FAQImportStatusCompleted { logger.Infof(ctx, "FAQ import already completed, skipping: %s", payload.TaskID) + if clearErr := s.clearRunningFAQImportInfoIfMatches(ctx, payload.KBID, payload.TaskID, payload.InstanceID, payload.EnqueuedAt); clearErr != nil { + logger.Warnf(ctx, "Failed to clear running FAQ import info for completed task: %v", clearErr) + } return nil // 幂等:已完成的任务直接返回 } // 获取已处理的数量(注意:这是相对于 validEntries 的索引) @@ -2223,7 +2394,7 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task) logger.Errorf(ctx, "Failed to delete unindexed chunks: %v", err) // 如果是最后一次重试,更新状态为失败 if isLastRetry { - if updateErr := s.updateFAQImportProgressStatus(ctx, payload.TaskID, types.FAQImportStatusFailed, 0, originalTotalEntries, 0, "清理未索引数据失败", err.Error()); updateErr != nil { + if updateErr := s.updateFAQImportProgressStatus(ctx, payload.TaskID, payload.InstanceID, payload.EnqueuedAt, types.FAQImportStatusFailed, 0, originalTotalEntries, 0, "清理未索引数据失败", err.Error()); updateErr != nil { logger.Errorf(ctx, "Failed to update task status to failed: %v", updateErr) } s.recordFAQImportKBActivity(ctx, &payload, progress, originalTotalEntries, types.AuditActionFAQImportFailed, types.AuditOutcomeFailed) @@ -2280,7 +2451,7 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task) logger.Errorf(ctx, "FAQ import task failed: %s, error: %v", payload.TaskID, err) // 如果是最后一次重试,更新状态为失败 if isLastRetry { - if updateErr := s.updateFAQImportProgressStatus(ctx, payload.TaskID, types.FAQImportStatusFailed, 0, originalTotalEntries, len(validEntries), "导入失败", err.Error()); updateErr != nil { + if updateErr := s.updateFAQImportProgressStatus(ctx, payload.TaskID, payload.InstanceID, payload.EnqueuedAt, types.FAQImportStatusFailed, 0, originalTotalEntries, len(validEntries), "导入失败", err.Error()); updateErr != nil { logger.Errorf(ctx, "Failed to update task status to failed: %v", updateErr) } s.recordFAQImportKBActivity(ctx, &payload, progress, originalTotalEntries, types.AuditActionFAQImportFailed, types.AuditOutcomeFailed) @@ -2385,7 +2556,7 @@ func (s *knowledgeService) finalizeFAQValidation(ctx context.Context, payload *t } // 然后调用状态更新来清理 running key - if err := s.updateFAQImportProgressStatus(ctx, payload.TaskID, types.FAQImportStatusCompleted, + if err := s.updateFAQImportProgressStatus(ctx, payload.TaskID, payload.InstanceID, payload.EnqueuedAt, types.FAQImportStatusCompleted, 100, originalTotalEntries, originalTotalEntries, progress.Message, ""); err != nil { logger.Warnf(ctx, "Failed to update final FAQ import status: %v", err) } diff --git a/internal/handler/faq.go b/internal/handler/faq.go index 8f4fb3ef..8cd8aa9a 100644 --- a/internal/handler/faq.go +++ b/internal/handler/faq.go @@ -3,6 +3,7 @@ package handler import ( "net/http" "strconv" + "strings" "github.com/gin-gonic/gin" @@ -381,19 +382,35 @@ func (h *FAQHandler) SearchFAQ(c *gin.Context) { // ExportEntries godoc // @Summary 导出FAQ条目 -// @Description 将所有FAQ条目导出为CSV文件 +// @Description 将所有FAQ条目导出为 CSV(默认)或 JSON。?format=json 返回与 FAQEntryPayload 结构兼容的数组。 // @Tags FAQ管理 // @Accept json // @Produce text/csv -// @Param id path string true "知识库ID" -// @Success 200 {file} file "CSV文件" -// @Failure 400 {object} errors.AppError "请求参数错误" +// @Produce application/json +// @Param id path string true "知识库ID" +// @Param format query string false "导出格式:csv(默认)或 json" +// @Success 200 {file} file "导出文件" +// @Failure 400 {object} errors.AppError "请求参数错误" // @Security Bearer // @Security ApiKeyAuth // @Router /knowledge-bases/{id}/faq/entries/export [get] func (h *FAQHandler) ExportEntries(c *gin.Context) { ctx := c.Request.Context() kbID := secutils.SanitizeForLog(c.Param("id")) + format := strings.ToLower(strings.TrimSpace(c.Query("format"))) + + if format == "json" { + jsonData, err := h.knowledgeService.ExportFAQEntriesJSON(ctx, kbID) + if err != nil { + logger.ErrorWithFields(ctx, err, nil) + c.Error(err) + return + } + c.Header("Content-Type", "application/json; charset=utf-8") + c.Header("Content-Disposition", "attachment; filename=faq_export.json") + c.Data(http.StatusOK, "application/json; charset=utf-8", jsonData) + return + } csvData, err := h.knowledgeService.ExportFAQEntries(ctx, kbID) if err != nil { diff --git a/internal/types/faq.go b/internal/types/faq.go index 90046609..cd8021a5 100644 --- a/internal/types/faq.go +++ b/internal/types/faq.go @@ -251,6 +251,21 @@ type FAQEntry struct { MatchedQuestion string `json:"matched_question,omitempty"` } +// FAQExportEntry 表示 JSON 导出的 FAQ 条目,与 FAQEntryPayload 的导入格式兼容, +// 便于"导出 → 编辑 → 重新导入"循环。新增字段时务必保留 omitempty,避免破坏 +// 历史导出文件的兼容性。 +type FAQExportEntry struct { + ID int64 `json:"id"` + TagName string `json:"tag_name,omitempty"` + StandardQuestion string `json:"standard_question"` + SimilarQuestions []string `json:"similar_questions,omitempty"` + NegativeQuestions []string `json:"negative_questions,omitempty"` + Answers []string `json:"answers,omitempty"` + AnswerStrategy AnswerStrategy `json:"answer_strategy,omitempty"` + IsEnabled bool `json:"is_enabled"` + IsRecommended bool `json:"is_recommended"` +} + // FAQEntryPayload 用于创建/更新 FAQ 条目的 payload type FAQEntryPayload struct { // ID 可选,用于数据迁移时指定 seq_id(必须小于自增起始值 100000000) diff --git a/internal/types/faq_sync.go b/internal/types/faq_sync.go new file mode 100644 index 00000000..d45b980a --- /dev/null +++ b/internal/types/faq_sync.go @@ -0,0 +1,77 @@ +package types + +import "encoding/json" + +// FAQChunkSyncPair links source/destination FAQ chunks sharing content_hash. +type FAQChunkSyncPair struct { + SrcChunkID string + DstChunkID string +} + +// FAQChunkDiffResult is the output of FAQChunkDiff. +type FAQChunkDiffResult struct { + ChunksToAdd []string + ChunksToDelete []string + MatchedPairs []FAQChunkSyncPair +} + +// FAQChunkStatus holds lightweight fields for clone status sync. +type FAQChunkStatus struct { + ID string + TagID string + IsEnabled bool + Flags ChunkFlags + AnswerStrategy AnswerStrategy + Metadata JSON +} + +// FAQChunkStatusFieldsEqual compares status fields (tag excluded). +func FAQChunkStatusFieldsEqual(a, b *FAQChunkStatus) bool { + if a == nil || b == nil { + return a == b + } + if a.IsEnabled != b.IsEnabled || a.Flags != b.Flags { + return false + } + return NormalizeAnswerStrategy(a.AnswerStrategy) == NormalizeAnswerStrategy(b.AnswerStrategy) +} + +// FAQChunkNeedsStatusSync reports whether a matched pair needs status sync. +func FAQChunkNeedsStatusSync(src, dst *FAQChunkStatus, mappedSrcTagID string) bool { + if src == nil || dst == nil { + return false + } + if mappedSrcTagID != dst.TagID { + return true + } + return !FAQChunkStatusFieldsEqual(src, dst) +} + +// NormalizeAnswerStrategy treats empty as "all". +func NormalizeAnswerStrategy(s AnswerStrategy) AnswerStrategy { + if s == "" { + return AnswerStrategyAll + } + return s +} + +// PatchFAQAnswerStrategy copies answer_strategy from src metadata onto dst metadata. +func PatchFAQAnswerStrategy(dstMeta, srcMeta JSON) (JSON, error) { + var dstM, srcM FAQChunkMetadata + if len(dstMeta) > 0 { + if err := json.Unmarshal(dstMeta, &dstM); err != nil { + return nil, err + } + } + if len(srcMeta) > 0 { + if err := json.Unmarshal(srcMeta, &srcM); err != nil { + return nil, err + } + } + dstM.AnswerStrategy = srcM.AnswerStrategy + bytes, err := json.Marshal(&dstM) + if err != nil { + return nil, err + } + return JSON(bytes), nil +} diff --git a/internal/types/faq_sync_test.go b/internal/types/faq_sync_test.go new file mode 100644 index 00000000..f6be2a77 --- /dev/null +++ b/internal/types/faq_sync_test.go @@ -0,0 +1,19 @@ +package types + +import "testing" + +func TestFAQChunkNeedsStatusSync(t *testing.T) { + src := &FAQChunkStatus{TagID: "a", IsEnabled: true, Flags: ChunkFlagRecommended, AnswerStrategy: AnswerStrategyAll} + dst := &FAQChunkStatus{TagID: "old", IsEnabled: true, Flags: ChunkFlagRecommended, AnswerStrategy: AnswerStrategyAll} + if !FAQChunkNeedsStatusSync(src, dst, "mapped") { + t.Fatal("tag mismatch should need sync") + } + dst.TagID = "mapped" + if FAQChunkNeedsStatusSync(src, dst, "mapped") { + t.Fatal("in sync after tag mapped") + } + dst.IsEnabled = false + if !FAQChunkNeedsStatusSync(src, dst, "mapped") { + t.Fatal("is_enabled mismatch should need sync") + } +} diff --git a/internal/types/interfaces/chunk.go b/internal/types/interfaces/chunk.go index 1e62f5e3..11b0fba7 100644 --- a/internal/types/interfaces/chunk.go +++ b/internal/types/interfaces/chunk.go @@ -97,8 +97,9 @@ type ChunkRepository interface { // newTagID: if not nil, updates tag_id to this value (empty string means uncategorized) UpdateChunkFieldsByTagID(ctx context.Context, tenantID uint64, kbID string, tagID string, isEnabled *bool, setFlags types.ChunkFlags, clearFlags types.ChunkFlags, newTagID *string, excludeIDs []string) ([]string, error) // FAQChunkDiff compares FAQ chunks between two knowledge bases and returns the differences. - // Returns: chunksToAdd (content_hash in src but not in dst), chunksToDelete (content_hash in dst but not in src) - FAQChunkDiff(ctx context.Context, srcTenantID uint64, srcKBID string, dstTenantID uint64, dstKBID string) (chunksToAdd []string, chunksToDelete []string, err error) + FAQChunkDiff(ctx context.Context, srcTenantID uint64, srcKBID string, dstTenantID uint64, dstKBID string) (*types.FAQChunkDiffResult, error) + // ListFAQChunkStatusByIDs loads status fields for FAQ clone sync. + ListFAQChunkStatusByIDs(ctx context.Context, tenantID uint64, ids []string) (map[string]*types.FAQChunkStatus, error) // ListRecommendedFAQChunks lists FAQ chunks with the recommended flag set. // Filter by explicitly selected kbIDs, knowledgeIDs, and/or FAQ tagIDs. diff --git a/internal/types/interfaces/knowledge.go b/internal/types/interfaces/knowledge.go index ae176da3..aea4499f 100644 --- a/internal/types/interfaces/knowledge.go +++ b/internal/types/interfaces/knowledge.go @@ -152,6 +152,8 @@ type KnowledgeService interface { SearchFAQEntries(ctx context.Context, kbID string, req *types.FAQSearchRequest) ([]*types.FAQEntry, error) // ExportFAQEntries exports all FAQ entries for a knowledge base as CSV data. ExportFAQEntries(ctx context.Context, kbID string) ([]byte, error) + // ExportFAQEntriesJSON exports all FAQ entries as a JSON array compatible with FAQEntryPayload. + ExportFAQEntriesJSON(ctx context.Context, kbID string) ([]byte, error) // UpdateKnowledgeTagBatch updates tag for document knowledge items in batch. // authorizedKBID restricts all updates to knowledge items belonging to this KB; // pass empty string to skip (caller must ensure authorization by other means). diff --git a/internal/types/task.go b/internal/types/task.go index 1461b7b2..fdcd916b 100644 --- a/internal/types/task.go +++ b/internal/types/task.go @@ -305,6 +305,7 @@ type FAQImportPayload struct { Mode string `json:"mode"` DryRun bool `json:"dry_run"` // dry run 模式只验证不导入 EnqueuedAt int64 `json:"enqueued_at"` // 任务入队时间戳,用于区分同一 TaskID 的不同次提交 + InstanceID string `json:"instance_id,omitempty"` Initiator TaskInitiator `json:"initiator,omitempty"` }