feat(knowledge): enhance FAQ export functionality

- Updated the exportFAQEntries function to support exporting FAQ entries in both CSV and JSON formats.
- Modified the frontend to include options for selecting the export format.
- Enhanced localization files to reflect the new export options in English, Russian, and Chinese.
- Implemented new tests to validate the export functionality for both formats.

This commit improves the flexibility of FAQ exports, allowing users to choose their preferred format for better integration with other tools.
This commit is contained in:
wizardchen
2026-07-22 20:57:31 +08:00
committed by lyingbug
parent 135033e6df
commit b6a3b347da
18 changed files with 1041 additions and 137 deletions
+5 -4
View File
@@ -445,10 +445,11 @@ export function searchFAQEntries(
return post(`/api/v1/knowledge-bases/${kbId}/faq/search`, data);
}
// Export FAQ entries as CSV file
export async function exportFAQEntries(kbId: string): Promise<Blob> {
const response = await getDown(`/api/v1/knowledge-bases/${kbId}/faq/entries/export`);
return response as unknown as Blob;
// Export FAQ entries as CSV or JSON file
export async function exportFAQEntries(kbId: string, format: 'csv' | 'json' = 'csv'): Promise<Blob> {
const suffix = format === 'json' ? '?format=json' : ''
const response = await getDown(`/api/v1/knowledge-bases/${kbId}/faq/entries/export${suffix}`)
return response as unknown as Blob
}
// FAQ Import Progress API
+3 -1
View File
@@ -2992,7 +2992,9 @@ export default {
downloadExampleExcel: 'Download Excel Example',
},
faqExport: {
exportButton: 'Export CSV',
exportButton: 'Export',
exportCSV: 'Export CSV',
exportJSON: 'Export JSON',
exportSuccess: 'Export successful',
exportFailed: 'Export failed',
},
+3 -1
View File
@@ -3689,7 +3689,9 @@ export default {
downloadExampleExcel: 'Скачать пример Excel'
},
faqExport: {
exportButton: 'Экспорт CSV',
exportButton: 'Экспорт',
exportCSV: 'Экспорт CSV',
exportJSON: 'Экспорт JSON',
exportSuccess: 'Экспорт успешен',
exportFailed: 'Ошибка экспорта'
},
+3 -1
View File
@@ -4281,7 +4281,9 @@ export default {
downloadExampleExcel: "下载 Excel 示例",
},
faqExport: {
exportButton: "导出 CSV",
exportButton: "导出",
exportCSV: "导出 CSV",
exportJSON: "导出 JSON",
exportSuccess: "导出成功",
exportFailed: "导出失败",
},
@@ -238,12 +238,15 @@
</t-tooltip>
</template>
<!-- 导出 -->
<t-tooltip :content="$t('knowledgeEditor.faqExport.exportButton')" placement="top">
<t-button variant="text" theme="default" class="content-bar-icon-btn" size="small"
@click="handleFaqAction({ value: 'export' })">
<template #icon><t-icon name="download" size="16px" /></template>
</t-button>
</t-tooltip>
<t-dropdown :options="faqExportOptions" trigger="click" placement="bottom-right"
@click="handleFaqAction">
<t-tooltip :content="$t('knowledgeEditor.faqExport.exportButton')" placement="top">
<t-button variant="text" theme="default" class="content-bar-icon-btn" size="small"
:loading="exportLoading">
<template #icon><t-icon name="download" size="16px" /></template>
</t-button>
</t-tooltip>
</t-dropdown>
<!-- 检索 -->
<t-tooltip :content="$t('knowledgeEditor.faq.searchTest')" placement="top">
<t-button variant="text" theme="default" class="content-bar-icon-btn" size="small"
@@ -998,6 +1001,11 @@ const canManage = computed(() => {
return orgStore.canManageKB(props.kbId, false)
})
const faqExportOptions = computed(() => [
{ content: t('knowledgeEditor.faqExport.exportCSV'), value: 'export_csv' },
{ content: t('knowledgeEditor.faqExport.exportJSON'), value: 'export_json' },
])
// FAQ 操作:新建组(新建条目 + 导入)
const faqCreateOptions = computed(() => {
if (!canEdit.value) return []
@@ -1019,6 +1027,12 @@ const handleFaqAction = (data: { value: string }) => {
case 'search':
searchDrawerVisible.value = true
break
case 'export_csv':
handleExportCSV()
break
case 'export_json':
handleExportJSON()
break
case 'export':
handleExportCSV()
break
@@ -2647,9 +2661,19 @@ const downloadExcelExample = () => {
XLSX.writeFile(workbook, 'faq_example.xlsx')
}
// 导出 FAQ 数据为 CSV
// 导出 FAQ 数据
const exportLoading = ref(false)
const handleExportCSV = async () => {
const downloadExportBlob = (blob: Blob, ext: 'csv' | 'json') => {
const url = URL.createObjectURL(blob)
const link = document.createElement('a')
link.href = url
link.download = `faq_export_${new Date().toISOString().slice(0, 10)}.${ext}`
document.body.appendChild(link)
link.click()
document.body.removeChild(link)
URL.revokeObjectURL(url)
}
const handleExportFAQ = async (format: 'csv' | 'json') => {
if (!props.kbId) {
MessagePlugin.warning(t('knowledgeBase.selectKnowledgeBase'))
return
@@ -2657,15 +2681,8 @@ const handleExportCSV = async () => {
exportLoading.value = true
try {
const blob = await exportFAQEntries(props.kbId)
const url = URL.createObjectURL(blob)
const link = document.createElement('a')
link.href = url
link.download = `faq_export_${new Date().toISOString().slice(0, 10)}.csv`
document.body.appendChild(link)
link.click()
document.body.removeChild(link)
URL.revokeObjectURL(url)
const blob = await exportFAQEntries(props.kbId, format)
downloadExportBlob(blob, format)
MessagePlugin.success(t('knowledgeEditor.faqExport.exportSuccess'))
} catch (error: any) {
console.error('Export failed:', error)
@@ -2674,6 +2691,8 @@ const handleExportCSV = async () => {
exportLoading.value = false
}
}
const handleExportCSV = () => handleExportFAQ('csv')
const handleExportJSON = () => handleExportFAQ('json')
watch(
() => props.kbId,
+153 -27
View File
@@ -931,44 +931,170 @@ func (r *chunkRepository) UpdateChunkFieldsByTagID(
return affectedIDs, nil
}
type chunkIDHash struct {
ID string `gorm:"column:id"`
ContentHash string `gorm:"column:content_hash"`
}
const faqChunkDiffBatchSize = 5000
// listFAQChunkIDHashesByKB loads id/content_hash pairs for all FAQ chunks in a KB.
func (r *chunkRepository) listFAQChunkIDHashesByKB(
ctx context.Context,
tenantID uint64,
kbID string,
) ([]chunkIDHash, error) {
var all []chunkIDHash
var lastID string
for {
var batch []chunkIDHash
query := r.db.WithContext(ctx).Model(&types.Chunk{}).
Select("id, content_hash").
Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?",
tenantID, kbID, types.ChunkTypeFAQ).
Order("id ASC").
Limit(faqChunkDiffBatchSize)
if lastID != "" {
query = query.Where("id > ?", lastID)
}
if err := query.Find(&batch).Error; err != nil {
return nil, err
}
if len(batch) == 0 {
break
}
all = append(all, batch...)
lastID = batch[len(batch)-1].ID
if len(batch) < faqChunkDiffBatchSize {
break
}
}
return all, nil
}
func diffFAQChunkIDsByContentHash(src, dst []chunkIDHash) (
chunksToAdd, chunksToDelete []string,
matched []types.FAQChunkSyncPair,
) {
dstHashes := make(map[string]struct{}, len(dst))
dstIDByHash := make(map[string]string, len(dst))
dstIDsByHash := make(map[string][]string, len(dst))
for _, pair := range dst {
dstHashes[pair.ContentHash] = struct{}{}
if _, ok := dstIDByHash[pair.ContentHash]; !ok {
dstIDByHash[pair.ContentHash] = pair.ID
}
dstIDsByHash[pair.ContentHash] = append(dstIDsByHash[pair.ContentHash], pair.ID)
}
srcHashes := make(map[string]struct{}, len(src))
for _, pair := range src {
srcHashes[pair.ContentHash] = struct{}{}
}
for _, pair := range src {
if _, exists := dstHashes[pair.ContentHash]; !exists {
chunksToAdd = append(chunksToAdd, pair.ID)
continue
}
if dstID, ok := dstIDByHash[pair.ContentHash]; ok {
matched = append(matched, types.FAQChunkSyncPair{
SrcChunkID: pair.ID,
DstChunkID: dstID,
})
}
}
for _, pair := range dst {
if _, exists := srcHashes[pair.ContentHash]; !exists {
chunksToDelete = append(chunksToDelete, pair.ID)
}
}
for hash, ids := range dstIDsByHash {
if hash == "" || len(ids) <= 1 {
continue
}
if _, inSrc := srcHashes[hash]; !inSrc {
continue
}
canonical := dstIDByHash[hash]
for _, id := range ids {
if id != canonical {
chunksToDelete = append(chunksToDelete, id)
}
}
}
return chunksToAdd, chunksToDelete, matched
}
// FAQChunkDiff compares FAQ chunks between two knowledge bases and returns the differences.
// Returns: chunksToAdd (IDs of chunks in src whose content_hash is not in dst),
//
// chunksToDelete (IDs of chunks in dst whose content_hash is not in src)
// chunksToDelete (IDs of chunks in dst whose content_hash is not in src, plus
// duplicate dst chunks that share a content_hash with another dst chunk when
// that hash still exists in src)
func (r *chunkRepository) FAQChunkDiff(
ctx context.Context,
srcTenantID uint64, srcKBID string,
dstTenantID uint64, dstKBID string,
) (chunksToAdd []string, chunksToDelete []string, err error) {
// Get content_hash set from destination KB
dstHashSubQuery := r.db.Model(&types.Chunk{}).
Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?", dstTenantID, dstKBID, types.ChunkTypeFAQ).
Select("content_hash")
// Find chunks in source that don't exist in destination (by content_hash)
err = r.db.WithContext(ctx).Model(&types.Chunk{}).
Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?", srcTenantID, srcKBID, types.ChunkTypeFAQ).
Where("content_hash NOT IN (?)", dstHashSubQuery).
Pluck("id", &chunksToAdd).Error
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil, fmt.Errorf("failed to get chunks to add: %w", err)
) (*types.FAQChunkDiffResult, error) {
srcPairs, err := r.listFAQChunkIDHashesByKB(ctx, srcTenantID, srcKBID)
if err != nil {
return nil, fmt.Errorf("failed to list source FAQ chunks: %w", err)
}
dstPairs, err := r.listFAQChunkIDHashesByKB(ctx, dstTenantID, dstKBID)
if err != nil {
return nil, fmt.Errorf("failed to list destination FAQ chunks: %w", err)
}
// Get content_hash set from source KB
srcHashSubQuery := r.db.Model(&types.Chunk{}).
Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?", srcTenantID, srcKBID, types.ChunkTypeFAQ).
Select("content_hash")
add, del, matched := diffFAQChunkIDsByContentHash(srcPairs, dstPairs)
return &types.FAQChunkDiffResult{
ChunksToAdd: add,
ChunksToDelete: del,
MatchedPairs: matched,
}, nil
}
// Find chunks in destination that don't exist in source (by content_hash)
err = r.db.WithContext(ctx).Model(&types.Chunk{}).
Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?", dstTenantID, dstKBID, types.ChunkTypeFAQ).
Where("content_hash NOT IN (?)", srcHashSubQuery).
Pluck("id", &chunksToDelete).Error
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil, fmt.Errorf("failed to get chunks to delete: %w", err)
// ListFAQChunkStatusByIDs loads status fields for FAQ clone sync.
func (r *chunkRepository) ListFAQChunkStatusByIDs(
ctx context.Context,
tenantID uint64,
ids []string,
) (map[string]*types.FAQChunkStatus, error) {
if len(ids) == 0 {
return map[string]*types.FAQChunkStatus{}, nil
}
return chunksToAdd, chunksToDelete, nil
const batchSize = 5000
var chunks []*types.Chunk
for i := 0; i < len(ids); i += batchSize {
end := i + batchSize
if end > len(ids) {
end = len(ids)
}
var batch []*types.Chunk
if err := r.db.WithContext(ctx).
Select("id, tag_id, is_enabled, flags, metadata").
Where("tenant_id = ? AND id IN ?", tenantID, ids[i:end]).
Find(&batch).Error; err != nil {
return nil, err
}
chunks = append(chunks, batch...)
}
out := make(map[string]*types.FAQChunkStatus, len(chunks))
for _, chunk := range chunks {
strategy := types.AnswerStrategyAll
if meta, err := chunk.FAQMetadata(); err == nil && meta != nil {
strategy = meta.AnswerStrategy
}
out[chunk.ID] = &types.FAQChunkStatus{
ID: chunk.ID,
TagID: chunk.TagID,
IsEnabled: chunk.IsEnabled,
Flags: chunk.Flags,
AnswerStrategy: strategy,
Metadata: chunk.Metadata,
}
}
return out, nil
}
// ListRecommendedFAQChunks lists FAQ chunks with the recommended flag set.
@@ -0,0 +1,117 @@
package repository
import (
"context"
"testing"
"github.com/Tencent/WeKnora/internal/types"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestDiffFAQChunkIDsByContentHash(t *testing.T) {
toAdd, toDelete, matched := diffFAQChunkIDsByContentHash(
[]chunkIDHash{
{ID: "src-1", ContentHash: "hash-a"},
{ID: "src-2", ContentHash: "hash-b"},
{ID: "src-3", ContentHash: "hash-only-src"},
},
[]chunkIDHash{
{ID: "dst-1", ContentHash: "hash-a"},
{ID: "dst-2", ContentHash: "hash-only-dst"},
},
)
assert.ElementsMatch(t, []string{"src-2", "src-3"}, toAdd)
assert.ElementsMatch(t, []string{"dst-2"}, toDelete)
require.Len(t, matched, 1)
assert.Equal(t, "src-1", matched[0].SrcChunkID)
assert.Equal(t, "dst-1", matched[0].DstChunkID)
}
func TestDiffFAQChunkIDsByContentHash_dstDuplicateHash(t *testing.T) {
// Two destination chunks share hash-a (same FAQ content, e.g. stale copy after
// answer_strategy-only change). Source has one chunk for hash-a → keep dst-1,
// delete dst-dup, match for status sync.
toAdd, toDelete, matched := diffFAQChunkIDsByContentHash(
[]chunkIDHash{{ID: "src-1", ContentHash: "hash-a"}},
[]chunkIDHash{
{ID: "dst-1", ContentHash: "hash-a"},
{ID: "dst-dup", ContentHash: "hash-a"},
},
)
assert.Empty(t, toAdd)
assert.ElementsMatch(t, []string{"dst-dup"}, toDelete)
require.Len(t, matched, 1)
assert.Equal(t, "src-1", matched[0].SrcChunkID)
assert.Equal(t, "dst-1", matched[0].DstChunkID)
}
func TestDiffFAQChunkIDsByContentHash_dstDuplicateHash_notInSrc(t *testing.T) {
// Duplicates whose hash is absent from source are all removed (orphan content).
toAdd, toDelete, matched := diffFAQChunkIDsByContentHash(
nil,
[]chunkIDHash{
{ID: "dst-orphan-1", ContentHash: "hash-orphan"},
{ID: "dst-orphan-2", ContentHash: "hash-orphan"},
},
)
assert.Empty(t, toAdd)
assert.ElementsMatch(t, []string{"dst-orphan-1", "dst-orphan-2"}, toDelete)
assert.Empty(t, matched)
}
func TestDiffFAQChunkIDsByContentHash_emptySide(t *testing.T) {
toAdd, toDelete, matched := diffFAQChunkIDsByContentHash(
[]chunkIDHash{{ID: "src-1", ContentHash: "hash-a"}},
nil,
)
assert.Equal(t, []string{"src-1"}, toAdd)
assert.Empty(t, toDelete)
assert.Empty(t, matched)
toAdd, toDelete, matched = diffFAQChunkIDsByContentHash(
nil,
[]chunkIDHash{{ID: "dst-1", ContentHash: "hash-a"}},
)
assert.Empty(t, toAdd)
assert.Equal(t, []string{"dst-1"}, toDelete)
assert.Empty(t, matched)
}
func TestFAQChunkDiff_SQLite(t *testing.T) {
db := setupChunkTestDB(t)
repo := NewChunkRepository(db)
ctx := context.Background()
srcKBID := uuid.New().String()
dstKBID := uuid.New().String()
srcKnowledgeID := uuid.New().String()
dstKnowledgeID := uuid.New().String()
require.NoError(t, db.Create(&types.Chunk{
ID: uuid.New().String(), TenantID: 1, KnowledgeBaseID: srcKBID, KnowledgeID: srcKnowledgeID,
ChunkType: types.ChunkTypeFAQ, ContentHash: "shared", Status: int(types.ChunkStatusIndexed),
}).Error)
require.NoError(t, db.Create(&types.Chunk{
ID: uuid.New().String(), TenantID: 1, KnowledgeBaseID: srcKBID, KnowledgeID: srcKnowledgeID,
ChunkType: types.ChunkTypeFAQ, ContentHash: "src-only", Status: int(types.ChunkStatusIndexed),
}).Error)
require.NoError(t, db.Create(&types.Chunk{
ID: uuid.New().String(), TenantID: 1, KnowledgeBaseID: dstKBID, KnowledgeID: dstKnowledgeID,
ChunkType: types.ChunkTypeFAQ, ContentHash: "shared", Status: int(types.ChunkStatusIndexed),
}).Error)
require.NoError(t, db.Create(&types.Chunk{
ID: uuid.New().String(), TenantID: 1, KnowledgeBaseID: dstKBID, KnowledgeID: dstKnowledgeID,
ChunkType: types.ChunkTypeFAQ, ContentHash: "dst-only", Status: int(types.ChunkStatusIndexed),
}).Error)
diff, err := repo.FAQChunkDiff(ctx, 1, srcKBID, 1, dstKBID)
require.NoError(t, err)
assert.Len(t, diff.ChunksToAdd, 1)
assert.Len(t, diff.ChunksToDelete, 1)
assert.Len(t, diff.MatchedPairs, 1)
}
@@ -0,0 +1,165 @@
package service
import (
"context"
"fmt"
"time"
"github.com/Tencent/WeKnora/internal/application/service/retriever"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
)
type faqStatusSyncPlan struct {
Pairs []types.FAQChunkSyncPair
SrcByID map[string]*types.FAQChunkStatus
DstByID map[string]*types.FAQChunkStatus
}
func (s *knowledgeService) buildFAQStatusSyncPlan(
ctx context.Context,
srcTenantID, dstTenantID uint64,
matched []types.FAQChunkSyncPair,
resolveTag func(srcTagID string) string,
) (*faqStatusSyncPlan, error) {
if len(matched) == 0 {
return &faqStatusSyncPlan{}, nil
}
srcIDs := make([]string, 0, len(matched))
dstIDs := make([]string, 0, len(matched))
for _, p := range matched {
srcIDs = append(srcIDs, p.SrcChunkID)
dstIDs = append(dstIDs, p.DstChunkID)
}
srcByID, err := s.chunkRepo.ListFAQChunkStatusByIDs(ctx, srcTenantID, srcIDs)
if err != nil {
return nil, err
}
dstByID, err := s.chunkRepo.ListFAQChunkStatusByIDs(ctx, dstTenantID, dstIDs)
if err != nil {
return nil, err
}
pairs := make([]types.FAQChunkSyncPair, 0)
for _, p := range matched {
src, dst := srcByID[p.SrcChunkID], dstByID[p.DstChunkID]
if src == nil || dst == nil {
continue
}
mappedTag := ""
if src.TagID != "" {
mappedTag = resolveTag(src.TagID)
}
if types.FAQChunkNeedsStatusSync(src, dst, mappedTag) {
pairs = append(pairs, p)
}
}
return &faqStatusSyncPlan{Pairs: pairs, SrcByID: srcByID, DstByID: dstByID}, nil
}
func (s *knowledgeService) syncFAQChunkStatusBatch(
ctx context.Context,
dstKB *types.KnowledgeBase,
pairs []types.FAQChunkSyncPair,
srcByID, dstByID map[string]*types.FAQChunkStatus,
resolveTag func(srcTagID string) string,
) error {
if len(pairs) == 0 {
return nil
}
tenantID := types.MustTenantIDFromContext(ctx)
enabledUpdates := make(map[string]bool)
recommendedUpdates := make(map[string]bool)
tagUpdates := make(map[string]string)
rows := make([]*types.Chunk, 0, len(pairs))
metadataIDs := make([]string, 0)
for _, p := range pairs {
src, dst := srcByID[p.SrcChunkID], dstByID[p.DstChunkID]
if src == nil || dst == nil {
continue
}
mappedTag := ""
if src.TagID != "" {
mappedTag = resolveTag(src.TagID)
}
if !types.FAQChunkNeedsStatusSync(src, dst, mappedTag) {
continue
}
rows = append(rows, &types.Chunk{
ID: dst.ID, IsEnabled: src.IsEnabled, Flags: src.Flags,
TagID: mappedTag, UpdatedAt: time.Now(),
})
if dst.IsEnabled != src.IsEnabled {
enabledUpdates[dst.ID] = src.IsEnabled
}
srcRec := src.Flags.HasFlag(types.ChunkFlagRecommended)
dstRec := dst.Flags.HasFlag(types.ChunkFlagRecommended)
if srcRec != dstRec {
recommendedUpdates[dst.ID] = srcRec
}
if mappedTag != dst.TagID {
tagUpdates[dst.ID] = mappedTag
}
if types.NormalizeAnswerStrategy(src.AnswerStrategy) != types.NormalizeAnswerStrategy(dst.AnswerStrategy) {
metadataIDs = append(metadataIDs, dst.ID)
}
}
if len(rows) > 0 {
if err := s.chunkRepo.UpdateChunks(ctx, rows); err != nil {
return err
}
}
for _, dstID := range metadataIDs {
src := srcForDst(pairs, srcByID, dstID)
dst := dstByID[dstID]
if src == nil || dst == nil {
continue
}
patched, err := types.PatchFAQAnswerStrategy(dst.Metadata, src.Metadata)
if err != nil {
return fmt.Errorf("patch answer_strategy for %s: %w", dstID, err)
}
full, err := s.chunkRepo.GetChunkByID(ctx, tenantID, dstID)
if err != nil {
return err
}
full.Metadata = patched
full.UpdatedAt = time.Now()
if err := s.chunkRepo.UpdateChunk(ctx, full); err != nil {
return err
}
}
if len(enabledUpdates) == 0 && len(tagUpdates) == 0 && len(recommendedUpdates) == 0 {
return nil
}
engine, err := retriever.CreateRetrieveEngineForKB(
ctx, s.retrieveEngine, s.ownership, tenantID, dstKB.VectorStoreID)
if err != nil {
return err
}
if len(enabledUpdates) > 0 {
if err := engine.BatchUpdateChunkEnabledStatus(ctx, enabledUpdates); err != nil {
return err
}
}
if len(tagUpdates) > 0 {
if err := engine.BatchUpdateChunkTagID(ctx, tagUpdates); err != nil {
return err
}
}
if len(recommendedUpdates) > 0 {
// Vector-store recommended flag sync is not yet available on all backends in
// this branch; DB flags were already updated via UpdateChunks above.
logger.Warnf(ctx, "FAQ clone sync: skipped vector recommended update for %d chunks (DB flags updated)", len(recommendedUpdates))
}
return nil
}
func srcForDst(pairs []types.FAQChunkSyncPair, srcByID map[string]*types.FAQChunkStatus, dstID string) *types.FAQChunkStatus {
for _, p := range pairs {
if p.DstChunkID == dstID {
return srcByID[p.SrcChunkID]
}
}
return nil
}
@@ -675,23 +675,47 @@ func (s *knowledgeService) cloneFAQKnowledgeBase(
}
srcKnowledge := srcKnowledgeList[0]
// Get chunk-level differences based on content_hash
chunksToAdd, chunksToDelete, err := s.chunkRepo.FAQChunkDiff(ctx, srcKB.TenantID, srcKB.ID, dstKB.TenantID, dstKB.ID)
// Get chunk-level differences based on content_hash.
diff, err := s.chunkRepo.FAQChunkDiff(ctx, srcKB.TenantID, srcKB.ID, dstKB.TenantID, dstKB.ID)
if err != nil {
logger.Errorf(ctx, "Failed to calculate FAQ chunk difference: %v", err)
handleError(progress, err, "Failed to calculate FAQ chunk difference")
return err
}
chunksToAdd := diff.ChunksToAdd
chunksToDelete := diff.ChunksToDelete
totalOperations := len(chunksToAdd) + len(chunksToDelete)
tagIDMapping := map[string]string{}
resolveFAQTag := func(srcTagID string) string {
if srcTagID == "" {
return ""
}
if id, ok := tagIDMapping[srcTagID]; ok {
return id
}
return s.getOrCreateTagInTarget(ctx, srcKB.TenantID, dstKB.TenantID, dstKB.ID, srcTagID, tagIDMapping)
}
syncPlan, err := s.buildFAQStatusSyncPlan(ctx, srcKB.TenantID, dstKB.TenantID, diff.MatchedPairs, resolveFAQTag)
if err != nil {
logger.Errorf(ctx, "Failed to build FAQ status sync plan: %v", err)
handleError(progress, err, "Failed to build FAQ status sync plan")
return err
}
chunksToUpdate := syncPlan.Pairs
totalOperations := len(chunksToAdd) + len(chunksToDelete) + len(chunksToUpdate)
progress.Total = totalOperations
progress.Message = fmt.Sprintf("Found %d FAQ entries to add, %d to delete", len(chunksToAdd), len(chunksToDelete))
progress.Message = fmt.Sprintf(
"Found %d FAQ entries to add, %d to delete, %d to update status",
len(chunksToAdd), len(chunksToDelete), len(chunksToUpdate),
)
progress.UpdatedAt = time.Now().Unix()
_ = s.saveKBCloneProgress(ctx, progress)
logger.Infof(ctx, "FAQ chunks to add: %d, delete: %d", len(chunksToAdd), len(chunksToDelete))
logger.Infof(ctx, "FAQ chunks to add: %d, delete: %d, update status: %d",
len(chunksToAdd), len(chunksToDelete), len(chunksToUpdate))
// If nothing to do, mark as completed
if totalOperations == 0 {
progress.Status = types.KBCloneStatusCompleted
progress.Progress = 100
@@ -760,7 +784,6 @@ func (s *knowledgeService) cloneFAQKnowledgeBase(
// Clone FAQ chunks from source to destination
batch := 50
tagIDMapping := map[string]string{} // srcTagID -> dstTagID
for i := 0; i < len(chunksToAdd); i += batch {
end := i + batch
if end > len(chunksToAdd) {
@@ -782,19 +805,11 @@ func (s *knowledgeService) cloneFAQKnowledgeBase(
// Map TagID to target knowledge base
targetTagID := ""
if srcChunk.TagID != "" {
if mappedTagID, ok := tagIDMapping[srcChunk.TagID]; ok {
targetTagID = mappedTagID
} else {
// Try to find or create the tag in target knowledge base
targetTagID = s.getOrCreateTagInTarget(ctx, srcKB.TenantID, dstKB.TenantID, dstKB.ID, srcChunk.TagID, tagIDMapping)
}
targetTagID = resolveFAQTag(srcChunk.TagID)
}
// Deep-copy extracted images into objects owned by the destination
// FAQ knowledge so deleting the source never breaks this clone. A FAQ
// chunk is self-contained (its own Content + image_info), so its
// ![](url) references can be rewritten immediately using the mapping
// just accumulated in imageURLCache.
// FAQ knowledge so deleting the source never breaks this clone.
newImageInfo, copied, copyErr := cloneChunkImageInfo(
ctx, dstSvc, srcChunk.ImageInfo, dstKB.TenantID, dstKnowledge.ID, imageURLCache)
if copyErr != nil {
@@ -860,6 +875,27 @@ func (s *knowledgeService) cloneFAQKnowledgeBase(
_ = s.saveKBCloneProgress(ctx, progress)
}
for i := 0; i < len(chunksToUpdate); i += batch {
end := i + batch
if end > len(chunksToUpdate) {
end = len(chunksToUpdate)
}
if err := s.syncFAQChunkStatusBatch(
ctx, dstKB, chunksToUpdate[i:end], syncPlan.SrcByID, syncPlan.DstByID, resolveFAQTag); err != nil {
logger.Errorf(ctx, "Failed to sync FAQ status fields: %v", err)
handleError(progress, err, "Failed to sync FAQ status fields")
return err
}
processedCount += end - i
if totalOperations > 0 {
progress.Progress = processedCount * 100 / totalOperations
}
progress.Processed = processedCount
progress.Message = fmt.Sprintf("Updated %d/%d FAQ status fields", processedCount-len(chunksToDelete)-len(chunksToAdd), len(chunksToUpdate))
progress.UpdatedAt = time.Now().Unix()
_ = s.saveKBCloneProgress(ctx, progress)
}
// Mark as completed
progress.Status = types.KBCloneStatusCompleted
progress.Progress = 100
@@ -2,6 +2,9 @@ package service
import (
"context"
"crypto/md5"
"encoding/hex"
"encoding/json"
"fmt"
"slices"
"strings"
@@ -1445,6 +1448,66 @@ func (s *knowledgeService) ExportFAQEntries(ctx context.Context, kbID string) ([
return s.buildFAQCSV(chunks, tagMap), nil
}
// ExportFAQEntriesJSON 以 JSON 数组形式导出 FAQ 知识库下的全部条目,
// 字段与 FAQEntryPayload 兼容,便于"导出 → 编辑 → 重新 append 导入"循环。
func (s *knowledgeService) ExportFAQEntriesJSON(ctx context.Context, kbID string) ([]byte, error) {
kb, err := s.validateFAQKnowledgeBase(ctx, kbID)
if err != nil {
return nil, err
}
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
faqKnowledge, err := s.findFAQKnowledge(ctx, tenantID, kb.ID)
if err != nil {
return nil, err
}
if faqKnowledge == nil {
return json.Marshal([]types.FAQExportEntry{})
}
chunks, err := s.chunkRepo.ListAllFAQChunksForExport(ctx, tenantID, faqKnowledge.ID)
if err != nil {
return nil, fmt.Errorf("failed to list FAQ chunks: %w", err)
}
tagMap, err := s.buildTagMap(ctx, tenantID, kbID)
if err != nil {
return nil, fmt.Errorf("failed to build tag map: %w", err)
}
return s.buildFAQJSON(chunks, tagMap)
}
func (s *knowledgeService) buildFAQJSON(chunks []*types.Chunk, tagMap map[string]string) ([]byte, error) {
entries := make([]types.FAQExportEntry, 0, len(chunks))
for _, chunk := range chunks {
meta, err := chunk.FAQMetadata()
if err != nil || meta == nil {
continue
}
tagName := ""
if chunk.TagID != "" && tagMap != nil {
if name, ok := tagMap[chunk.TagID]; ok {
tagName = name
}
}
entries = append(entries, types.FAQExportEntry{
ID: chunk.SeqID,
TagName: tagName,
StandardQuestion: meta.StandardQuestion,
SimilarQuestions: meta.SimilarQuestions,
NegativeQuestions: meta.NegativeQuestions,
Answers: meta.Answers,
AnswerStrategy: meta.AnswerStrategy,
IsEnabled: chunk.IsEnabled,
IsRecommended: chunk.Flags.HasFlag(types.ChunkFlagRecommended),
})
}
return json.Marshal(entries)
}
// buildTagMap builds a map from tag_id to tag_name for the given knowledge base.
func (s *knowledgeService) buildTagMap(ctx context.Context, tenantID uint64, kbID string) (map[string]string, error) {
const pageSize = 1000
@@ -1787,6 +1850,74 @@ func (s *knowledgeService) checkFAQQuestionDuplicate(
return werrors.NewBadRequestError("标准问或相似问与已有条目重复")
}
// faqTagResolver resolves a payload's TagID/TagName to the tag's internal UUID.
type faqTagResolver func(*types.FAQEntryPayload) (string, error)
// buildFAQTagResolver 预加载 entries 引用的 tag UUID 映射,避免逐条 resolveTagID。
func (s *knowledgeService) buildFAQTagResolver(
ctx context.Context, kbID string, entries []types.FAQEntryPayload,
) faqTagResolver {
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
var seqIDs []int64
var names []string
seqSeen := make(map[int64]struct{})
nameSeen := make(map[string]struct{})
for i := range entries {
e := &entries[i]
switch {
case e.TagID != 0:
if _, ok := seqSeen[e.TagID]; !ok {
seqSeen[e.TagID] = struct{}{}
seqIDs = append(seqIDs, e.TagID)
}
case e.TagName != "":
if _, ok := nameSeen[e.TagName]; !ok {
nameSeen[e.TagName] = struct{}{}
names = append(names, e.TagName)
}
}
}
seqIDToUUID := make(map[int64]string)
if len(seqIDs) > 0 {
if tags, err := s.tagRepo.GetBySeqIDs(ctx, tenantID, seqIDs); err == nil {
for _, t := range tags {
seqIDToUUID[t.SeqID] = t.ID
}
} else {
logger.Warnf(ctx, "buildFAQTagResolver: batch GetBySeqIDs failed (%d ids), fallback per-entry: %v",
len(seqIDs), err)
}
}
nameToUUID := make(map[string]string)
for _, name := range names {
if tag, err := s.tagRepo.GetByName(ctx, tenantID, kbID, name); err == nil && tag != nil {
nameToUUID[name] = tag.ID
}
}
return func(e *types.FAQEntryPayload) (string, error) {
if e.TagID != 0 {
if uuid, ok := seqIDToUUID[e.TagID]; ok {
return uuid, nil
}
} else if e.TagName != "" {
if uuid, ok := nameToUUID[e.TagName]; ok {
return uuid, nil
}
}
return s.resolveTagID(ctx, kbID, e)
}
}
// hashQuestion 计算问题内容的哈希值,用于生成稳定的 sourceID。
func hashQuestion(question string) string {
h := md5.Sum([]byte(question))
return hex.EncodeToString(h[:4])
}
// resolveTagID resolves tag ID (UUID) from payload, prioritizing tag_id (seq_id) over tag_name
// If no tag is specified, creates or finds the "未分类" tag
// Returns the internal UUID of the tag
@@ -73,15 +73,29 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context,
// 记录任务入队时间
enqueuedAt := time.Now().Unix()
instanceID := uuid.NewString()
runningInfoSet := false
enqueueSucceeded := false
// 设置 KB 的运行中任务信息
if err := s.setRunningFAQImportInfo(ctx, kbID, &runningFAQImportInfo{
TaskID: taskID,
EnqueuedAt: enqueuedAt,
InstanceID: instanceID,
}); err != nil {
logger.Errorf(ctx, "Failed to set running FAQ import task info: %v", err)
// 不影响任务执行,继续
} else {
runningInfoSet = true
}
defer func() {
if !runningInfoSet || enqueueSucceeded {
return
}
if clearErr := s.clearRunningFAQImportInfoIfMatches(ctx, kbID, taskID, instanceID, enqueuedAt); clearErr != nil {
logger.Warnf(ctx, "Failed to clear FAQ import running info after setup failure: %v", clearErr)
}
}()
// 初始化导入任务状态到Redis
progress := &types.FAQImportProgress{
@@ -120,6 +134,7 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context,
Mode: payload.Mode,
DryRun: payload.DryRun,
EnqueuedAt: enqueuedAt,
InstanceID: instanceID,
Initiator: types.TaskInitiatorFromContext(ctx),
}
@@ -189,9 +204,8 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context,
maxRetry = 3 // dry run 重试次数少一些
}
// 使用 taskID:enqueuedAt 作为 asynq 的唯一任务标识
// 这样同一个用户 TaskID 的不同次提交不会冲突
asynqTaskID := fmt.Sprintf("%s:%d", taskID, enqueuedAt)
// 使用 taskID:instanceID 作为 asynq 的唯一任务标识
asynqTaskID := fmt.Sprintf("%s:%s", taskID, instanceID)
task := asynq.NewTask(
types.TypeFAQImport,
@@ -207,6 +221,7 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context,
return "", fmt.Errorf("failed to enqueue task: %w", err)
}
logger.Infof(ctx, "Enqueued FAQ import task: id=%s queue=%s task_id=%s dry_run=%v", info.ID, info.Queue, taskID, payload.DryRun)
enqueueSucceeded = true
if !payload.DryRun {
recordKBActivity(ctx, s.audit, tenantID, kbID, types.AuditActionFAQImportStarted,
@@ -1215,34 +1230,118 @@ func (s *knowledgeService) calculateReplaceOperations(ctx context.Context,
}
}
// 批量预加载 tag 信息,避免循环内逐条查询数据库
// 收集所有需要查询的 tag_id 和 tag_name
tagSeqIDSet := make(map[int64]bool)
tagNameSet := make(map[string]bool)
for _, ewh := range entriesWithHash {
if ewh.entry.TagID != 0 {
tagSeqIDSet[ewh.entry.TagID] = true
} else if ewh.entry.TagName != "" {
tagNameSet[ewh.entry.TagName] = true
} else {
tagNameSet[types.UntaggedTagName] = true
}
}
// 批量查询 tag by seq_id
tagSeqIDToUUID := make(map[int64]string)
if len(tagSeqIDSet) > 0 {
seqIDs := make([]int64, 0, len(tagSeqIDSet))
for seqID := range tagSeqIDSet {
seqIDs = append(seqIDs, seqID)
}
tags, err := s.tagRepo.GetBySeqIDs(ctx, tenantID, seqIDs)
if err != nil {
logger.Warnf(ctx, "Failed to batch load tags by seq_ids: %v", err)
} else {
for _, tag := range tags {
tagSeqIDToUUID[tag.SeqID] = tag.ID
}
}
}
// 批量查询 tag by name
tagNameToUUID := make(map[string]string)
if len(tagNameSet) > 0 && kbID != "" {
for name := range tagNameSet {
if tag, err := s.tagRepo.GetByName(ctx, tenantID, kbID, name); err == nil && tag != nil {
tagNameToUUID[name] = tag.ID
}
}
}
logger.Infof(ctx, "Preloaded %d tags by seq_id, %d tags by name for %d entries",
len(tagSeqIDToUUID), len(tagNameToUUID), len(entriesWithHash))
// resolveTagIDFromCache 从缓存中解析 tag ID,缓存未命中时回退到数据库查询
resolveTagIDFromCache := func(entry *types.FAQEntryPayload) (string, error) {
if entry.TagID != 0 {
if uuid, ok := tagSeqIDToUUID[entry.TagID]; ok {
return uuid, nil
}
// 缓存未命中,回退到数据库查询(可能需要创建)
return s.resolveTagID(ctx, kbID, entry)
}
tagName := entry.TagName
if tagName == "" {
tagName = types.UntaggedTagName
}
if uuid, ok := tagNameToUUID[tagName]; ok {
return uuid, nil
}
// 缓存未命中,回退到数据库查询(可能需要创建)
return s.resolveTagID(ctx, kbID, entry)
}
// 计算需要创建的条目(利用已经计算好的hash,避免重复计算)
entriesToProcess := make([]types.FAQEntryPayload, 0, len(entriesWithHash))
skippedCount := batchSkippedCount
for _, ewh := range entriesWithHash {
for idx, ewh := range entriesWithHash {
// 每处理 1000 条打印一次进度日志
if idx > 0 && idx%1000 == 0 {
logger.Infof(ctx, "calculateReplaceOperations progress: %d/%d entries processed", idx, len(entriesWithHash))
}
existingChunk := existingHashMap[ewh.hash]
if existingChunk != nil {
// hash 匹配,检查 tag 是否变化
newTagID, err := s.resolveTagID(ctx, kbID, &ewh.entry)
newTagID, err := resolveTagIDFromCache(&ewh.entry)
if err != nil {
logger.Warnf(ctx, "Failed to resolve tag for entry, treating as new: %v", err)
entriesToProcess = append(entriesToProcess, ewh.entry)
continue
}
if existingChunk.TagID != newTagID {
// tag 变化了,需要删除旧的并创建新的
logger.Infof(ctx, "FAQ entry tag changed from %s to %s, will update", existingChunk.TagID, newTagID)
enabledChanged := ewh.entry.IsEnabled != nil && *ewh.entry.IsEnabled != existingChunk.IsEnabled
recommendedChanged := ewh.entry.IsRecommended != nil &&
*ewh.entry.IsRecommended != existingChunk.Flags.HasFlag(types.ChunkFlagRecommended)
answerStrategyChanged := false
if existingMeta, metaErr := existingChunk.FAQMetadata(); metaErr == nil && existingMeta != nil {
answerStrategyChanged = ewh.meta.AnswerStrategy != existingMeta.AnswerStrategy
}
if existingChunk.TagID != newTagID || enabledChanged || recommendedChanged || answerStrategyChanged {
if existingChunk.TagID != newTagID {
logger.Infof(ctx, "FAQ entry tag changed from %s to %s, will update", existingChunk.TagID, newTagID)
}
if enabledChanged || recommendedChanged || answerStrategyChanged {
logger.Infof(ctx, "FAQ entry operational fields changed (enabled: %v->%v, recommended: %v->%v, answerStrategy changed: %v), will update",
existingChunk.IsEnabled, ewh.entry.IsEnabled,
existingChunk.Flags.HasFlag(types.ChunkFlagRecommended), ewh.entry.IsRecommended,
answerStrategyChanged)
}
chunksToDelete = append(chunksToDelete, existingChunk)
entriesToProcess = append(entriesToProcess, ewh.entry)
} else {
// hash 和 tag 都相同,跳过
// hash、tag、运营状态都相同,跳过
skippedCount++
}
continue
}
// hash不匹配或不存在,需要创建
// hash 不匹配或不存在,需要创建
entriesToProcess = append(entriesToProcess, ewh.entry)
}
@@ -1512,7 +1611,7 @@ func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string,
actualProcessed += len(batch)
// 更新任务进度
progress := int(float64(actualProcessed) / float64(totalEntries) * 100)
if err := s.updateFAQImportProgressStatus(ctx, taskID, types.FAQImportStatusProcessing, progress, totalEntries, actualProcessed, fmt.Sprintf("正在处理第 %d/%d 条", actualProcessed, totalEntries), ""); err != nil {
if err := s.updateFAQImportProgressStatus(ctx, taskID, "", 0, types.FAQImportStatusProcessing, progress, totalEntries, actualProcessed, fmt.Sprintf("正在处理第 %d/%d 条", actualProcessed, totalEntries), ""); err != nil {
logger.Errorf(ctx, "Failed to update task progress: %v", err)
}
@@ -1555,6 +1654,8 @@ func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string,
func (s *knowledgeService) updateFAQImportProgressStatus(
ctx context.Context,
taskID string,
instanceID string,
enqueuedAt int64,
status types.FAQImportTaskStatus,
progress, total, processed int,
message, errorMsg string,
@@ -1585,7 +1686,7 @@ func (s *knowledgeService) updateFAQImportProgressStatus(
// 任务完成或失败时,清除 running key
if status == types.FAQImportStatusCompleted || status == types.FAQImportStatusFailed {
if existingProgress.KBID != "" {
if clearErr := s.clearRunningFAQImportTaskID(ctx, existingProgress.KBID); clearErr != nil {
if clearErr := s.clearRunningFAQImportInfoIfMatches(ctx, existingProgress.KBID, taskID, instanceID, enqueuedAt); clearErr != nil {
logger.Errorf(ctx, "Failed to clear running FAQ import task ID: %v", clearErr)
}
}
@@ -1611,6 +1712,7 @@ func (s *knowledgeService) cleanupFAQEntriesFileOnFinalFailure(ctx context.Conte
type runningFAQImportInfo struct {
TaskID string `json:"task_id"`
EnqueuedAt int64 `json:"enqueued_at"`
InstanceID string `json:"instance_id,omitempty"`
}
// getRunningFAQImportInfo checks if there's a running FAQ import task for the given KB
@@ -1677,6 +1779,39 @@ func (s *knowledgeService) clearRunningFAQImportTaskID(ctx context.Context, kbID
return s.redisClient.Del(ctx, key).Err()
}
func (s *knowledgeService) clearRunningFAQImportInfoIfMatches(ctx context.Context, kbID, taskID, instanceID string, enqueuedAt int64) error {
if s.redisClient == nil {
if v, ok := s.memFAQRunningImport.Load(kbID); ok {
info, _ := v.(*runningFAQImportInfo)
if runningFAQImportInfoMatches(info, taskID, instanceID, enqueuedAt) {
s.memFAQRunningImport.Delete(kbID)
}
}
return nil
}
info, err := s.getRunningFAQImportInfo(ctx, kbID)
if err != nil {
return err
}
if !runningFAQImportInfoMatches(info, taskID, instanceID, enqueuedAt) {
return nil
}
key := getFAQImportRunningKey(kbID)
return s.redisClient.Del(ctx, key).Err()
}
func runningFAQImportInfoMatches(info *runningFAQImportInfo, taskID, instanceID string, enqueuedAt int64) bool {
if info == nil || info.TaskID != taskID {
return false
}
if info.InstanceID != "" && instanceID != "" {
return info.InstanceID == instanceID
}
return enqueuedAt == 0 || info.EnqueuedAt == 0 || info.EnqueuedAt == enqueuedAt
}
// incrementalIndexFAQEntry 增量更新FAQ条目的索引
// 只对内容变化的部分进行embedding计算和索引更新,跳过未变化的部分
func (s *knowledgeService) incrementalIndexFAQEntry(
@@ -1691,6 +1826,8 @@ func (s *knowledgeService) incrementalIndexFAQEntry(
newMeta *types.FAQChunkMetadata,
) error {
indexStartTime := time.Now()
logger.Debugf(ctx, "incrementalIndexFAQEntry: starting for chunk=%s, oldSimilarQuestions=%d, newSimilarQuestions=%d",
chunk.ID, len(oldSimilarQuestions), len(newMeta.SimilarQuestions))
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
ctx, s.retrieveEngine, s.ownership, types.MustTenantIDFromContext(ctx), kb.VectorStoreID)
@@ -1703,12 +1840,26 @@ func (s *knowledgeService) incrementalIndexFAQEntry(
indexMode = kb.FAQConfig.IndexMode
}
// 构建旧的内容(用于比较)
buildOldContent := func(question string) string {
if indexMode == types.FAQIndexModeQuestionAnswer && len(oldAnswers) > 0 {
// 对新旧数据进行归一化处理,确保与 buildFAQIndexInfoList 的行为一致
// 旧数据归一化
oldStandardQuestion = types.NormalizeQuestion(oldStandardQuestion)
normalizedOldSimilarQuestions := make([]string, 0, len(oldSimilarQuestions))
for _, q := range oldSimilarQuestions {
if nq := types.NormalizeQuestion(q); nq != "" {
normalizedOldSimilarQuestions = append(normalizedOldSimilarQuestions, nq)
}
}
oldSimilarQuestions = normalizedOldSimilarQuestions
oldAnswers = types.SanitizeStrings(oldAnswers)
// 新数据归一化
normalizedNewMeta := newMeta.Normalize()
// 构建索引内容
buildContent := func(question string, answers []string) string {
if indexMode == types.FAQIndexModeQuestionAnswer && len(answers) > 0 {
var builder strings.Builder
builder.WriteString(question)
for _, ans := range oldAnswers {
for _, ans := range answers {
builder.WriteString("\n")
builder.WriteString(ans)
}
@@ -1717,30 +1868,20 @@ func (s *knowledgeService) incrementalIndexFAQEntry(
return question
}
// 构建新的内容
buildNewContent := func(question string) string {
if indexMode == types.FAQIndexModeQuestionAnswer && len(newMeta.Answers) > 0 {
var builder strings.Builder
builder.WriteString(question)
for _, ans := range newMeta.Answers {
builder.WriteString("\n")
builder.WriteString(ans)
}
return builder.String()
}
return question
}
// 检查答案是否变化
answersChanged := !slices.Equal(oldAnswers, newMeta.Answers)
// 检查答案是否变化(仅在 QuestionAnswer 模式下才影响索引)
answersChanged := indexMode == types.FAQIndexModeQuestionAnswer && !slices.Equal(oldAnswers, normalizedNewMeta.Answers)
logger.Debugf(ctx, "incrementalIndexFAQEntry: answersChanged=%v (indexMode=%s), oldAnswers=%d, newAnswers=%d",
answersChanged, indexMode, len(oldAnswers), len(normalizedNewMeta.Answers))
// 收集需要更新的索引项
var indexInfoToUpdate []*types.IndexInfo
// 1. 检查标准问是否需要更新
oldStdContent := buildOldContent(oldStandardQuestion)
newStdContent := buildNewContent(newMeta.StandardQuestion)
if oldStdContent != newStdContent {
oldStdContent := buildContent(oldStandardQuestion, oldAnswers)
newStdContent := buildContent(normalizedNewMeta.StandardQuestion, normalizedNewMeta.Answers)
stdQuestionChanged := oldStdContent != newStdContent
if stdQuestionChanged {
logger.Debugf(ctx, "incrementalIndexFAQEntry: standard question changed, sourceID=%s", chunk.ID)
indexInfoToUpdate = append(indexInfoToUpdate, &types.IndexInfo{
Content: newStdContent,
SourceID: chunk.ID,
@@ -1755,27 +1896,41 @@ func (s *knowledgeService) incrementalIndexFAQEntry(
})
}
// 2. 检查每个相似问是否需要更新
oldCount := len(oldSimilarQuestions)
newCount := len(newMeta.SimilarQuestions)
// 2. 基于内容哈希处理相似问的增删改
// 构建旧问题集合 (问题 -> 是否存在)
oldQuestionsSet := make(map[string]struct{}, len(oldSimilarQuestions))
for _, q := range oldSimilarQuestions {
oldQuestionsSet[q] = struct{}{}
}
for i, newQ := range newMeta.SimilarQuestions {
needUpdate := false
if i >= oldCount {
// 新增的相似问
needUpdate = true
} else {
// 已存在的相似问,检查内容是否变化
oldQ := oldSimilarQuestions[i]
if oldQ != newQ || answersChanged {
needUpdate = true
}
// 构建新问题集合
newQuestionsSet := make(map[string]struct{}, len(normalizedNewMeta.SimilarQuestions))
for _, q := range normalizedNewMeta.SimilarQuestions {
newQuestionsSet[q] = struct{}{}
}
// 找出需要删除的问题(在旧集合中但不在新集合中)
var sourceIDsToDelete []string
var deletedQuestions []string
for oldQ := range oldQuestionsSet {
if _, exists := newQuestionsSet[oldQ]; !exists {
sourceID := fmt.Sprintf("%s-%s", chunk.ID, hashQuestion(oldQ))
sourceIDsToDelete = append(sourceIDsToDelete, sourceID)
deletedQuestions = append(deletedQuestions, oldQ)
}
}
if needUpdate {
sourceID := fmt.Sprintf("%s-%d", chunk.ID, i)
// 找出需要新增或更新的问题
var addedQuestions, updatedQuestions []string
for newQ := range newQuestionsSet {
_, existedBefore := oldQuestionsSet[newQ]
// 需要更新的条件:
// 1. 新问题(之前不存在)
// 2. 答案变化(需要重新embedding)
if !existedBefore || answersChanged {
sourceID := fmt.Sprintf("%s-%s", chunk.ID, hashQuestion(newQ))
indexInfoToUpdate = append(indexInfoToUpdate, &types.IndexInfo{
Content: buildNewContent(newQ),
Content: buildContent(newQ, normalizedNewMeta.Answers),
SourceID: sourceID,
SourceType: types.ChunkSourceType,
ChunkID: chunk.ID,
@@ -1786,22 +1941,35 @@ func (s *knowledgeService) incrementalIndexFAQEntry(
IsEnabled: chunk.IsEnabled,
IsRecommended: chunk.Flags.HasFlag(types.ChunkFlagRecommended),
})
if !existedBefore {
addedQuestions = append(addedQuestions, newQ)
} else {
updatedQuestions = append(updatedQuestions, newQ)
}
}
}
// 3. 删除多余的旧相似问索引
if oldCount > newCount {
sourceIDsToDelete := make([]string, 0, oldCount-newCount)
for i := newCount; i < oldCount; i++ {
sourceIDsToDelete = append(sourceIDsToDelete, fmt.Sprintf("%s-%d", chunk.ID, i))
}
logger.Debugf(ctx, "incrementalIndexFAQEntry: deleting %d obsolete source IDs", len(sourceIDsToDelete))
// 输出详细的变化日志
if len(deletedQuestions) > 0 {
logger.Debugf(ctx, "incrementalIndexFAQEntry: deleted similar questions: %v", deletedQuestions)
}
if len(addedQuestions) > 0 {
logger.Debugf(ctx, "incrementalIndexFAQEntry: added similar questions: %v", addedQuestions)
}
if len(updatedQuestions) > 0 {
logger.Debugf(ctx, "incrementalIndexFAQEntry: updated similar questions (answers changed): %v", updatedQuestions)
}
// 3. 删除不再存在的相似问索引
if len(sourceIDsToDelete) > 0 {
logger.Debugf(ctx, "incrementalIndexFAQEntry: deleting %d obsolete sourceIDs: %v", len(sourceIDsToDelete), sourceIDsToDelete)
if delErr := retrieveEngine.DeleteBySourceIDList(ctx, sourceIDsToDelete, embeddingModel.GetDimensions(), types.KnowledgeTypeFAQ); delErr != nil {
logger.Warnf(ctx, "incrementalIndexFAQEntry: failed to delete obsolete source IDs: %v", delErr)
}
}
// 4. 批量索引需要更新的内容
newCount := len(normalizedNewMeta.SimilarQuestions)
if len(indexInfoToUpdate) > 0 {
logger.Debugf(ctx, "incrementalIndexFAQEntry: updating %d index entries (skipped %d unchanged)",
len(indexInfoToUpdate), 1+newCount-len(indexInfoToUpdate))
@@ -2193,7 +2361,7 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
logger.Errorf(ctx, "Failed to get knowledge base: %v", err)
// 如果是最后一次重试,更新状态为失败
if isLastRetry {
if updateErr := s.updateFAQImportProgressStatus(ctx, payload.TaskID, types.FAQImportStatusFailed, 0, originalTotalEntries, 0, "获取知识库失败", err.Error()); updateErr != nil {
if updateErr := s.updateFAQImportProgressStatus(ctx, payload.TaskID, payload.InstanceID, payload.EnqueuedAt, types.FAQImportStatusFailed, 0, originalTotalEntries, 0, "获取知识库失败", err.Error()); updateErr != nil {
logger.Errorf(ctx, "Failed to update task status to failed: %v", updateErr)
}
s.recordFAQImportKBActivity(ctx, &payload, progress, originalTotalEntries, types.AuditActionFAQImportFailed, types.AuditOutcomeFailed)
@@ -2207,6 +2375,9 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
if existingProgress != nil {
if existingProgress.Status == types.FAQImportStatusCompleted {
logger.Infof(ctx, "FAQ import already completed, skipping: %s", payload.TaskID)
if clearErr := s.clearRunningFAQImportInfoIfMatches(ctx, payload.KBID, payload.TaskID, payload.InstanceID, payload.EnqueuedAt); clearErr != nil {
logger.Warnf(ctx, "Failed to clear running FAQ import info for completed task: %v", clearErr)
}
return nil // 幂等:已完成的任务直接返回
}
// 获取已处理的数量(注意:这是相对于 validEntries 的索引)
@@ -2223,7 +2394,7 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
logger.Errorf(ctx, "Failed to delete unindexed chunks: %v", err)
// 如果是最后一次重试,更新状态为失败
if isLastRetry {
if updateErr := s.updateFAQImportProgressStatus(ctx, payload.TaskID, types.FAQImportStatusFailed, 0, originalTotalEntries, 0, "清理未索引数据失败", err.Error()); updateErr != nil {
if updateErr := s.updateFAQImportProgressStatus(ctx, payload.TaskID, payload.InstanceID, payload.EnqueuedAt, types.FAQImportStatusFailed, 0, originalTotalEntries, 0, "清理未索引数据失败", err.Error()); updateErr != nil {
logger.Errorf(ctx, "Failed to update task status to failed: %v", updateErr)
}
s.recordFAQImportKBActivity(ctx, &payload, progress, originalTotalEntries, types.AuditActionFAQImportFailed, types.AuditOutcomeFailed)
@@ -2280,7 +2451,7 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
logger.Errorf(ctx, "FAQ import task failed: %s, error: %v", payload.TaskID, err)
// 如果是最后一次重试,更新状态为失败
if isLastRetry {
if updateErr := s.updateFAQImportProgressStatus(ctx, payload.TaskID, types.FAQImportStatusFailed, 0, originalTotalEntries, len(validEntries), "导入失败", err.Error()); updateErr != nil {
if updateErr := s.updateFAQImportProgressStatus(ctx, payload.TaskID, payload.InstanceID, payload.EnqueuedAt, types.FAQImportStatusFailed, 0, originalTotalEntries, len(validEntries), "导入失败", err.Error()); updateErr != nil {
logger.Errorf(ctx, "Failed to update task status to failed: %v", updateErr)
}
s.recordFAQImportKBActivity(ctx, &payload, progress, originalTotalEntries, types.AuditActionFAQImportFailed, types.AuditOutcomeFailed)
@@ -2385,7 +2556,7 @@ func (s *knowledgeService) finalizeFAQValidation(ctx context.Context, payload *t
}
// 然后调用状态更新来清理 running key
if err := s.updateFAQImportProgressStatus(ctx, payload.TaskID, types.FAQImportStatusCompleted,
if err := s.updateFAQImportProgressStatus(ctx, payload.TaskID, payload.InstanceID, payload.EnqueuedAt, types.FAQImportStatusCompleted,
100, originalTotalEntries, originalTotalEntries, progress.Message, ""); err != nil {
logger.Warnf(ctx, "Failed to update final FAQ import status: %v", err)
}
+21 -4
View File
@@ -3,6 +3,7 @@ package handler
import (
"net/http"
"strconv"
"strings"
"github.com/gin-gonic/gin"
@@ -381,19 +382,35 @@ func (h *FAQHandler) SearchFAQ(c *gin.Context) {
// ExportEntries godoc
// @Summary 导出FAQ条目
// @Description 将所有FAQ条目导出为CSV文件
// @Description 将所有FAQ条目导出为 CSV(默认)或 JSON。?format=json 返回与 FAQEntryPayload 结构兼容的数组。
// @Tags FAQ管理
// @Accept json
// @Produce text/csv
// @Param id path string true "知识库ID"
// @Success 200 {file} file "CSV文件"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Produce application/json
// @Param id path string true "知识库ID"
// @Param format query string false "导出格式:csv(默认)或 json"
// @Success 200 {file} file "导出文件"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Security Bearer
// @Security ApiKeyAuth
// @Router /knowledge-bases/{id}/faq/entries/export [get]
func (h *FAQHandler) ExportEntries(c *gin.Context) {
ctx := c.Request.Context()
kbID := secutils.SanitizeForLog(c.Param("id"))
format := strings.ToLower(strings.TrimSpace(c.Query("format")))
if format == "json" {
jsonData, err := h.knowledgeService.ExportFAQEntriesJSON(ctx, kbID)
if err != nil {
logger.ErrorWithFields(ctx, err, nil)
c.Error(err)
return
}
c.Header("Content-Type", "application/json; charset=utf-8")
c.Header("Content-Disposition", "attachment; filename=faq_export.json")
c.Data(http.StatusOK, "application/json; charset=utf-8", jsonData)
return
}
csvData, err := h.knowledgeService.ExportFAQEntries(ctx, kbID)
if err != nil {
+15
View File
@@ -251,6 +251,21 @@ type FAQEntry struct {
MatchedQuestion string `json:"matched_question,omitempty"`
}
// FAQExportEntry 表示 JSON 导出的 FAQ 条目,与 FAQEntryPayload 的导入格式兼容,
// 便于"导出 → 编辑 → 重新导入"循环。新增字段时务必保留 omitempty,避免破坏
// 历史导出文件的兼容性。
type FAQExportEntry struct {
ID int64 `json:"id"`
TagName string `json:"tag_name,omitempty"`
StandardQuestion string `json:"standard_question"`
SimilarQuestions []string `json:"similar_questions,omitempty"`
NegativeQuestions []string `json:"negative_questions,omitempty"`
Answers []string `json:"answers,omitempty"`
AnswerStrategy AnswerStrategy `json:"answer_strategy,omitempty"`
IsEnabled bool `json:"is_enabled"`
IsRecommended bool `json:"is_recommended"`
}
// FAQEntryPayload 用于创建/更新 FAQ 条目的 payload
type FAQEntryPayload struct {
// ID 可选,用于数据迁移时指定 seq_id(必须小于自增起始值 100000000)
+77
View File
@@ -0,0 +1,77 @@
package types
import "encoding/json"
// FAQChunkSyncPair links source/destination FAQ chunks sharing content_hash.
type FAQChunkSyncPair struct {
SrcChunkID string
DstChunkID string
}
// FAQChunkDiffResult is the output of FAQChunkDiff.
type FAQChunkDiffResult struct {
ChunksToAdd []string
ChunksToDelete []string
MatchedPairs []FAQChunkSyncPair
}
// FAQChunkStatus holds lightweight fields for clone status sync.
type FAQChunkStatus struct {
ID string
TagID string
IsEnabled bool
Flags ChunkFlags
AnswerStrategy AnswerStrategy
Metadata JSON
}
// FAQChunkStatusFieldsEqual compares status fields (tag excluded).
func FAQChunkStatusFieldsEqual(a, b *FAQChunkStatus) bool {
if a == nil || b == nil {
return a == b
}
if a.IsEnabled != b.IsEnabled || a.Flags != b.Flags {
return false
}
return NormalizeAnswerStrategy(a.AnswerStrategy) == NormalizeAnswerStrategy(b.AnswerStrategy)
}
// FAQChunkNeedsStatusSync reports whether a matched pair needs status sync.
func FAQChunkNeedsStatusSync(src, dst *FAQChunkStatus, mappedSrcTagID string) bool {
if src == nil || dst == nil {
return false
}
if mappedSrcTagID != dst.TagID {
return true
}
return !FAQChunkStatusFieldsEqual(src, dst)
}
// NormalizeAnswerStrategy treats empty as "all".
func NormalizeAnswerStrategy(s AnswerStrategy) AnswerStrategy {
if s == "" {
return AnswerStrategyAll
}
return s
}
// PatchFAQAnswerStrategy copies answer_strategy from src metadata onto dst metadata.
func PatchFAQAnswerStrategy(dstMeta, srcMeta JSON) (JSON, error) {
var dstM, srcM FAQChunkMetadata
if len(dstMeta) > 0 {
if err := json.Unmarshal(dstMeta, &dstM); err != nil {
return nil, err
}
}
if len(srcMeta) > 0 {
if err := json.Unmarshal(srcMeta, &srcM); err != nil {
return nil, err
}
}
dstM.AnswerStrategy = srcM.AnswerStrategy
bytes, err := json.Marshal(&dstM)
if err != nil {
return nil, err
}
return JSON(bytes), nil
}
+19
View File
@@ -0,0 +1,19 @@
package types
import "testing"
func TestFAQChunkNeedsStatusSync(t *testing.T) {
src := &FAQChunkStatus{TagID: "a", IsEnabled: true, Flags: ChunkFlagRecommended, AnswerStrategy: AnswerStrategyAll}
dst := &FAQChunkStatus{TagID: "old", IsEnabled: true, Flags: ChunkFlagRecommended, AnswerStrategy: AnswerStrategyAll}
if !FAQChunkNeedsStatusSync(src, dst, "mapped") {
t.Fatal("tag mismatch should need sync")
}
dst.TagID = "mapped"
if FAQChunkNeedsStatusSync(src, dst, "mapped") {
t.Fatal("in sync after tag mapped")
}
dst.IsEnabled = false
if !FAQChunkNeedsStatusSync(src, dst, "mapped") {
t.Fatal("is_enabled mismatch should need sync")
}
}
+3 -2
View File
@@ -97,8 +97,9 @@ type ChunkRepository interface {
// newTagID: if not nil, updates tag_id to this value (empty string means uncategorized)
UpdateChunkFieldsByTagID(ctx context.Context, tenantID uint64, kbID string, tagID string, isEnabled *bool, setFlags types.ChunkFlags, clearFlags types.ChunkFlags, newTagID *string, excludeIDs []string) ([]string, error)
// FAQChunkDiff compares FAQ chunks between two knowledge bases and returns the differences.
// Returns: chunksToAdd (content_hash in src but not in dst), chunksToDelete (content_hash in dst but not in src)
FAQChunkDiff(ctx context.Context, srcTenantID uint64, srcKBID string, dstTenantID uint64, dstKBID string) (chunksToAdd []string, chunksToDelete []string, err error)
FAQChunkDiff(ctx context.Context, srcTenantID uint64, srcKBID string, dstTenantID uint64, dstKBID string) (*types.FAQChunkDiffResult, error)
// ListFAQChunkStatusByIDs loads status fields for FAQ clone sync.
ListFAQChunkStatusByIDs(ctx context.Context, tenantID uint64, ids []string) (map[string]*types.FAQChunkStatus, error)
// ListRecommendedFAQChunks lists FAQ chunks with the recommended flag set.
// Filter by explicitly selected kbIDs, knowledgeIDs, and/or FAQ tagIDs.
+2
View File
@@ -152,6 +152,8 @@ type KnowledgeService interface {
SearchFAQEntries(ctx context.Context, kbID string, req *types.FAQSearchRequest) ([]*types.FAQEntry, error)
// ExportFAQEntries exports all FAQ entries for a knowledge base as CSV data.
ExportFAQEntries(ctx context.Context, kbID string) ([]byte, error)
// ExportFAQEntriesJSON exports all FAQ entries as a JSON array compatible with FAQEntryPayload.
ExportFAQEntriesJSON(ctx context.Context, kbID string) ([]byte, error)
// UpdateKnowledgeTagBatch updates tag for document knowledge items in batch.
// authorizedKBID restricts all updates to knowledge items belonging to this KB;
// pass empty string to skip (caller must ensure authorization by other means).
+1
View File
@@ -305,6 +305,7 @@ type FAQImportPayload struct {
Mode string `json:"mode"`
DryRun bool `json:"dry_run"` // dry run 模式只验证不导入
EnqueuedAt int64 `json:"enqueued_at"` // 任务入队时间戳,用于区分同一 TaskID 的不同次提交
InstanceID string `json:"instance_id,omitempty"`
Initiator TaskInitiator `json:"initiator,omitempty"`
}