feat(knowledge): enhance FAQ export functionality
- Updated the exportFAQEntries function to support exporting FAQ entries in both CSV and JSON formats. - Modified the frontend to include options for selecting the export format. - Enhanced localization files to reflect the new export options in English, Russian, and Chinese. - Implemented new tests to validate the export functionality for both formats. This commit improves the flexibility of FAQ exports, allowing users to choose their preferred format for better integration with other tools.
This commit is contained in:
@@ -445,10 +445,11 @@ export function searchFAQEntries(
|
||||
return post(`/api/v1/knowledge-bases/${kbId}/faq/search`, data);
|
||||
}
|
||||
|
||||
// Export FAQ entries as CSV file
|
||||
export async function exportFAQEntries(kbId: string): Promise<Blob> {
|
||||
const response = await getDown(`/api/v1/knowledge-bases/${kbId}/faq/entries/export`);
|
||||
return response as unknown as Blob;
|
||||
// Export FAQ entries as CSV or JSON file
|
||||
export async function exportFAQEntries(kbId: string, format: 'csv' | 'json' = 'csv'): Promise<Blob> {
|
||||
const suffix = format === 'json' ? '?format=json' : ''
|
||||
const response = await getDown(`/api/v1/knowledge-bases/${kbId}/faq/entries/export${suffix}`)
|
||||
return response as unknown as Blob
|
||||
}
|
||||
|
||||
// FAQ Import Progress API
|
||||
|
||||
@@ -2992,7 +2992,9 @@ export default {
|
||||
downloadExampleExcel: 'Download Excel Example',
|
||||
},
|
||||
faqExport: {
|
||||
exportButton: 'Export CSV',
|
||||
exportButton: 'Export',
|
||||
exportCSV: 'Export CSV',
|
||||
exportJSON: 'Export JSON',
|
||||
exportSuccess: 'Export successful',
|
||||
exportFailed: 'Export failed',
|
||||
},
|
||||
|
||||
@@ -3689,7 +3689,9 @@ export default {
|
||||
downloadExampleExcel: 'Скачать пример Excel'
|
||||
},
|
||||
faqExport: {
|
||||
exportButton: 'Экспорт CSV',
|
||||
exportButton: 'Экспорт',
|
||||
exportCSV: 'Экспорт CSV',
|
||||
exportJSON: 'Экспорт JSON',
|
||||
exportSuccess: 'Экспорт успешен',
|
||||
exportFailed: 'Ошибка экспорта'
|
||||
},
|
||||
|
||||
@@ -4281,7 +4281,9 @@ export default {
|
||||
downloadExampleExcel: "下载 Excel 示例",
|
||||
},
|
||||
faqExport: {
|
||||
exportButton: "导出 CSV",
|
||||
exportButton: "导出",
|
||||
exportCSV: "导出 CSV",
|
||||
exportJSON: "导出 JSON",
|
||||
exportSuccess: "导出成功",
|
||||
exportFailed: "导出失败",
|
||||
},
|
||||
|
||||
@@ -238,12 +238,15 @@
|
||||
</t-tooltip>
|
||||
</template>
|
||||
<!-- 导出 -->
|
||||
<t-tooltip :content="$t('knowledgeEditor.faqExport.exportButton')" placement="top">
|
||||
<t-button variant="text" theme="default" class="content-bar-icon-btn" size="small"
|
||||
@click="handleFaqAction({ value: 'export' })">
|
||||
<template #icon><t-icon name="download" size="16px" /></template>
|
||||
</t-button>
|
||||
</t-tooltip>
|
||||
<t-dropdown :options="faqExportOptions" trigger="click" placement="bottom-right"
|
||||
@click="handleFaqAction">
|
||||
<t-tooltip :content="$t('knowledgeEditor.faqExport.exportButton')" placement="top">
|
||||
<t-button variant="text" theme="default" class="content-bar-icon-btn" size="small"
|
||||
:loading="exportLoading">
|
||||
<template #icon><t-icon name="download" size="16px" /></template>
|
||||
</t-button>
|
||||
</t-tooltip>
|
||||
</t-dropdown>
|
||||
<!-- 检索 -->
|
||||
<t-tooltip :content="$t('knowledgeEditor.faq.searchTest')" placement="top">
|
||||
<t-button variant="text" theme="default" class="content-bar-icon-btn" size="small"
|
||||
@@ -998,6 +1001,11 @@ const canManage = computed(() => {
|
||||
return orgStore.canManageKB(props.kbId, false)
|
||||
})
|
||||
|
||||
const faqExportOptions = computed(() => [
|
||||
{ content: t('knowledgeEditor.faqExport.exportCSV'), value: 'export_csv' },
|
||||
{ content: t('knowledgeEditor.faqExport.exportJSON'), value: 'export_json' },
|
||||
])
|
||||
|
||||
// FAQ 操作:新建组(新建条目 + 导入)
|
||||
const faqCreateOptions = computed(() => {
|
||||
if (!canEdit.value) return []
|
||||
@@ -1019,6 +1027,12 @@ const handleFaqAction = (data: { value: string }) => {
|
||||
case 'search':
|
||||
searchDrawerVisible.value = true
|
||||
break
|
||||
case 'export_csv':
|
||||
handleExportCSV()
|
||||
break
|
||||
case 'export_json':
|
||||
handleExportJSON()
|
||||
break
|
||||
case 'export':
|
||||
handleExportCSV()
|
||||
break
|
||||
@@ -2647,9 +2661,19 @@ const downloadExcelExample = () => {
|
||||
XLSX.writeFile(workbook, 'faq_example.xlsx')
|
||||
}
|
||||
|
||||
// 导出 FAQ 数据为 CSV
|
||||
// 导出 FAQ 数据
|
||||
const exportLoading = ref(false)
|
||||
const handleExportCSV = async () => {
|
||||
const downloadExportBlob = (blob: Blob, ext: 'csv' | 'json') => {
|
||||
const url = URL.createObjectURL(blob)
|
||||
const link = document.createElement('a')
|
||||
link.href = url
|
||||
link.download = `faq_export_${new Date().toISOString().slice(0, 10)}.${ext}`
|
||||
document.body.appendChild(link)
|
||||
link.click()
|
||||
document.body.removeChild(link)
|
||||
URL.revokeObjectURL(url)
|
||||
}
|
||||
const handleExportFAQ = async (format: 'csv' | 'json') => {
|
||||
if (!props.kbId) {
|
||||
MessagePlugin.warning(t('knowledgeBase.selectKnowledgeBase'))
|
||||
return
|
||||
@@ -2657,15 +2681,8 @@ const handleExportCSV = async () => {
|
||||
|
||||
exportLoading.value = true
|
||||
try {
|
||||
const blob = await exportFAQEntries(props.kbId)
|
||||
const url = URL.createObjectURL(blob)
|
||||
const link = document.createElement('a')
|
||||
link.href = url
|
||||
link.download = `faq_export_${new Date().toISOString().slice(0, 10)}.csv`
|
||||
document.body.appendChild(link)
|
||||
link.click()
|
||||
document.body.removeChild(link)
|
||||
URL.revokeObjectURL(url)
|
||||
const blob = await exportFAQEntries(props.kbId, format)
|
||||
downloadExportBlob(blob, format)
|
||||
MessagePlugin.success(t('knowledgeEditor.faqExport.exportSuccess'))
|
||||
} catch (error: any) {
|
||||
console.error('Export failed:', error)
|
||||
@@ -2674,6 +2691,8 @@ const handleExportCSV = async () => {
|
||||
exportLoading.value = false
|
||||
}
|
||||
}
|
||||
const handleExportCSV = () => handleExportFAQ('csv')
|
||||
const handleExportJSON = () => handleExportFAQ('json')
|
||||
|
||||
watch(
|
||||
() => props.kbId,
|
||||
|
||||
@@ -931,44 +931,170 @@ func (r *chunkRepository) UpdateChunkFieldsByTagID(
|
||||
return affectedIDs, nil
|
||||
}
|
||||
|
||||
type chunkIDHash struct {
|
||||
ID string `gorm:"column:id"`
|
||||
ContentHash string `gorm:"column:content_hash"`
|
||||
}
|
||||
|
||||
const faqChunkDiffBatchSize = 5000
|
||||
|
||||
// listFAQChunkIDHashesByKB loads id/content_hash pairs for all FAQ chunks in a KB.
|
||||
func (r *chunkRepository) listFAQChunkIDHashesByKB(
|
||||
ctx context.Context,
|
||||
tenantID uint64,
|
||||
kbID string,
|
||||
) ([]chunkIDHash, error) {
|
||||
var all []chunkIDHash
|
||||
var lastID string
|
||||
|
||||
for {
|
||||
var batch []chunkIDHash
|
||||
query := r.db.WithContext(ctx).Model(&types.Chunk{}).
|
||||
Select("id, content_hash").
|
||||
Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?",
|
||||
tenantID, kbID, types.ChunkTypeFAQ).
|
||||
Order("id ASC").
|
||||
Limit(faqChunkDiffBatchSize)
|
||||
if lastID != "" {
|
||||
query = query.Where("id > ?", lastID)
|
||||
}
|
||||
if err := query.Find(&batch).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(batch) == 0 {
|
||||
break
|
||||
}
|
||||
all = append(all, batch...)
|
||||
lastID = batch[len(batch)-1].ID
|
||||
if len(batch) < faqChunkDiffBatchSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
return all, nil
|
||||
}
|
||||
|
||||
func diffFAQChunkIDsByContentHash(src, dst []chunkIDHash) (
|
||||
chunksToAdd, chunksToDelete []string,
|
||||
matched []types.FAQChunkSyncPair,
|
||||
) {
|
||||
dstHashes := make(map[string]struct{}, len(dst))
|
||||
dstIDByHash := make(map[string]string, len(dst))
|
||||
dstIDsByHash := make(map[string][]string, len(dst))
|
||||
for _, pair := range dst {
|
||||
dstHashes[pair.ContentHash] = struct{}{}
|
||||
if _, ok := dstIDByHash[pair.ContentHash]; !ok {
|
||||
dstIDByHash[pair.ContentHash] = pair.ID
|
||||
}
|
||||
dstIDsByHash[pair.ContentHash] = append(dstIDsByHash[pair.ContentHash], pair.ID)
|
||||
}
|
||||
srcHashes := make(map[string]struct{}, len(src))
|
||||
for _, pair := range src {
|
||||
srcHashes[pair.ContentHash] = struct{}{}
|
||||
}
|
||||
|
||||
for _, pair := range src {
|
||||
if _, exists := dstHashes[pair.ContentHash]; !exists {
|
||||
chunksToAdd = append(chunksToAdd, pair.ID)
|
||||
continue
|
||||
}
|
||||
if dstID, ok := dstIDByHash[pair.ContentHash]; ok {
|
||||
matched = append(matched, types.FAQChunkSyncPair{
|
||||
SrcChunkID: pair.ID,
|
||||
DstChunkID: dstID,
|
||||
})
|
||||
}
|
||||
}
|
||||
for _, pair := range dst {
|
||||
if _, exists := srcHashes[pair.ContentHash]; !exists {
|
||||
chunksToDelete = append(chunksToDelete, pair.ID)
|
||||
}
|
||||
}
|
||||
for hash, ids := range dstIDsByHash {
|
||||
if hash == "" || len(ids) <= 1 {
|
||||
continue
|
||||
}
|
||||
if _, inSrc := srcHashes[hash]; !inSrc {
|
||||
continue
|
||||
}
|
||||
canonical := dstIDByHash[hash]
|
||||
for _, id := range ids {
|
||||
if id != canonical {
|
||||
chunksToDelete = append(chunksToDelete, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
return chunksToAdd, chunksToDelete, matched
|
||||
}
|
||||
|
||||
// FAQChunkDiff compares FAQ chunks between two knowledge bases and returns the differences.
|
||||
// Returns: chunksToAdd (IDs of chunks in src whose content_hash is not in dst),
|
||||
//
|
||||
// chunksToDelete (IDs of chunks in dst whose content_hash is not in src)
|
||||
// chunksToDelete (IDs of chunks in dst whose content_hash is not in src, plus
|
||||
// duplicate dst chunks that share a content_hash with another dst chunk when
|
||||
// that hash still exists in src)
|
||||
func (r *chunkRepository) FAQChunkDiff(
|
||||
ctx context.Context,
|
||||
srcTenantID uint64, srcKBID string,
|
||||
dstTenantID uint64, dstKBID string,
|
||||
) (chunksToAdd []string, chunksToDelete []string, err error) {
|
||||
// Get content_hash set from destination KB
|
||||
dstHashSubQuery := r.db.Model(&types.Chunk{}).
|
||||
Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?", dstTenantID, dstKBID, types.ChunkTypeFAQ).
|
||||
Select("content_hash")
|
||||
|
||||
// Find chunks in source that don't exist in destination (by content_hash)
|
||||
err = r.db.WithContext(ctx).Model(&types.Chunk{}).
|
||||
Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?", srcTenantID, srcKBID, types.ChunkTypeFAQ).
|
||||
Where("content_hash NOT IN (?)", dstHashSubQuery).
|
||||
Pluck("id", &chunksToAdd).Error
|
||||
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, nil, fmt.Errorf("failed to get chunks to add: %w", err)
|
||||
) (*types.FAQChunkDiffResult, error) {
|
||||
srcPairs, err := r.listFAQChunkIDHashesByKB(ctx, srcTenantID, srcKBID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list source FAQ chunks: %w", err)
|
||||
}
|
||||
dstPairs, err := r.listFAQChunkIDHashesByKB(ctx, dstTenantID, dstKBID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list destination FAQ chunks: %w", err)
|
||||
}
|
||||
|
||||
// Get content_hash set from source KB
|
||||
srcHashSubQuery := r.db.Model(&types.Chunk{}).
|
||||
Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?", srcTenantID, srcKBID, types.ChunkTypeFAQ).
|
||||
Select("content_hash")
|
||||
add, del, matched := diffFAQChunkIDsByContentHash(srcPairs, dstPairs)
|
||||
return &types.FAQChunkDiffResult{
|
||||
ChunksToAdd: add,
|
||||
ChunksToDelete: del,
|
||||
MatchedPairs: matched,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Find chunks in destination that don't exist in source (by content_hash)
|
||||
err = r.db.WithContext(ctx).Model(&types.Chunk{}).
|
||||
Where("tenant_id = ? AND knowledge_base_id = ? AND chunk_type = ?", dstTenantID, dstKBID, types.ChunkTypeFAQ).
|
||||
Where("content_hash NOT IN (?)", srcHashSubQuery).
|
||||
Pluck("id", &chunksToDelete).Error
|
||||
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, nil, fmt.Errorf("failed to get chunks to delete: %w", err)
|
||||
// ListFAQChunkStatusByIDs loads status fields for FAQ clone sync.
|
||||
func (r *chunkRepository) ListFAQChunkStatusByIDs(
|
||||
ctx context.Context,
|
||||
tenantID uint64,
|
||||
ids []string,
|
||||
) (map[string]*types.FAQChunkStatus, error) {
|
||||
if len(ids) == 0 {
|
||||
return map[string]*types.FAQChunkStatus{}, nil
|
||||
}
|
||||
|
||||
return chunksToAdd, chunksToDelete, nil
|
||||
const batchSize = 5000
|
||||
var chunks []*types.Chunk
|
||||
for i := 0; i < len(ids); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(ids) {
|
||||
end = len(ids)
|
||||
}
|
||||
var batch []*types.Chunk
|
||||
if err := r.db.WithContext(ctx).
|
||||
Select("id, tag_id, is_enabled, flags, metadata").
|
||||
Where("tenant_id = ? AND id IN ?", tenantID, ids[i:end]).
|
||||
Find(&batch).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
chunks = append(chunks, batch...)
|
||||
}
|
||||
out := make(map[string]*types.FAQChunkStatus, len(chunks))
|
||||
for _, chunk := range chunks {
|
||||
strategy := types.AnswerStrategyAll
|
||||
if meta, err := chunk.FAQMetadata(); err == nil && meta != nil {
|
||||
strategy = meta.AnswerStrategy
|
||||
}
|
||||
out[chunk.ID] = &types.FAQChunkStatus{
|
||||
ID: chunk.ID,
|
||||
TagID: chunk.TagID,
|
||||
IsEnabled: chunk.IsEnabled,
|
||||
Flags: chunk.Flags,
|
||||
AnswerStrategy: strategy,
|
||||
Metadata: chunk.Metadata,
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ListRecommendedFAQChunks lists FAQ chunks with the recommended flag set.
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestDiffFAQChunkIDsByContentHash(t *testing.T) {
|
||||
toAdd, toDelete, matched := diffFAQChunkIDsByContentHash(
|
||||
[]chunkIDHash{
|
||||
{ID: "src-1", ContentHash: "hash-a"},
|
||||
{ID: "src-2", ContentHash: "hash-b"},
|
||||
{ID: "src-3", ContentHash: "hash-only-src"},
|
||||
},
|
||||
[]chunkIDHash{
|
||||
{ID: "dst-1", ContentHash: "hash-a"},
|
||||
{ID: "dst-2", ContentHash: "hash-only-dst"},
|
||||
},
|
||||
)
|
||||
|
||||
assert.ElementsMatch(t, []string{"src-2", "src-3"}, toAdd)
|
||||
assert.ElementsMatch(t, []string{"dst-2"}, toDelete)
|
||||
require.Len(t, matched, 1)
|
||||
assert.Equal(t, "src-1", matched[0].SrcChunkID)
|
||||
assert.Equal(t, "dst-1", matched[0].DstChunkID)
|
||||
}
|
||||
|
||||
func TestDiffFAQChunkIDsByContentHash_dstDuplicateHash(t *testing.T) {
|
||||
// Two destination chunks share hash-a (same FAQ content, e.g. stale copy after
|
||||
// answer_strategy-only change). Source has one chunk for hash-a → keep dst-1,
|
||||
// delete dst-dup, match for status sync.
|
||||
toAdd, toDelete, matched := diffFAQChunkIDsByContentHash(
|
||||
[]chunkIDHash{{ID: "src-1", ContentHash: "hash-a"}},
|
||||
[]chunkIDHash{
|
||||
{ID: "dst-1", ContentHash: "hash-a"},
|
||||
{ID: "dst-dup", ContentHash: "hash-a"},
|
||||
},
|
||||
)
|
||||
|
||||
assert.Empty(t, toAdd)
|
||||
assert.ElementsMatch(t, []string{"dst-dup"}, toDelete)
|
||||
require.Len(t, matched, 1)
|
||||
assert.Equal(t, "src-1", matched[0].SrcChunkID)
|
||||
assert.Equal(t, "dst-1", matched[0].DstChunkID)
|
||||
}
|
||||
|
||||
func TestDiffFAQChunkIDsByContentHash_dstDuplicateHash_notInSrc(t *testing.T) {
|
||||
// Duplicates whose hash is absent from source are all removed (orphan content).
|
||||
toAdd, toDelete, matched := diffFAQChunkIDsByContentHash(
|
||||
nil,
|
||||
[]chunkIDHash{
|
||||
{ID: "dst-orphan-1", ContentHash: "hash-orphan"},
|
||||
{ID: "dst-orphan-2", ContentHash: "hash-orphan"},
|
||||
},
|
||||
)
|
||||
|
||||
assert.Empty(t, toAdd)
|
||||
assert.ElementsMatch(t, []string{"dst-orphan-1", "dst-orphan-2"}, toDelete)
|
||||
assert.Empty(t, matched)
|
||||
}
|
||||
|
||||
func TestDiffFAQChunkIDsByContentHash_emptySide(t *testing.T) {
|
||||
toAdd, toDelete, matched := diffFAQChunkIDsByContentHash(
|
||||
[]chunkIDHash{{ID: "src-1", ContentHash: "hash-a"}},
|
||||
nil,
|
||||
)
|
||||
assert.Equal(t, []string{"src-1"}, toAdd)
|
||||
assert.Empty(t, toDelete)
|
||||
assert.Empty(t, matched)
|
||||
|
||||
toAdd, toDelete, matched = diffFAQChunkIDsByContentHash(
|
||||
nil,
|
||||
[]chunkIDHash{{ID: "dst-1", ContentHash: "hash-a"}},
|
||||
)
|
||||
assert.Empty(t, toAdd)
|
||||
assert.Equal(t, []string{"dst-1"}, toDelete)
|
||||
assert.Empty(t, matched)
|
||||
}
|
||||
|
||||
func TestFAQChunkDiff_SQLite(t *testing.T) {
|
||||
db := setupChunkTestDB(t)
|
||||
repo := NewChunkRepository(db)
|
||||
ctx := context.Background()
|
||||
|
||||
srcKBID := uuid.New().String()
|
||||
dstKBID := uuid.New().String()
|
||||
srcKnowledgeID := uuid.New().String()
|
||||
dstKnowledgeID := uuid.New().String()
|
||||
|
||||
require.NoError(t, db.Create(&types.Chunk{
|
||||
ID: uuid.New().String(), TenantID: 1, KnowledgeBaseID: srcKBID, KnowledgeID: srcKnowledgeID,
|
||||
ChunkType: types.ChunkTypeFAQ, ContentHash: "shared", Status: int(types.ChunkStatusIndexed),
|
||||
}).Error)
|
||||
require.NoError(t, db.Create(&types.Chunk{
|
||||
ID: uuid.New().String(), TenantID: 1, KnowledgeBaseID: srcKBID, KnowledgeID: srcKnowledgeID,
|
||||
ChunkType: types.ChunkTypeFAQ, ContentHash: "src-only", Status: int(types.ChunkStatusIndexed),
|
||||
}).Error)
|
||||
require.NoError(t, db.Create(&types.Chunk{
|
||||
ID: uuid.New().String(), TenantID: 1, KnowledgeBaseID: dstKBID, KnowledgeID: dstKnowledgeID,
|
||||
ChunkType: types.ChunkTypeFAQ, ContentHash: "shared", Status: int(types.ChunkStatusIndexed),
|
||||
}).Error)
|
||||
require.NoError(t, db.Create(&types.Chunk{
|
||||
ID: uuid.New().String(), TenantID: 1, KnowledgeBaseID: dstKBID, KnowledgeID: dstKnowledgeID,
|
||||
ChunkType: types.ChunkTypeFAQ, ContentHash: "dst-only", Status: int(types.ChunkStatusIndexed),
|
||||
}).Error)
|
||||
|
||||
diff, err := repo.FAQChunkDiff(ctx, 1, srcKBID, 1, dstKBID)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, diff.ChunksToAdd, 1)
|
||||
assert.Len(t, diff.ChunksToDelete, 1)
|
||||
assert.Len(t, diff.MatchedPairs, 1)
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/application/service/retriever"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
type faqStatusSyncPlan struct {
|
||||
Pairs []types.FAQChunkSyncPair
|
||||
SrcByID map[string]*types.FAQChunkStatus
|
||||
DstByID map[string]*types.FAQChunkStatus
|
||||
}
|
||||
|
||||
func (s *knowledgeService) buildFAQStatusSyncPlan(
|
||||
ctx context.Context,
|
||||
srcTenantID, dstTenantID uint64,
|
||||
matched []types.FAQChunkSyncPair,
|
||||
resolveTag func(srcTagID string) string,
|
||||
) (*faqStatusSyncPlan, error) {
|
||||
if len(matched) == 0 {
|
||||
return &faqStatusSyncPlan{}, nil
|
||||
}
|
||||
srcIDs := make([]string, 0, len(matched))
|
||||
dstIDs := make([]string, 0, len(matched))
|
||||
for _, p := range matched {
|
||||
srcIDs = append(srcIDs, p.SrcChunkID)
|
||||
dstIDs = append(dstIDs, p.DstChunkID)
|
||||
}
|
||||
srcByID, err := s.chunkRepo.ListFAQChunkStatusByIDs(ctx, srcTenantID, srcIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dstByID, err := s.chunkRepo.ListFAQChunkStatusByIDs(ctx, dstTenantID, dstIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pairs := make([]types.FAQChunkSyncPair, 0)
|
||||
for _, p := range matched {
|
||||
src, dst := srcByID[p.SrcChunkID], dstByID[p.DstChunkID]
|
||||
if src == nil || dst == nil {
|
||||
continue
|
||||
}
|
||||
mappedTag := ""
|
||||
if src.TagID != "" {
|
||||
mappedTag = resolveTag(src.TagID)
|
||||
}
|
||||
if types.FAQChunkNeedsStatusSync(src, dst, mappedTag) {
|
||||
pairs = append(pairs, p)
|
||||
}
|
||||
}
|
||||
return &faqStatusSyncPlan{Pairs: pairs, SrcByID: srcByID, DstByID: dstByID}, nil
|
||||
}
|
||||
|
||||
func (s *knowledgeService) syncFAQChunkStatusBatch(
|
||||
ctx context.Context,
|
||||
dstKB *types.KnowledgeBase,
|
||||
pairs []types.FAQChunkSyncPair,
|
||||
srcByID, dstByID map[string]*types.FAQChunkStatus,
|
||||
resolveTag func(srcTagID string) string,
|
||||
) error {
|
||||
if len(pairs) == 0 {
|
||||
return nil
|
||||
}
|
||||
tenantID := types.MustTenantIDFromContext(ctx)
|
||||
enabledUpdates := make(map[string]bool)
|
||||
recommendedUpdates := make(map[string]bool)
|
||||
tagUpdates := make(map[string]string)
|
||||
rows := make([]*types.Chunk, 0, len(pairs))
|
||||
metadataIDs := make([]string, 0)
|
||||
|
||||
for _, p := range pairs {
|
||||
src, dst := srcByID[p.SrcChunkID], dstByID[p.DstChunkID]
|
||||
if src == nil || dst == nil {
|
||||
continue
|
||||
}
|
||||
mappedTag := ""
|
||||
if src.TagID != "" {
|
||||
mappedTag = resolveTag(src.TagID)
|
||||
}
|
||||
if !types.FAQChunkNeedsStatusSync(src, dst, mappedTag) {
|
||||
continue
|
||||
}
|
||||
rows = append(rows, &types.Chunk{
|
||||
ID: dst.ID, IsEnabled: src.IsEnabled, Flags: src.Flags,
|
||||
TagID: mappedTag, UpdatedAt: time.Now(),
|
||||
})
|
||||
if dst.IsEnabled != src.IsEnabled {
|
||||
enabledUpdates[dst.ID] = src.IsEnabled
|
||||
}
|
||||
srcRec := src.Flags.HasFlag(types.ChunkFlagRecommended)
|
||||
dstRec := dst.Flags.HasFlag(types.ChunkFlagRecommended)
|
||||
if srcRec != dstRec {
|
||||
recommendedUpdates[dst.ID] = srcRec
|
||||
}
|
||||
if mappedTag != dst.TagID {
|
||||
tagUpdates[dst.ID] = mappedTag
|
||||
}
|
||||
if types.NormalizeAnswerStrategy(src.AnswerStrategy) != types.NormalizeAnswerStrategy(dst.AnswerStrategy) {
|
||||
metadataIDs = append(metadataIDs, dst.ID)
|
||||
}
|
||||
}
|
||||
if len(rows) > 0 {
|
||||
if err := s.chunkRepo.UpdateChunks(ctx, rows); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, dstID := range metadataIDs {
|
||||
src := srcForDst(pairs, srcByID, dstID)
|
||||
dst := dstByID[dstID]
|
||||
if src == nil || dst == nil {
|
||||
continue
|
||||
}
|
||||
patched, err := types.PatchFAQAnswerStrategy(dst.Metadata, src.Metadata)
|
||||
if err != nil {
|
||||
return fmt.Errorf("patch answer_strategy for %s: %w", dstID, err)
|
||||
}
|
||||
full, err := s.chunkRepo.GetChunkByID(ctx, tenantID, dstID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
full.Metadata = patched
|
||||
full.UpdatedAt = time.Now()
|
||||
if err := s.chunkRepo.UpdateChunk(ctx, full); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(enabledUpdates) == 0 && len(tagUpdates) == 0 && len(recommendedUpdates) == 0 {
|
||||
return nil
|
||||
}
|
||||
engine, err := retriever.CreateRetrieveEngineForKB(
|
||||
ctx, s.retrieveEngine, s.ownership, tenantID, dstKB.VectorStoreID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(enabledUpdates) > 0 {
|
||||
if err := engine.BatchUpdateChunkEnabledStatus(ctx, enabledUpdates); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(tagUpdates) > 0 {
|
||||
if err := engine.BatchUpdateChunkTagID(ctx, tagUpdates); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(recommendedUpdates) > 0 {
|
||||
// Vector-store recommended flag sync is not yet available on all backends in
|
||||
// this branch; DB flags were already updated via UpdateChunks above.
|
||||
logger.Warnf(ctx, "FAQ clone sync: skipped vector recommended update for %d chunks (DB flags updated)", len(recommendedUpdates))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func srcForDst(pairs []types.FAQChunkSyncPair, srcByID map[string]*types.FAQChunkStatus, dstID string) *types.FAQChunkStatus {
|
||||
for _, p := range pairs {
|
||||
if p.DstChunkID == dstID {
|
||||
return srcByID[p.SrcChunkID]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -675,23 +675,47 @@ func (s *knowledgeService) cloneFAQKnowledgeBase(
|
||||
}
|
||||
srcKnowledge := srcKnowledgeList[0]
|
||||
|
||||
// Get chunk-level differences based on content_hash
|
||||
chunksToAdd, chunksToDelete, err := s.chunkRepo.FAQChunkDiff(ctx, srcKB.TenantID, srcKB.ID, dstKB.TenantID, dstKB.ID)
|
||||
// Get chunk-level differences based on content_hash.
|
||||
diff, err := s.chunkRepo.FAQChunkDiff(ctx, srcKB.TenantID, srcKB.ID, dstKB.TenantID, dstKB.ID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to calculate FAQ chunk difference: %v", err)
|
||||
handleError(progress, err, "Failed to calculate FAQ chunk difference")
|
||||
return err
|
||||
}
|
||||
chunksToAdd := diff.ChunksToAdd
|
||||
chunksToDelete := diff.ChunksToDelete
|
||||
|
||||
totalOperations := len(chunksToAdd) + len(chunksToDelete)
|
||||
tagIDMapping := map[string]string{}
|
||||
resolveFAQTag := func(srcTagID string) string {
|
||||
if srcTagID == "" {
|
||||
return ""
|
||||
}
|
||||
if id, ok := tagIDMapping[srcTagID]; ok {
|
||||
return id
|
||||
}
|
||||
return s.getOrCreateTagInTarget(ctx, srcKB.TenantID, dstKB.TenantID, dstKB.ID, srcTagID, tagIDMapping)
|
||||
}
|
||||
|
||||
syncPlan, err := s.buildFAQStatusSyncPlan(ctx, srcKB.TenantID, dstKB.TenantID, diff.MatchedPairs, resolveFAQTag)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to build FAQ status sync plan: %v", err)
|
||||
handleError(progress, err, "Failed to build FAQ status sync plan")
|
||||
return err
|
||||
}
|
||||
chunksToUpdate := syncPlan.Pairs
|
||||
|
||||
totalOperations := len(chunksToAdd) + len(chunksToDelete) + len(chunksToUpdate)
|
||||
progress.Total = totalOperations
|
||||
progress.Message = fmt.Sprintf("Found %d FAQ entries to add, %d to delete", len(chunksToAdd), len(chunksToDelete))
|
||||
progress.Message = fmt.Sprintf(
|
||||
"Found %d FAQ entries to add, %d to delete, %d to update status",
|
||||
len(chunksToAdd), len(chunksToDelete), len(chunksToUpdate),
|
||||
)
|
||||
progress.UpdatedAt = time.Now().Unix()
|
||||
_ = s.saveKBCloneProgress(ctx, progress)
|
||||
|
||||
logger.Infof(ctx, "FAQ chunks to add: %d, delete: %d", len(chunksToAdd), len(chunksToDelete))
|
||||
logger.Infof(ctx, "FAQ chunks to add: %d, delete: %d, update status: %d",
|
||||
len(chunksToAdd), len(chunksToDelete), len(chunksToUpdate))
|
||||
|
||||
// If nothing to do, mark as completed
|
||||
if totalOperations == 0 {
|
||||
progress.Status = types.KBCloneStatusCompleted
|
||||
progress.Progress = 100
|
||||
@@ -760,7 +784,6 @@ func (s *knowledgeService) cloneFAQKnowledgeBase(
|
||||
|
||||
// Clone FAQ chunks from source to destination
|
||||
batch := 50
|
||||
tagIDMapping := map[string]string{} // srcTagID -> dstTagID
|
||||
for i := 0; i < len(chunksToAdd); i += batch {
|
||||
end := i + batch
|
||||
if end > len(chunksToAdd) {
|
||||
@@ -782,19 +805,11 @@ func (s *knowledgeService) cloneFAQKnowledgeBase(
|
||||
// Map TagID to target knowledge base
|
||||
targetTagID := ""
|
||||
if srcChunk.TagID != "" {
|
||||
if mappedTagID, ok := tagIDMapping[srcChunk.TagID]; ok {
|
||||
targetTagID = mappedTagID
|
||||
} else {
|
||||
// Try to find or create the tag in target knowledge base
|
||||
targetTagID = s.getOrCreateTagInTarget(ctx, srcKB.TenantID, dstKB.TenantID, dstKB.ID, srcChunk.TagID, tagIDMapping)
|
||||
}
|
||||
targetTagID = resolveFAQTag(srcChunk.TagID)
|
||||
}
|
||||
|
||||
// Deep-copy extracted images into objects owned by the destination
|
||||
// FAQ knowledge so deleting the source never breaks this clone. A FAQ
|
||||
// chunk is self-contained (its own Content + image_info), so its
|
||||
//  references can be rewritten immediately using the mapping
|
||||
// just accumulated in imageURLCache.
|
||||
// FAQ knowledge so deleting the source never breaks this clone.
|
||||
newImageInfo, copied, copyErr := cloneChunkImageInfo(
|
||||
ctx, dstSvc, srcChunk.ImageInfo, dstKB.TenantID, dstKnowledge.ID, imageURLCache)
|
||||
if copyErr != nil {
|
||||
@@ -860,6 +875,27 @@ func (s *knowledgeService) cloneFAQKnowledgeBase(
|
||||
_ = s.saveKBCloneProgress(ctx, progress)
|
||||
}
|
||||
|
||||
for i := 0; i < len(chunksToUpdate); i += batch {
|
||||
end := i + batch
|
||||
if end > len(chunksToUpdate) {
|
||||
end = len(chunksToUpdate)
|
||||
}
|
||||
if err := s.syncFAQChunkStatusBatch(
|
||||
ctx, dstKB, chunksToUpdate[i:end], syncPlan.SrcByID, syncPlan.DstByID, resolveFAQTag); err != nil {
|
||||
logger.Errorf(ctx, "Failed to sync FAQ status fields: %v", err)
|
||||
handleError(progress, err, "Failed to sync FAQ status fields")
|
||||
return err
|
||||
}
|
||||
processedCount += end - i
|
||||
if totalOperations > 0 {
|
||||
progress.Progress = processedCount * 100 / totalOperations
|
||||
}
|
||||
progress.Processed = processedCount
|
||||
progress.Message = fmt.Sprintf("Updated %d/%d FAQ status fields", processedCount-len(chunksToDelete)-len(chunksToAdd), len(chunksToUpdate))
|
||||
progress.UpdatedAt = time.Now().Unix()
|
||||
_ = s.saveKBCloneProgress(ctx, progress)
|
||||
}
|
||||
|
||||
// Mark as completed
|
||||
progress.Status = types.KBCloneStatusCompleted
|
||||
progress.Progress = 100
|
||||
|
||||
@@ -2,6 +2,9 @@ package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
@@ -1445,6 +1448,66 @@ func (s *knowledgeService) ExportFAQEntries(ctx context.Context, kbID string) ([
|
||||
return s.buildFAQCSV(chunks, tagMap), nil
|
||||
}
|
||||
|
||||
// ExportFAQEntriesJSON 以 JSON 数组形式导出 FAQ 知识库下的全部条目,
|
||||
// 字段与 FAQEntryPayload 兼容,便于"导出 → 编辑 → 重新 append 导入"循环。
|
||||
func (s *knowledgeService) ExportFAQEntriesJSON(ctx context.Context, kbID string) ([]byte, error) {
|
||||
kb, err := s.validateFAQKnowledgeBase(ctx, kbID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
|
||||
faqKnowledge, err := s.findFAQKnowledge(ctx, tenantID, kb.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if faqKnowledge == nil {
|
||||
return json.Marshal([]types.FAQExportEntry{})
|
||||
}
|
||||
|
||||
chunks, err := s.chunkRepo.ListAllFAQChunksForExport(ctx, tenantID, faqKnowledge.ID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list FAQ chunks: %w", err)
|
||||
}
|
||||
|
||||
tagMap, err := s.buildTagMap(ctx, tenantID, kbID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to build tag map: %w", err)
|
||||
}
|
||||
|
||||
return s.buildFAQJSON(chunks, tagMap)
|
||||
}
|
||||
|
||||
func (s *knowledgeService) buildFAQJSON(chunks []*types.Chunk, tagMap map[string]string) ([]byte, error) {
|
||||
entries := make([]types.FAQExportEntry, 0, len(chunks))
|
||||
for _, chunk := range chunks {
|
||||
meta, err := chunk.FAQMetadata()
|
||||
if err != nil || meta == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
tagName := ""
|
||||
if chunk.TagID != "" && tagMap != nil {
|
||||
if name, ok := tagMap[chunk.TagID]; ok {
|
||||
tagName = name
|
||||
}
|
||||
}
|
||||
|
||||
entries = append(entries, types.FAQExportEntry{
|
||||
ID: chunk.SeqID,
|
||||
TagName: tagName,
|
||||
StandardQuestion: meta.StandardQuestion,
|
||||
SimilarQuestions: meta.SimilarQuestions,
|
||||
NegativeQuestions: meta.NegativeQuestions,
|
||||
Answers: meta.Answers,
|
||||
AnswerStrategy: meta.AnswerStrategy,
|
||||
IsEnabled: chunk.IsEnabled,
|
||||
IsRecommended: chunk.Flags.HasFlag(types.ChunkFlagRecommended),
|
||||
})
|
||||
}
|
||||
return json.Marshal(entries)
|
||||
}
|
||||
|
||||
// buildTagMap builds a map from tag_id to tag_name for the given knowledge base.
|
||||
func (s *knowledgeService) buildTagMap(ctx context.Context, tenantID uint64, kbID string) (map[string]string, error) {
|
||||
const pageSize = 1000
|
||||
@@ -1787,6 +1850,74 @@ func (s *knowledgeService) checkFAQQuestionDuplicate(
|
||||
return werrors.NewBadRequestError("标准问或相似问与已有条目重复")
|
||||
}
|
||||
|
||||
// faqTagResolver resolves a payload's TagID/TagName to the tag's internal UUID.
|
||||
type faqTagResolver func(*types.FAQEntryPayload) (string, error)
|
||||
|
||||
// buildFAQTagResolver 预加载 entries 引用的 tag UUID 映射,避免逐条 resolveTagID。
|
||||
func (s *knowledgeService) buildFAQTagResolver(
|
||||
ctx context.Context, kbID string, entries []types.FAQEntryPayload,
|
||||
) faqTagResolver {
|
||||
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
|
||||
|
||||
var seqIDs []int64
|
||||
var names []string
|
||||
seqSeen := make(map[int64]struct{})
|
||||
nameSeen := make(map[string]struct{})
|
||||
for i := range entries {
|
||||
e := &entries[i]
|
||||
switch {
|
||||
case e.TagID != 0:
|
||||
if _, ok := seqSeen[e.TagID]; !ok {
|
||||
seqSeen[e.TagID] = struct{}{}
|
||||
seqIDs = append(seqIDs, e.TagID)
|
||||
}
|
||||
case e.TagName != "":
|
||||
if _, ok := nameSeen[e.TagName]; !ok {
|
||||
nameSeen[e.TagName] = struct{}{}
|
||||
names = append(names, e.TagName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
seqIDToUUID := make(map[int64]string)
|
||||
if len(seqIDs) > 0 {
|
||||
if tags, err := s.tagRepo.GetBySeqIDs(ctx, tenantID, seqIDs); err == nil {
|
||||
for _, t := range tags {
|
||||
seqIDToUUID[t.SeqID] = t.ID
|
||||
}
|
||||
} else {
|
||||
logger.Warnf(ctx, "buildFAQTagResolver: batch GetBySeqIDs failed (%d ids), fallback per-entry: %v",
|
||||
len(seqIDs), err)
|
||||
}
|
||||
}
|
||||
|
||||
nameToUUID := make(map[string]string)
|
||||
for _, name := range names {
|
||||
if tag, err := s.tagRepo.GetByName(ctx, tenantID, kbID, name); err == nil && tag != nil {
|
||||
nameToUUID[name] = tag.ID
|
||||
}
|
||||
}
|
||||
|
||||
return func(e *types.FAQEntryPayload) (string, error) {
|
||||
if e.TagID != 0 {
|
||||
if uuid, ok := seqIDToUUID[e.TagID]; ok {
|
||||
return uuid, nil
|
||||
}
|
||||
} else if e.TagName != "" {
|
||||
if uuid, ok := nameToUUID[e.TagName]; ok {
|
||||
return uuid, nil
|
||||
}
|
||||
}
|
||||
return s.resolveTagID(ctx, kbID, e)
|
||||
}
|
||||
}
|
||||
|
||||
// hashQuestion 计算问题内容的哈希值,用于生成稳定的 sourceID。
|
||||
func hashQuestion(question string) string {
|
||||
h := md5.Sum([]byte(question))
|
||||
return hex.EncodeToString(h[:4])
|
||||
}
|
||||
|
||||
// resolveTagID resolves tag ID (UUID) from payload, prioritizing tag_id (seq_id) over tag_name
|
||||
// If no tag is specified, creates or finds the "未分类" tag
|
||||
// Returns the internal UUID of the tag
|
||||
|
||||
@@ -73,15 +73,29 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context,
|
||||
|
||||
// 记录任务入队时间
|
||||
enqueuedAt := time.Now().Unix()
|
||||
instanceID := uuid.NewString()
|
||||
runningInfoSet := false
|
||||
enqueueSucceeded := false
|
||||
|
||||
// 设置 KB 的运行中任务信息
|
||||
if err := s.setRunningFAQImportInfo(ctx, kbID, &runningFAQImportInfo{
|
||||
TaskID: taskID,
|
||||
EnqueuedAt: enqueuedAt,
|
||||
InstanceID: instanceID,
|
||||
}); err != nil {
|
||||
logger.Errorf(ctx, "Failed to set running FAQ import task info: %v", err)
|
||||
// 不影响任务执行,继续
|
||||
} else {
|
||||
runningInfoSet = true
|
||||
}
|
||||
defer func() {
|
||||
if !runningInfoSet || enqueueSucceeded {
|
||||
return
|
||||
}
|
||||
if clearErr := s.clearRunningFAQImportInfoIfMatches(ctx, kbID, taskID, instanceID, enqueuedAt); clearErr != nil {
|
||||
logger.Warnf(ctx, "Failed to clear FAQ import running info after setup failure: %v", clearErr)
|
||||
}
|
||||
}()
|
||||
|
||||
// 初始化导入任务状态到Redis
|
||||
progress := &types.FAQImportProgress{
|
||||
@@ -120,6 +134,7 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context,
|
||||
Mode: payload.Mode,
|
||||
DryRun: payload.DryRun,
|
||||
EnqueuedAt: enqueuedAt,
|
||||
InstanceID: instanceID,
|
||||
Initiator: types.TaskInitiatorFromContext(ctx),
|
||||
}
|
||||
|
||||
@@ -189,9 +204,8 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context,
|
||||
maxRetry = 3 // dry run 重试次数少一些
|
||||
}
|
||||
|
||||
// 使用 taskID:enqueuedAt 作为 asynq 的唯一任务标识
|
||||
// 这样同一个用户 TaskID 的不同次提交不会冲突
|
||||
asynqTaskID := fmt.Sprintf("%s:%d", taskID, enqueuedAt)
|
||||
// 使用 taskID:instanceID 作为 asynq 的唯一任务标识
|
||||
asynqTaskID := fmt.Sprintf("%s:%s", taskID, instanceID)
|
||||
|
||||
task := asynq.NewTask(
|
||||
types.TypeFAQImport,
|
||||
@@ -207,6 +221,7 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context,
|
||||
return "", fmt.Errorf("failed to enqueue task: %w", err)
|
||||
}
|
||||
logger.Infof(ctx, "Enqueued FAQ import task: id=%s queue=%s task_id=%s dry_run=%v", info.ID, info.Queue, taskID, payload.DryRun)
|
||||
enqueueSucceeded = true
|
||||
|
||||
if !payload.DryRun {
|
||||
recordKBActivity(ctx, s.audit, tenantID, kbID, types.AuditActionFAQImportStarted,
|
||||
@@ -1215,34 +1230,118 @@ func (s *knowledgeService) calculateReplaceOperations(ctx context.Context,
|
||||
}
|
||||
}
|
||||
|
||||
// 批量预加载 tag 信息,避免循环内逐条查询数据库
|
||||
// 收集所有需要查询的 tag_id 和 tag_name
|
||||
tagSeqIDSet := make(map[int64]bool)
|
||||
tagNameSet := make(map[string]bool)
|
||||
for _, ewh := range entriesWithHash {
|
||||
if ewh.entry.TagID != 0 {
|
||||
tagSeqIDSet[ewh.entry.TagID] = true
|
||||
} else if ewh.entry.TagName != "" {
|
||||
tagNameSet[ewh.entry.TagName] = true
|
||||
} else {
|
||||
tagNameSet[types.UntaggedTagName] = true
|
||||
}
|
||||
}
|
||||
|
||||
// 批量查询 tag by seq_id
|
||||
tagSeqIDToUUID := make(map[int64]string)
|
||||
if len(tagSeqIDSet) > 0 {
|
||||
seqIDs := make([]int64, 0, len(tagSeqIDSet))
|
||||
for seqID := range tagSeqIDSet {
|
||||
seqIDs = append(seqIDs, seqID)
|
||||
}
|
||||
tags, err := s.tagRepo.GetBySeqIDs(ctx, tenantID, seqIDs)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "Failed to batch load tags by seq_ids: %v", err)
|
||||
} else {
|
||||
for _, tag := range tags {
|
||||
tagSeqIDToUUID[tag.SeqID] = tag.ID
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 批量查询 tag by name
|
||||
tagNameToUUID := make(map[string]string)
|
||||
if len(tagNameSet) > 0 && kbID != "" {
|
||||
for name := range tagNameSet {
|
||||
if tag, err := s.tagRepo.GetByName(ctx, tenantID, kbID, name); err == nil && tag != nil {
|
||||
tagNameToUUID[name] = tag.ID
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Preloaded %d tags by seq_id, %d tags by name for %d entries",
|
||||
len(tagSeqIDToUUID), len(tagNameToUUID), len(entriesWithHash))
|
||||
|
||||
// resolveTagIDFromCache 从缓存中解析 tag ID,缓存未命中时回退到数据库查询
|
||||
resolveTagIDFromCache := func(entry *types.FAQEntryPayload) (string, error) {
|
||||
if entry.TagID != 0 {
|
||||
if uuid, ok := tagSeqIDToUUID[entry.TagID]; ok {
|
||||
return uuid, nil
|
||||
}
|
||||
// 缓存未命中,回退到数据库查询(可能需要创建)
|
||||
return s.resolveTagID(ctx, kbID, entry)
|
||||
}
|
||||
tagName := entry.TagName
|
||||
if tagName == "" {
|
||||
tagName = types.UntaggedTagName
|
||||
}
|
||||
if uuid, ok := tagNameToUUID[tagName]; ok {
|
||||
return uuid, nil
|
||||
}
|
||||
// 缓存未命中,回退到数据库查询(可能需要创建)
|
||||
return s.resolveTagID(ctx, kbID, entry)
|
||||
}
|
||||
|
||||
// 计算需要创建的条目(利用已经计算好的hash,避免重复计算)
|
||||
entriesToProcess := make([]types.FAQEntryPayload, 0, len(entriesWithHash))
|
||||
skippedCount := batchSkippedCount
|
||||
|
||||
for _, ewh := range entriesWithHash {
|
||||
for idx, ewh := range entriesWithHash {
|
||||
// 每处理 1000 条打印一次进度日志
|
||||
if idx > 0 && idx%1000 == 0 {
|
||||
logger.Infof(ctx, "calculateReplaceOperations progress: %d/%d entries processed", idx, len(entriesWithHash))
|
||||
}
|
||||
|
||||
existingChunk := existingHashMap[ewh.hash]
|
||||
if existingChunk != nil {
|
||||
// hash 匹配,检查 tag 是否变化
|
||||
newTagID, err := s.resolveTagID(ctx, kbID, &ewh.entry)
|
||||
newTagID, err := resolveTagIDFromCache(&ewh.entry)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "Failed to resolve tag for entry, treating as new: %v", err)
|
||||
entriesToProcess = append(entriesToProcess, ewh.entry)
|
||||
continue
|
||||
}
|
||||
|
||||
if existingChunk.TagID != newTagID {
|
||||
// tag 变化了,需要删除旧的并创建新的
|
||||
logger.Infof(ctx, "FAQ entry tag changed from %s to %s, will update", existingChunk.TagID, newTagID)
|
||||
enabledChanged := ewh.entry.IsEnabled != nil && *ewh.entry.IsEnabled != existingChunk.IsEnabled
|
||||
recommendedChanged := ewh.entry.IsRecommended != nil &&
|
||||
*ewh.entry.IsRecommended != existingChunk.Flags.HasFlag(types.ChunkFlagRecommended)
|
||||
answerStrategyChanged := false
|
||||
if existingMeta, metaErr := existingChunk.FAQMetadata(); metaErr == nil && existingMeta != nil {
|
||||
answerStrategyChanged = ewh.meta.AnswerStrategy != existingMeta.AnswerStrategy
|
||||
}
|
||||
|
||||
if existingChunk.TagID != newTagID || enabledChanged || recommendedChanged || answerStrategyChanged {
|
||||
if existingChunk.TagID != newTagID {
|
||||
logger.Infof(ctx, "FAQ entry tag changed from %s to %s, will update", existingChunk.TagID, newTagID)
|
||||
}
|
||||
if enabledChanged || recommendedChanged || answerStrategyChanged {
|
||||
logger.Infof(ctx, "FAQ entry operational fields changed (enabled: %v->%v, recommended: %v->%v, answerStrategy changed: %v), will update",
|
||||
existingChunk.IsEnabled, ewh.entry.IsEnabled,
|
||||
existingChunk.Flags.HasFlag(types.ChunkFlagRecommended), ewh.entry.IsRecommended,
|
||||
answerStrategyChanged)
|
||||
}
|
||||
chunksToDelete = append(chunksToDelete, existingChunk)
|
||||
entriesToProcess = append(entriesToProcess, ewh.entry)
|
||||
} else {
|
||||
// hash 和 tag 都相同,跳过
|
||||
// hash、tag、运营状态都相同,跳过
|
||||
skippedCount++
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// hash不匹配或不存在,需要创建
|
||||
// hash 不匹配或不存在,需要创建
|
||||
entriesToProcess = append(entriesToProcess, ewh.entry)
|
||||
}
|
||||
|
||||
@@ -1512,7 +1611,7 @@ func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string,
|
||||
actualProcessed += len(batch)
|
||||
// 更新任务进度
|
||||
progress := int(float64(actualProcessed) / float64(totalEntries) * 100)
|
||||
if err := s.updateFAQImportProgressStatus(ctx, taskID, types.FAQImportStatusProcessing, progress, totalEntries, actualProcessed, fmt.Sprintf("正在处理第 %d/%d 条", actualProcessed, totalEntries), ""); err != nil {
|
||||
if err := s.updateFAQImportProgressStatus(ctx, taskID, "", 0, types.FAQImportStatusProcessing, progress, totalEntries, actualProcessed, fmt.Sprintf("正在处理第 %d/%d 条", actualProcessed, totalEntries), ""); err != nil {
|
||||
logger.Errorf(ctx, "Failed to update task progress: %v", err)
|
||||
}
|
||||
|
||||
@@ -1555,6 +1654,8 @@ func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string,
|
||||
func (s *knowledgeService) updateFAQImportProgressStatus(
|
||||
ctx context.Context,
|
||||
taskID string,
|
||||
instanceID string,
|
||||
enqueuedAt int64,
|
||||
status types.FAQImportTaskStatus,
|
||||
progress, total, processed int,
|
||||
message, errorMsg string,
|
||||
@@ -1585,7 +1686,7 @@ func (s *knowledgeService) updateFAQImportProgressStatus(
|
||||
// 任务完成或失败时,清除 running key
|
||||
if status == types.FAQImportStatusCompleted || status == types.FAQImportStatusFailed {
|
||||
if existingProgress.KBID != "" {
|
||||
if clearErr := s.clearRunningFAQImportTaskID(ctx, existingProgress.KBID); clearErr != nil {
|
||||
if clearErr := s.clearRunningFAQImportInfoIfMatches(ctx, existingProgress.KBID, taskID, instanceID, enqueuedAt); clearErr != nil {
|
||||
logger.Errorf(ctx, "Failed to clear running FAQ import task ID: %v", clearErr)
|
||||
}
|
||||
}
|
||||
@@ -1611,6 +1712,7 @@ func (s *knowledgeService) cleanupFAQEntriesFileOnFinalFailure(ctx context.Conte
|
||||
type runningFAQImportInfo struct {
|
||||
TaskID string `json:"task_id"`
|
||||
EnqueuedAt int64 `json:"enqueued_at"`
|
||||
InstanceID string `json:"instance_id,omitempty"`
|
||||
}
|
||||
|
||||
// getRunningFAQImportInfo checks if there's a running FAQ import task for the given KB
|
||||
@@ -1677,6 +1779,39 @@ func (s *knowledgeService) clearRunningFAQImportTaskID(ctx context.Context, kbID
|
||||
return s.redisClient.Del(ctx, key).Err()
|
||||
}
|
||||
|
||||
func (s *knowledgeService) clearRunningFAQImportInfoIfMatches(ctx context.Context, kbID, taskID, instanceID string, enqueuedAt int64) error {
|
||||
if s.redisClient == nil {
|
||||
if v, ok := s.memFAQRunningImport.Load(kbID); ok {
|
||||
info, _ := v.(*runningFAQImportInfo)
|
||||
if runningFAQImportInfoMatches(info, taskID, instanceID, enqueuedAt) {
|
||||
s.memFAQRunningImport.Delete(kbID)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
info, err := s.getRunningFAQImportInfo(ctx, kbID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !runningFAQImportInfoMatches(info, taskID, instanceID, enqueuedAt) {
|
||||
return nil
|
||||
}
|
||||
|
||||
key := getFAQImportRunningKey(kbID)
|
||||
return s.redisClient.Del(ctx, key).Err()
|
||||
}
|
||||
|
||||
func runningFAQImportInfoMatches(info *runningFAQImportInfo, taskID, instanceID string, enqueuedAt int64) bool {
|
||||
if info == nil || info.TaskID != taskID {
|
||||
return false
|
||||
}
|
||||
if info.InstanceID != "" && instanceID != "" {
|
||||
return info.InstanceID == instanceID
|
||||
}
|
||||
return enqueuedAt == 0 || info.EnqueuedAt == 0 || info.EnqueuedAt == enqueuedAt
|
||||
}
|
||||
|
||||
// incrementalIndexFAQEntry 增量更新FAQ条目的索引
|
||||
// 只对内容变化的部分进行embedding计算和索引更新,跳过未变化的部分
|
||||
func (s *knowledgeService) incrementalIndexFAQEntry(
|
||||
@@ -1691,6 +1826,8 @@ func (s *knowledgeService) incrementalIndexFAQEntry(
|
||||
newMeta *types.FAQChunkMetadata,
|
||||
) error {
|
||||
indexStartTime := time.Now()
|
||||
logger.Debugf(ctx, "incrementalIndexFAQEntry: starting for chunk=%s, oldSimilarQuestions=%d, newSimilarQuestions=%d",
|
||||
chunk.ID, len(oldSimilarQuestions), len(newMeta.SimilarQuestions))
|
||||
|
||||
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
|
||||
ctx, s.retrieveEngine, s.ownership, types.MustTenantIDFromContext(ctx), kb.VectorStoreID)
|
||||
@@ -1703,12 +1840,26 @@ func (s *knowledgeService) incrementalIndexFAQEntry(
|
||||
indexMode = kb.FAQConfig.IndexMode
|
||||
}
|
||||
|
||||
// 构建旧的内容(用于比较)
|
||||
buildOldContent := func(question string) string {
|
||||
if indexMode == types.FAQIndexModeQuestionAnswer && len(oldAnswers) > 0 {
|
||||
// 对新旧数据进行归一化处理,确保与 buildFAQIndexInfoList 的行为一致
|
||||
// 旧数据归一化
|
||||
oldStandardQuestion = types.NormalizeQuestion(oldStandardQuestion)
|
||||
normalizedOldSimilarQuestions := make([]string, 0, len(oldSimilarQuestions))
|
||||
for _, q := range oldSimilarQuestions {
|
||||
if nq := types.NormalizeQuestion(q); nq != "" {
|
||||
normalizedOldSimilarQuestions = append(normalizedOldSimilarQuestions, nq)
|
||||
}
|
||||
}
|
||||
oldSimilarQuestions = normalizedOldSimilarQuestions
|
||||
oldAnswers = types.SanitizeStrings(oldAnswers)
|
||||
// 新数据归一化
|
||||
normalizedNewMeta := newMeta.Normalize()
|
||||
|
||||
// 构建索引内容
|
||||
buildContent := func(question string, answers []string) string {
|
||||
if indexMode == types.FAQIndexModeQuestionAnswer && len(answers) > 0 {
|
||||
var builder strings.Builder
|
||||
builder.WriteString(question)
|
||||
for _, ans := range oldAnswers {
|
||||
for _, ans := range answers {
|
||||
builder.WriteString("\n")
|
||||
builder.WriteString(ans)
|
||||
}
|
||||
@@ -1717,30 +1868,20 @@ func (s *knowledgeService) incrementalIndexFAQEntry(
|
||||
return question
|
||||
}
|
||||
|
||||
// 构建新的内容
|
||||
buildNewContent := func(question string) string {
|
||||
if indexMode == types.FAQIndexModeQuestionAnswer && len(newMeta.Answers) > 0 {
|
||||
var builder strings.Builder
|
||||
builder.WriteString(question)
|
||||
for _, ans := range newMeta.Answers {
|
||||
builder.WriteString("\n")
|
||||
builder.WriteString(ans)
|
||||
}
|
||||
return builder.String()
|
||||
}
|
||||
return question
|
||||
}
|
||||
|
||||
// 检查答案是否变化
|
||||
answersChanged := !slices.Equal(oldAnswers, newMeta.Answers)
|
||||
// 检查答案是否变化(仅在 QuestionAnswer 模式下才影响索引)
|
||||
answersChanged := indexMode == types.FAQIndexModeQuestionAnswer && !slices.Equal(oldAnswers, normalizedNewMeta.Answers)
|
||||
logger.Debugf(ctx, "incrementalIndexFAQEntry: answersChanged=%v (indexMode=%s), oldAnswers=%d, newAnswers=%d",
|
||||
answersChanged, indexMode, len(oldAnswers), len(normalizedNewMeta.Answers))
|
||||
|
||||
// 收集需要更新的索引项
|
||||
var indexInfoToUpdate []*types.IndexInfo
|
||||
|
||||
// 1. 检查标准问是否需要更新
|
||||
oldStdContent := buildOldContent(oldStandardQuestion)
|
||||
newStdContent := buildNewContent(newMeta.StandardQuestion)
|
||||
if oldStdContent != newStdContent {
|
||||
oldStdContent := buildContent(oldStandardQuestion, oldAnswers)
|
||||
newStdContent := buildContent(normalizedNewMeta.StandardQuestion, normalizedNewMeta.Answers)
|
||||
stdQuestionChanged := oldStdContent != newStdContent
|
||||
if stdQuestionChanged {
|
||||
logger.Debugf(ctx, "incrementalIndexFAQEntry: standard question changed, sourceID=%s", chunk.ID)
|
||||
indexInfoToUpdate = append(indexInfoToUpdate, &types.IndexInfo{
|
||||
Content: newStdContent,
|
||||
SourceID: chunk.ID,
|
||||
@@ -1755,27 +1896,41 @@ func (s *knowledgeService) incrementalIndexFAQEntry(
|
||||
})
|
||||
}
|
||||
|
||||
// 2. 检查每个相似问是否需要更新
|
||||
oldCount := len(oldSimilarQuestions)
|
||||
newCount := len(newMeta.SimilarQuestions)
|
||||
// 2. 基于内容哈希处理相似问的增删改
|
||||
// 构建旧问题集合 (问题 -> 是否存在)
|
||||
oldQuestionsSet := make(map[string]struct{}, len(oldSimilarQuestions))
|
||||
for _, q := range oldSimilarQuestions {
|
||||
oldQuestionsSet[q] = struct{}{}
|
||||
}
|
||||
|
||||
for i, newQ := range newMeta.SimilarQuestions {
|
||||
needUpdate := false
|
||||
if i >= oldCount {
|
||||
// 新增的相似问
|
||||
needUpdate = true
|
||||
} else {
|
||||
// 已存在的相似问,检查内容是否变化
|
||||
oldQ := oldSimilarQuestions[i]
|
||||
if oldQ != newQ || answersChanged {
|
||||
needUpdate = true
|
||||
}
|
||||
// 构建新问题集合
|
||||
newQuestionsSet := make(map[string]struct{}, len(normalizedNewMeta.SimilarQuestions))
|
||||
for _, q := range normalizedNewMeta.SimilarQuestions {
|
||||
newQuestionsSet[q] = struct{}{}
|
||||
}
|
||||
|
||||
// 找出需要删除的问题(在旧集合中但不在新集合中)
|
||||
var sourceIDsToDelete []string
|
||||
var deletedQuestions []string
|
||||
for oldQ := range oldQuestionsSet {
|
||||
if _, exists := newQuestionsSet[oldQ]; !exists {
|
||||
sourceID := fmt.Sprintf("%s-%s", chunk.ID, hashQuestion(oldQ))
|
||||
sourceIDsToDelete = append(sourceIDsToDelete, sourceID)
|
||||
deletedQuestions = append(deletedQuestions, oldQ)
|
||||
}
|
||||
}
|
||||
|
||||
if needUpdate {
|
||||
sourceID := fmt.Sprintf("%s-%d", chunk.ID, i)
|
||||
// 找出需要新增或更新的问题
|
||||
var addedQuestions, updatedQuestions []string
|
||||
for newQ := range newQuestionsSet {
|
||||
_, existedBefore := oldQuestionsSet[newQ]
|
||||
// 需要更新的条件:
|
||||
// 1. 新问题(之前不存在)
|
||||
// 2. 答案变化(需要重新embedding)
|
||||
if !existedBefore || answersChanged {
|
||||
sourceID := fmt.Sprintf("%s-%s", chunk.ID, hashQuestion(newQ))
|
||||
indexInfoToUpdate = append(indexInfoToUpdate, &types.IndexInfo{
|
||||
Content: buildNewContent(newQ),
|
||||
Content: buildContent(newQ, normalizedNewMeta.Answers),
|
||||
SourceID: sourceID,
|
||||
SourceType: types.ChunkSourceType,
|
||||
ChunkID: chunk.ID,
|
||||
@@ -1786,22 +1941,35 @@ func (s *knowledgeService) incrementalIndexFAQEntry(
|
||||
IsEnabled: chunk.IsEnabled,
|
||||
IsRecommended: chunk.Flags.HasFlag(types.ChunkFlagRecommended),
|
||||
})
|
||||
if !existedBefore {
|
||||
addedQuestions = append(addedQuestions, newQ)
|
||||
} else {
|
||||
updatedQuestions = append(updatedQuestions, newQ)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 3. 删除多余的旧相似问索引
|
||||
if oldCount > newCount {
|
||||
sourceIDsToDelete := make([]string, 0, oldCount-newCount)
|
||||
for i := newCount; i < oldCount; i++ {
|
||||
sourceIDsToDelete = append(sourceIDsToDelete, fmt.Sprintf("%s-%d", chunk.ID, i))
|
||||
}
|
||||
logger.Debugf(ctx, "incrementalIndexFAQEntry: deleting %d obsolete source IDs", len(sourceIDsToDelete))
|
||||
// 输出详细的变化日志
|
||||
if len(deletedQuestions) > 0 {
|
||||
logger.Debugf(ctx, "incrementalIndexFAQEntry: deleted similar questions: %v", deletedQuestions)
|
||||
}
|
||||
if len(addedQuestions) > 0 {
|
||||
logger.Debugf(ctx, "incrementalIndexFAQEntry: added similar questions: %v", addedQuestions)
|
||||
}
|
||||
if len(updatedQuestions) > 0 {
|
||||
logger.Debugf(ctx, "incrementalIndexFAQEntry: updated similar questions (answers changed): %v", updatedQuestions)
|
||||
}
|
||||
|
||||
// 3. 删除不再存在的相似问索引
|
||||
if len(sourceIDsToDelete) > 0 {
|
||||
logger.Debugf(ctx, "incrementalIndexFAQEntry: deleting %d obsolete sourceIDs: %v", len(sourceIDsToDelete), sourceIDsToDelete)
|
||||
if delErr := retrieveEngine.DeleteBySourceIDList(ctx, sourceIDsToDelete, embeddingModel.GetDimensions(), types.KnowledgeTypeFAQ); delErr != nil {
|
||||
logger.Warnf(ctx, "incrementalIndexFAQEntry: failed to delete obsolete source IDs: %v", delErr)
|
||||
}
|
||||
}
|
||||
|
||||
// 4. 批量索引需要更新的内容
|
||||
newCount := len(normalizedNewMeta.SimilarQuestions)
|
||||
if len(indexInfoToUpdate) > 0 {
|
||||
logger.Debugf(ctx, "incrementalIndexFAQEntry: updating %d index entries (skipped %d unchanged)",
|
||||
len(indexInfoToUpdate), 1+newCount-len(indexInfoToUpdate))
|
||||
@@ -2193,7 +2361,7 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
|
||||
logger.Errorf(ctx, "Failed to get knowledge base: %v", err)
|
||||
// 如果是最后一次重试,更新状态为失败
|
||||
if isLastRetry {
|
||||
if updateErr := s.updateFAQImportProgressStatus(ctx, payload.TaskID, types.FAQImportStatusFailed, 0, originalTotalEntries, 0, "获取知识库失败", err.Error()); updateErr != nil {
|
||||
if updateErr := s.updateFAQImportProgressStatus(ctx, payload.TaskID, payload.InstanceID, payload.EnqueuedAt, types.FAQImportStatusFailed, 0, originalTotalEntries, 0, "获取知识库失败", err.Error()); updateErr != nil {
|
||||
logger.Errorf(ctx, "Failed to update task status to failed: %v", updateErr)
|
||||
}
|
||||
s.recordFAQImportKBActivity(ctx, &payload, progress, originalTotalEntries, types.AuditActionFAQImportFailed, types.AuditOutcomeFailed)
|
||||
@@ -2207,6 +2375,9 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
|
||||
if existingProgress != nil {
|
||||
if existingProgress.Status == types.FAQImportStatusCompleted {
|
||||
logger.Infof(ctx, "FAQ import already completed, skipping: %s", payload.TaskID)
|
||||
if clearErr := s.clearRunningFAQImportInfoIfMatches(ctx, payload.KBID, payload.TaskID, payload.InstanceID, payload.EnqueuedAt); clearErr != nil {
|
||||
logger.Warnf(ctx, "Failed to clear running FAQ import info for completed task: %v", clearErr)
|
||||
}
|
||||
return nil // 幂等:已完成的任务直接返回
|
||||
}
|
||||
// 获取已处理的数量(注意:这是相对于 validEntries 的索引)
|
||||
@@ -2223,7 +2394,7 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
|
||||
logger.Errorf(ctx, "Failed to delete unindexed chunks: %v", err)
|
||||
// 如果是最后一次重试,更新状态为失败
|
||||
if isLastRetry {
|
||||
if updateErr := s.updateFAQImportProgressStatus(ctx, payload.TaskID, types.FAQImportStatusFailed, 0, originalTotalEntries, 0, "清理未索引数据失败", err.Error()); updateErr != nil {
|
||||
if updateErr := s.updateFAQImportProgressStatus(ctx, payload.TaskID, payload.InstanceID, payload.EnqueuedAt, types.FAQImportStatusFailed, 0, originalTotalEntries, 0, "清理未索引数据失败", err.Error()); updateErr != nil {
|
||||
logger.Errorf(ctx, "Failed to update task status to failed: %v", updateErr)
|
||||
}
|
||||
s.recordFAQImportKBActivity(ctx, &payload, progress, originalTotalEntries, types.AuditActionFAQImportFailed, types.AuditOutcomeFailed)
|
||||
@@ -2280,7 +2451,7 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
|
||||
logger.Errorf(ctx, "FAQ import task failed: %s, error: %v", payload.TaskID, err)
|
||||
// 如果是最后一次重试,更新状态为失败
|
||||
if isLastRetry {
|
||||
if updateErr := s.updateFAQImportProgressStatus(ctx, payload.TaskID, types.FAQImportStatusFailed, 0, originalTotalEntries, len(validEntries), "导入失败", err.Error()); updateErr != nil {
|
||||
if updateErr := s.updateFAQImportProgressStatus(ctx, payload.TaskID, payload.InstanceID, payload.EnqueuedAt, types.FAQImportStatusFailed, 0, originalTotalEntries, len(validEntries), "导入失败", err.Error()); updateErr != nil {
|
||||
logger.Errorf(ctx, "Failed to update task status to failed: %v", updateErr)
|
||||
}
|
||||
s.recordFAQImportKBActivity(ctx, &payload, progress, originalTotalEntries, types.AuditActionFAQImportFailed, types.AuditOutcomeFailed)
|
||||
@@ -2385,7 +2556,7 @@ func (s *knowledgeService) finalizeFAQValidation(ctx context.Context, payload *t
|
||||
}
|
||||
|
||||
// 然后调用状态更新来清理 running key
|
||||
if err := s.updateFAQImportProgressStatus(ctx, payload.TaskID, types.FAQImportStatusCompleted,
|
||||
if err := s.updateFAQImportProgressStatus(ctx, payload.TaskID, payload.InstanceID, payload.EnqueuedAt, types.FAQImportStatusCompleted,
|
||||
100, originalTotalEntries, originalTotalEntries, progress.Message, ""); err != nil {
|
||||
logger.Warnf(ctx, "Failed to update final FAQ import status: %v", err)
|
||||
}
|
||||
|
||||
+21
-4
@@ -3,6 +3,7 @@ package handler
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
@@ -381,19 +382,35 @@ func (h *FAQHandler) SearchFAQ(c *gin.Context) {
|
||||
|
||||
// ExportEntries godoc
|
||||
// @Summary 导出FAQ条目
|
||||
// @Description 将所有FAQ条目导出为CSV文件
|
||||
// @Description 将所有FAQ条目导出为 CSV(默认)或 JSON。?format=json 返回与 FAQEntryPayload 结构兼容的数组。
|
||||
// @Tags FAQ管理
|
||||
// @Accept json
|
||||
// @Produce text/csv
|
||||
// @Param id path string true "知识库ID"
|
||||
// @Success 200 {file} file "CSV文件"
|
||||
// @Failure 400 {object} errors.AppError "请求参数错误"
|
||||
// @Produce application/json
|
||||
// @Param id path string true "知识库ID"
|
||||
// @Param format query string false "导出格式:csv(默认)或 json"
|
||||
// @Success 200 {file} file "导出文件"
|
||||
// @Failure 400 {object} errors.AppError "请求参数错误"
|
||||
// @Security Bearer
|
||||
// @Security ApiKeyAuth
|
||||
// @Router /knowledge-bases/{id}/faq/entries/export [get]
|
||||
func (h *FAQHandler) ExportEntries(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
kbID := secutils.SanitizeForLog(c.Param("id"))
|
||||
format := strings.ToLower(strings.TrimSpace(c.Query("format")))
|
||||
|
||||
if format == "json" {
|
||||
jsonData, err := h.knowledgeService.ExportFAQEntriesJSON(ctx, kbID)
|
||||
if err != nil {
|
||||
logger.ErrorWithFields(ctx, err, nil)
|
||||
c.Error(err)
|
||||
return
|
||||
}
|
||||
c.Header("Content-Type", "application/json; charset=utf-8")
|
||||
c.Header("Content-Disposition", "attachment; filename=faq_export.json")
|
||||
c.Data(http.StatusOK, "application/json; charset=utf-8", jsonData)
|
||||
return
|
||||
}
|
||||
|
||||
csvData, err := h.knowledgeService.ExportFAQEntries(ctx, kbID)
|
||||
if err != nil {
|
||||
|
||||
@@ -251,6 +251,21 @@ type FAQEntry struct {
|
||||
MatchedQuestion string `json:"matched_question,omitempty"`
|
||||
}
|
||||
|
||||
// FAQExportEntry 表示 JSON 导出的 FAQ 条目,与 FAQEntryPayload 的导入格式兼容,
|
||||
// 便于"导出 → 编辑 → 重新导入"循环。新增字段时务必保留 omitempty,避免破坏
|
||||
// 历史导出文件的兼容性。
|
||||
type FAQExportEntry struct {
|
||||
ID int64 `json:"id"`
|
||||
TagName string `json:"tag_name,omitempty"`
|
||||
StandardQuestion string `json:"standard_question"`
|
||||
SimilarQuestions []string `json:"similar_questions,omitempty"`
|
||||
NegativeQuestions []string `json:"negative_questions,omitempty"`
|
||||
Answers []string `json:"answers,omitempty"`
|
||||
AnswerStrategy AnswerStrategy `json:"answer_strategy,omitempty"`
|
||||
IsEnabled bool `json:"is_enabled"`
|
||||
IsRecommended bool `json:"is_recommended"`
|
||||
}
|
||||
|
||||
// FAQEntryPayload 用于创建/更新 FAQ 条目的 payload
|
||||
type FAQEntryPayload struct {
|
||||
// ID 可选,用于数据迁移时指定 seq_id(必须小于自增起始值 100000000)
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
package types
|
||||
|
||||
import "encoding/json"
|
||||
|
||||
// FAQChunkSyncPair links source/destination FAQ chunks sharing content_hash.
|
||||
type FAQChunkSyncPair struct {
|
||||
SrcChunkID string
|
||||
DstChunkID string
|
||||
}
|
||||
|
||||
// FAQChunkDiffResult is the output of FAQChunkDiff.
|
||||
type FAQChunkDiffResult struct {
|
||||
ChunksToAdd []string
|
||||
ChunksToDelete []string
|
||||
MatchedPairs []FAQChunkSyncPair
|
||||
}
|
||||
|
||||
// FAQChunkStatus holds lightweight fields for clone status sync.
|
||||
type FAQChunkStatus struct {
|
||||
ID string
|
||||
TagID string
|
||||
IsEnabled bool
|
||||
Flags ChunkFlags
|
||||
AnswerStrategy AnswerStrategy
|
||||
Metadata JSON
|
||||
}
|
||||
|
||||
// FAQChunkStatusFieldsEqual compares status fields (tag excluded).
|
||||
func FAQChunkStatusFieldsEqual(a, b *FAQChunkStatus) bool {
|
||||
if a == nil || b == nil {
|
||||
return a == b
|
||||
}
|
||||
if a.IsEnabled != b.IsEnabled || a.Flags != b.Flags {
|
||||
return false
|
||||
}
|
||||
return NormalizeAnswerStrategy(a.AnswerStrategy) == NormalizeAnswerStrategy(b.AnswerStrategy)
|
||||
}
|
||||
|
||||
// FAQChunkNeedsStatusSync reports whether a matched pair needs status sync.
|
||||
func FAQChunkNeedsStatusSync(src, dst *FAQChunkStatus, mappedSrcTagID string) bool {
|
||||
if src == nil || dst == nil {
|
||||
return false
|
||||
}
|
||||
if mappedSrcTagID != dst.TagID {
|
||||
return true
|
||||
}
|
||||
return !FAQChunkStatusFieldsEqual(src, dst)
|
||||
}
|
||||
|
||||
// NormalizeAnswerStrategy treats empty as "all".
|
||||
func NormalizeAnswerStrategy(s AnswerStrategy) AnswerStrategy {
|
||||
if s == "" {
|
||||
return AnswerStrategyAll
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// PatchFAQAnswerStrategy copies answer_strategy from src metadata onto dst metadata.
|
||||
func PatchFAQAnswerStrategy(dstMeta, srcMeta JSON) (JSON, error) {
|
||||
var dstM, srcM FAQChunkMetadata
|
||||
if len(dstMeta) > 0 {
|
||||
if err := json.Unmarshal(dstMeta, &dstM); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if len(srcMeta) > 0 {
|
||||
if err := json.Unmarshal(srcMeta, &srcM); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
dstM.AnswerStrategy = srcM.AnswerStrategy
|
||||
bytes, err := json.Marshal(&dstM)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return JSON(bytes), nil
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package types
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestFAQChunkNeedsStatusSync(t *testing.T) {
|
||||
src := &FAQChunkStatus{TagID: "a", IsEnabled: true, Flags: ChunkFlagRecommended, AnswerStrategy: AnswerStrategyAll}
|
||||
dst := &FAQChunkStatus{TagID: "old", IsEnabled: true, Flags: ChunkFlagRecommended, AnswerStrategy: AnswerStrategyAll}
|
||||
if !FAQChunkNeedsStatusSync(src, dst, "mapped") {
|
||||
t.Fatal("tag mismatch should need sync")
|
||||
}
|
||||
dst.TagID = "mapped"
|
||||
if FAQChunkNeedsStatusSync(src, dst, "mapped") {
|
||||
t.Fatal("in sync after tag mapped")
|
||||
}
|
||||
dst.IsEnabled = false
|
||||
if !FAQChunkNeedsStatusSync(src, dst, "mapped") {
|
||||
t.Fatal("is_enabled mismatch should need sync")
|
||||
}
|
||||
}
|
||||
@@ -97,8 +97,9 @@ type ChunkRepository interface {
|
||||
// newTagID: if not nil, updates tag_id to this value (empty string means uncategorized)
|
||||
UpdateChunkFieldsByTagID(ctx context.Context, tenantID uint64, kbID string, tagID string, isEnabled *bool, setFlags types.ChunkFlags, clearFlags types.ChunkFlags, newTagID *string, excludeIDs []string) ([]string, error)
|
||||
// FAQChunkDiff compares FAQ chunks between two knowledge bases and returns the differences.
|
||||
// Returns: chunksToAdd (content_hash in src but not in dst), chunksToDelete (content_hash in dst but not in src)
|
||||
FAQChunkDiff(ctx context.Context, srcTenantID uint64, srcKBID string, dstTenantID uint64, dstKBID string) (chunksToAdd []string, chunksToDelete []string, err error)
|
||||
FAQChunkDiff(ctx context.Context, srcTenantID uint64, srcKBID string, dstTenantID uint64, dstKBID string) (*types.FAQChunkDiffResult, error)
|
||||
// ListFAQChunkStatusByIDs loads status fields for FAQ clone sync.
|
||||
ListFAQChunkStatusByIDs(ctx context.Context, tenantID uint64, ids []string) (map[string]*types.FAQChunkStatus, error)
|
||||
|
||||
// ListRecommendedFAQChunks lists FAQ chunks with the recommended flag set.
|
||||
// Filter by explicitly selected kbIDs, knowledgeIDs, and/or FAQ tagIDs.
|
||||
|
||||
@@ -152,6 +152,8 @@ type KnowledgeService interface {
|
||||
SearchFAQEntries(ctx context.Context, kbID string, req *types.FAQSearchRequest) ([]*types.FAQEntry, error)
|
||||
// ExportFAQEntries exports all FAQ entries for a knowledge base as CSV data.
|
||||
ExportFAQEntries(ctx context.Context, kbID string) ([]byte, error)
|
||||
// ExportFAQEntriesJSON exports all FAQ entries as a JSON array compatible with FAQEntryPayload.
|
||||
ExportFAQEntriesJSON(ctx context.Context, kbID string) ([]byte, error)
|
||||
// UpdateKnowledgeTagBatch updates tag for document knowledge items in batch.
|
||||
// authorizedKBID restricts all updates to knowledge items belonging to this KB;
|
||||
// pass empty string to skip (caller must ensure authorization by other means).
|
||||
|
||||
@@ -305,6 +305,7 @@ type FAQImportPayload struct {
|
||||
Mode string `json:"mode"`
|
||||
DryRun bool `json:"dry_run"` // dry run 模式只验证不导入
|
||||
EnqueuedAt int64 `json:"enqueued_at"` // 任务入队时间戳,用于区分同一 TaskID 的不同次提交
|
||||
InstanceID string `json:"instance_id,omitempty"`
|
||||
Initiator TaskInitiator `json:"initiator,omitempty"`
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user