feat: Enhance chunk management and FAQ import functionality

- Updated chunk repository methods to support deletion of unindexed chunks and listing all FAQ chunks by knowledge ID.
- Introduced batch update functionality for chunks in the chunk service.
- Enhanced FAQ import logic to handle append and replace operations, improving efficiency and data integrity.
- Added content hash calculation for FAQ entries to facilitate quick matching and deduplication.
- Refactored related methods to streamline the import process and improve logging for better tracking of import tasks.
This commit is contained in:
wizardchen
2025-11-24 15:47:42 +08:00
parent 385108ce22
commit e56233ef11
9 changed files with 353 additions and 118 deletions
+44 -7
View File
@@ -81,7 +81,8 @@ func (r *chunkRepository) ListPagedChunksByKnowledgeID(
var total int64
query := r.db.WithContext(ctx).Model(&types.Chunk{}).
Where("tenant_id = ? AND knowledge_id = ? AND chunk_type IN (?)", tenantID, knowledgeID, chunkType)
Where("tenant_id = ? AND knowledge_id = ? AND chunk_type IN (?) AND status in (?)",
tenantID, knowledgeID, chunkType, []types.ChunkStatus{types.ChunkStatusStored, types.ChunkStatusDefault})
if tagID != "" {
query = query.Where("tag_id = ?", tagID)
}
@@ -94,7 +95,8 @@ func (r *chunkRepository) ListPagedChunksByKnowledgeID(
// Then query the paginated data
dataQuery := r.db.WithContext(ctx).
Select("id, content, knowledge_id, knowledge_base_id, start_at, end_at, chunk_index, is_enabled, chunk_type, parent_chunk_id, image_info, metadata, tag_id")
dataQuery = dataQuery.Where("tenant_id = ? AND knowledge_id = ? AND chunk_type IN (?)", tenantID, knowledgeID, chunkType)
dataQuery = dataQuery.Where("tenant_id = ? AND knowledge_id = ? AND chunk_type IN (?) AND status in (?)",
tenantID, knowledgeID, chunkType, []types.ChunkStatus{types.ChunkStatusStored, types.ChunkStatusDefault})
if tagID != "" {
dataQuery = dataQuery.Where("tag_id = ?", tagID)
}
@@ -172,21 +174,56 @@ func (r *chunkRepository) CountChunksByKnowledgeBaseID(ctx context.Context, tena
return count, err
}
func (r *chunkRepository) DeleteChunksByChunkIndexRange(ctx context.Context, tenantID uint, knowledgeID string, startChunkIndex int, endChunkIndex int) ([]*types.Chunk, error) {
// DeleteUnindexedChunks by knowledge id and chunk index range
func (r *chunkRepository) DeleteUnindexedChunks(ctx context.Context, tenantID uint, knowledgeID string) ([]*types.Chunk, error) {
var chunks []*types.Chunk
// 先查询要删除的chunks
if err := r.db.WithContext(ctx).
Where("tenant_id = ? AND knowledge_id = ? AND chunk_index >= ? AND chunk_index <= ? AND deleted_at IS NULL", tenantID, knowledgeID, startChunkIndex, endChunkIndex).
Where("tenant_id = ? AND knowledge_id = ? AND status = ?", tenantID, knowledgeID, types.ChunkStatusStored).
Find(&chunks).Error; err != nil {
return nil, err
}
// 然后删除它们
if len(chunks) > 0 {
if err := r.db.WithContext(ctx).
Where("tenant_id = ? AND knowledge_id = ? AND chunk_index >= ? AND chunk_index <= ? AND deleted_at IS NULL", tenantID, knowledgeID, startChunkIndex, endChunkIndex).
Where("tenant_id = ? AND knowledge_id = ? AND status = ?", tenantID, knowledgeID, types.ChunkStatusStored).
Delete(&types.Chunk{}).Error; err != nil {
return nil, err
}
}
return chunks, nil
}
// ListAllFAQChunksByKnowledgeID lists all FAQ chunks for a knowledge ID (only essential fields for efficiency)
// Uses batch query to handle large datasets
func (r *chunkRepository) ListAllFAQChunksByKnowledgeID(ctx context.Context, tenantID uint, knowledgeID string) ([]*types.Chunk, error) {
const batchSize = 1000 // 每批查询1000条
var allChunks []*types.Chunk
offset := 0
for {
var batchChunks []*types.Chunk
if err := r.db.WithContext(ctx).
Select("id, content_hash").
Where("tenant_id = ? AND knowledge_id = ? AND chunk_type = ?", tenantID, knowledgeID, types.ChunkTypeFAQ).
Offset(offset).
Limit(batchSize).
Find(&batchChunks).Error; err != nil {
return nil, err
}
// 如果没有查询到数据,说明已经查询完毕
if len(batchChunks) == 0 {
break
}
allChunks = append(allChunks, batchChunks...)
// 如果返回的数据少于批次大小,说明已经是最后一批
if len(batchChunks) < batchSize {
break
}
offset += batchSize
}
return allChunks, nil
}
+20
View File
@@ -186,6 +186,26 @@ func (s *chunkService) UpdateChunk(ctx context.Context, chunk *types.Chunk) erro
return nil
}
// UpdateChunks updates chunks in batch
func (s *chunkService) UpdateChunks(ctx context.Context, chunks []*types.Chunk) error {
if len(chunks) == 0 {
return nil
}
logger.Infof(ctx, "Updating %d chunks in batch", len(chunks))
// Update the chunks in the repository
err := s.chunkRepository.UpdateChunks(ctx, chunks)
if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{
"chunk_count": len(chunks),
})
return err
}
logger.Infof(ctx, "Successfully updated %d chunks", len(chunks))
return nil
}
// DeleteChunk deletes a chunk by ID
// This method removes a specific chunk from the repository
// Parameters:
+181 -101
View File
@@ -2207,47 +2207,24 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context,
// 初始化导入任务状态到Knowledge表
taskID := faqKnowledge.ID // 使用Knowledge ID作为taskID
// 从Knowledge Metadata中读取当前的NextChunkIndex(自增计数器)
// 如果不存在,则初始化为1
existingMeta, _ := types.ParseFAQImportMetadata(faqKnowledge)
nextChunkIndex := 1
if existingMeta != nil && existingMeta.NextChunkIndex > 0 {
nextChunkIndex = existingMeta.NextChunkIndex
}
startChunkIndex := nextChunkIndex
endChunkIndex := startChunkIndex + len(payload.Entries) - 1
// 更新NextChunkIndex为下一个要分配的值
newNextChunkIndex := nextChunkIndex + len(payload.Entries)
// 更新Metadata,包含区间信息和新的NextChunkIndex
importMeta := &types.FAQImportMetadata{
ImportProgress: 0,
ImportTotal: len(payload.Entries),
ImportProcessed: 0,
NextChunkIndex: newNextChunkIndex,
}
if err := s.updateFAQImportStatusWithRanges(ctx, taskID, types.FAQImportStatusPending, 0, len(payload.Entries), 0, "", importMeta.NextChunkIndex); err != nil {
if err := s.updateFAQImportStatusWithRanges(ctx, taskID, types.FAQImportStatusPending,
0, len(payload.Entries), 0, ""); err != nil {
logger.Errorf(ctx, "Failed to initialize FAQ import task status: %v", err)
return "", fmt.Errorf("failed to initialize task: %w", err)
}
logger.Infof(ctx, "Allocated ChunkIndex range [%d, %d] for FAQ import task %s, next ChunkIndex will be %d",
startChunkIndex, endChunkIndex, taskID, newNextChunkIndex)
taskID)
// Enqueue FAQ import task to Asynq
logger.Info(ctx, "Enqueuing FAQ import task to Asynq")
taskPayload := types.FAQImportPayload{
TenantID: tenantID,
TaskID: taskID,
KBID: kbID,
KnowledgeID: faqKnowledge.ID,
Entries: payload.Entries,
Mode: payload.Mode,
StartChunkIndex: startChunkIndex,
EndChunkIndex: endChunkIndex,
TenantID: tenantID,
TaskID: taskID,
KBID: kbID,
KnowledgeID: faqKnowledge.ID,
Entries: payload.Entries,
Mode: payload.Mode,
}
payloadBytes, err := json.Marshal(taskPayload)
@@ -2267,12 +2244,89 @@ func (s *knowledgeService) UpsertFAQEntries(ctx context.Context,
return taskID, nil
}
// calculateAppendOperations 计算Append模式下需要处理的条目,跳过已存在且内容相同的条目
func (s *knowledgeService) calculateAppendOperations(ctx context.Context,
tenantID uint, knowledgeID string, entries []types.FAQEntryPayload) ([]types.FAQEntryPayload, int, error) {
if len(entries) == 0 {
return []types.FAQEntryPayload{}, 0, nil
}
return entries, 0, nil
}
// calculateReplaceOperations 计算Replace模式下需要删除、创建、更新的条目
func (s *knowledgeService) calculateReplaceOperations(ctx context.Context,
tenantID uint, knowledgeID string, newEntries []types.FAQEntryPayload) ([]types.FAQEntryPayload, []*types.Chunk, int, error) {
// 计算所有新条目的 content hash,并同时构建 hash 到 entry 的映射
type entryWithHash struct {
entry types.FAQEntryPayload
hash string
}
entriesWithHash := make([]entryWithHash, 0, len(newEntries))
newHashSet := make(map[string]bool)
for _, entry := range newEntries {
meta, err := sanitizeFAQEntryPayload(&entry)
if err != nil {
return nil, nil, 0, fmt.Errorf("failed to sanitize entry: %w", err)
}
hash := types.CalculateFAQContentHash(meta)
if hash != "" {
entriesWithHash = append(entriesWithHash, entryWithHash{entry: entry, hash: hash})
newHashSet[hash] = true
}
}
// 查询所有已存在的chunks
allExistingChunks, err := s.chunkRepo.ListAllFAQChunksByKnowledgeID(ctx, tenantID, knowledgeID)
if err != nil {
return nil, nil, 0, fmt.Errorf("failed to list existing chunks: %w", err)
}
// 在内存中过滤出匹配新条目hash的chunks,并构建map
existingHashMap := make(map[string]*types.Chunk)
for _, chunk := range allExistingChunks {
if chunk.ContentHash != "" && newHashSet[chunk.ContentHash] {
existingHashMap[chunk.ContentHash] = chunk
}
}
// 计算需要删除的chunks(数据库中有但新批次中没有的,或hash不匹配的)
chunksToDelete := make([]*types.Chunk, 0)
for _, chunk := range allExistingChunks {
if chunk.ContentHash == "" {
// 如果没有hash,需要删除(可能是旧数据)
chunksToDelete = append(chunksToDelete, chunk)
} else if !newHashSet[chunk.ContentHash] {
// hash不在新条目中,需要删除
chunksToDelete = append(chunksToDelete, chunk)
}
}
// 计算需要创建的条目(利用已经计算好的hash,避免重复计算)
entriesToProcess := make([]types.FAQEntryPayload, 0, len(entriesWithHash))
skippedCount := 0
for _, ewh := range entriesWithHash {
if existingHashMap[ewh.hash] != nil {
// hash匹配,说明内容完全相同,跳过(不需要删除和创建)
skippedCount++
continue
}
// hash不匹配或不存在,需要创建
entriesToProcess = append(entriesToProcess, ewh.entry)
}
return entriesToProcess, chunksToDelete, skippedCount, nil
}
// executeFAQImport 执行实际的FAQ导入逻辑
func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string, kbID string,
payload *types.FAQBatchUpsertPayload, tenantID uint, startChunkIndex int, processed int, totalEntries int) (err error) {
payload *types.FAQBatchUpsertPayload, tenantID uint, processedCount int) (err error) {
// 保存知识库和embedding模型信息,用于清理索引
var kb *types.KnowledgeBase
var embeddingModel embedding.Embedder
totalEntries := len(payload.Entries) + processedCount
// Recovery机制:如果发生任何错误或panic,回滚所有已创建的chunks和索引数据
defer func() {
@@ -2303,24 +2357,60 @@ func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string,
return err
}
// 如果模式为replace,则清理知识库
if payload.Mode == types.FAQBatchModeReplace {
if err := s.cleanupFAQKnowledge(ctx, faqKnowledge); err != nil {
return err
}
}
// 获取索引模式
indexMode := types.FAQIndexModeQuestionOnly
if kb.FAQConfig != nil && kb.FAQConfig.IndexMode != "" {
indexMode = kb.FAQConfig.IndexMode
}
// 分批处理
remainingEntries := len(payload.Entries)
// 增量更新逻辑:计算需要处理的条目
var entriesToProcess []types.FAQEntryPayload
var chunksToDelete []*types.Chunk
var skippedCount int
if payload.Mode == types.FAQBatchModeReplace {
// Replace模式:计算需要删除、创建、更新的条目
entriesToProcess, chunksToDelete, skippedCount, err =
s.calculateReplaceOperations(ctx, tenantID, faqKnowledge.ID, payload.Entries)
if err != nil {
return fmt.Errorf("failed to calculate replace operations: %w", err)
}
// 删除需要删除的chunks(包括需要更新的旧chunks)
if len(chunksToDelete) > 0 {
chunkIDsToDelete := make([]string, 0, len(chunksToDelete))
for _, chunk := range chunksToDelete {
chunkIDsToDelete = append(chunkIDsToDelete, chunk.ID)
}
if err := s.chunkRepo.DeleteChunks(ctx, tenantID, chunkIDsToDelete); err != nil {
return fmt.Errorf("failed to delete chunks: %w", err)
}
// 删除索引
if err := s.deleteFAQChunkVectors(ctx, kb, faqKnowledge, chunksToDelete); err != nil {
return fmt.Errorf("failed to delete chunk vectors: %w", err)
}
logger.Infof(ctx, "FAQ import task %s: deleted %d chunks (including updates)", taskID, len(chunksToDelete))
}
} else {
// Append模式:查询已存在的条目,跳过未变化的
entriesToProcess, skippedCount, err = s.calculateAppendOperations(ctx, tenantID, faqKnowledge.ID, payload.Entries)
if err != nil {
return fmt.Errorf("failed to calculate append operations: %w", err)
}
}
logger.Infof(ctx, "FAQ import task %s: total entries: %d, to process: %d, skipped: %d", taskID, len(payload.Entries), len(entriesToProcess), skippedCount)
// 如果没有需要处理的条目,直接返回
if len(entriesToProcess) == 0 {
logger.Infof(ctx, "FAQ import task %s: no entries to process, all skipped", taskID)
return nil
}
// 分批处理需要创建的条目
remainingEntries := len(entriesToProcess)
totalStartTime := time.Now()
// 保存初始的processed值,用于计算ChunkIndex(i+idx是剩余条目中的索引,加上初始processed得到原始位置)
initialProcessed := processed
actualProcessed := skippedCount + processedCount
logger.Infof(ctx, "FAQ import task %s: starting batch processing, remaining entries: %d, total entries: %d, batch size: %d", taskID, remainingEntries, totalEntries, faqImportBatchSize)
@@ -2331,7 +2421,7 @@ func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string,
end = remainingEntries
}
batch := payload.Entries[i:end]
batch := entriesToProcess[i:end]
logger.Infof(ctx, "FAQ import task %s: processing batch %d-%d (%d entries)", taskID, i+1, end, len(batch))
// 构建chunks
@@ -2352,17 +2442,17 @@ func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string,
isEnabled = *entry.IsEnabled
}
// ChunkIndex计算:startChunkIndex + (i+idx) + initialProcessed
// i+idx是剩余条目中的索引,加上initialProcessed得到原始条目中的位置
chunk := &types.Chunk{
ID: uuid.New().String(),
TenantID: tenantID,
KnowledgeID: faqKnowledge.ID,
KnowledgeBaseID: kb.ID,
Content: buildFAQChunkContent(meta, indexMode),
ChunkIndex: startChunkIndex + i + idx + initialProcessed,
IsEnabled: isEnabled,
ChunkType: types.ChunkTypeFAQ,
TagID: entry.TagID,
// ChunkIndex: 0,
IsEnabled: isEnabled,
ChunkType: types.ChunkTypeFAQ,
TagID: entry.TagID,
Status: int(types.ChunkStatusStored), // store but not indexed
}
if err := chunk.SetFAQMetadata(meta); err != nil {
return fmt.Errorf("failed to set FAQ metadata: %w", err)
@@ -2390,22 +2480,31 @@ func (s *knowledgeService) executeFAQImport(ctx context.Context, taskID string,
indexDuration := time.Since(indexStartTime)
logger.Infof(ctx, "FAQ import task %s: batch %d-%d indexed %d chunks in %v", taskID, i+1, end, len(chunks), indexDuration)
processed = processed + len(batch)
// 更新chunks的Status为已索引
chunksToUpdate := make([]*types.Chunk, 0, len(chunks))
for _, chunk := range chunks {
chunk.Status = int(types.ChunkStatusIndexed) // indexed
chunksToUpdate = append(chunksToUpdate, chunk)
}
if err := s.chunkService.UpdateChunks(ctx, chunksToUpdate); err != nil {
return fmt.Errorf("failed to update chunks status: %w", err)
}
actualProcessed += len(batch)
// 更新任务进度
progress := int(float64(processed) / float64(totalEntries) * 100)
if err := s.updateFAQImportStatus(ctx, taskID, types.FAQImportStatusProcessing, progress, totalEntries, processed, ""); err != nil {
progress := int(float64(actualProcessed) / float64(totalEntries) * 100)
if err := s.updateFAQImportStatus(ctx, taskID, types.FAQImportStatusProcessing, progress, totalEntries, actualProcessed, ""); err != nil {
logger.Errorf(ctx, "Failed to update task progress: %v", err)
}
batchDuration := time.Since(batchStartTime)
logger.Infof(ctx, "FAQ import task %s: batch %d-%d completed in %v (build: %v, create: %v, index: %v), total progress: %d/%d (%d%%)",
taskID, i+1, end, batchDuration, buildDuration, createDuration, indexDuration, processed, totalEntries, progress)
taskID, i+1, end, batchDuration, buildDuration, createDuration, indexDuration, actualProcessed, totalEntries, progress)
}
totalDuration := time.Since(totalStartTime)
logger.Infof(ctx, "FAQ import task %s: all batches completed, processed: %d entries in %v, avg: %v per entry",
taskID, remainingEntries, totalDuration, totalDuration/time.Duration(remainingEntries))
logger.Infof(ctx, "FAQ import task %s: all batches completed, processed: %d entries (skipped: %d) in %v, avg: %v per entry",
taskID, actualProcessed, skippedCount, totalDuration, totalDuration/time.Duration(actualProcessed))
return nil
}
@@ -2988,12 +3087,12 @@ func (s *knowledgeService) ensureFAQKnowledge(ctx context.Context, tenantID uint
func (s *knowledgeService) updateFAQImportStatus(ctx context.Context, knowledgeID string, status types.FAQImportTaskStatus,
progress, total, processed int, errorMsg string) error {
return s.updateFAQImportStatusWithRanges(ctx, knowledgeID, status, progress, total, processed, errorMsg, 0)
return s.updateFAQImportStatusWithRanges(ctx, knowledgeID, status, progress, total, processed, errorMsg)
}
// updateFAQImportStatusWithRanges 更新FAQ Knowledge的导入任务状态,包含NextChunkIndex
func (s *knowledgeService) updateFAQImportStatusWithRanges(ctx context.Context, knowledgeID string, status types.FAQImportTaskStatus,
progress, total, processed int, errorMsg string, nextChunkIndex int) error {
progress, total, processed int, errorMsg string) error {
tenantID := ctx.Value(types.TenantIDContextKey).(uint)
knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID)
if err != nil {
@@ -3020,9 +3119,6 @@ func (s *knowledgeService) updateFAQImportStatusWithRanges(ctx context.Context,
meta.ImportProgress = progress
meta.ImportTotal = total
meta.ImportProcessed = processed
if nextChunkIndex > 0 {
meta.NextChunkIndex = nextChunkIndex
}
metaJSON, err := meta.ToJSON()
if err != nil {
return fmt.Errorf("failed to marshal import metadata: %w", err)
@@ -3809,7 +3905,8 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
}
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo)
logger.Infof(ctx, "Processing FAQ import task: task_id=%s, kb_id=%s, total_entries=%d", payload.TaskID, payload.KBID, len(payload.Entries))
logger.Infof(ctx, "Processing FAQ import task: task_id=%s, kb_id=%s, total_entries=%d",
payload.TaskID, payload.KBID, len(payload.Entries))
// 幂等性检查:获取knowledge记录(FAQ任务使用knowledge ID作为taskID)
knowledge, err := s.repo.GetKnowledgeByID(ctx, payload.TenantID, payload.TaskID)
@@ -3822,6 +3919,12 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
return nil
}
kb, err := s.kbService.GetKnowledgeBaseByID(ctx, payload.KBID)
if err != nil {
logger.Errorf(ctx, "Failed to get knowledge base: %v", err)
return fmt.Errorf("failed to get knowledge base: %w", err)
}
// 检查任务状态 - 幂等性处理
if knowledge.ParseStatus == "completed" {
logger.Infof(ctx, "FAQ import already completed, skipping: %s", payload.TaskID)
@@ -3841,65 +3944,42 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
// 如果已经处理了一部分,需要从该位置继续
if processedCount < originalTotalEntries {
// 幂等性处理:清理processedCount之后可能已部分处理的chunks和索引数据
// 因为任务可能在处理过程中中断,导致部分chunks已入库或已写入索引
logger.Infof(ctx, "Cleaning up potentially partially processed chunks after entry %d", processedCount)
// 计算需要清理的ChunkIndex范围
// 已处理的chunks的ChunkIndex范围是 [payload.StartChunkIndex, payload.StartChunkIndex + processedCount - 1]
// 需要清理的chunks是从已处理位置之后开始,到分配的结束位置
minChunkIndex := payload.StartChunkIndex + processedCount
maxChunkIndex := payload.EndChunkIndex
logger.Infof(ctx, "Cleaning chunks with ChunkIndex in range [%d, %d] (allocated range: [%d, %d], processedCount=%d), knowledge_id=%s, kb_id=%s",
minChunkIndex, maxChunkIndex, payload.StartChunkIndex, payload.EndChunkIndex, processedCount, payload.KnowledgeID, payload.KBID)
// 查询并删除需要清理的chunks(使用ChunkIndex范围查询)
// 注意:只清理processedCount之后的部分,避免删除已成功处理的chunks
chunksToDelete, err := s.chunkRepo.DeleteChunksByChunkIndexRange(ctx,
payload.TenantID,
payload.KnowledgeID,
minChunkIndex,
maxChunkIndex,
)
// 幂等性处理:清理可能已部分处理的chunks和索引数据
chunksDeleted, err := s.chunkRepo.DeleteUnindexedChunks(ctx, payload.TenantID, payload.KnowledgeID)
if err != nil {
logger.Errorf(ctx, "Failed to query chunks for cleanup: %v", err)
return fmt.Errorf("failed to query chunks: %w", err)
}
logger.Infof(ctx, "Cleaned %d chunks, from knowledge: %s, range: [%d, %d]",
len(chunksToDelete), payload.KnowledgeID, minChunkIndex, maxChunkIndex)
// 获取KB信息以删除索引数据
kb, err := s.kbService.GetKnowledgeBaseByID(ctx, payload.KBID)
if err != nil {
logger.Errorf(ctx, "Failed to get knowledge base: %v", err)
return fmt.Errorf("failed to get knowledge base: %w", err)
logger.Errorf(ctx, "Failed to delete unindexed chunks: %v", err)
return fmt.Errorf("failed to delete unindexed chunks: %w", err)
}
logger.Infof(ctx, "Deleted unindexed chunks: %d", len(chunksDeleted))
// 删除索引数据
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID)
if err == nil {
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines)
if err == nil {
chunkIDs := make([]string, 0, len(chunksToDelete))
for _, chunk := range chunksToDelete {
chunkIDs := make([]string, 0, len(chunksDeleted))
for _, chunk := range chunksDeleted {
chunkIDs = append(chunkIDs, chunk.ID)
}
if err := retrieveEngine.DeleteByChunkIDList(ctx, chunkIDs, embeddingModel.GetDimensions()); err != nil {
logger.Warnf(ctx, "Failed to delete index data for chunks (may not exist): %v", err)
} else {
logger.Infof(ctx, "Successfully deleted index data for %d chunks", len(chunksToDelete))
logger.Infof(ctx, "Successfully deleted index data for %d chunks", len(chunksDeleted))
}
}
}
// 从已处理的位置继续
payload.Entries = payload.Entries[processedCount:]
// Append模式从已处理的位置继续
if payload.Mode == types.FAQBatchModeAppend {
payload.Entries = payload.Entries[processedCount:]
}
// Replace 模式使用hash去重,不截断payload.Entries
logger.Infof(ctx, "Continuing FAQ import from entry %d, remaining: %d entries", processedCount, len(payload.Entries))
}
// 更新任务状态为运行中
if err := s.updateFAQImportStatusWithRanges(ctx, payload.TaskID, types.FAQImportStatusProcessing, 0,
originalTotalEntries, processedCount, "", importMeta.NextChunkIndex); err != nil {
originalTotalEntries, processedCount, ""); err != nil {
logger.Errorf(ctx, "Failed to update task status to running: %v", err)
}
@@ -3909,8 +3989,8 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
Mode: payload.Mode,
}
// 执行FAQ导入(传入原始总数量)
if err := s.executeFAQImport(ctx, payload.TaskID, payload.KBID, faqPayload, payload.TenantID, payload.StartChunkIndex, processedCount, originalTotalEntries); err != nil {
// 执行FAQ导入
if err := s.executeFAQImport(ctx, payload.TaskID, payload.KBID, faqPayload, payload.TenantID, originalTotalEntries-len(payload.Entries)); err != nil {
logger.Errorf(ctx, "FAQ import task failed: %s, error: %v", payload.TaskID, err)
return fmt.Errorf("FAQ import failed: %w", err)
}
+15
View File
@@ -30,6 +30,17 @@ const (
ChunkTypeWebSearch ChunkType = "web_search"
)
// ChunkStatus 定义了不同状态的 Chunk
type ChunkStatus int
const (
ChunkStatusDefault ChunkStatus = 0
// ChunkStatusStored 表示已存储的 Chunk
ChunkStatusStored ChunkStatus = 1
// ChunkStatusIndexed 表示已索引的 Chunk
ChunkStatusIndexed ChunkStatus = 2
)
// ImageInfo 表示与 Chunk 关联的图片信息
type ImageInfo struct {
// 图片URL(COS)
@@ -69,6 +80,8 @@ type Chunk struct {
ChunkIndex int `json:"chunk_index"`
// Whether the chunk is enabled, can be used to temporarily disable certain chunks
IsEnabled bool `json:"is_enabled" gorm:"default:true"`
// Status of the chunk
Status int `json:"status" gorm:"default:0"`
// Starting character position in the original text
StartAt int `json:"start_at" `
// Ending character position in the original text
@@ -87,6 +100,8 @@ type Chunk struct {
IndirectRelationChunks JSON `json:"indirect_relation_chunks" gorm:"type:json"`
// Metadata 存储 chunk 级别的扩展信息,例如 FAQ 元数据
Metadata JSON `json:"metadata" gorm:"type:json"`
// ContentHash 存储内容的 hash 值,用于快速匹配(主要用于 FAQ)
ContentHash string `json:"content_hash" gorm:"type:varchar(64);index"`
// 图片信息,存储为 JSON
ImageInfo string `json:"image_info" gorm:"type:text"`
// Chunk creation time
+6 -8
View File
@@ -28,14 +28,12 @@ type DocumentProcessPayload struct {
// FAQImportPayload FAQ导入任务payload
type FAQImportPayload struct {
TenantID uint `json:"tenant_id"`
TaskID string `json:"task_id"`
KBID string `json:"kb_id"`
KnowledgeID string `json:"knowledge_id"`
Entries []FAQEntryPayload `json:"entries"`
Mode string `json:"mode"`
StartChunkIndex int `json:"start_chunk_index"`
EndChunkIndex int `json:"end_chunk_index"`
TenantID uint `json:"tenant_id"`
TaskID string `json:"task_id"`
KBID string `json:"kb_id"`
KnowledgeID string `json:"knowledge_id"`
Entries []FAQEntryPayload `json:"entries"`
Mode string `json:"mode"`
}
type PromptTemplateStructured struct {
+46 -1
View File
@@ -1,7 +1,10 @@
package types
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"sort"
"strings"
"time"
)
@@ -50,6 +53,7 @@ func (c *Chunk) SetFAQMetadata(meta *FAQChunkMetadata) error {
}
if meta == nil {
c.Metadata = nil
c.ContentHash = ""
return nil
}
meta.Normalize()
@@ -58,9 +62,51 @@ func (c *Chunk) SetFAQMetadata(meta *FAQChunkMetadata) error {
return err
}
c.Metadata = JSON(bytes)
// 计算并设置 ContentHash
c.ContentHash = CalculateFAQContentHash(meta)
return nil
}
// CalculateFAQContentHash 计算 FAQ 内容的 hash 值
// hash 基于:标准问 + 相似问(排序后)+ 反例(排序后)+ 答案(排序后)
// 用于快速匹配和去重
func CalculateFAQContentHash(meta *FAQChunkMetadata) string {
if meta == nil {
return ""
}
// 创建副本并标准化
normalized := *meta
normalized.Normalize()
// 对数组进行排序(确保相同内容产生相同 hash)
similarQuestions := make([]string, len(normalized.SimilarQuestions))
copy(similarQuestions, normalized.SimilarQuestions)
sort.Strings(similarQuestions)
negativeQuestions := make([]string, len(normalized.NegativeQuestions))
copy(negativeQuestions, normalized.NegativeQuestions)
sort.Strings(negativeQuestions)
answers := make([]string, len(normalized.Answers))
copy(answers, normalized.Answers)
sort.Strings(answers)
// 构建用于 hash 的字符串:标准问 + 相似问 + 反例 + 答案
var builder strings.Builder
builder.WriteString(normalized.StandardQuestion)
builder.WriteString("|")
builder.WriteString(strings.Join(similarQuestions, ","))
builder.WriteString("|")
builder.WriteString(strings.Join(negativeQuestions, ","))
builder.WriteString("|")
builder.WriteString(strings.Join(answers, ","))
// 计算 SHA256 hash
hash := sha256.Sum256([]byte(builder.String()))
return hex.EncodeToString(hash[:])
}
// FAQEntry 表示返回给前端的 FAQ 条目
type FAQEntry struct {
ID string `json:"id"`
@@ -125,7 +171,6 @@ type FAQImportMetadata struct {
ImportProgress int `json:"import_progress"` // 0-100
ImportTotal int `json:"import_total"`
ImportProcessed int `json:"import_processed"`
NextChunkIndex int `json:"next_chunk_index,omitempty"` // 下一个要分配的ChunkIndex(自增计数器)
}
// ToJSON converts the metadata to JSON type.
+6 -1
View File
@@ -41,7 +41,10 @@ type ChunkRepository interface {
DeleteByKnowledgeList(ctx context.Context, tenantID uint, knowledgeIDs []string) error
// CountChunksByKnowledgeBaseID counts the number of chunks in a knowledge base.
CountChunksByKnowledgeBaseID(ctx context.Context, tenantID uint, kbID string) (int64, error)
DeleteChunksByChunkIndexRange(ctx context.Context, tenantID uint, knowledgeID string, startChunkIndex int, endChunkIndex int) ([]*types.Chunk, error)
// DeleteUnindexedChunks deletes unindexed chunks by knowledge id and chunk index range
DeleteUnindexedChunks(ctx context.Context, tenantID uint, knowledgeID string) ([]*types.Chunk, error)
// ListAllFAQChunksByKnowledgeID lists all FAQ chunks for a knowledge ID (only ID and ContentHash fields for efficiency)
ListAllFAQChunksByKnowledgeID(ctx context.Context, tenantID uint, knowledgeID string) ([]*types.Chunk, error)
}
// ChunkService defines the interface for chunk service operations
@@ -61,6 +64,8 @@ type ChunkService interface {
) (*types.PageResult, error)
// UpdateChunk updates a chunk
UpdateChunk(ctx context.Context, chunk *types.Chunk) error
// UpdateChunks updates chunks in batch
UpdateChunks(ctx context.Context, chunks []*types.Chunk) error
// DeleteChunk deletes a chunk
DeleteChunk(ctx context.Context, id string) error
// DeleteChunks deletes chunks by IDs in batch
@@ -0,0 +1,17 @@
-- 000013_add_chunk_status_and_hash.down.sql
-- Rollback: Remove status and content_hash fields from chunks table
BEGIN;
-- Drop index on content_hash
DROP INDEX IF EXISTS idx_chunks_content_hash;
-- Drop content_hash column
ALTER TABLE chunks
DROP COLUMN IF EXISTS content_hash;
-- Drop status column
ALTER TABLE chunks
DROP COLUMN IF EXISTS status;
COMMIT;
@@ -0,0 +1,18 @@
-- 000013_add_chunk_status_and_hash.up.sql
-- Add status and content_hash fields to chunks table
BEGIN;
-- Add status field to track chunk processing state
ALTER TABLE chunks
ADD COLUMN IF NOT EXISTS status INT NOT NULL DEFAULT 0;
-- Add content_hash field for quick content matching (primarily for FAQ)
ALTER TABLE chunks
ADD COLUMN IF NOT EXISTS content_hash VARCHAR(64);
-- Create index on content_hash for efficient lookup
CREATE INDEX IF NOT EXISTS idx_chunks_content_hash
ON chunks(content_hash);
COMMIT;