From efe7b5d0f17975d46d450935cf0f3d023c20971d Mon Sep 17 00:00:00 2001 From: wizardchen Date: Mon, 15 Dec 2025 21:08:17 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E4=BC=98=E5=8C=96=E5=BC=82=E6=AD=A5?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E9=87=8D=E8=AF=95=E9=80=BB=E8=BE=91=EF=BC=8C?= =?UTF-8?q?=E4=BB=85=E5=9C=A8=E6=9C=80=E5=90=8E=E4=B8=80=E6=AC=A1=E9=87=8D?= =?UTF-8?q?=E8=AF=95=E6=97=B6=E6=9B=B4=E6=96=B0=E5=A4=B1=E8=B4=A5=E7=8A=B6?= =?UTF-8?q?=E6=80=81=EF=BC=8C=E6=94=B9=E8=BF=9B=E6=B7=B7=E5=90=88=E6=90=9C?= =?UTF-8?q?=E7=B4=A2=E7=BB=93=E6=9E=9C=E8=9E=8D=E5=90=88=E7=AD=96=E7=95=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/application/service/knowledge.go | 86 +++++++--- internal/application/service/knowledgebase.go | 149 +++++++++++------- 2 files changed, 155 insertions(+), 80 deletions(-) diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index 472d8b81..77152ed0 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -4617,6 +4617,12 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e ctx = logger.WithRequestID(ctx, payload.RequestId) ctx = logger.WithField(ctx, "document_process", payload.KnowledgeID) ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID) + + // 获取任务重试信息,用于判断是否是最后一次重试 + retryCount, _ := asynq.GetRetryCount(ctx) + maxRetry, _ := asynq.GetMaxRetry(ctx) + isLastRetry := retryCount >= maxRetry + tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID) if err != nil { logger.Errorf(ctx, "failed to get tenant: %v", err) @@ -4624,7 +4630,8 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e } ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo) - logger.Infof(ctx, "Processing document task: knowledge_id=%s, file_path=%s", payload.KnowledgeID, payload.FilePath) + logger.Infof(ctx, "Processing document task: knowledge_id=%s, file_path=%s, retry=%d/%d", + payload.KnowledgeID, payload.FilePath, retryCount, maxRetry) // 幂等性检查:获取knowledge记录 knowledge, err := s.repo.GetKnowledgeByID(ctx, payload.TenantID, payload.KnowledgeID) @@ -4739,10 +4746,13 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e RequestId: payload.RequestId, }) if err != nil { - knowledge.ParseStatus = "failed" - knowledge.ErrorMessage = err.Error() - knowledge.UpdatedAt = time.Now() - s.repo.UpdateKnowledge(ctx, knowledge) + // 如果是最后一次重试,更新状态为失败 + if isLastRetry { + knowledge.ParseStatus = "failed" + knowledge.ErrorMessage = err.Error() + knowledge.UpdatedAt = time.Now() + s.repo.UpdateKnowledge(ctx, knowledge) + } return fmt.Errorf("failed to read from URL: %w", err) } chunks = urlResp.Chunks @@ -4773,10 +4783,13 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e if err != nil { logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). WithField("error", err).Errorf("processDocument get file failed") - knowledge.ParseStatus = "failed" - knowledge.ErrorMessage = err.Error() - knowledge.UpdatedAt = time.Now() - s.repo.UpdateKnowledge(ctx, knowledge) + // 如果是最后一次重试,更新状态为失败 + if isLastRetry { + knowledge.ParseStatus = "failed" + knowledge.ErrorMessage = err.Error() + knowledge.UpdatedAt = time.Now() + s.repo.UpdateKnowledge(ctx, knowledge) + } return fmt.Errorf("failed to get file: %w", err) } defer fileReader.Close() @@ -4784,10 +4797,13 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e // 读取文件内容 contentBytes, err := io.ReadAll(fileReader) if err != nil { - knowledge.ParseStatus = "failed" - knowledge.ErrorMessage = err.Error() - knowledge.UpdatedAt = time.Now() - s.repo.UpdateKnowledge(ctx, knowledge) + // 如果是最后一次重试,更新状态为失败 + if isLastRetry { + knowledge.ParseStatus = "failed" + knowledge.ErrorMessage = err.Error() + knowledge.UpdatedAt = time.Now() + s.repo.UpdateKnowledge(ctx, knowledge) + } return fmt.Errorf("failed to read file: %w", err) } @@ -4817,10 +4833,13 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e if err != nil { logger.GetLogger(ctx).WithField("knowledge_id", knowledge.ID). WithField("error", err).Errorf("processDocument read file failed") - knowledge.ParseStatus = "failed" - knowledge.ErrorMessage = err.Error() - knowledge.UpdatedAt = time.Now() - s.repo.UpdateKnowledge(ctx, knowledge) + // 如果是最后一次重试,更新状态为失败 + if isLastRetry { + knowledge.ParseStatus = "failed" + knowledge.ErrorMessage = err.Error() + knowledge.UpdatedAt = time.Now() + s.repo.UpdateKnowledge(ctx, knowledge) + } return fmt.Errorf("failed to read file from docreader: %w", err) } chunks = fileResp.Chunks @@ -4847,6 +4866,11 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task) ctx = logger.WithField(ctx, "faq_import", payload.TaskID) ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID) + // 获取任务重试信息,用于判断是否是最后一次重试 + retryCount, _ := asynq.GetRetryCount(ctx) + maxRetry, _ := asynq.GetMaxRetry(ctx) + isLastRetry := retryCount >= maxRetry + tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID) if err != nil { logger.Errorf(ctx, "failed to get tenant: %v", err) @@ -4854,8 +4878,8 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task) } ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo) - logger.Infof(ctx, "Processing FAQ import task: task_id=%s, kb_id=%s, total_entries=%d", - payload.TaskID, payload.KBID, len(payload.Entries)) + logger.Infof(ctx, "Processing FAQ import task: task_id=%s, kb_id=%s, total_entries=%d, retry=%d/%d", + payload.TaskID, payload.KBID, len(payload.Entries), retryCount, maxRetry) // 幂等性检查:获取knowledge记录(FAQ任务使用knowledge ID作为taskID) knowledge, err := s.repo.GetKnowledgeByID(ctx, payload.TenantID, payload.TaskID) @@ -4871,6 +4895,12 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task) kb, err := s.kbService.GetKnowledgeBaseByID(ctx, payload.KBID) if err != nil { logger.Errorf(ctx, "Failed to get knowledge base: %v", err) + // 如果是最后一次重试,更新状态为失败 + if isLastRetry { + if updateErr := s.updateFAQImportStatus(ctx, payload.TaskID, types.FAQImportStatusFailed, 0, len(payload.Entries), 0, err.Error()); updateErr != nil { + logger.Errorf(ctx, "Failed to update task status to failed: %v", updateErr) + } + } return fmt.Errorf("failed to get knowledge base: %w", err) } @@ -4897,6 +4927,12 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task) chunksDeleted, err := s.chunkRepo.DeleteUnindexedChunks(ctx, payload.TenantID, payload.KnowledgeID) if err != nil { logger.Errorf(ctx, "Failed to delete unindexed chunks: %v", err) + // 如果是最后一次重试,更新状态为失败 + if isLastRetry { + if updateErr := s.updateFAQImportStatus(ctx, payload.TaskID, types.FAQImportStatusFailed, 0, originalTotalEntries, processedCount, err.Error()); updateErr != nil { + logger.Errorf(ctx, "Failed to update task status to failed: %v", updateErr) + } + } return fmt.Errorf("failed to delete unindexed chunks: %w", err) } logger.Infof(ctx, "Deleted unindexed chunks: %d", len(chunksDeleted)) @@ -4949,6 +4985,18 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task) // 执行FAQ导入 if err := s.executeFAQImport(ctx, payload.TaskID, payload.KBID, faqPayload, payload.TenantID, originalTotalEntries-len(payload.Entries)); err != nil { logger.Errorf(ctx, "FAQ import task failed: %s, error: %v", payload.TaskID, err) + // 如果是最后一次重试,更新状态为失败 + if isLastRetry { + // 获取当前已处理的进度 + currentMeta, _ := types.ParseFAQImportMetadata(knowledge) + currentProcessed := 0 + if currentMeta != nil { + currentProcessed = currentMeta.ImportProcessed + } + if updateErr := s.updateFAQImportStatus(ctx, payload.TaskID, types.FAQImportStatusFailed, 0, originalTotalEntries, currentProcessed, err.Error()); updateErr != nil { + logger.Errorf(ctx, "Failed to update task status to failed: %v", updateErr) + } + } return fmt.Errorf("FAQ import failed: %w", err) } diff --git a/internal/application/service/knowledgebase.go b/internal/application/service/knowledgebase.go index 1ea1d11d..3bcfc41a 100644 --- a/internal/application/service/knowledgebase.go +++ b/internal/application/service/knowledgebase.go @@ -544,81 +544,108 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context, logger.Info(ctx, "No search results found") return nil, nil } - logger.Infof(ctx, "Result count before RRF fusion: vector=%d, keyword=%d", len(vectorResults), len(keywordResults)) + logger.Infof(ctx, "Result count before fusion: vector=%d, keyword=%d", len(vectorResults), len(keywordResults)) - // Use RRF (Reciprocal Rank Fusion) to merge results - // RRF score = sum(1 / (k + rank)) for each retriever where the chunk appears - // k=60 is a common choice that works well in practice - const rrfK = 60 + var deduplicatedChunks []*types.IndexWithScore - // Build rank maps for each retriever (already sorted by score from retriever) - vectorRanks := make(map[string]int) - for i, r := range vectorResults { - if _, exists := vectorRanks[r.ChunkID]; !exists { - vectorRanks[r.ChunkID] = i + 1 // 1-indexed rank + // If only vector results (no keyword results), keep original embedding scores + // This is important for FAQ search which only uses vector retrieval + if len(keywordResults) == 0 { + logger.Info(ctx, "Only vector results, keeping original embedding scores") + chunkInfoMap := make(map[string]*types.IndexWithScore) + for _, r := range vectorResults { + if _, exists := chunkInfoMap[r.ChunkID]; !exists { + chunkInfoMap[r.ChunkID] = r + } } - } - keywordRanks := make(map[string]int) - for i, r := range keywordResults { - if _, exists := keywordRanks[r.ChunkID]; !exists { - keywordRanks[r.ChunkID] = i + 1 // 1-indexed rank + deduplicatedChunks = make([]*types.IndexWithScore, 0, len(chunkInfoMap)) + for _, info := range chunkInfoMap { + deduplicatedChunks = append(deduplicatedChunks, info) } - } + slices.SortFunc(deduplicatedChunks, func(a, b *types.IndexWithScore) int { + if a.Score > b.Score { + return -1 + } else if a.Score < b.Score { + return 1 + } + return 0 + }) + logger.Infof(ctx, "Result count after deduplication: %d", len(deduplicatedChunks)) + } else { + // Use RRF (Reciprocal Rank Fusion) to merge results from multiple retrievers + // RRF score = sum(1 / (k + rank)) for each retriever where the chunk appears + // k=60 is a common choice that works well in practice + const rrfK = 60 - // Collect all unique chunks and compute RRF scores - chunkInfoMap := make(map[string]*types.IndexWithScore) - rrfScores := make(map[string]float64) - - // Process vector results - for _, r := range vectorResults { - if _, exists := chunkInfoMap[r.ChunkID]; !exists { - chunkInfoMap[r.ChunkID] = r + // Build rank maps for each retriever (already sorted by score from retriever) + vectorRanks := make(map[string]int) + for i, r := range vectorResults { + if _, exists := vectorRanks[r.ChunkID]; !exists { + vectorRanks[r.ChunkID] = i + 1 // 1-indexed rank + } } - } - // Process keyword results - for _, r := range keywordResults { - if _, exists := chunkInfoMap[r.ChunkID]; !exists { - chunkInfoMap[r.ChunkID] = r + keywordRanks := make(map[string]int) + for i, r := range keywordResults { + if _, exists := keywordRanks[r.ChunkID]; !exists { + keywordRanks[r.ChunkID] = i + 1 // 1-indexed rank + } } - } - // Compute RRF scores - for chunkID := range chunkInfoMap { - rrfScore := 0.0 - if rank, ok := vectorRanks[chunkID]; ok { - rrfScore += 1.0 / float64(rrfK+rank) + // Collect all unique chunks and compute RRF scores + chunkInfoMap := make(map[string]*types.IndexWithScore) + rrfScores := make(map[string]float64) + + // Process vector results + for _, r := range vectorResults { + if _, exists := chunkInfoMap[r.ChunkID]; !exists { + chunkInfoMap[r.ChunkID] = r + } } - if rank, ok := keywordRanks[chunkID]; ok { - rrfScore += 1.0 / float64(rrfK+rank) + // Process keyword results + for _, r := range keywordResults { + if _, exists := chunkInfoMap[r.ChunkID]; !exists { + chunkInfoMap[r.ChunkID] = r + } } - rrfScores[chunkID] = rrfScore - } - // Convert to slice and sort by RRF score - deduplicatedChunks := make([]*types.IndexWithScore, 0, len(chunkInfoMap)) - for chunkID, info := range chunkInfoMap { - // Store RRF score in the Score field for downstream processing - info.Score = rrfScores[chunkID] - deduplicatedChunks = append(deduplicatedChunks, info) - } - slices.SortFunc(deduplicatedChunks, func(a, b *types.IndexWithScore) int { - if a.Score > b.Score { - return -1 - } else if a.Score < b.Score { - return 1 + // Compute RRF scores + for chunkID := range chunkInfoMap { + rrfScore := 0.0 + if rank, ok := vectorRanks[chunkID]; ok { + rrfScore += 1.0 / float64(rrfK+rank) + } + if rank, ok := keywordRanks[chunkID]; ok { + rrfScore += 1.0 / float64(rrfK+rank) + } + rrfScores[chunkID] = rrfScore } - return 0 - }) - logger.Infof(ctx, "Result count after RRF fusion: %d", len(deduplicatedChunks)) + // Convert to slice and sort by RRF score + deduplicatedChunks = make([]*types.IndexWithScore, 0, len(chunkInfoMap)) + for chunkID, info := range chunkInfoMap { + // Store RRF score in the Score field for downstream processing + info.Score = rrfScores[chunkID] + deduplicatedChunks = append(deduplicatedChunks, info) + } + slices.SortFunc(deduplicatedChunks, func(a, b *types.IndexWithScore) int { + if a.Score > b.Score { + return -1 + } else if a.Score < b.Score { + return 1 + } + return 0 + }) - // Log top results after RRF fusion for debugging - for i, chunk := range deduplicatedChunks { - if i < 15 { - vRank, vOk := vectorRanks[chunk.ChunkID] - kRank, kOk := keywordRanks[chunk.ChunkID] - logger.Debugf(ctx, "RRF rank %d: chunk_id=%s, rrf_score=%.6f, vector_rank=%v(%v), keyword_rank=%v(%v)", - i, chunk.ChunkID, chunk.Score, vRank, vOk, kRank, kOk) + logger.Infof(ctx, "Result count after RRF fusion: %d", len(deduplicatedChunks)) + + // Log top results after RRF fusion for debugging + for i, chunk := range deduplicatedChunks { + if i < 15 { + vRank, vOk := vectorRanks[chunk.ChunkID] + kRank, kOk := keywordRanks[chunk.ChunkID] + logger.Debugf(ctx, "RRF rank %d: chunk_id=%s, rrf_score=%.6f, vector_rank=%v(%v), keyword_rank=%v(%v)", + i, chunk.ChunkID, chunk.Score, vRank, vOk, kRank, kOk) + } } }