From f0ca9883fc30c11a398da9d876e8f316ea2432a1 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Wed, 3 Dec 2025 19:43:48 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E4=BC=98=E5=8C=96=E7=9F=A5=E8=AF=86?= =?UTF-8?q?=E5=BA=93=E5=88=A0=E9=99=A4=E6=B5=81=E7=A8=8B=EF=BC=8C=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E7=8A=B6=E6=80=81=E6=A0=87=E8=AE=B0=E9=98=B2=E6=AD=A2?= =?UTF-8?q?=E5=BC=82=E6=AD=A5=E4=BB=BB=E5=8A=A1=E5=86=B2=E7=AA=81=E5=B9=B6?= =?UTF-8?q?=E5=AE=8C=E5=96=84=E7=8A=B6=E6=80=81=E5=B8=B8=E9=87=8F=E5=AE=9A?= =?UTF-8?q?=E4=B9=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/application/service/knowledge.go | 101 ++++++++++++++++++++-- internal/types/knowledge.go | 14 +++ 2 files changed, 107 insertions(+), 8 deletions(-) diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index 6a22e2a3..d3346611 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -117,6 +117,21 @@ func (s *knowledgeService) GetRepository() interfaces.KnowledgeRepository { return s.repo } +// isKnowledgeDeleting checks if a knowledge entry is being deleted. +// This is used to prevent async tasks from conflicting with deletion operations. +func (s *knowledgeService) isKnowledgeDeleting(ctx context.Context, tenantID uint64, knowledgeID string) bool { + knowledge, err := s.repo.GetKnowledgeByID(ctx, tenantID, knowledgeID) + if err != nil { + // If we can't find the knowledge, assume it's deleted + logger.Warnf(ctx, "Failed to check knowledge deletion status (assuming deleted): %v", err) + return true + } + if knowledge == nil { + return true + } + return knowledge.ParseStatus == types.ParseStatusDeleting +} + // CreateKnowledgeFromFile creates a knowledge entry from an uploaded file func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context, kbID string, file *multipart.FileHeader, metadata map[string]string, enableMultimodel *bool, customFileName string, @@ -702,6 +717,19 @@ func (s *knowledgeService) DeleteKnowledge(ctx context.Context, id string) error if err != nil { return err } + + // Mark as deleting first to prevent async task conflicts + // This ensures that any running async tasks will detect the deletion and abort + originalStatus := knowledge.ParseStatus + knowledge.ParseStatus = types.ParseStatusDeleting + knowledge.UpdatedAt = time.Now() + if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil { + logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge failed to mark as deleting") + // Continue with deletion even if marking fails + } else { + logger.Infof(ctx, "Marked knowledge %s as deleting (previous status: %s)", id, originalStatus) + } + wg := errgroup.Group{} // Delete knowledge embeddings from vector store wg.Go(func() error { @@ -767,7 +795,7 @@ func (s *knowledgeService) DeleteKnowledge(ctx context.Context, id string) error return s.repo.DeleteKnowledge(ctx, ctx.Value(types.TenantIDContextKey).(uint64), id) } -// DeleteKnowledge deletes a knowledge entry and all related resources +// DeleteKnowledgeList deletes a knowledge entry and all related resources func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string) error { if len(ids) == 0 { return nil @@ -779,6 +807,18 @@ func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string return err } + // Mark all as deleting first to prevent async task conflicts + for _, knowledge := range knowledgeList { + knowledge.ParseStatus = types.ParseStatusDeleting + knowledge.UpdatedAt = time.Now() + if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil { + logger.GetLogger(ctx).WithField("error", err).WithField("knowledge_id", knowledge.ID). + Errorf("DeleteKnowledgeList failed to mark as deleting") + // Continue with deletion even if marking fails + } + } + logger.Infof(ctx, "Marked %d knowledge entries as deleting", len(knowledgeList)) + wg := errgroup.Group{} // 2. Delete knowledge embeddings from vector store wg.Go(func() error { @@ -980,6 +1020,13 @@ func (s *knowledgeService) processChunks(ctx context.Context, attribute.Int("chunk_count", len(chunks)), ) + // Check if knowledge is being deleted before processing + if s.isKnowledgeDeleting(ctx, knowledge.TenantID, knowledge.ID) { + logger.Infof(ctx, "Knowledge is being deleted, aborting chunk processing: %s", knowledge.ID) + span.AddEvent("aborted: knowledge is being deleted") + return + } + // Get embedding model for vectorization embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID) if err != nil { @@ -1182,7 +1229,7 @@ func (s *knowledgeService) processChunks(ctx context.Context, // Re-fetch tenant storage information tenantInfo, err = s.tenantRepo.GetTenantByID(ctx, tenantInfo.ID) if err != nil { - knowledge.ParseStatus = "failed" + knowledge.ParseStatus = types.ParseStatusFailed knowledge.ErrorMessage = err.Error() knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) @@ -1191,7 +1238,7 @@ func (s *knowledgeService) processChunks(ctx context.Context, } // Check if there's enough storage quota available if tenantInfo.StorageUsed+totalStorageSize > tenantInfo.StorageQuota { - knowledge.ParseStatus = "failed" + knowledge.ParseStatus = types.ParseStatusFailed knowledge.ErrorMessage = "存储空间不足" knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) @@ -1200,10 +1247,17 @@ func (s *knowledgeService) processChunks(ctx context.Context, } } + // Check again if knowledge is being deleted before writing to database + if s.isKnowledgeDeleting(ctx, knowledge.TenantID, knowledge.ID) { + logger.Infof(ctx, "Knowledge is being deleted, aborting before saving chunks: %s", knowledge.ID) + span.AddEvent("aborted: knowledge is being deleted before saving") + return + } + // Save chunks to database span.AddEvent("create chunks") if err := s.chunkService.CreateChunks(ctx, insertChunks); err != nil { - knowledge.ParseStatus = "failed" + knowledge.ParseStatus = types.ParseStatusFailed knowledge.ErrorMessage = err.Error() knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) @@ -1211,10 +1265,21 @@ func (s *knowledgeService) processChunks(ctx context.Context, return } + // Check again before batch indexing (this is a heavy operation) + if s.isKnowledgeDeleting(ctx, knowledge.TenantID, knowledge.ID) { + logger.Infof(ctx, "Knowledge is being deleted, cleaning up and aborting before indexing: %s", knowledge.ID) + // Clean up the chunks we just created + if err := s.chunkService.DeleteChunksByKnowledgeID(ctx, knowledge.ID); err != nil { + logger.Warnf(ctx, "Failed to cleanup chunks after deletion detected: %v", err) + } + span.AddEvent("aborted: knowledge is being deleted before indexing") + return + } + span.AddEvent("batch index") err = retrieveEngine.BatchIndex(ctx, embeddingModel, indexInfoList) if err != nil { - knowledge.ParseStatus = "failed" + knowledge.ParseStatus = types.ParseStatusFailed knowledge.ErrorMessage = err.Error() knowledge.UpdatedAt = time.Now() s.repo.UpdateKnowledge(ctx, knowledge) @@ -1246,8 +1311,22 @@ func (s *knowledgeService) processChunks(ctx context.Context, } } + // Final check before marking as completed - if deleted during processing, don't update status + if s.isKnowledgeDeleting(ctx, knowledge.TenantID, knowledge.ID) { + logger.Infof(ctx, "Knowledge was deleted during processing, skipping completion update: %s", knowledge.ID) + // Clean up the data we just created since the knowledge is being deleted + if err := s.chunkService.DeleteChunksByKnowledgeID(ctx, knowledge.ID); err != nil { + logger.Warnf(ctx, "Failed to cleanup chunks after deletion detected: %v", err) + } + if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions()); err != nil { + logger.Warnf(ctx, "Failed to cleanup index after deletion detected: %v", err) + } + span.AddEvent("aborted: knowledge was deleted during processing") + return + } + // Update knowledge status to completed - knowledge.ParseStatus = "completed" + knowledge.ParseStatus = types.ParseStatusCompleted knowledge.EnableStatus = "enabled" knowledge.StorageSize = totalStorageSize now := time.Now() @@ -4361,13 +4440,19 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e return nil } + // 检查是否正在删除 - 如果是则直接退出,避免与删除操作冲突 + if knowledge.ParseStatus == types.ParseStatusDeleting { + logger.Infof(ctx, "Knowledge is being deleted, aborting processing: %s", payload.KnowledgeID) + return nil + } + // 检查任务状态 - 幂等性处理 - if knowledge.ParseStatus == "completed" { + if knowledge.ParseStatus == types.ParseStatusCompleted { logger.Infof(ctx, "Document already completed, skipping: %s", payload.KnowledgeID) return nil // 幂等:已完成的任务直接返回 } - if knowledge.ParseStatus == "failed" { + if knowledge.ParseStatus == types.ParseStatusFailed { // 检查是否可恢复(例如:超时、临时错误等) // 对于不可恢复的错误,直接返回 logger.Warnf( diff --git a/internal/types/knowledge.go b/internal/types/knowledge.go index 2dc82436..bd8e6b5c 100644 --- a/internal/types/knowledge.go +++ b/internal/types/knowledge.go @@ -16,6 +16,20 @@ const ( KnowledgeTypeFAQ = "faq" ) +// Knowledge parse status constants +const ( + // ParseStatusPending indicates the knowledge is waiting to be processed + ParseStatusPending = "pending" + // ParseStatusProcessing indicates the knowledge is being processed + ParseStatusProcessing = "processing" + // ParseStatusCompleted indicates the knowledge has been processed successfully + ParseStatusCompleted = "completed" + // ParseStatusFailed indicates the knowledge processing failed + ParseStatusFailed = "failed" + // ParseStatusDeleting indicates the knowledge is being deleted (used to prevent async task conflicts) + ParseStatusDeleting = "deleting" +) + // ManualKnowledgeFormat represents the format of the manual knowledge const ( ManualKnowledgeFormatMarkdown = "markdown"