From 4c0846f9c7ed0e81f2ec5658edac0eaf6e5ff01b Mon Sep 17 00:00:00 2001 From: wizardchen Date: Thu, 25 Dec 2025 15:35:57 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=A2=9E=E5=BC=BA=E6=A0=87=E7=AD=BE?= =?UTF-8?q?=E5=88=A0=E9=99=A4=E5=8A=9F=E8=83=BD=EF=BC=8C=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E6=8E=92=E9=99=A4=E6=8C=87=E5=AE=9AID=EF=BC=9B=E6=96=B0?= =?UTF-8?q?=E5=A2=9E=E5=BC=82=E6=AD=A5=E7=B4=A2=E5=BC=95=E5=88=A0=E9=99=A4?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E4=BC=98=E5=8C=96=E5=88=A0=E9=99=A4=E6=B5=81?= =?UTF-8?q?=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/tag.go | 12 +- internal/application/repository/chunk.go | 48 +++++++- internal/application/service/tag.go | 145 ++++++++++++++++++++--- internal/handler/tag.go | 20 +++- internal/router/task.go | 4 + internal/types/extract_graph.go | 11 ++ internal/types/interfaces/chunk.go | 3 +- internal/types/interfaces/tag.go | 6 +- 8 files changed, 221 insertions(+), 28 deletions(-) diff --git a/client/tag.go b/client/tag.go index 4e21e5c7..a4e89093 100644 --- a/client/tag.go +++ b/client/tag.go @@ -140,8 +140,9 @@ func (c *Client) UpdateTag(ctx context.Context, // DeleteTag deletes a tag. // Set force to true to delete even if the tag is referenced. // Set contentOnly to true to only delete the content under the tag but keep the tag itself. +// excludeIDs: IDs of chunks to exclude from deletion. func (c *Client) DeleteTag(ctx context.Context, - knowledgeBaseID, tagID string, force bool, contentOnly bool, + knowledgeBaseID, tagID string, force bool, contentOnly bool, excludeIDs []string, ) error { path := fmt.Sprintf("/api/v1/knowledge-bases/%s/tags/%s", knowledgeBaseID, tagID) query := url.Values{} @@ -152,7 +153,14 @@ func (c *Client) DeleteTag(ctx context.Context, query.Add("content_only", "true") } - resp, err := c.doRequest(ctx, http.MethodDelete, path, nil, query) + var body interface{} + if len(excludeIDs) > 0 { + body = map[string]interface{}{ + "exclude_ids": excludeIDs, + } + } + + resp, err := c.doRequest(ctx, http.MethodDelete, path, body, query) if err != nil { return err } diff --git a/internal/application/repository/chunk.go b/internal/application/repository/chunk.go index 7cb423c3..1bfd16ca 100644 --- a/internal/application/repository/chunk.go +++ b/internal/application/repository/chunk.go @@ -313,10 +313,50 @@ func (r *chunkRepository) DeleteByKnowledgeList(ctx context.Context, tenantID ui } // DeleteChunksByTagID deletes all chunks with the specified tag ID -func (r *chunkRepository) DeleteChunksByTagID(ctx context.Context, tenantID uint64, kbID string, tagID string) error { - return r.db.WithContext(ctx).Where( - "tenant_id = ? AND knowledge_base_id = ? AND tag_id = ?", tenantID, kbID, tagID, - ).Delete(&types.Chunk{}).Error +// Returns the IDs of deleted chunks for index cleanup +func (r *chunkRepository) DeleteChunksByTagID(ctx context.Context, tenantID uint64, kbID string, tagID string, excludeIDs []string) ([]string, error) { + // Build exclude set for O(1) lookup + excludeSet := make(map[string]struct{}, len(excludeIDs)) + for _, id := range excludeIDs { + excludeSet[id] = struct{}{} + } + + // Get all chunk IDs for this tag + var allIDs []string + if err := r.db.WithContext(ctx).Model(&types.Chunk{}). + Where("tenant_id = ? AND knowledge_base_id = ? AND tag_id = ?", tenantID, kbID, tagID). + Pluck("id", &allIDs).Error; err != nil { + return nil, err + } + + // Filter out excluded IDs + toDelete := make([]string, 0, len(allIDs)) + for _, id := range allIDs { + if _, excluded := excludeSet[id]; !excluded { + toDelete = append(toDelete, id) + } + } + + if len(toDelete) == 0 { + return nil, nil + } + + // Delete in batches + const batchSize = 1000 + for i := 0; i < len(toDelete); i += batchSize { + end := i + batchSize + if end > len(toDelete) { + end = len(toDelete) + } + batch := toDelete[i:end] + + if err := r.db.WithContext(ctx).Where("id IN ?", batch).Delete(&types.Chunk{}).Error; err != nil { + // Return already planned deletions up to this point for index cleanup + return toDelete[:i], err + } + } + + return toDelete, nil } // CountChunksByKnowledgeBaseID counts the number of chunks in a knowledge base diff --git a/internal/application/service/tag.go b/internal/application/service/tag.go index d9a7d5d4..e886d568 100644 --- a/internal/application/service/tag.go +++ b/internal/application/service/tag.go @@ -2,23 +2,30 @@ package service import ( "context" + "encoding/json" "errors" "strings" "time" + "github.com/Tencent/WeKnora/internal/application/service/retriever" werrors "github.com/Tencent/WeKnora/internal/errors" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/Tencent/WeKnora/internal/utils" "github.com/google/uuid" + "github.com/hibiken/asynq" "gorm.io/gorm" ) // knowledgeTagService implements KnowledgeTagService. type knowledgeTagService struct { - kbService interfaces.KnowledgeBaseService - repo interfaces.KnowledgeTagRepository - chunkRepo interfaces.ChunkRepository + kbService interfaces.KnowledgeBaseService + repo interfaces.KnowledgeTagRepository + chunkRepo interfaces.ChunkRepository + retrieveEngine interfaces.RetrieveEngineRegistry + modelService interfaces.ModelService + task *asynq.Client } // NewKnowledgeTagService creates a new tag service. @@ -26,11 +33,17 @@ func NewKnowledgeTagService( kbService interfaces.KnowledgeBaseService, repo interfaces.KnowledgeTagRepository, chunkRepo interfaces.ChunkRepository, + retrieveEngine interfaces.RetrieveEngineRegistry, + modelService interfaces.ModelService, + task *asynq.Client, ) (interfaces.KnowledgeTagService, error) { return &knowledgeTagService{ - kbService: kbService, - repo: repo, - chunkRepo: chunkRepo, + kbService: kbService, + repo: repo, + chunkRepo: chunkRepo, + retrieveEngine: retrieveEngine, + modelService: modelService, + task: task, }, nil } @@ -154,7 +167,7 @@ func (s *knowledgeTagService) UpdateTag( // DeleteTag deletes a tag. When force=true, also deletes all chunks under this tag. // When contentOnly=true, only deletes the content under the tag but keeps the tag itself. -func (s *knowledgeTagService) DeleteTag(ctx context.Context, id string, force bool, contentOnly bool) error { +func (s *knowledgeTagService) DeleteTag(ctx context.Context, id string, force bool, contentOnly bool, excludeIDs []string) error { if id == "" { return werrors.NewBadRequestError("标签ID不能为空") } @@ -163,19 +176,45 @@ func (s *knowledgeTagService) DeleteTag(ctx context.Context, id string, force bo if err != nil { return err } + + // Get KB info for embedding model + kb, err := s.kbService.GetKnowledgeBaseByID(ctx, tag.KnowledgeBaseID) + if err != nil { + return err + } + kCount, cCount, err := s.repo.CountReferences(ctx, tenantID, tag.KnowledgeBaseID, tag.ID) if err != nil { return err } + // Get tenant info for effective engines + tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) + + // Helper function to delete chunks and enqueue index deletion task + deleteChunksAndEnqueueIndexDelete := func() error { + // Delete chunks and get their IDs + deletedIDs, err := s.chunkRepo.DeleteChunksByTagID(ctx, tenantID, tag.KnowledgeBaseID, tag.ID, excludeIDs) + if err != nil { + logger.Errorf(ctx, "Failed to delete chunks by tag ID %s: %v", tag.ID, err) + return werrors.NewInternalServerError("删除标签下的数据失败") + } + + // Enqueue async index deletion task for the deleted chunks + if len(deletedIDs) > 0 { + s.enqueueIndexDeleteTask(ctx, tenantID, kb.ID, kb.EmbeddingModelID, string(kb.Type), deletedIDs, tenantInfo.GetEffectiveEngines()) + } + + logger.Infof(ctx, "Deleted %d chunks under tag %s", len(deletedIDs), tag.ID) + return nil + } + // contentOnly mode: only delete content, keep the tag if contentOnly { if cCount > 0 { - if err := s.chunkRepo.DeleteChunksByTagID(ctx, tenantID, tag.KnowledgeBaseID, tag.ID); err != nil { - logger.Errorf(ctx, "Failed to delete chunks by tag ID %s: %v", tag.ID, err) - return werrors.NewInternalServerError("删除标签下的数据失败") + if err := deleteChunksAndEnqueueIndexDelete(); err != nil { + return err } - logger.Infof(ctx, "Deleted %d chunks under tag %s (content_only mode)", cCount, tag.ID) } return nil } @@ -185,15 +224,91 @@ func (s *knowledgeTagService) DeleteTag(ctx context.Context, id string, force bo } // When force=true, delete all chunks under this tag first if force && cCount > 0 { - if err := s.chunkRepo.DeleteChunksByTagID(ctx, tenantID, tag.KnowledgeBaseID, tag.ID); err != nil { - logger.Errorf(ctx, "Failed to delete chunks by tag ID %s: %v", tag.ID, err) - return werrors.NewInternalServerError("删除标签下的数据失败") + if err := deleteChunksAndEnqueueIndexDelete(); err != nil { + return err } - logger.Infof(ctx, "Deleted %d chunks under tag %s", cCount, tag.ID) + } + // If there are excludeIDs, we cannot delete the tag itself as it still has content + if len(excludeIDs) > 0 { + return nil } return s.repo.Delete(ctx, tenantID, id) } +// enqueueIndexDeleteTask enqueues an async task for index deletion (low priority) +func (s *knowledgeTagService) enqueueIndexDeleteTask(ctx context.Context, + tenantID uint64, kbID, embeddingModelID, kbType string, chunkIDs []string, effectiveEngines []types.RetrieverEngineParams, +) { + payload := types.IndexDeletePayload{ + TenantID: tenantID, + KnowledgeBaseID: kbID, + EmbeddingModelID: embeddingModelID, + KBType: kbType, + ChunkIDs: chunkIDs, + EffectiveEngines: effectiveEngines, + } + payloadBytes, err := json.Marshal(payload) + if err != nil { + logger.Errorf(ctx, "Failed to marshal index delete payload: %v", err) + return + } + + task := asynq.NewTask(types.TypeIndexDelete, payloadBytes, asynq.Queue(utils.GetAsynqQueueName("low")), asynq.MaxRetry(10)) + info, err := s.task.Enqueue(task) + if err != nil { + logger.Errorf(ctx, "Failed to enqueue index delete task: %v", err) + return + } + logger.Infof(ctx, "Enqueued index delete task: %s for %d chunks", info.ID, len(chunkIDs)) +} + +// ProcessIndexDelete handles async index deletion task +func (s *knowledgeTagService) ProcessIndexDelete(ctx context.Context, t *asynq.Task) error { + var payload types.IndexDeletePayload + if err := json.Unmarshal(t.Payload(), &payload); err != nil { + logger.Errorf(ctx, "Failed to unmarshal index delete payload: %v", err) + return err + } + + logger.Infof(ctx, "Processing index delete task for %d chunks in KB %s", len(payload.ChunkIDs), payload.KnowledgeBaseID) + + // Create retrieve engine + retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, payload.EffectiveEngines) + if err != nil { + logger.Warnf(ctx, "Failed to create retrieve engine for index cleanup: %v", err) + return err + } + + // Get embedding model dimensions + embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, payload.EmbeddingModelID) + if err != nil { + logger.Warnf(ctx, "Failed to get embedding model for index cleanup: %v", err) + return err + } + + // Delete indices in batches to avoid overwhelming the backend + const batchSize = 100 + chunkIDs := payload.ChunkIDs + dimension := embeddingModel.GetDimensions() + + for i := 0; i < len(chunkIDs); i += batchSize { + end := i + batchSize + if end > len(chunkIDs) { + end = len(chunkIDs) + } + batch := chunkIDs[i:end] + + if err := retrieveEngine.DeleteByChunkIDList(ctx, batch, dimension, payload.KBType); err != nil { + logger.Warnf(ctx, "Failed to delete indices for chunks batch [%d-%d]: %v", i, end, err) + return err + } + logger.Debugf(ctx, "Deleted indices batch [%d-%d] of %d chunks", i, end, len(chunkIDs)) + } + + logger.Infof(ctx, "Successfully deleted indices for %d chunks", len(payload.ChunkIDs)) + return nil +} + // FindOrCreateTagByName finds a tag by name or creates it if not exists. func (s *knowledgeTagService) FindOrCreateTagByName(ctx context.Context, kbID string, name string) (*types.KnowledgeTag, error) { name = strings.TrimSpace(name) diff --git a/internal/handler/tag.go b/internal/handler/tag.go index 6bc7c032..3bd8d8f3 100644 --- a/internal/handler/tag.go +++ b/internal/handler/tag.go @@ -17,6 +17,11 @@ type TagHandler struct { tagService interfaces.KnowledgeTagService } +// DeleteTagRequest represents the request body for deleting a tag +type DeleteTagRequest struct { + ExcludeIDs []string `json:"exclude_ids"` // Chunk IDs to exclude from deletion +} + // NewTagHandler creates a new TagHandler. func NewTagHandler(tagService interfaces.KnowledgeTagService) *TagHandler { return &TagHandler{tagService: tagService} @@ -161,10 +166,11 @@ func (h *TagHandler) UpdateTag(c *gin.Context) { // @Tags 标签管理 // @Accept json // @Produce json -// @Param id path string true "知识库ID" -// @Param tag_id path string true "标签ID" -// @Param force query bool false "强制删除" -// @Param content_only query bool false "仅删除内容,保留标签" +// @Param id path string true "知识库ID" +// @Param tag_id path string true "标签ID" +// @Param force query bool false "强制删除" +// @Param content_only query bool false "仅删除内容,保留标签" +// @Param body body DeleteTagRequest false "删除选项" // @Success 200 {object} map[string]interface{} "删除成功" // @Failure 400 {object} errors.AppError "请求参数错误" // @Security Bearer @@ -177,7 +183,11 @@ func (h *TagHandler) DeleteTag(c *gin.Context) { force := c.Query("force") == "true" contentOnly := c.Query("content_only") == "true" - if err := h.tagService.DeleteTag(ctx, tagID, force, contentOnly); err != nil { + var req DeleteTagRequest + // Ignore bind error since body is optional + _ = c.ShouldBindJSON(&req) + + if err := h.tagService.DeleteTag(ctx, tagID, force, contentOnly, req.ExcludeIDs); err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{ "tag_id": tagID, }) diff --git a/internal/router/task.go b/internal/router/task.go index c04aa6de..1b925a9a 100644 --- a/internal/router/task.go +++ b/internal/router/task.go @@ -17,6 +17,7 @@ type AsynqTaskParams struct { Server *asynq.Server Extracter interfaces.Extracter KnowledgeService interfaces.KnowledgeService + TagService interfaces.KnowledgeTagService } func getAsynqRedisClientOpt() *asynq.RedisClientOpt { @@ -76,6 +77,9 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux { // Register KB clone handler mux.HandleFunc(types.TypeKBClone, params.KnowledgeService.ProcessKBClone) + // Register index delete handler + mux.HandleFunc(types.TypeIndexDelete, params.TagService.ProcessIndexDelete) + go func() { // Start the server if err := params.Server.Run(mux); err != nil { diff --git a/internal/types/extract_graph.go b/internal/types/extract_graph.go index 082967f2..49fb4580 100644 --- a/internal/types/extract_graph.go +++ b/internal/types/extract_graph.go @@ -7,6 +7,7 @@ const ( TypeQuestionGeneration = "question:generation" // 问题生成任务 TypeSummaryGeneration = "summary:generation" // 摘要生成任务 TypeKBClone = "kb:clone" // 知识库复制任务 + TypeIndexDelete = "index:delete" // 索引删除任务 ) // ExtractChunkPayload represents the extract chunk task payload @@ -65,6 +66,16 @@ type KBClonePayload struct { TargetID string `json:"target_id"` } +// IndexDeletePayload represents the index delete task payload +type IndexDeletePayload struct { + TenantID uint64 `json:"tenant_id"` + KnowledgeBaseID string `json:"knowledge_base_id"` + EmbeddingModelID string `json:"embedding_model_id"` + KBType string `json:"kb_type"` + ChunkIDs []string `json:"chunk_ids"` + EffectiveEngines []RetrieverEngineParams `json:"effective_engines"` +} + // KBCloneTaskStatus represents the status of a knowledge base clone task type KBCloneTaskStatus string diff --git a/internal/types/interfaces/chunk.go b/internal/types/interfaces/chunk.go index 475a3c0f..1e6d09bd 100644 --- a/internal/types/interfaces/chunk.go +++ b/internal/types/interfaces/chunk.go @@ -45,7 +45,8 @@ type ChunkRepository interface { // DeleteByKnowledgeList deletes all chunks for a knowledge list DeleteByKnowledgeList(ctx context.Context, tenantID uint64, knowledgeIDs []string) error // DeleteChunksByTagID deletes all chunks with the specified tag ID - DeleteChunksByTagID(ctx context.Context, tenantID uint64, kbID string, tagID string) error + // Returns the IDs of deleted chunks for index cleanup + DeleteChunksByTagID(ctx context.Context, tenantID uint64, kbID string, tagID string, excludeIDs []string) ([]string, error) // CountChunksByKnowledgeBaseID counts the number of chunks in a knowledge base. CountChunksByKnowledgeBaseID(ctx context.Context, tenantID uint64, kbID string) (int64, error) // DeleteUnindexedChunks deletes unindexed chunks by knowledge id and chunk index range diff --git a/internal/types/interfaces/tag.go b/internal/types/interfaces/tag.go index 9f6e0411..fd578bba 100644 --- a/internal/types/interfaces/tag.go +++ b/internal/types/interfaces/tag.go @@ -4,6 +4,7 @@ import ( "context" "github.com/Tencent/WeKnora/internal/types" + "github.com/hibiken/asynq" ) // KnowledgeTagService defines operations on knowledge base scoped tags. @@ -16,9 +17,12 @@ type KnowledgeTagService interface { UpdateTag(ctx context.Context, id string, name *string, color *string, sortOrder *int) (*types.KnowledgeTag, error) // DeleteTag deletes a tag. // When contentOnly=true, only deletes the content under the tag but keeps the tag itself. - DeleteTag(ctx context.Context, id string, force bool, contentOnly bool) error + // excludeIDs: IDs of chunks to exclude from deletion (only valid when deleting chunks) + DeleteTag(ctx context.Context, id string, force bool, contentOnly bool, excludeIDs []string) error // FindOrCreateTagByName finds a tag by name or creates it if not exists. FindOrCreateTagByName(ctx context.Context, kbID string, name string) (*types.KnowledgeTag, error) + // ProcessIndexDelete handles async index deletion task + ProcessIndexDelete(ctx context.Context, t *asynq.Task) error } // KnowledgeTagRepository defines persistence operations for tags.