feat: 增强标签删除功能,支持排除指定ID;新增异步索引删除任务优化删除流程

This commit is contained in:
wizardchen
2025-12-25 15:35:57 +08:00
committed by lyingbug
parent 87ead9dec4
commit 4c0846f9c7
8 changed files with 221 additions and 28 deletions
+10 -2
View File
@@ -140,8 +140,9 @@ func (c *Client) UpdateTag(ctx context.Context,
// DeleteTag deletes a tag.
// Set force to true to delete even if the tag is referenced.
// Set contentOnly to true to only delete the content under the tag but keep the tag itself.
// excludeIDs: IDs of chunks to exclude from deletion.
func (c *Client) DeleteTag(ctx context.Context,
knowledgeBaseID, tagID string, force bool, contentOnly bool,
knowledgeBaseID, tagID string, force bool, contentOnly bool, excludeIDs []string,
) error {
path := fmt.Sprintf("/api/v1/knowledge-bases/%s/tags/%s", knowledgeBaseID, tagID)
query := url.Values{}
@@ -152,7 +153,14 @@ func (c *Client) DeleteTag(ctx context.Context,
query.Add("content_only", "true")
}
resp, err := c.doRequest(ctx, http.MethodDelete, path, nil, query)
var body interface{}
if len(excludeIDs) > 0 {
body = map[string]interface{}{
"exclude_ids": excludeIDs,
}
}
resp, err := c.doRequest(ctx, http.MethodDelete, path, body, query)
if err != nil {
return err
}
+44 -4
View File
@@ -313,10 +313,50 @@ func (r *chunkRepository) DeleteByKnowledgeList(ctx context.Context, tenantID ui
}
// DeleteChunksByTagID deletes all chunks with the specified tag ID
func (r *chunkRepository) DeleteChunksByTagID(ctx context.Context, tenantID uint64, kbID string, tagID string) error {
return r.db.WithContext(ctx).Where(
"tenant_id = ? AND knowledge_base_id = ? AND tag_id = ?", tenantID, kbID, tagID,
).Delete(&types.Chunk{}).Error
// Returns the IDs of deleted chunks for index cleanup
func (r *chunkRepository) DeleteChunksByTagID(ctx context.Context, tenantID uint64, kbID string, tagID string, excludeIDs []string) ([]string, error) {
// Build exclude set for O(1) lookup
excludeSet := make(map[string]struct{}, len(excludeIDs))
for _, id := range excludeIDs {
excludeSet[id] = struct{}{}
}
// Get all chunk IDs for this tag
var allIDs []string
if err := r.db.WithContext(ctx).Model(&types.Chunk{}).
Where("tenant_id = ? AND knowledge_base_id = ? AND tag_id = ?", tenantID, kbID, tagID).
Pluck("id", &allIDs).Error; err != nil {
return nil, err
}
// Filter out excluded IDs
toDelete := make([]string, 0, len(allIDs))
for _, id := range allIDs {
if _, excluded := excludeSet[id]; !excluded {
toDelete = append(toDelete, id)
}
}
if len(toDelete) == 0 {
return nil, nil
}
// Delete in batches
const batchSize = 1000
for i := 0; i < len(toDelete); i += batchSize {
end := i + batchSize
if end > len(toDelete) {
end = len(toDelete)
}
batch := toDelete[i:end]
if err := r.db.WithContext(ctx).Where("id IN ?", batch).Delete(&types.Chunk{}).Error; err != nil {
// Return already planned deletions up to this point for index cleanup
return toDelete[:i], err
}
}
return toDelete, nil
}
// CountChunksByKnowledgeBaseID counts the number of chunks in a knowledge base
+130 -15
View File
@@ -2,23 +2,30 @@ package service
import (
"context"
"encoding/json"
"errors"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/application/service/retriever"
werrors "github.com/Tencent/WeKnora/internal/errors"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/Tencent/WeKnora/internal/utils"
"github.com/google/uuid"
"github.com/hibiken/asynq"
"gorm.io/gorm"
)
// knowledgeTagService implements KnowledgeTagService.
type knowledgeTagService struct {
kbService interfaces.KnowledgeBaseService
repo interfaces.KnowledgeTagRepository
chunkRepo interfaces.ChunkRepository
kbService interfaces.KnowledgeBaseService
repo interfaces.KnowledgeTagRepository
chunkRepo interfaces.ChunkRepository
retrieveEngine interfaces.RetrieveEngineRegistry
modelService interfaces.ModelService
task *asynq.Client
}
// NewKnowledgeTagService creates a new tag service.
@@ -26,11 +33,17 @@ func NewKnowledgeTagService(
kbService interfaces.KnowledgeBaseService,
repo interfaces.KnowledgeTagRepository,
chunkRepo interfaces.ChunkRepository,
retrieveEngine interfaces.RetrieveEngineRegistry,
modelService interfaces.ModelService,
task *asynq.Client,
) (interfaces.KnowledgeTagService, error) {
return &knowledgeTagService{
kbService: kbService,
repo: repo,
chunkRepo: chunkRepo,
kbService: kbService,
repo: repo,
chunkRepo: chunkRepo,
retrieveEngine: retrieveEngine,
modelService: modelService,
task: task,
}, nil
}
@@ -154,7 +167,7 @@ func (s *knowledgeTagService) UpdateTag(
// DeleteTag deletes a tag. When force=true, also deletes all chunks under this tag.
// When contentOnly=true, only deletes the content under the tag but keeps the tag itself.
func (s *knowledgeTagService) DeleteTag(ctx context.Context, id string, force bool, contentOnly bool) error {
func (s *knowledgeTagService) DeleteTag(ctx context.Context, id string, force bool, contentOnly bool, excludeIDs []string) error {
if id == "" {
return werrors.NewBadRequestError("标签ID不能为空")
}
@@ -163,19 +176,45 @@ func (s *knowledgeTagService) DeleteTag(ctx context.Context, id string, force bo
if err != nil {
return err
}
// Get KB info for embedding model
kb, err := s.kbService.GetKnowledgeBaseByID(ctx, tag.KnowledgeBaseID)
if err != nil {
return err
}
kCount, cCount, err := s.repo.CountReferences(ctx, tenantID, tag.KnowledgeBaseID, tag.ID)
if err != nil {
return err
}
// Get tenant info for effective engines
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
// Helper function to delete chunks and enqueue index deletion task
deleteChunksAndEnqueueIndexDelete := func() error {
// Delete chunks and get their IDs
deletedIDs, err := s.chunkRepo.DeleteChunksByTagID(ctx, tenantID, tag.KnowledgeBaseID, tag.ID, excludeIDs)
if err != nil {
logger.Errorf(ctx, "Failed to delete chunks by tag ID %s: %v", tag.ID, err)
return werrors.NewInternalServerError("删除标签下的数据失败")
}
// Enqueue async index deletion task for the deleted chunks
if len(deletedIDs) > 0 {
s.enqueueIndexDeleteTask(ctx, tenantID, kb.ID, kb.EmbeddingModelID, string(kb.Type), deletedIDs, tenantInfo.GetEffectiveEngines())
}
logger.Infof(ctx, "Deleted %d chunks under tag %s", len(deletedIDs), tag.ID)
return nil
}
// contentOnly mode: only delete content, keep the tag
if contentOnly {
if cCount > 0 {
if err := s.chunkRepo.DeleteChunksByTagID(ctx, tenantID, tag.KnowledgeBaseID, tag.ID); err != nil {
logger.Errorf(ctx, "Failed to delete chunks by tag ID %s: %v", tag.ID, err)
return werrors.NewInternalServerError("删除标签下的数据失败")
if err := deleteChunksAndEnqueueIndexDelete(); err != nil {
return err
}
logger.Infof(ctx, "Deleted %d chunks under tag %s (content_only mode)", cCount, tag.ID)
}
return nil
}
@@ -185,15 +224,91 @@ func (s *knowledgeTagService) DeleteTag(ctx context.Context, id string, force bo
}
// When force=true, delete all chunks under this tag first
if force && cCount > 0 {
if err := s.chunkRepo.DeleteChunksByTagID(ctx, tenantID, tag.KnowledgeBaseID, tag.ID); err != nil {
logger.Errorf(ctx, "Failed to delete chunks by tag ID %s: %v", tag.ID, err)
return werrors.NewInternalServerError("删除标签下的数据失败")
if err := deleteChunksAndEnqueueIndexDelete(); err != nil {
return err
}
logger.Infof(ctx, "Deleted %d chunks under tag %s", cCount, tag.ID)
}
// If there are excludeIDs, we cannot delete the tag itself as it still has content
if len(excludeIDs) > 0 {
return nil
}
return s.repo.Delete(ctx, tenantID, id)
}
// enqueueIndexDeleteTask enqueues an async task for index deletion (low priority)
func (s *knowledgeTagService) enqueueIndexDeleteTask(ctx context.Context,
tenantID uint64, kbID, embeddingModelID, kbType string, chunkIDs []string, effectiveEngines []types.RetrieverEngineParams,
) {
payload := types.IndexDeletePayload{
TenantID: tenantID,
KnowledgeBaseID: kbID,
EmbeddingModelID: embeddingModelID,
KBType: kbType,
ChunkIDs: chunkIDs,
EffectiveEngines: effectiveEngines,
}
payloadBytes, err := json.Marshal(payload)
if err != nil {
logger.Errorf(ctx, "Failed to marshal index delete payload: %v", err)
return
}
task := asynq.NewTask(types.TypeIndexDelete, payloadBytes, asynq.Queue(utils.GetAsynqQueueName("low")), asynq.MaxRetry(10))
info, err := s.task.Enqueue(task)
if err != nil {
logger.Errorf(ctx, "Failed to enqueue index delete task: %v", err)
return
}
logger.Infof(ctx, "Enqueued index delete task: %s for %d chunks", info.ID, len(chunkIDs))
}
// ProcessIndexDelete handles async index deletion task
func (s *knowledgeTagService) ProcessIndexDelete(ctx context.Context, t *asynq.Task) error {
var payload types.IndexDeletePayload
if err := json.Unmarshal(t.Payload(), &payload); err != nil {
logger.Errorf(ctx, "Failed to unmarshal index delete payload: %v", err)
return err
}
logger.Infof(ctx, "Processing index delete task for %d chunks in KB %s", len(payload.ChunkIDs), payload.KnowledgeBaseID)
// Create retrieve engine
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, payload.EffectiveEngines)
if err != nil {
logger.Warnf(ctx, "Failed to create retrieve engine for index cleanup: %v", err)
return err
}
// Get embedding model dimensions
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, payload.EmbeddingModelID)
if err != nil {
logger.Warnf(ctx, "Failed to get embedding model for index cleanup: %v", err)
return err
}
// Delete indices in batches to avoid overwhelming the backend
const batchSize = 100
chunkIDs := payload.ChunkIDs
dimension := embeddingModel.GetDimensions()
for i := 0; i < len(chunkIDs); i += batchSize {
end := i + batchSize
if end > len(chunkIDs) {
end = len(chunkIDs)
}
batch := chunkIDs[i:end]
if err := retrieveEngine.DeleteByChunkIDList(ctx, batch, dimension, payload.KBType); err != nil {
logger.Warnf(ctx, "Failed to delete indices for chunks batch [%d-%d]: %v", i, end, err)
return err
}
logger.Debugf(ctx, "Deleted indices batch [%d-%d] of %d chunks", i, end, len(chunkIDs))
}
logger.Infof(ctx, "Successfully deleted indices for %d chunks", len(payload.ChunkIDs))
return nil
}
// FindOrCreateTagByName finds a tag by name or creates it if not exists.
func (s *knowledgeTagService) FindOrCreateTagByName(ctx context.Context, kbID string, name string) (*types.KnowledgeTag, error) {
name = strings.TrimSpace(name)
+15 -5
View File
@@ -17,6 +17,11 @@ type TagHandler struct {
tagService interfaces.KnowledgeTagService
}
// DeleteTagRequest represents the request body for deleting a tag
type DeleteTagRequest struct {
ExcludeIDs []string `json:"exclude_ids"` // Chunk IDs to exclude from deletion
}
// NewTagHandler creates a new TagHandler.
func NewTagHandler(tagService interfaces.KnowledgeTagService) *TagHandler {
return &TagHandler{tagService: tagService}
@@ -161,10 +166,11 @@ func (h *TagHandler) UpdateTag(c *gin.Context) {
// @Tags 标签管理
// @Accept json
// @Produce json
// @Param id path string true "知识库ID"
// @Param tag_id path string true "标签ID"
// @Param force query bool false "强制删除"
// @Param content_only query bool false "仅删除内容,保留标签"
// @Param id path string true "知识库ID"
// @Param tag_id path string true "标签ID"
// @Param force query bool false "强制删除"
// @Param content_only query bool false "仅删除内容,保留标签"
// @Param body body DeleteTagRequest false "删除选项"
// @Success 200 {object} map[string]interface{} "删除成功"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Security Bearer
@@ -177,7 +183,11 @@ func (h *TagHandler) DeleteTag(c *gin.Context) {
force := c.Query("force") == "true"
contentOnly := c.Query("content_only") == "true"
if err := h.tagService.DeleteTag(ctx, tagID, force, contentOnly); err != nil {
var req DeleteTagRequest
// Ignore bind error since body is optional
_ = c.ShouldBindJSON(&req)
if err := h.tagService.DeleteTag(ctx, tagID, force, contentOnly, req.ExcludeIDs); err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{
"tag_id": tagID,
})
+4
View File
@@ -17,6 +17,7 @@ type AsynqTaskParams struct {
Server *asynq.Server
Extracter interfaces.Extracter
KnowledgeService interfaces.KnowledgeService
TagService interfaces.KnowledgeTagService
}
func getAsynqRedisClientOpt() *asynq.RedisClientOpt {
@@ -76,6 +77,9 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux {
// Register KB clone handler
mux.HandleFunc(types.TypeKBClone, params.KnowledgeService.ProcessKBClone)
// Register index delete handler
mux.HandleFunc(types.TypeIndexDelete, params.TagService.ProcessIndexDelete)
go func() {
// Start the server
if err := params.Server.Run(mux); err != nil {
+11
View File
@@ -7,6 +7,7 @@ const (
TypeQuestionGeneration = "question:generation" // 问题生成任务
TypeSummaryGeneration = "summary:generation" // 摘要生成任务
TypeKBClone = "kb:clone" // 知识库复制任务
TypeIndexDelete = "index:delete" // 索引删除任务
)
// ExtractChunkPayload represents the extract chunk task payload
@@ -65,6 +66,16 @@ type KBClonePayload struct {
TargetID string `json:"target_id"`
}
// IndexDeletePayload represents the index delete task payload
type IndexDeletePayload struct {
TenantID uint64 `json:"tenant_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
EmbeddingModelID string `json:"embedding_model_id"`
KBType string `json:"kb_type"`
ChunkIDs []string `json:"chunk_ids"`
EffectiveEngines []RetrieverEngineParams `json:"effective_engines"`
}
// KBCloneTaskStatus represents the status of a knowledge base clone task
type KBCloneTaskStatus string
+2 -1
View File
@@ -45,7 +45,8 @@ type ChunkRepository interface {
// DeleteByKnowledgeList deletes all chunks for a knowledge list
DeleteByKnowledgeList(ctx context.Context, tenantID uint64, knowledgeIDs []string) error
// DeleteChunksByTagID deletes all chunks with the specified tag ID
DeleteChunksByTagID(ctx context.Context, tenantID uint64, kbID string, tagID string) error
// Returns the IDs of deleted chunks for index cleanup
DeleteChunksByTagID(ctx context.Context, tenantID uint64, kbID string, tagID string, excludeIDs []string) ([]string, error)
// CountChunksByKnowledgeBaseID counts the number of chunks in a knowledge base.
CountChunksByKnowledgeBaseID(ctx context.Context, tenantID uint64, kbID string) (int64, error)
// DeleteUnindexedChunks deletes unindexed chunks by knowledge id and chunk index range
+5 -1
View File
@@ -4,6 +4,7 @@ import (
"context"
"github.com/Tencent/WeKnora/internal/types"
"github.com/hibiken/asynq"
)
// KnowledgeTagService defines operations on knowledge base scoped tags.
@@ -16,9 +17,12 @@ type KnowledgeTagService interface {
UpdateTag(ctx context.Context, id string, name *string, color *string, sortOrder *int) (*types.KnowledgeTag, error)
// DeleteTag deletes a tag.
// When contentOnly=true, only deletes the content under the tag but keeps the tag itself.
DeleteTag(ctx context.Context, id string, force bool, contentOnly bool) error
// excludeIDs: IDs of chunks to exclude from deletion (only valid when deleting chunks)
DeleteTag(ctx context.Context, id string, force bool, contentOnly bool, excludeIDs []string) error
// FindOrCreateTagByName finds a tag by name or creates it if not exists.
FindOrCreateTagByName(ctx context.Context, kbID string, name string) (*types.KnowledgeTag, error)
// ProcessIndexDelete handles async index deletion task
ProcessIndexDelete(ctx context.Context, t *asynq.Task) error
}
// KnowledgeTagRepository defines persistence operations for tags.