feat: 增强标签删除功能,支持排除指定ID;新增异步索引删除任务优化删除流程
This commit is contained in:
+10
-2
@@ -140,8 +140,9 @@ func (c *Client) UpdateTag(ctx context.Context,
|
||||
// DeleteTag deletes a tag.
|
||||
// Set force to true to delete even if the tag is referenced.
|
||||
// Set contentOnly to true to only delete the content under the tag but keep the tag itself.
|
||||
// excludeIDs: IDs of chunks to exclude from deletion.
|
||||
func (c *Client) DeleteTag(ctx context.Context,
|
||||
knowledgeBaseID, tagID string, force bool, contentOnly bool,
|
||||
knowledgeBaseID, tagID string, force bool, contentOnly bool, excludeIDs []string,
|
||||
) error {
|
||||
path := fmt.Sprintf("/api/v1/knowledge-bases/%s/tags/%s", knowledgeBaseID, tagID)
|
||||
query := url.Values{}
|
||||
@@ -152,7 +153,14 @@ func (c *Client) DeleteTag(ctx context.Context,
|
||||
query.Add("content_only", "true")
|
||||
}
|
||||
|
||||
resp, err := c.doRequest(ctx, http.MethodDelete, path, nil, query)
|
||||
var body interface{}
|
||||
if len(excludeIDs) > 0 {
|
||||
body = map[string]interface{}{
|
||||
"exclude_ids": excludeIDs,
|
||||
}
|
||||
}
|
||||
|
||||
resp, err := c.doRequest(ctx, http.MethodDelete, path, body, query)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -313,10 +313,50 @@ func (r *chunkRepository) DeleteByKnowledgeList(ctx context.Context, tenantID ui
|
||||
}
|
||||
|
||||
// DeleteChunksByTagID deletes all chunks with the specified tag ID
|
||||
func (r *chunkRepository) DeleteChunksByTagID(ctx context.Context, tenantID uint64, kbID string, tagID string) error {
|
||||
return r.db.WithContext(ctx).Where(
|
||||
"tenant_id = ? AND knowledge_base_id = ? AND tag_id = ?", tenantID, kbID, tagID,
|
||||
).Delete(&types.Chunk{}).Error
|
||||
// Returns the IDs of deleted chunks for index cleanup
|
||||
func (r *chunkRepository) DeleteChunksByTagID(ctx context.Context, tenantID uint64, kbID string, tagID string, excludeIDs []string) ([]string, error) {
|
||||
// Build exclude set for O(1) lookup
|
||||
excludeSet := make(map[string]struct{}, len(excludeIDs))
|
||||
for _, id := range excludeIDs {
|
||||
excludeSet[id] = struct{}{}
|
||||
}
|
||||
|
||||
// Get all chunk IDs for this tag
|
||||
var allIDs []string
|
||||
if err := r.db.WithContext(ctx).Model(&types.Chunk{}).
|
||||
Where("tenant_id = ? AND knowledge_base_id = ? AND tag_id = ?", tenantID, kbID, tagID).
|
||||
Pluck("id", &allIDs).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Filter out excluded IDs
|
||||
toDelete := make([]string, 0, len(allIDs))
|
||||
for _, id := range allIDs {
|
||||
if _, excluded := excludeSet[id]; !excluded {
|
||||
toDelete = append(toDelete, id)
|
||||
}
|
||||
}
|
||||
|
||||
if len(toDelete) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Delete in batches
|
||||
const batchSize = 1000
|
||||
for i := 0; i < len(toDelete); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(toDelete) {
|
||||
end = len(toDelete)
|
||||
}
|
||||
batch := toDelete[i:end]
|
||||
|
||||
if err := r.db.WithContext(ctx).Where("id IN ?", batch).Delete(&types.Chunk{}).Error; err != nil {
|
||||
// Return already planned deletions up to this point for index cleanup
|
||||
return toDelete[:i], err
|
||||
}
|
||||
}
|
||||
|
||||
return toDelete, nil
|
||||
}
|
||||
|
||||
// CountChunksByKnowledgeBaseID counts the number of chunks in a knowledge base
|
||||
|
||||
@@ -2,23 +2,30 @@ package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/application/service/retriever"
|
||||
werrors "github.com/Tencent/WeKnora/internal/errors"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/google/uuid"
|
||||
"github.com/hibiken/asynq"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// knowledgeTagService implements KnowledgeTagService.
|
||||
type knowledgeTagService struct {
|
||||
kbService interfaces.KnowledgeBaseService
|
||||
repo interfaces.KnowledgeTagRepository
|
||||
chunkRepo interfaces.ChunkRepository
|
||||
kbService interfaces.KnowledgeBaseService
|
||||
repo interfaces.KnowledgeTagRepository
|
||||
chunkRepo interfaces.ChunkRepository
|
||||
retrieveEngine interfaces.RetrieveEngineRegistry
|
||||
modelService interfaces.ModelService
|
||||
task *asynq.Client
|
||||
}
|
||||
|
||||
// NewKnowledgeTagService creates a new tag service.
|
||||
@@ -26,11 +33,17 @@ func NewKnowledgeTagService(
|
||||
kbService interfaces.KnowledgeBaseService,
|
||||
repo interfaces.KnowledgeTagRepository,
|
||||
chunkRepo interfaces.ChunkRepository,
|
||||
retrieveEngine interfaces.RetrieveEngineRegistry,
|
||||
modelService interfaces.ModelService,
|
||||
task *asynq.Client,
|
||||
) (interfaces.KnowledgeTagService, error) {
|
||||
return &knowledgeTagService{
|
||||
kbService: kbService,
|
||||
repo: repo,
|
||||
chunkRepo: chunkRepo,
|
||||
kbService: kbService,
|
||||
repo: repo,
|
||||
chunkRepo: chunkRepo,
|
||||
retrieveEngine: retrieveEngine,
|
||||
modelService: modelService,
|
||||
task: task,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -154,7 +167,7 @@ func (s *knowledgeTagService) UpdateTag(
|
||||
|
||||
// DeleteTag deletes a tag. When force=true, also deletes all chunks under this tag.
|
||||
// When contentOnly=true, only deletes the content under the tag but keeps the tag itself.
|
||||
func (s *knowledgeTagService) DeleteTag(ctx context.Context, id string, force bool, contentOnly bool) error {
|
||||
func (s *knowledgeTagService) DeleteTag(ctx context.Context, id string, force bool, contentOnly bool, excludeIDs []string) error {
|
||||
if id == "" {
|
||||
return werrors.NewBadRequestError("标签ID不能为空")
|
||||
}
|
||||
@@ -163,19 +176,45 @@ func (s *knowledgeTagService) DeleteTag(ctx context.Context, id string, force bo
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Get KB info for embedding model
|
||||
kb, err := s.kbService.GetKnowledgeBaseByID(ctx, tag.KnowledgeBaseID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
kCount, cCount, err := s.repo.CountReferences(ctx, tenantID, tag.KnowledgeBaseID, tag.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Get tenant info for effective engines
|
||||
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
|
||||
// Helper function to delete chunks and enqueue index deletion task
|
||||
deleteChunksAndEnqueueIndexDelete := func() error {
|
||||
// Delete chunks and get their IDs
|
||||
deletedIDs, err := s.chunkRepo.DeleteChunksByTagID(ctx, tenantID, tag.KnowledgeBaseID, tag.ID, excludeIDs)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to delete chunks by tag ID %s: %v", tag.ID, err)
|
||||
return werrors.NewInternalServerError("删除标签下的数据失败")
|
||||
}
|
||||
|
||||
// Enqueue async index deletion task for the deleted chunks
|
||||
if len(deletedIDs) > 0 {
|
||||
s.enqueueIndexDeleteTask(ctx, tenantID, kb.ID, kb.EmbeddingModelID, string(kb.Type), deletedIDs, tenantInfo.GetEffectiveEngines())
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Deleted %d chunks under tag %s", len(deletedIDs), tag.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// contentOnly mode: only delete content, keep the tag
|
||||
if contentOnly {
|
||||
if cCount > 0 {
|
||||
if err := s.chunkRepo.DeleteChunksByTagID(ctx, tenantID, tag.KnowledgeBaseID, tag.ID); err != nil {
|
||||
logger.Errorf(ctx, "Failed to delete chunks by tag ID %s: %v", tag.ID, err)
|
||||
return werrors.NewInternalServerError("删除标签下的数据失败")
|
||||
if err := deleteChunksAndEnqueueIndexDelete(); err != nil {
|
||||
return err
|
||||
}
|
||||
logger.Infof(ctx, "Deleted %d chunks under tag %s (content_only mode)", cCount, tag.ID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -185,15 +224,91 @@ func (s *knowledgeTagService) DeleteTag(ctx context.Context, id string, force bo
|
||||
}
|
||||
// When force=true, delete all chunks under this tag first
|
||||
if force && cCount > 0 {
|
||||
if err := s.chunkRepo.DeleteChunksByTagID(ctx, tenantID, tag.KnowledgeBaseID, tag.ID); err != nil {
|
||||
logger.Errorf(ctx, "Failed to delete chunks by tag ID %s: %v", tag.ID, err)
|
||||
return werrors.NewInternalServerError("删除标签下的数据失败")
|
||||
if err := deleteChunksAndEnqueueIndexDelete(); err != nil {
|
||||
return err
|
||||
}
|
||||
logger.Infof(ctx, "Deleted %d chunks under tag %s", cCount, tag.ID)
|
||||
}
|
||||
// If there are excludeIDs, we cannot delete the tag itself as it still has content
|
||||
if len(excludeIDs) > 0 {
|
||||
return nil
|
||||
}
|
||||
return s.repo.Delete(ctx, tenantID, id)
|
||||
}
|
||||
|
||||
// enqueueIndexDeleteTask enqueues an async task for index deletion (low priority)
|
||||
func (s *knowledgeTagService) enqueueIndexDeleteTask(ctx context.Context,
|
||||
tenantID uint64, kbID, embeddingModelID, kbType string, chunkIDs []string, effectiveEngines []types.RetrieverEngineParams,
|
||||
) {
|
||||
payload := types.IndexDeletePayload{
|
||||
TenantID: tenantID,
|
||||
KnowledgeBaseID: kbID,
|
||||
EmbeddingModelID: embeddingModelID,
|
||||
KBType: kbType,
|
||||
ChunkIDs: chunkIDs,
|
||||
EffectiveEngines: effectiveEngines,
|
||||
}
|
||||
payloadBytes, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to marshal index delete payload: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
task := asynq.NewTask(types.TypeIndexDelete, payloadBytes, asynq.Queue(utils.GetAsynqQueueName("low")), asynq.MaxRetry(10))
|
||||
info, err := s.task.Enqueue(task)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to enqueue index delete task: %v", err)
|
||||
return
|
||||
}
|
||||
logger.Infof(ctx, "Enqueued index delete task: %s for %d chunks", info.ID, len(chunkIDs))
|
||||
}
|
||||
|
||||
// ProcessIndexDelete handles async index deletion task
|
||||
func (s *knowledgeTagService) ProcessIndexDelete(ctx context.Context, t *asynq.Task) error {
|
||||
var payload types.IndexDeletePayload
|
||||
if err := json.Unmarshal(t.Payload(), &payload); err != nil {
|
||||
logger.Errorf(ctx, "Failed to unmarshal index delete payload: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Processing index delete task for %d chunks in KB %s", len(payload.ChunkIDs), payload.KnowledgeBaseID)
|
||||
|
||||
// Create retrieve engine
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, payload.EffectiveEngines)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "Failed to create retrieve engine for index cleanup: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Get embedding model dimensions
|
||||
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, payload.EmbeddingModelID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "Failed to get embedding model for index cleanup: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Delete indices in batches to avoid overwhelming the backend
|
||||
const batchSize = 100
|
||||
chunkIDs := payload.ChunkIDs
|
||||
dimension := embeddingModel.GetDimensions()
|
||||
|
||||
for i := 0; i < len(chunkIDs); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(chunkIDs) {
|
||||
end = len(chunkIDs)
|
||||
}
|
||||
batch := chunkIDs[i:end]
|
||||
|
||||
if err := retrieveEngine.DeleteByChunkIDList(ctx, batch, dimension, payload.KBType); err != nil {
|
||||
logger.Warnf(ctx, "Failed to delete indices for chunks batch [%d-%d]: %v", i, end, err)
|
||||
return err
|
||||
}
|
||||
logger.Debugf(ctx, "Deleted indices batch [%d-%d] of %d chunks", i, end, len(chunkIDs))
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Successfully deleted indices for %d chunks", len(payload.ChunkIDs))
|
||||
return nil
|
||||
}
|
||||
|
||||
// FindOrCreateTagByName finds a tag by name or creates it if not exists.
|
||||
func (s *knowledgeTagService) FindOrCreateTagByName(ctx context.Context, kbID string, name string) (*types.KnowledgeTag, error) {
|
||||
name = strings.TrimSpace(name)
|
||||
|
||||
+15
-5
@@ -17,6 +17,11 @@ type TagHandler struct {
|
||||
tagService interfaces.KnowledgeTagService
|
||||
}
|
||||
|
||||
// DeleteTagRequest represents the request body for deleting a tag
|
||||
type DeleteTagRequest struct {
|
||||
ExcludeIDs []string `json:"exclude_ids"` // Chunk IDs to exclude from deletion
|
||||
}
|
||||
|
||||
// NewTagHandler creates a new TagHandler.
|
||||
func NewTagHandler(tagService interfaces.KnowledgeTagService) *TagHandler {
|
||||
return &TagHandler{tagService: tagService}
|
||||
@@ -161,10 +166,11 @@ func (h *TagHandler) UpdateTag(c *gin.Context) {
|
||||
// @Tags 标签管理
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param id path string true "知识库ID"
|
||||
// @Param tag_id path string true "标签ID"
|
||||
// @Param force query bool false "强制删除"
|
||||
// @Param content_only query bool false "仅删除内容,保留标签"
|
||||
// @Param id path string true "知识库ID"
|
||||
// @Param tag_id path string true "标签ID"
|
||||
// @Param force query bool false "强制删除"
|
||||
// @Param content_only query bool false "仅删除内容,保留标签"
|
||||
// @Param body body DeleteTagRequest false "删除选项"
|
||||
// @Success 200 {object} map[string]interface{} "删除成功"
|
||||
// @Failure 400 {object} errors.AppError "请求参数错误"
|
||||
// @Security Bearer
|
||||
@@ -177,7 +183,11 @@ func (h *TagHandler) DeleteTag(c *gin.Context) {
|
||||
force := c.Query("force") == "true"
|
||||
contentOnly := c.Query("content_only") == "true"
|
||||
|
||||
if err := h.tagService.DeleteTag(ctx, tagID, force, contentOnly); err != nil {
|
||||
var req DeleteTagRequest
|
||||
// Ignore bind error since body is optional
|
||||
_ = c.ShouldBindJSON(&req)
|
||||
|
||||
if err := h.tagService.DeleteTag(ctx, tagID, force, contentOnly, req.ExcludeIDs); err != nil {
|
||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{
|
||||
"tag_id": tagID,
|
||||
})
|
||||
|
||||
@@ -17,6 +17,7 @@ type AsynqTaskParams struct {
|
||||
Server *asynq.Server
|
||||
Extracter interfaces.Extracter
|
||||
KnowledgeService interfaces.KnowledgeService
|
||||
TagService interfaces.KnowledgeTagService
|
||||
}
|
||||
|
||||
func getAsynqRedisClientOpt() *asynq.RedisClientOpt {
|
||||
@@ -76,6 +77,9 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux {
|
||||
// Register KB clone handler
|
||||
mux.HandleFunc(types.TypeKBClone, params.KnowledgeService.ProcessKBClone)
|
||||
|
||||
// Register index delete handler
|
||||
mux.HandleFunc(types.TypeIndexDelete, params.TagService.ProcessIndexDelete)
|
||||
|
||||
go func() {
|
||||
// Start the server
|
||||
if err := params.Server.Run(mux); err != nil {
|
||||
|
||||
@@ -7,6 +7,7 @@ const (
|
||||
TypeQuestionGeneration = "question:generation" // 问题生成任务
|
||||
TypeSummaryGeneration = "summary:generation" // 摘要生成任务
|
||||
TypeKBClone = "kb:clone" // 知识库复制任务
|
||||
TypeIndexDelete = "index:delete" // 索引删除任务
|
||||
)
|
||||
|
||||
// ExtractChunkPayload represents the extract chunk task payload
|
||||
@@ -65,6 +66,16 @@ type KBClonePayload struct {
|
||||
TargetID string `json:"target_id"`
|
||||
}
|
||||
|
||||
// IndexDeletePayload represents the index delete task payload
|
||||
type IndexDeletePayload struct {
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||||
EmbeddingModelID string `json:"embedding_model_id"`
|
||||
KBType string `json:"kb_type"`
|
||||
ChunkIDs []string `json:"chunk_ids"`
|
||||
EffectiveEngines []RetrieverEngineParams `json:"effective_engines"`
|
||||
}
|
||||
|
||||
// KBCloneTaskStatus represents the status of a knowledge base clone task
|
||||
type KBCloneTaskStatus string
|
||||
|
||||
|
||||
@@ -45,7 +45,8 @@ type ChunkRepository interface {
|
||||
// DeleteByKnowledgeList deletes all chunks for a knowledge list
|
||||
DeleteByKnowledgeList(ctx context.Context, tenantID uint64, knowledgeIDs []string) error
|
||||
// DeleteChunksByTagID deletes all chunks with the specified tag ID
|
||||
DeleteChunksByTagID(ctx context.Context, tenantID uint64, kbID string, tagID string) error
|
||||
// Returns the IDs of deleted chunks for index cleanup
|
||||
DeleteChunksByTagID(ctx context.Context, tenantID uint64, kbID string, tagID string, excludeIDs []string) ([]string, error)
|
||||
// CountChunksByKnowledgeBaseID counts the number of chunks in a knowledge base.
|
||||
CountChunksByKnowledgeBaseID(ctx context.Context, tenantID uint64, kbID string) (int64, error)
|
||||
// DeleteUnindexedChunks deletes unindexed chunks by knowledge id and chunk index range
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/hibiken/asynq"
|
||||
)
|
||||
|
||||
// KnowledgeTagService defines operations on knowledge base scoped tags.
|
||||
@@ -16,9 +17,12 @@ type KnowledgeTagService interface {
|
||||
UpdateTag(ctx context.Context, id string, name *string, color *string, sortOrder *int) (*types.KnowledgeTag, error)
|
||||
// DeleteTag deletes a tag.
|
||||
// When contentOnly=true, only deletes the content under the tag but keeps the tag itself.
|
||||
DeleteTag(ctx context.Context, id string, force bool, contentOnly bool) error
|
||||
// excludeIDs: IDs of chunks to exclude from deletion (only valid when deleting chunks)
|
||||
DeleteTag(ctx context.Context, id string, force bool, contentOnly bool, excludeIDs []string) error
|
||||
// FindOrCreateTagByName finds a tag by name or creates it if not exists.
|
||||
FindOrCreateTagByName(ctx context.Context, kbID string, name string) (*types.KnowledgeTag, error)
|
||||
// ProcessIndexDelete handles async index deletion task
|
||||
ProcessIndexDelete(ctx context.Context, t *asynq.Task) error
|
||||
}
|
||||
|
||||
// KnowledgeTagRepository defines persistence operations for tags.
|
||||
|
||||
Reference in New Issue
Block a user