feat: 增强标签删除功能,支持排除指定ID;新增异步索引删除任务优化删除流程

This commit is contained in:
wizardchen
2025-12-25 15:35:57 +08:00
committed by lyingbug
parent 87ead9dec4
commit 4c0846f9c7
8 changed files with 221 additions and 28 deletions
+44 -4
View File
@@ -313,10 +313,50 @@ func (r *chunkRepository) DeleteByKnowledgeList(ctx context.Context, tenantID ui
}
// DeleteChunksByTagID deletes all chunks with the specified tag ID
func (r *chunkRepository) DeleteChunksByTagID(ctx context.Context, tenantID uint64, kbID string, tagID string) error {
return r.db.WithContext(ctx).Where(
"tenant_id = ? AND knowledge_base_id = ? AND tag_id = ?", tenantID, kbID, tagID,
).Delete(&types.Chunk{}).Error
// Returns the IDs of deleted chunks for index cleanup
func (r *chunkRepository) DeleteChunksByTagID(ctx context.Context, tenantID uint64, kbID string, tagID string, excludeIDs []string) ([]string, error) {
// Build exclude set for O(1) lookup
excludeSet := make(map[string]struct{}, len(excludeIDs))
for _, id := range excludeIDs {
excludeSet[id] = struct{}{}
}
// Get all chunk IDs for this tag
var allIDs []string
if err := r.db.WithContext(ctx).Model(&types.Chunk{}).
Where("tenant_id = ? AND knowledge_base_id = ? AND tag_id = ?", tenantID, kbID, tagID).
Pluck("id", &allIDs).Error; err != nil {
return nil, err
}
// Filter out excluded IDs
toDelete := make([]string, 0, len(allIDs))
for _, id := range allIDs {
if _, excluded := excludeSet[id]; !excluded {
toDelete = append(toDelete, id)
}
}
if len(toDelete) == 0 {
return nil, nil
}
// Delete in batches
const batchSize = 1000
for i := 0; i < len(toDelete); i += batchSize {
end := i + batchSize
if end > len(toDelete) {
end = len(toDelete)
}
batch := toDelete[i:end]
if err := r.db.WithContext(ctx).Where("id IN ?", batch).Delete(&types.Chunk{}).Error; err != nil {
// Return already planned deletions up to this point for index cleanup
return toDelete[:i], err
}
}
return toDelete, nil
}
// CountChunksByKnowledgeBaseID counts the number of chunks in a knowledge base
+130 -15
View File
@@ -2,23 +2,30 @@ package service
import (
"context"
"encoding/json"
"errors"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/application/service/retriever"
werrors "github.com/Tencent/WeKnora/internal/errors"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/Tencent/WeKnora/internal/utils"
"github.com/google/uuid"
"github.com/hibiken/asynq"
"gorm.io/gorm"
)
// knowledgeTagService implements KnowledgeTagService.
type knowledgeTagService struct {
kbService interfaces.KnowledgeBaseService
repo interfaces.KnowledgeTagRepository
chunkRepo interfaces.ChunkRepository
kbService interfaces.KnowledgeBaseService
repo interfaces.KnowledgeTagRepository
chunkRepo interfaces.ChunkRepository
retrieveEngine interfaces.RetrieveEngineRegistry
modelService interfaces.ModelService
task *asynq.Client
}
// NewKnowledgeTagService creates a new tag service.
@@ -26,11 +33,17 @@ func NewKnowledgeTagService(
kbService interfaces.KnowledgeBaseService,
repo interfaces.KnowledgeTagRepository,
chunkRepo interfaces.ChunkRepository,
retrieveEngine interfaces.RetrieveEngineRegistry,
modelService interfaces.ModelService,
task *asynq.Client,
) (interfaces.KnowledgeTagService, error) {
return &knowledgeTagService{
kbService: kbService,
repo: repo,
chunkRepo: chunkRepo,
kbService: kbService,
repo: repo,
chunkRepo: chunkRepo,
retrieveEngine: retrieveEngine,
modelService: modelService,
task: task,
}, nil
}
@@ -154,7 +167,7 @@ func (s *knowledgeTagService) UpdateTag(
// DeleteTag deletes a tag. When force=true, also deletes all chunks under this tag.
// When contentOnly=true, only deletes the content under the tag but keeps the tag itself.
func (s *knowledgeTagService) DeleteTag(ctx context.Context, id string, force bool, contentOnly bool) error {
func (s *knowledgeTagService) DeleteTag(ctx context.Context, id string, force bool, contentOnly bool, excludeIDs []string) error {
if id == "" {
return werrors.NewBadRequestError("标签ID不能为空")
}
@@ -163,19 +176,45 @@ func (s *knowledgeTagService) DeleteTag(ctx context.Context, id string, force bo
if err != nil {
return err
}
// Get KB info for embedding model
kb, err := s.kbService.GetKnowledgeBaseByID(ctx, tag.KnowledgeBaseID)
if err != nil {
return err
}
kCount, cCount, err := s.repo.CountReferences(ctx, tenantID, tag.KnowledgeBaseID, tag.ID)
if err != nil {
return err
}
// Get tenant info for effective engines
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
// Helper function to delete chunks and enqueue index deletion task
deleteChunksAndEnqueueIndexDelete := func() error {
// Delete chunks and get their IDs
deletedIDs, err := s.chunkRepo.DeleteChunksByTagID(ctx, tenantID, tag.KnowledgeBaseID, tag.ID, excludeIDs)
if err != nil {
logger.Errorf(ctx, "Failed to delete chunks by tag ID %s: %v", tag.ID, err)
return werrors.NewInternalServerError("删除标签下的数据失败")
}
// Enqueue async index deletion task for the deleted chunks
if len(deletedIDs) > 0 {
s.enqueueIndexDeleteTask(ctx, tenantID, kb.ID, kb.EmbeddingModelID, string(kb.Type), deletedIDs, tenantInfo.GetEffectiveEngines())
}
logger.Infof(ctx, "Deleted %d chunks under tag %s", len(deletedIDs), tag.ID)
return nil
}
// contentOnly mode: only delete content, keep the tag
if contentOnly {
if cCount > 0 {
if err := s.chunkRepo.DeleteChunksByTagID(ctx, tenantID, tag.KnowledgeBaseID, tag.ID); err != nil {
logger.Errorf(ctx, "Failed to delete chunks by tag ID %s: %v", tag.ID, err)
return werrors.NewInternalServerError("删除标签下的数据失败")
if err := deleteChunksAndEnqueueIndexDelete(); err != nil {
return err
}
logger.Infof(ctx, "Deleted %d chunks under tag %s (content_only mode)", cCount, tag.ID)
}
return nil
}
@@ -185,15 +224,91 @@ func (s *knowledgeTagService) DeleteTag(ctx context.Context, id string, force bo
}
// When force=true, delete all chunks under this tag first
if force && cCount > 0 {
if err := s.chunkRepo.DeleteChunksByTagID(ctx, tenantID, tag.KnowledgeBaseID, tag.ID); err != nil {
logger.Errorf(ctx, "Failed to delete chunks by tag ID %s: %v", tag.ID, err)
return werrors.NewInternalServerError("删除标签下的数据失败")
if err := deleteChunksAndEnqueueIndexDelete(); err != nil {
return err
}
logger.Infof(ctx, "Deleted %d chunks under tag %s", cCount, tag.ID)
}
// If there are excludeIDs, we cannot delete the tag itself as it still has content
if len(excludeIDs) > 0 {
return nil
}
return s.repo.Delete(ctx, tenantID, id)
}
// enqueueIndexDeleteTask enqueues an async task for index deletion (low priority)
func (s *knowledgeTagService) enqueueIndexDeleteTask(ctx context.Context,
tenantID uint64, kbID, embeddingModelID, kbType string, chunkIDs []string, effectiveEngines []types.RetrieverEngineParams,
) {
payload := types.IndexDeletePayload{
TenantID: tenantID,
KnowledgeBaseID: kbID,
EmbeddingModelID: embeddingModelID,
KBType: kbType,
ChunkIDs: chunkIDs,
EffectiveEngines: effectiveEngines,
}
payloadBytes, err := json.Marshal(payload)
if err != nil {
logger.Errorf(ctx, "Failed to marshal index delete payload: %v", err)
return
}
task := asynq.NewTask(types.TypeIndexDelete, payloadBytes, asynq.Queue(utils.GetAsynqQueueName("low")), asynq.MaxRetry(10))
info, err := s.task.Enqueue(task)
if err != nil {
logger.Errorf(ctx, "Failed to enqueue index delete task: %v", err)
return
}
logger.Infof(ctx, "Enqueued index delete task: %s for %d chunks", info.ID, len(chunkIDs))
}
// ProcessIndexDelete handles async index deletion task
func (s *knowledgeTagService) ProcessIndexDelete(ctx context.Context, t *asynq.Task) error {
var payload types.IndexDeletePayload
if err := json.Unmarshal(t.Payload(), &payload); err != nil {
logger.Errorf(ctx, "Failed to unmarshal index delete payload: %v", err)
return err
}
logger.Infof(ctx, "Processing index delete task for %d chunks in KB %s", len(payload.ChunkIDs), payload.KnowledgeBaseID)
// Create retrieve engine
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, payload.EffectiveEngines)
if err != nil {
logger.Warnf(ctx, "Failed to create retrieve engine for index cleanup: %v", err)
return err
}
// Get embedding model dimensions
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, payload.EmbeddingModelID)
if err != nil {
logger.Warnf(ctx, "Failed to get embedding model for index cleanup: %v", err)
return err
}
// Delete indices in batches to avoid overwhelming the backend
const batchSize = 100
chunkIDs := payload.ChunkIDs
dimension := embeddingModel.GetDimensions()
for i := 0; i < len(chunkIDs); i += batchSize {
end := i + batchSize
if end > len(chunkIDs) {
end = len(chunkIDs)
}
batch := chunkIDs[i:end]
if err := retrieveEngine.DeleteByChunkIDList(ctx, batch, dimension, payload.KBType); err != nil {
logger.Warnf(ctx, "Failed to delete indices for chunks batch [%d-%d]: %v", i, end, err)
return err
}
logger.Debugf(ctx, "Deleted indices batch [%d-%d] of %d chunks", i, end, len(chunkIDs))
}
logger.Infof(ctx, "Successfully deleted indices for %d chunks", len(payload.ChunkIDs))
return nil
}
// FindOrCreateTagByName finds a tag by name or creates it if not exists.
func (s *knowledgeTagService) FindOrCreateTagByName(ctx context.Context, kbID string, name string) (*types.KnowledgeTag, error) {
name = strings.TrimSpace(name)