From 37de0afad04e4c9f3e740d19a336ffc0ca0d7f54 Mon Sep 17 00:00:00 2001 From: zhangzherui Date: Thu, 23 Jul 2026 04:51:40 +0800 Subject: [PATCH] fix(tasks): dequeue work for deleted knowledge bases --- internal/application/repository/task_queue.go | 63 ++- .../application/repository/task_queue_test.go | 105 ++++ internal/application/service/knowledgebase.go | 94 +++- .../service/knowledgebase_task_cancel_test.go | 293 +++++++++++ .../service/wiki_deleted_kb_guard_test.go | 175 +++++++ internal/application/service/wiki_ingest.go | 116 +++-- .../application/service/wiki_ingest_batch.go | 14 + .../container/recover_pending_wiki_tasks.go | 26 +- .../container/reset_pending_tasks_test.go | 22 + internal/router/task_inspector.go | 477 ++++++++++++------ .../router/task_inspector_kb_cancel_test.go | 348 +++++++++++++ internal/types/interfaces/task_inspector.go | 18 + internal/types/interfaces/task_queue.go | 17 + internal/types/task.go | 1 + 14 files changed, 1575 insertions(+), 194 deletions(-) create mode 100644 internal/application/service/knowledgebase_task_cancel_test.go create mode 100644 internal/application/service/wiki_deleted_kb_guard_test.go create mode 100644 internal/router/task_inspector_kb_cancel_test.go diff --git a/internal/application/repository/task_queue.go b/internal/application/repository/task_queue.go index 328bd533..fa0360d8 100644 --- a/internal/application/repository/task_queue.go +++ b/internal/application/repository/task_queue.go @@ -10,6 +10,7 @@ import ( "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" "gorm.io/gorm" + "gorm.io/gorm/clause" ) // taskPendingOpsRepository implements interfaces.TaskPendingOpsRepository. @@ -26,6 +27,13 @@ func NewTaskPendingOpsRepository(db *gorm.DB) interfaces.TaskPendingOpsRepositor // Scope/ScopeID/Op (Payload optional). ID, FailCount default to zero; // EnqueuedAt is filled with the DB-side default if left zero. func (r *taskPendingOpsRepository) Enqueue(ctx context.Context, op *types.TaskPendingOp) error { + if err := preparePendingOp(op); err != nil { + return err + } + return r.db.WithContext(ctx).Create(op).Error +} + +func preparePendingOp(op *types.TaskPendingOp) error { if op == nil { return errors.New("task pending ops: nil op") } @@ -41,7 +49,48 @@ func (r *taskPendingOpsRepository) Enqueue(ctx context.Context, op *types.TaskPe // driver-level default handling. op.Payload = []byte("{}") } - return r.db.WithContext(ctx).Create(op).Error + return nil +} + +// EnqueueIfKnowledgeBaseActive prevents detached wiki cleanup from writing new +// durable work after a KB was soft-deleted. On Postgres the share lock +// serializes this check+insert transaction against the row update performed by +// soft deletion: whichever operation acquires the row first determines the +// order, and the deletion path's subsequent scope scrub removes any insert +// that committed before it. +func (r *taskPendingOpsRepository) EnqueueIfKnowledgeBaseActive( + ctx context.Context, + op *types.TaskPendingOp, +) (bool, error) { + if err := preparePendingOp(op); err != nil { + return false, err + } + if op.Scope != types.TaskScopeKnowledgeBase { + return false, errors.New("task pending ops: guarded enqueue requires knowledge_base scope") + } + accepted := false + err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + query := tx.Model(&types.KnowledgeBase{}). + Select("id"). + Where("id = ? AND tenant_id = ?", op.ScopeID, op.TenantID) + dialector := tx.Dialector + if dialector.Name() == "postgres" { + query = query.Clauses(clause.Locking{Strength: "SHARE"}) + } + var kb types.KnowledgeBase + if err := query.Take(&kb).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil + } + return err + } + if err := tx.Create(op).Error; err != nil { + return err + } + accepted = true + return nil + }) + return accepted, err } // PeekBatch returns up to `limit` rows for the (task_type, scope, scope_id) @@ -228,6 +277,18 @@ func (r *taskPendingOpsRepository) DeleteByIDs(ctx context.Context, ids []int64) Delete(&types.TaskPendingOp{}).Error } +// DeleteByScope removes every pending operation owned by a scope, regardless +// of task type. Both scope fields are required so a malformed lifecycle call +// can never turn into an unbounded queue deletion. +func (r *taskPendingOpsRepository) DeleteByScope(ctx context.Context, scope, scopeID string) error { + if scope == "" || scopeID == "" { + return errors.New("task pending ops: scope and scope_id are required") + } + return r.db.WithContext(ctx). + Where("scope = ? AND scope_id = ?", scope, scopeID). + Delete(&types.TaskPendingOp{}).Error +} + // IncrFailCount atomically bumps fail_count for one row and returns the // new value. We use UPDATE ... RETURNING so the read+write happens in // one round trip and races between concurrent IncrFailCount callers diff --git a/internal/application/repository/task_queue_test.go b/internal/application/repository/task_queue_test.go index a3aece6c..1ca90e51 100644 --- a/internal/application/repository/task_queue_test.go +++ b/internal/application/repository/task_queue_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gorm.io/driver/sqlite" @@ -182,6 +183,110 @@ func TestTaskPendingOps_DeleteByIDs_RemovesOnlyTargets(t *testing.T) { assert.Equal(t, "b", got[0].DedupKey) } +func TestTaskPendingOps_DeleteByScope_RemovesAllTaskTypesAndIsolatesScopes(t *testing.T) { + db := setupTaskQueueTestDB(t) + repo := NewTaskPendingOpsRepository(db) + cleaner, ok := repo.(interfaces.TaskPendingOpsScopeCleaner) + require.True(t, ok) + ctx := context.Background() + + // The deleted KB can have durable work in more than one wiki queue. + require.NoError(t, repo.Enqueue(ctx, + makePendingOp(types.TypeWikiIngest, types.TaskScopeKnowledgeBase, "kb-delete", "ingest", "k1", nil))) + require.NoError(t, repo.Enqueue(ctx, + makePendingOp(types.TypeWikiFinalize, types.TaskScopeKnowledgeBase, "kb-delete", "finalize", "k2", nil))) + + // Rows in another KB or another scope must survive even when their IDs or + // task types overlap with the deleted KB. + require.NoError(t, repo.Enqueue(ctx, + makePendingOp(types.TypeWikiIngest, types.TaskScopeKnowledgeBase, "kb-keep", "ingest", "k3", nil))) + require.NoError(t, repo.Enqueue(ctx, + makePendingOp(types.TypeWikiFinalize, types.TaskScopeKnowledge, "kb-delete", "finalize", "k4", nil))) + + require.NoError(t, cleaner.DeleteByScope(ctx, types.TaskScopeKnowledgeBase, "kb-delete")) + + var remaining []*types.TaskPendingOp + require.NoError(t, db.Order("id ASC").Find(&remaining).Error) + require.Len(t, remaining, 2) + assert.Equal(t, "kb-keep", remaining[0].ScopeID) + assert.Equal(t, types.TaskScopeKnowledge, remaining[1].Scope) + assert.Equal(t, "kb-delete", remaining[1].ScopeID) +} + +func TestTaskPendingOps_DeleteByScope_RejectsMissingScope(t *testing.T) { + db := setupTaskQueueTestDB(t) + repo := NewTaskPendingOpsRepository(db) + cleaner, ok := repo.(interfaces.TaskPendingOpsScopeCleaner) + require.True(t, ok) + ctx := context.Background() + + require.NoError(t, repo.Enqueue(ctx, + makePendingOp(types.TypeWikiIngest, types.TaskScopeKnowledgeBase, "kb", "ingest", "k1", nil))) + + assert.Error(t, cleaner.DeleteByScope(ctx, "", "kb")) + assert.Error(t, cleaner.DeleteByScope(ctx, types.TaskScopeKnowledgeBase, "")) + + var count int64 + require.NoError(t, db.Model(&types.TaskPendingOp{}).Count(&count).Error) + assert.Equal(t, int64(1), count) +} + +func TestTaskPendingOps_EnqueueIfKnowledgeBaseActive(t *testing.T) { + db := setupTaskQueueTestDB(t) + require.NoError(t, db.Exec(`CREATE TABLE knowledge_bases ( + id VARCHAR(64) PRIMARY KEY, + tenant_id INTEGER NOT NULL, + deleted_at DATETIME + )`).Error) + require.NoError(t, db.Exec( + "INSERT INTO knowledge_bases (id, tenant_id, deleted_at) VALUES (?, ?, NULL), (?, ?, ?)", + "kb-active", 1, "kb-deleted", 1, time.Now(), + ).Error) + + repo := NewTaskPendingOpsRepository(db) + guard, ok := repo.(interfaces.TaskPendingOpsKnowledgeBaseGuard) + require.True(t, ok) + ctx := context.Background() + + accepted, err := guard.EnqueueIfKnowledgeBaseActive(ctx, + makePendingOp(types.TypeWikiIngest, types.TaskScopeKnowledgeBase, "kb-active", "ingest", "active", nil)) + require.NoError(t, err) + assert.True(t, accepted) + + for _, tc := range []struct { + name string + op *types.TaskPendingOp + }{ + { + name: "soft deleted", + op: makePendingOp( + types.TypeWikiIngest, types.TaskScopeKnowledgeBase, "kb-deleted", "ingest", "deleted", nil, + ), + }, + { + name: "missing", + op: makePendingOp( + types.TypeWikiIngest, types.TaskScopeKnowledgeBase, "kb-missing", "ingest", "missing", nil, + ), + }, + {name: "tenant mismatch", op: &types.TaskPendingOp{ + TenantID: 2, TaskType: types.TypeWikiIngest, Scope: types.TaskScopeKnowledgeBase, + ScopeID: "kb-active", Op: "ingest", DedupKey: "wrong-tenant", + }}, + } { + t.Run(tc.name, func(t *testing.T) { + accepted, err := guard.EnqueueIfKnowledgeBaseActive(ctx, tc.op) + require.NoError(t, err) + assert.False(t, accepted) + }) + } + + var rows []*types.TaskPendingOp + require.NoError(t, db.Find(&rows).Error) + require.Len(t, rows, 1) + assert.Equal(t, "active", rows[0].DedupKey) +} + // TestTaskPendingOps_IncrFailCount_ReturnsNewValueAndPersists exercises // the UPDATE...RETURNING flow. Successive bumps should observe monotonic // counts. diff --git a/internal/application/service/knowledgebase.go b/internal/application/service/knowledgebase.go index 183b02eb..5eb8f0e9 100644 --- a/internal/application/service/knowledgebase.go +++ b/internal/application/service/knowledgebase.go @@ -25,6 +25,8 @@ import ( // ErrInvalidTenantID represents an error for invalid tenant ID var ErrInvalidTenantID = errors.New("invalid tenant ID") +const kbTaskCleanupTimeout = 5 * time.Second + // knowledgeBaseService implements the knowledge base service interface type knowledgeBaseService struct { repo interfaces.KnowledgeBaseRepository @@ -40,6 +42,8 @@ type knowledgeBaseService struct { storageResolver interfaces.StorageBackendResolver graphEngine interfaces.RetrieveGraphRepository asynqClient interfaces.TaskEnqueuer + taskInspector interfaces.TaskInspector + taskPendingRepo interfaces.TaskPendingOpsRepository dsRepo interfaces.DataSourceRepository syncLogRepo interfaces.SyncLogRepository dsScheduler *datasource.Scheduler @@ -60,6 +64,8 @@ func NewKnowledgeBaseService(repo interfaces.KnowledgeBaseRepository, storageResolver interfaces.StorageBackendResolver, graphEngine interfaces.RetrieveGraphRepository, asynqClient interfaces.TaskEnqueuer, + taskInspector interfaces.TaskInspector, + taskPendingRepo interfaces.TaskPendingOpsRepository, dsRepo interfaces.DataSourceRepository, syncLogRepo interfaces.SyncLogRepository, dsScheduler *datasource.Scheduler, @@ -79,6 +85,8 @@ func NewKnowledgeBaseService(repo interfaces.KnowledgeBaseRepository, storageResolver: storageResolver, graphEngine: graphEngine, asynqClient: asynqClient, + taskInspector: taskInspector, + taskPendingRepo: taskPendingRepo, dsRepo: dsRepo, syncLogRepo: syncLogRepo, dsScheduler: dsScheduler, @@ -720,6 +728,11 @@ func (s *knowledgeBaseService) DeleteKnowledgeBase(ctx context.Context, id strin recordKBActivity(ctx, s.audit, tenantID, id, types.AuditActionKBDeleted, "knowledge_base", id, types.AuditOutcomeSuccess, map[string]any{"name": deletedName}) + // Stop both ephemeral queue work and durable wiki operations that target + // the now-deleted KB. ProcessKBDelete repeats this with document IDs and + // performs one final scrub after heavy cleanup to close enqueue races. + s.cleanupTasksForKnowledgeBase(ctx, id, nil, nil) + // Step 1b: Remove all organization shares for this KB so org settings no longer show them if s.shareRepo != nil { if delErr := s.shareRepo.DeleteByKnowledgeBaseID(ctx, id); delErr != nil { @@ -729,12 +742,16 @@ func (s *knowledgeBaseService) DeleteKnowledgeBase(ctx context.Context, id strin // Step 1c: Stop and soft-delete all data sources bound to this KB so cron // schedules and in-flight sync logs do not keep running against a deleted KB. - s.deleteDataSourcesForKnowledgeBase(ctx, id) + dataSourceIDs := s.deleteDataSourcesForKnowledgeBase(ctx, id) + if len(dataSourceIDs) > 0 { + s.cancelTasksForKnowledgeBase(ctx, id, nil, dataSourceIDs) + } // Step 2: Enqueue async task for heavy cleanup operations payload := types.KBDeletePayload{ TenantID: tenantID, KnowledgeBaseID: id, + DataSourceIDs: dataSourceIDs, EffectiveEngines: tenantInfo.GetEffectiveEngines(), VectorStoreID: vectorStoreIDSnapshot, // snapshot taken before soft-delete } @@ -772,9 +789,18 @@ func (s *knowledgeBaseService) ProcessKBDelete(ctx context.Context, t *asynq.Tas tenantID := payload.TenantID kbID := payload.KnowledgeBaseID + var knowledgeIDs []string // Set tenant context for downstream services ctx = context.WithValue(ctx, types.TenantIDContextKey, tenantID) + defer func() { + // Workers may enqueue downstream work while the delete task performs + // heavy storage cleanup. A detached, bounded final scrub runs on every + // return path, including retryable failures and cancellation. + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), kbTaskCleanupTimeout) + defer cancel() + s.cleanupTasksForKnowledgeBase(cleanupCtx, kbID, knowledgeIDs, payload.DataSourceIDs) + }() logger.Infof(ctx, "Processing KB delete task for knowledge base: %s", kbID) @@ -788,14 +814,18 @@ func (s *knowledgeBaseService) ProcessKBDelete(ctx context.Context, t *asynq.Tas return err } logger.Infof(ctx, "Found %d knowledge entries to delete", len(knowledgeList)) + knowledgeIDs = make([]string, 0, len(knowledgeList)) + for _, knowledge := range knowledgeList { + knowledgeIDs = append(knowledgeIDs, knowledge.ID) + } + + // Repeat the best-effort queue scrub with document IDs. Some batch tasks + // only carry knowledge_id(s), and active work from the first pass may have + // enqueued another downstream task before cancellation reached it. + s.cleanupTasksForKnowledgeBase(ctx, kbID, knowledgeIDs, payload.DataSourceIDs) // Step 2: Delete all knowledge entries and their resources if len(knowledgeList) > 0 { - knowledgeIDs := make([]string, 0, len(knowledgeList)) - for _, knowledge := range knowledgeList { - knowledgeIDs = append(knowledgeIDs, knowledge.ID) - } - logger.Infof(ctx, "Deleting all knowledge entries and their resources") // Delete embeddings from vector store. @@ -911,23 +941,68 @@ func (s *knowledgeBaseService) ProcessKBDelete(ctx context.Context, t *asynq.Tas return nil } +// cancelTasksForKnowledgeBase removes queue work for a deleted KB when the +// configured task backend supports knowledge-base-wide inspection. Queue +// cleanup is an optimization: the soft-deleted database row remains the +// durable source of truth, so backend failures must not fail KB deletion. +func (s *knowledgeBaseService) cancelTasksForKnowledgeBase( + ctx context.Context, + kbID string, + knowledgeIDs []string, + dataSourceIDs []string, +) { + canceller, ok := s.taskInspector.(interfaces.KnowledgeBaseTaskCanceller) + if !ok || kbID == "" { + return + } + if _, _, err := canceller.CancelTasksForKnowledgeBase(ctx, kbID, knowledgeIDs, dataSourceIDs); err != nil { + logger.Warnf(ctx, "Failed to cancel queued tasks for deleted KB %s: %v", kbID, err) + } +} + +// cleanupTasksForKnowledgeBase removes both asynq records and durable wiki +// operations. The latter must be cleared as well or startup recovery can +// recreate Redis triggers for a KB that no longer exists. +func (s *knowledgeBaseService) cleanupTasksForKnowledgeBase( + ctx context.Context, + kbID string, + knowledgeIDs []string, + dataSourceIDs []string, +) { + if kbID == "" { + return + } + cleaner, ok := s.taskPendingRepo.(interfaces.TaskPendingOpsScopeCleaner) + if ok { + // Clear durable work before scanning Redis. A large or degraded queue + // must not consume the caller's entire deadline and starve the database + // fence that prevents startup recovery from reviving this KB. + if err := cleaner.DeleteByScope(ctx, types.TaskScopeKnowledgeBase, kbID); err != nil { + logger.Warnf(ctx, "Failed to clear durable tasks for deleted KB %s: %v", kbID, err) + } + } + s.cancelTasksForKnowledgeBase(ctx, kbID, knowledgeIDs, dataSourceIDs) +} + // deleteDataSourcesForKnowledgeBase mirrors DataSourceService.DeleteDataSource for // every data source attached to the KB. Errors on individual sources are logged // but do not fail KB deletion — the KB record is already soft-deleted. -func (s *knowledgeBaseService) deleteDataSourcesForKnowledgeBase(ctx context.Context, kbID string) { +func (s *knowledgeBaseService) deleteDataSourcesForKnowledgeBase(ctx context.Context, kbID string) []string { if s.dsRepo == nil { - return + return nil } dataSources, err := s.dsRepo.FindByKnowledgeBase(ctx, kbID) if err != nil { logger.Warnf(ctx, "Failed to list data sources for deleted KB %s: %v", kbID, err) - return + return nil } + dataSourceIDs := make([]string, 0, len(dataSources)) for _, ds := range dataSources { if ds == nil || ds.ID == "" { continue } + dataSourceIDs = append(dataSourceIDs, ds.ID) if err := s.dsRepo.Delete(ctx, ds.ID); err != nil { logger.Warnf(ctx, "Failed to delete data source %s for KB %s: %v", ds.ID, kbID, err) continue @@ -942,6 +1017,7 @@ func (s *knowledgeBaseService) deleteDataSourcesForKnowledgeBase(ctx context.Con } logger.Infof(ctx, "Data source deleted with knowledge base: ds=%s kb=%s", ds.ID, kbID) } + return dataSourceIDs } // SetEmbeddingModel sets the embedding model for a knowledge base diff --git a/internal/application/service/knowledgebase_task_cancel_test.go b/internal/application/service/knowledgebase_task_cancel_test.go new file mode 100644 index 00000000..ee18ad18 --- /dev/null +++ b/internal/application/service/knowledgebase_task_cancel_test.go @@ -0,0 +1,293 @@ +package service + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "github.com/Tencent/WeKnora/internal/models/embedding" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/hibiken/asynq" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type kbTaskCancelCall struct { + kbID string + knowledgeIDs []string + dataSourceIDs []string +} + +type recordingKBTaskInspector struct { + repo *kbDeleteKBRepo + calls []kbTaskCancelCall + cancelErr error + sawSoftDeletedRecord bool +} + +func (r *recordingKBTaskInspector) CancelTasksForKnowledge( + context.Context, + string, +) (int, int, error) { + return 0, 0, nil +} + +func (r *recordingKBTaskInspector) HasQueuedTasksForKnowledge(context.Context, string) (bool, error) { + return false, nil +} + +func (r *recordingKBTaskInspector) QueueStats(context.Context) ([]types.QueueStat, bool, error) { + return nil, true, nil +} + +func (r *recordingKBTaskInspector) WorkerServerStats(context.Context) ([]types.WorkerServerStat, bool, error) { + return nil, true, nil +} + +func (r *recordingKBTaskInspector) CancelTasksForKnowledgeBase( + _ context.Context, + kbID string, + knowledgeIDs []string, + dataSourceIDs []string, +) (int, int, error) { + r.calls = append(r.calls, kbTaskCancelCall{ + kbID: kbID, + knowledgeIDs: append([]string(nil), knowledgeIDs...), + dataSourceIDs: append([]string(nil), dataSourceIDs...), + }) + if r.repo != nil && r.repo.deletedID == kbID { + r.sawSoftDeletedRecord = true + } + return 0, 0, r.cancelErr +} + +var ( + _ interfaces.TaskInspector = (*recordingKBTaskInspector)(nil) + _ interfaces.KnowledgeBaseTaskCanceller = (*recordingKBTaskInspector)(nil) +) + +type recordingKBDeleteEnqueuer struct { + calls int + task *asynq.Task +} + +type recordingKBPendingRepo struct { + interfaces.TaskPendingOpsRepository + scopeIDs []string + deleteErr error +} + +func (r *recordingKBPendingRepo) DeleteByScope(_ context.Context, scope, scopeID string) error { + if scope == types.TaskScopeKnowledgeBase { + r.scopeIDs = append(r.scopeIDs, scopeID) + } + return r.deleteErr +} + +func (r *recordingKBDeleteEnqueuer) Enqueue( + task *asynq.Task, + _ ...asynq.Option, +) (*asynq.TaskInfo, error) { + r.calls++ + r.task = task + return &asynq.TaskInfo{ID: "kb-delete-task"}, nil +} + +func TestDeleteKnowledgeBaseForwardsDataSourceTaskScope(t *testing.T) { + const kbID = "kb-with-datasource" + kbRepo := &kbDeleteKBRepo{fakeKBRepo: *newFakeKBRepo()} + kbRepo.rows[kbID] = &types.KnowledgeBase{ID: kbID, TenantID: 1, Name: "test"} + inspector := &recordingKBTaskInspector{repo: kbRepo} + enqueuer := &recordingKBDeleteEnqueuer{} + dsRepo := newKBDeleteDSRepo(kbID, &types.DataSource{ID: "datasource-1", KnowledgeBaseID: kbID}) + svc := &knowledgeBaseService{ + repo: kbRepo, + asynqClient: enqueuer, + taskInspector: inspector, + dsRepo: dsRepo, + } + + err := svc.DeleteKnowledgeBase(ctxWithTenantStorage(1, "local"), kbID) + + require.NoError(t, err) + require.Len(t, inspector.calls, 2) + assert.Empty(t, inspector.calls[0].dataSourceIDs) + assert.Equal(t, []string{"datasource-1"}, inspector.calls[1].dataSourceIDs) + require.NotNil(t, enqueuer.task) + var payload types.KBDeletePayload + require.NoError(t, json.Unmarshal(enqueuer.task.Payload(), &payload)) + assert.Equal(t, []string{"datasource-1"}, payload.DataSourceIDs) +} + +func TestDeleteKnowledgeBaseCancelsQueuedTasksBestEffort(t *testing.T) { + tests := []struct { + name string + cancelErr error + pendingErr error + }{ + {name: "success"}, + {name: "inspector failure", cancelErr: errors.New("redis unavailable")}, + {name: "durable queue failure", pendingErr: errors.New("database unavailable")}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + const kbID = "kb-task-cleanup" + kbRepo := &kbDeleteKBRepo{fakeKBRepo: *newFakeKBRepo()} + kbRepo.rows[kbID] = &types.KnowledgeBase{ID: kbID, TenantID: 1, Name: "test"} + inspector := &recordingKBTaskInspector{repo: kbRepo, cancelErr: tt.cancelErr} + pendingRepo := &recordingKBPendingRepo{deleteErr: tt.pendingErr} + enqueuer := &recordingKBDeleteEnqueuer{} + svc := &knowledgeBaseService{ + repo: kbRepo, + asynqClient: enqueuer, + taskInspector: inspector, + taskPendingRepo: pendingRepo, + } + + err := svc.DeleteKnowledgeBase(ctxWithTenantStorage(1, "local"), kbID) + + require.NoError(t, err) + require.Len(t, inspector.calls, 1) + assert.Equal(t, kbID, inspector.calls[0].kbID) + assert.Empty(t, inspector.calls[0].knowledgeIDs) + assert.True(t, inspector.sawSoftDeletedRecord) + assert.Equal(t, []string{kbID}, pendingRepo.scopeIDs) + assert.Equal(t, 1, enqueuer.calls) + }) + } +} + +type emptyKBKnowledgeRepo struct { + interfaces.KnowledgeRepository +} + +func (emptyKBKnowledgeRepo) ListKnowledgeByKnowledgeBaseID( + context.Context, + uint64, + string, +) ([]*types.Knowledge, error) { + return nil, nil +} + +func TestProcessKBDeleteRepeatsQueueCleanup(t *testing.T) { + inspector := &recordingKBTaskInspector{} + pendingRepo := &recordingKBPendingRepo{} + svc := &knowledgeBaseService{ + kgRepo: emptyKBKnowledgeRepo{}, + taskInspector: inspector, + taskPendingRepo: pendingRepo, + } + payload, err := json.Marshal(types.KBDeletePayload{TenantID: 1, KnowledgeBaseID: "kb-race"}) + require.NoError(t, err) + + err = svc.ProcessKBDelete(context.Background(), asynq.NewTask(types.TypeKBDelete, payload)) + + require.NoError(t, err) + require.Len(t, inspector.calls, 2) + for _, call := range inspector.calls { + assert.Equal(t, "kb-race", call.kbID) + assert.Empty(t, call.knowledgeIDs) + } + assert.Equal(t, []string{"kb-race", "kb-race"}, pendingRepo.scopeIDs) +} + +type populatedKBKnowledgeRepo struct { + interfaces.KnowledgeRepository + items []*types.Knowledge +} + +func (r populatedKBKnowledgeRepo) ListKnowledgeByKnowledgeBaseID( + context.Context, + uint64, + string, +) ([]*types.Knowledge, error) { + return r.items, nil +} + +func (populatedKBKnowledgeRepo) DeleteKnowledgeList(context.Context, uint64, []string) error { + return nil +} + +type kbCleanupChunkRepo struct { + interfaces.ChunkRepository +} + +func (kbCleanupChunkRepo) ListImageInfoByKnowledgeIDs( + context.Context, + uint64, + []string, +) ([]interfaces.ChunkImageInfo, error) { + return nil, nil +} + +func (kbCleanupChunkRepo) DeleteChunksByKnowledgeID(context.Context, uint64, string) error { + return nil +} + +type kbCleanupModelService struct { + interfaces.ModelService +} + +func (kbCleanupModelService) GetEmbeddingModel(context.Context, string) (embedding.Embedder, error) { + return kbCleanupEmbedder{}, nil +} + +type kbCleanupEmbedder struct{} + +func (kbCleanupEmbedder) Embed(context.Context, string) ([]float32, error) { return nil, nil } +func (kbCleanupEmbedder) BatchEmbed(context.Context, []string) ([][]float32, error) { + return nil, nil +} +func (kbCleanupEmbedder) GetModelName() string { return "test" } +func (kbCleanupEmbedder) GetDimensions() int { return 1 } +func (kbCleanupEmbedder) GetModelID() string { return "test" } +func (kbCleanupEmbedder) BatchEmbedWithPool( + context.Context, + embedding.Embedder, + []string, +) ([][]float32, error) { + return nil, nil +} + +func TestProcessKBDeleteCollectsKnowledgeIDsForEveryScrub(t *testing.T) { + inspector := &recordingKBTaskInspector{} + svc := &knowledgeBaseService{ + kgRepo: populatedKBKnowledgeRepo{items: []*types.Knowledge{ + {ID: "knowledge-1", KnowledgeBaseID: "kb-1", EmbeddingModelID: "model-1"}, + {ID: "knowledge-2", KnowledgeBaseID: "kb-1", EmbeddingModelID: "model-1"}, + }}, + chunkRepo: kbCleanupChunkRepo{}, + modelService: kbCleanupModelService{}, + taskInspector: inspector, + } + payload, err := json.Marshal(types.KBDeletePayload{TenantID: 1, KnowledgeBaseID: "kb-1"}) + require.NoError(t, err) + + err = svc.ProcessKBDelete(context.Background(), asynq.NewTask(types.TypeKBDelete, payload)) + + require.NoError(t, err) + require.Len(t, inspector.calls, 2) + for _, call := range inspector.calls { + assert.Equal(t, []string{"knowledge-1", "knowledge-2"}, call.knowledgeIDs) + } +} + +func TestCancelTasksForKnowledgeBaseForwardsKnowledgeIDs(t *testing.T) { + inspector := &recordingKBTaskInspector{} + svc := &knowledgeBaseService{taskInspector: inspector} + + svc.cancelTasksForKnowledgeBase( + context.Background(), + "kb-1", + []string{"knowledge-1", "knowledge-2"}, + []string{"datasource-1"}, + ) + + require.Len(t, inspector.calls, 1) + assert.Equal(t, "kb-1", inspector.calls[0].kbID) + assert.Equal(t, []string{"knowledge-1", "knowledge-2"}, inspector.calls[0].knowledgeIDs) + assert.Equal(t, []string{"datasource-1"}, inspector.calls[0].dataSourceIDs) +} diff --git a/internal/application/service/wiki_deleted_kb_guard_test.go b/internal/application/service/wiki_deleted_kb_guard_test.go new file mode 100644 index 00000000..bd3f05d9 --- /dev/null +++ b/internal/application/service/wiki_deleted_kb_guard_test.go @@ -0,0 +1,175 @@ +package service + +import ( + "context" + "encoding/json" + "errors" + "testing" + + apprepo "github.com/Tencent/WeKnora/internal/application/repository" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/hibiken/asynq" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type wikiKBGuardPendingRepo struct { + interfaces.TaskPendingOpsRepository + accepted bool + guardErr error + guardedOps []*types.TaskPendingOp + deleteErr error + deletedKBs []string + rows []*types.TaskPendingOp +} + +func (r *wikiKBGuardPendingRepo) EnqueueIfKnowledgeBaseActive( + _ context.Context, + op *types.TaskPendingOp, +) (bool, error) { + r.guardedOps = append(r.guardedOps, op) + return r.accepted, r.guardErr +} + +func (r *wikiKBGuardPendingRepo) DeleteByScope(_ context.Context, scope, scopeID string) error { + if scope == types.TaskScopeKnowledgeBase { + r.deletedKBs = append(r.deletedKBs, scopeID) + } + return r.deleteErr +} + +func (r *wikiKBGuardPendingRepo) PeekBatch( + context.Context, + string, + string, + string, + int, +) ([]*types.TaskPendingOp, error) { + return r.rows, nil +} + +type wikiGuardTaskQueue struct { + interfaces.TaskEnqueuer + tasks []*asynq.Task +} + +func (q *wikiGuardTaskQueue) Enqueue(task *asynq.Task, _ ...asynq.Option) (*asynq.TaskInfo, error) { + q.tasks = append(q.tasks, task) + return &asynq.TaskInfo{ID: "guard-test", Type: task.Type()}, nil +} + +type wikiGuardKBService struct { + interfaces.KnowledgeBaseService + kb *types.KnowledgeBase + err error +} + +func (s *wikiGuardKBService) GetKnowledgeBaseByIDOnly( + context.Context, + string, +) (*types.KnowledgeBase, error) { + return s.kb, s.err +} + +func TestEnqueueWikiWorkSkipsDeletedKnowledgeBase(t *testing.T) { + tests := []struct { + name string + run func(context.Context, interfaces.TaskEnqueuer, interfaces.TaskPendingOpsRepository) + }{ + { + name: "ingest", + run: func(ctx context.Context, task interfaces.TaskEnqueuer, repo interfaces.TaskPendingOpsRepository) { + EnqueueWikiIngest(ctx, task, repo, 7, "kb-deleted", "knowledge-1") + }, + }, + { + name: "retract", + run: func(ctx context.Context, task interfaces.TaskEnqueuer, repo interfaces.TaskPendingOpsRepository) { + EnqueueWikiRetract(ctx, task, repo, WikiRetractPayload{ + TenantID: 7, KnowledgeBaseID: "kb-deleted", KnowledgeID: "knowledge-1", + }) + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + repo := &wikiKBGuardPendingRepo{accepted: false} + queue := &wikiGuardTaskQueue{} + test.run(context.Background(), queue, repo) + + require.Len(t, repo.guardedOps, 1) + assert.Equal(t, "kb-deleted", repo.guardedOps[0].ScopeID) + assert.Empty(t, queue.tasks) + }) + } +} + +func TestEnqueueWikiFinalizeOnlySchedulesAcceptedRows(t *testing.T) { + for _, accepted := range []bool{false, true} { + t.Run(map[bool]string{false: "deleted", true: "active"}[accepted], func(t *testing.T) { + repo := &wikiKBGuardPendingRepo{accepted: accepted} + queue := &wikiGuardTaskQueue{} + svc := &wikiIngestService{pendingRepo: repo, task: queue} + + svc.enqueueFinalize( + context.Background(), + WikiIngestPayload{TenantID: 7, KnowledgeBaseID: "kb-1"}, + []string{"slug-1"}, + map[string]string{"slug-1": "Title"}, + []wikiFinalizeChange{{Action: wikiFinalizeAdded, DocTitle: "Document"}}, + []string{"folder-1"}, + ) + + require.Len(t, repo.guardedOps, 3) + if accepted { + require.Len(t, queue.tasks, 1) + assert.Equal(t, types.TypeWikiFinalize, queue.tasks[0].Type()) + } else { + assert.Empty(t, queue.tasks) + } + }) + } +} + +func TestWikiHandlersDrainDeletedKnowledgeBaseQueue(t *testing.T) { + payload, err := json.Marshal(WikiIngestPayload{TenantID: 7, KnowledgeBaseID: "kb-deleted"}) + require.NoError(t, err) + + for _, taskType := range []string{types.TypeWikiIngest, types.TypeWikiFinalize} { + t.Run(taskType, func(t *testing.T) { + repo := &wikiKBGuardPendingRepo{ + rows: []*types.TaskPendingOp{{ID: 1, ScopeID: "kb-deleted"}}, + } + svc := &wikiIngestService{ + kbService: &wikiGuardKBService{err: apprepo.ErrKnowledgeBaseNotFound}, + pendingRepo: repo, + } + + err := svc.Handle(context.Background(), asynq.NewTask(taskType, payload)) + + require.NoError(t, err) + assert.Equal(t, []string{"kb-deleted"}, repo.deletedKBs) + }) + } +} + +func TestWikiDeletedKnowledgeBaseCleanupFailureRetries(t *testing.T) { + payload, err := json.Marshal(WikiIngestPayload{TenantID: 7, KnowledgeBaseID: "kb-deleted"}) + require.NoError(t, err) + wantErr := errors.New("cleanup failed") + repo := &wikiKBGuardPendingRepo{deleteErr: wantErr} + svc := &wikiIngestService{ + kbService: &wikiGuardKBService{err: apprepo.ErrKnowledgeBaseNotFound}, + pendingRepo: repo, + } + + err = svc.ProcessWikiIngest( + context.Background(), + asynq.NewTask(types.TypeWikiIngest, payload), + ) + + require.ErrorIs(t, err, wantErr) + assert.Equal(t, []string{"kb-deleted"}, repo.deletedKBs) +} diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go index 4b207a3c..45748ef5 100644 --- a/internal/application/service/wiki_ingest.go +++ b/internal/application/service/wiki_ingest.go @@ -426,6 +426,23 @@ func (s *wikiIngestService) beginWikiSubspan(ctx context.Context, knowledgeID st // Lite mode (no Redis) still works as long as Postgres is reachable — // the queue lives in PG, only the active-batch lock is Redis-only and // has a process-local fallback (liteLocks) inside the worker. +func enqueueWikiPendingOp( + ctx context.Context, + pendingRepo interfaces.TaskPendingOpsRepository, + op *types.TaskPendingOp, +) (bool, error) { + if pendingRepo == nil { + return true, nil + } + if guard, ok := pendingRepo.(interfaces.TaskPendingOpsKnowledgeBaseGuard); ok { + return guard.EnqueueIfKnowledgeBaseActive(ctx, op) + } + if err := pendingRepo.Enqueue(ctx, op); err != nil { + return false, err + } + return true, nil +} + func EnqueueWikiIngest( ctx context.Context, task interfaces.TaskEnqueuer, @@ -450,20 +467,22 @@ func EnqueueWikiIngest( logger.Warnf(ctx, "wiki ingest: failed to marshal pending op for %s: %v", knowledgeID, err) return } - if pendingRepo != nil { - if err := pendingRepo.Enqueue(ctx, &types.TaskPendingOp{ - TenantID: tenantID, - TaskType: wikiTaskType, - Scope: wikiTaskScope, - ScopeID: kbID, - Op: WikiOpIngest, - DedupKey: knowledgeID, - Payload: payloadBytes, - }); err != nil { - logger.Warnf(ctx, "wiki ingest: failed to enqueue pending op for %s: %v", knowledgeID, err) - // Fall through and still schedule the trigger task — the - // next upload (or the next retry pass) will catch the gap. - } + accepted, err := enqueueWikiPendingOp(ctx, pendingRepo, &types.TaskPendingOp{ + TenantID: tenantID, + TaskType: wikiTaskType, + Scope: wikiTaskScope, + ScopeID: kbID, + Op: WikiOpIngest, + DedupKey: knowledgeID, + Payload: payloadBytes, + }) + if err != nil { + logger.Warnf(ctx, "wiki ingest: failed to enqueue pending op for %s: %v", knowledgeID, err) + return + } + if !accepted { + logger.Infof(ctx, "wiki ingest: skip enqueue for deleted KB %s", kbID) + return } trigger := WikiIngestPayload{ @@ -512,18 +531,22 @@ func EnqueueWikiRetract( logger.Warnf(ctx, "wiki retract: failed to marshal pending op: %v", err) return } - if pendingRepo != nil { - if err := pendingRepo.Enqueue(ctx, &types.TaskPendingOp{ - TenantID: payload.TenantID, - TaskType: wikiTaskType, - Scope: wikiTaskScope, - ScopeID: payload.KnowledgeBaseID, - Op: WikiOpRetract, - DedupKey: payload.KnowledgeID, - Payload: payloadBytes, - }); err != nil { - logger.Warnf(ctx, "wiki retract: failed to enqueue pending op: %v", err) - } + accepted, err := enqueueWikiPendingOp(ctx, pendingRepo, &types.TaskPendingOp{ + TenantID: payload.TenantID, + TaskType: wikiTaskType, + Scope: wikiTaskScope, + ScopeID: payload.KnowledgeBaseID, + Op: WikiOpRetract, + DedupKey: payload.KnowledgeID, + Payload: payloadBytes, + }) + if err != nil { + logger.Warnf(ctx, "wiki retract: failed to enqueue pending op: %v", err) + return + } + if !accepted { + logger.Infof(ctx, "wiki retract: skip enqueue for deleted KB %s", payload.KnowledgeBaseID) + return } trigger := WikiIngestPayload{ @@ -564,6 +587,25 @@ func wikiIngestCleanupContext(ctx context.Context) (context.Context, context.Can return context.WithTimeout(context.WithoutCancel(ctx), wikiIngestCleanupTimeout) } +func (s *wikiIngestService) clearDeletedKnowledgeBasePendingOps(ctx context.Context, kbID string) error { + cleaner, ok := s.pendingRepo.(interfaces.TaskPendingOpsScopeCleaner) + if !ok || kbID == "" { + return nil + } + cleanupCtx, cancel := wikiIngestCleanupContext(ctx) + defer cancel() + return cleaner.DeleteByScope(cleanupCtx, types.TaskScopeKnowledgeBase, kbID) +} + +func (s *wikiIngestService) enqueueFinalizeRow(ctx context.Context, op *types.TaskPendingOp) bool { + accepted, err := enqueueWikiPendingOp(ctx, s.pendingRepo, op) + if err != nil { + logger.Warnf(ctx, "wiki finalize: enqueue %s row failed: %v", op.Op, err) + return false + } + return accepted +} + // enqueueFinalize persists this batch's KB-global convergence work into the // finalize lane of task_pending_ops and schedules a debounced trigger. One // "slug" row per affected page (carrying its fresh title when this batch wrote @@ -580,13 +622,14 @@ func (s *wikiIngestService) enqueueFinalize( if s.pendingRepo == nil { return } + acceptedAny := false for _, slug := range affectedSlugs { row := wikiFinalizeRow{Slug: slug, Title: freshTitleBySlug[slug]} b, err := json.Marshal(row) if err != nil { continue } - if err := s.pendingRepo.Enqueue(ctx, &types.TaskPendingOp{ + if s.enqueueFinalizeRow(ctx, &types.TaskPendingOp{ TenantID: payload.TenantID, TaskType: wikiFinalizeTaskType, Scope: wikiTaskScope, @@ -594,8 +637,8 @@ func (s *wikiIngestService) enqueueFinalize( Op: wikiFinalizeOpSlug, DedupKey: slug, Payload: b, - }); err != nil { - logger.Warnf(ctx, "wiki finalize: enqueue slug row for %s failed: %v", slug, err) + }) { + acceptedAny = true } } for i := range changes { @@ -604,7 +647,7 @@ func (s *wikiIngestService) enqueueFinalize( if err != nil { continue } - if err := s.pendingRepo.Enqueue(ctx, &types.TaskPendingOp{ + if s.enqueueFinalizeRow(ctx, &types.TaskPendingOp{ TenantID: payload.TenantID, TaskType: wikiFinalizeTaskType, Scope: wikiTaskScope, @@ -612,14 +655,14 @@ func (s *wikiIngestService) enqueueFinalize( Op: wikiFinalizeOpChange, DedupKey: "", Payload: b, - }); err != nil { - logger.Warnf(ctx, "wiki finalize: enqueue change row failed: %v", err) + }) { + acceptedAny = true } } if len(folderIDs) > 0 { row := wikiFinalizeRow{FolderIDs: uniqueWikiFolderIDs(folderIDs)} if b, err := json.Marshal(row); err == nil { - if err := s.pendingRepo.Enqueue(ctx, &types.TaskPendingOp{ + if s.enqueueFinalizeRow(ctx, &types.TaskPendingOp{ TenantID: payload.TenantID, TaskType: wikiFinalizeTaskType, Scope: wikiTaskScope, @@ -627,11 +670,14 @@ func (s *wikiIngestService) enqueueFinalize( Op: wikiFinalizeOpFolderPrune, DedupKey: "", Payload: b, - }); err != nil { - logger.Warnf(ctx, "wiki finalize: enqueue folder prune row failed: %v", err) + }) { + acceptedAny = true } } } + if !acceptedAny { + return + } s.scheduleFinalize(ctx, payload) } diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go index f533e5b6..04a6bba7 100644 --- a/internal/application/service/wiki_ingest_batch.go +++ b/internal/application/service/wiki_ingest_batch.go @@ -11,6 +11,7 @@ import ( "unicode/utf8" "github.com/Tencent/WeKnora/internal/agent" + apprepo "github.com/Tencent/WeKnora/internal/application/repository" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/models/chat" "github.com/Tencent/WeKnora/internal/tracing/langfuse" @@ -280,6 +281,13 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task } kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID) + if errors.Is(err, apprepo.ErrKnowledgeBaseNotFound) || (err == nil && kb == nil) { + exitStatus = "kb_deleted" + if cleanupErr := s.clearDeletedKnowledgeBasePendingOps(ctx, payload.KnowledgeBaseID); cleanupErr != nil { + return fmt.Errorf("wiki ingest: clear deleted KB queue: %w", cleanupErr) + } + return nil + } if err != nil { exitStatus = "get_kb_failed" return fmt.Errorf("wiki ingest: get KB: %w", err) @@ -1006,6 +1014,12 @@ func (s *wikiIngestService) ProcessWikiFinalize(ctx context.Context, t *asynq.Ta } kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID) + if errors.Is(err, apprepo.ErrKnowledgeBaseNotFound) || (err == nil && kb == nil) { + if cleanupErr := s.clearDeletedKnowledgeBasePendingOps(ctx, payload.KnowledgeBaseID); cleanupErr != nil { + return fmt.Errorf("wiki finalize: clear deleted KB queue: %w", cleanupErr) + } + return nil + } if err != nil { return fmt.Errorf("wiki finalize: get KB: %w", err) } diff --git a/internal/container/recover_pending_wiki_tasks.go b/internal/container/recover_pending_wiki_tasks.go index 6d9d529a..f72d994a 100644 --- a/internal/container/recover_pending_wiki_tasks.go +++ b/internal/container/recover_pending_wiki_tasks.go @@ -34,12 +34,34 @@ func recoverPendingWikiTasks(db *gorm.DB, task interfaces.TaskEnqueuer) { return } ctx := context.Background() + const activeKnowledgeBase = `EXISTS ( + SELECT 1 FROM knowledge_bases kb + WHERE kb.id = task_pending_ops.scope_id + AND kb.tenant_id = task_pending_ops.tenant_id + AND kb.deleted_at IS NULL + )` + wikiTaskTypes := []string{types.TypeWikiIngest, types.TypeWikiFinalize} + + // Durable rows for a deleted/missing KB must not recreate ephemeral + // triggers at startup. Fail closed if this cleanup cannot be verified. + cleanup := db.WithContext(ctx). + Where("scope = ? AND task_type IN ?", types.TaskScopeKnowledgeBase, wikiTaskTypes). + Where("NOT " + activeKnowledgeBase). + Delete(&types.TaskPendingOp{}) + if cleanup.Error != nil { + logger.Warnf(ctx, "[WikiRecovery] failed to clear deleted KB queues: %v", cleanup.Error) + return + } + if cleanup.RowsAffected > 0 { + logger.Infof(ctx, "[WikiRecovery] removed %d pending row(s) for deleted knowledge bases", cleanup.RowsAffected) + } + var scopes []pendingWikiScope if err := db.WithContext(ctx). Model(&types.TaskPendingOp{}). Distinct("tenant_id", "task_type", "scope_id"). - Where("scope = ? AND task_type IN ?", types.TaskScopeKnowledgeBase, - []string{types.TypeWikiIngest, types.TypeWikiFinalize}). + Where("scope = ? AND task_type IN ?", types.TaskScopeKnowledgeBase, wikiTaskTypes). + Where(activeKnowledgeBase). Find(&scopes).Error; err != nil { logger.Warnf(ctx, "[WikiRecovery] failed to list pending queues: %v", err) return diff --git a/internal/container/reset_pending_tasks_test.go b/internal/container/reset_pending_tasks_test.go index 5c4b2aeb..814c7bd9 100644 --- a/internal/container/reset_pending_tasks_test.go +++ b/internal/container/reset_pending_tasks_test.go @@ -78,6 +78,14 @@ CREATE TABLE IF NOT EXISTS task_pending_ops ( ); ` +const resetPendingKnowledgeBasesDDL = ` +CREATE TABLE IF NOT EXISTS knowledge_bases ( + id VARCHAR(64) PRIMARY KEY, + tenant_id INTEGER NOT NULL DEFAULT 0, + deleted_at DATETIME +); +` + func setupResetPendingDB(t *testing.T) *gorm.DB { t.Helper() db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) @@ -86,6 +94,7 @@ func setupResetPendingDB(t *testing.T) *gorm.DB { require.NoError(t, db.Exec(resetPendingSyncLogDDL).Error) require.NoError(t, db.Exec(resetPendingSpansDDL).Error) require.NoError(t, db.Exec(resetPendingOpsDDL).Error) + require.NoError(t, db.Exec(resetPendingKnowledgeBasesDDL).Error) return db } @@ -295,6 +304,11 @@ func (r *recordingTaskEnqueuer) Enqueue(task *asynq.Task, _ ...asynq.Option) (*a func TestRecoverPendingWikiTasks_RecreatesOneTriggerPerLaneAndKB(t *testing.T) { db := setupResetPendingDB(t) + require.NoError(t, db.Exec( + `INSERT INTO knowledge_bases (id, tenant_id, deleted_at) + VALUES (?, ?, NULL), (?, ?, NULL), (?, ?, ?)`, + "kb-a", 7, "kb-b", 8, "kb-deleted", 9, time.Now(), + ).Error) rows := []struct { tenantID uint64 taskType string @@ -305,6 +319,8 @@ func TestRecoverPendingWikiTasks_RecreatesOneTriggerPerLaneAndKB(t *testing.T) { {7, types.TypeWikiIngest, "kb-a", "k-2"}, // same lane: one trigger {7, types.TypeWikiFinalize, "kb-a", "slug-a"}, {8, types.TypeWikiIngest, "kb-b", "k-3"}, + {9, types.TypeWikiIngest, "kb-deleted", "k-deleted"}, + {10, types.TypeWikiFinalize, "kb-missing", "k-missing"}, } for _, row := range rows { require.NoError(t, db.Exec( @@ -328,4 +344,10 @@ func TestRecoverPendingWikiTasks_RecreatesOneTriggerPerLaneAndKB(t *testing.T) { assert.Equal(t, uint64(7), seen[types.TypeWikiIngest+":kb-a"].TenantID) assert.Equal(t, uint64(7), seen[types.TypeWikiFinalize+":kb-a"].TenantID) assert.Equal(t, uint64(8), seen[types.TypeWikiIngest+":kb-b"].TenantID) + + var orphaned int64 + require.NoError(t, db.Model(&types.TaskPendingOp{}). + Where("scope_id IN ?", []string{"kb-deleted", "kb-missing"}). + Count(&orphaned).Error) + assert.Zero(t, orphaned) } diff --git a/internal/router/task_inspector.go b/internal/router/task_inspector.go index d230d6d6..34063738 100644 --- a/internal/router/task_inspector.go +++ b/internal/router/task_inspector.go @@ -96,7 +96,12 @@ var taskTypesForKnowledgeCancel = map[string]struct{}{ // listPageSize caps each Redis LIST call. Asynq pages tasks, so we // loop until a short page comes back. 100 matches asynq's default. -const listPageSize = 100 +const ( + listPageSize = 100 + maxQueueMutationPasses = 1000 + activeCancellationSettleTimeout = time.Second + activeCancellationPollInterval = 25 * time.Millisecond +) // CancelTasksForKnowledge removes queued tasks whose payload references // the given knowledge_id and signals active workers running such tasks @@ -107,18 +112,9 @@ func (a *asynqTaskInspector) CancelTasksForKnowledge( if a == nil || a.inspector == nil || knowledgeID == "" { return 0, 0, nil } - deleted := 0 - cancelled := 0 - - for _, queue := range queuesScanned { - // Pending / Scheduled / Retry can all be deleted by task ID. - // Archived tasks are NOT touched: dead-letter rows are - // already final and should remain visible to operators. - deleted += a.deletePendingMatches(ctx, queue, knowledgeID) - deleted += a.deleteScheduledMatches(ctx, queue, knowledgeID) - deleted += a.deleteRetryMatches(ctx, queue, knowledgeID) - cancelled += a.cancelActiveMatches(ctx, queue, knowledgeID) - } + deleted, cancelled := a.cancelMatchingTasks(ctx, func(taskType string, payload []byte) bool { + return matchesKnowledge(taskType, payload, knowledgeID) + }) logger.Infof(ctx, "[TaskInspector] knowledge=%s cancel summary: deleted_from_queue=%d active_cancel_signaled=%d", @@ -127,6 +123,42 @@ func (a *asynqTaskInspector) CancelTasksForKnowledge( return deleted, cancelled, nil } +// CancelTasksForKnowledgeBase removes tasks that still reference a knowledge +// base (or one of its knowledges) after the knowledge base has been deleted. +// The kb:delete task is deliberately excluded because it performs the durable +// storage cleanup and must remain queued. +func (a *asynqTaskInspector) CancelTasksForKnowledgeBase( + ctx context.Context, knowledgeBaseID string, knowledgeIDs []string, dataSourceIDs []string, +) (int, int, error) { + if a == nil || a.inspector == nil { + return 0, 0, nil + } + knowledgeIDSet := make(map[string]struct{}, len(knowledgeIDs)) + for _, knowledgeID := range knowledgeIDs { + if knowledgeID != "" { + knowledgeIDSet[knowledgeID] = struct{}{} + } + } + dataSourceIDSet := make(map[string]struct{}, len(dataSourceIDs)) + for _, dataSourceID := range dataSourceIDs { + if dataSourceID != "" { + dataSourceIDSet[dataSourceID] = struct{}{} + } + } + if knowledgeBaseID == "" && len(knowledgeIDSet) == 0 && len(dataSourceIDSet) == 0 { + return 0, 0, nil + } + + deleted, cancelled := a.cancelMatchingTasks(ctx, func(taskType string, payload []byte) bool { + return matchesKnowledgeBase(taskType, payload, knowledgeBaseID, knowledgeIDSet, dataSourceIDSet) + }) + logger.Infof(ctx, + "[TaskInspector] knowledge_base=%s cancel summary: deleted_from_queue=%d active_cancel_signaled=%d", + knowledgeBaseID, deleted, cancelled, + ) + return deleted, cancelled, nil +} + // HasQueuedTasksForKnowledge reports whether any pending / scheduled / // retry / active task referencing knowledgeID still lives in the queue. // Read-only counterpart of CancelTasksForKnowledge — the housekeeping @@ -138,18 +170,12 @@ func (a *asynqTaskInspector) HasQueuedTasksForKnowledge( if a == nil || a.inspector == nil || knowledgeID == "" { return false, nil } - listers := []struct { - state string - list func(string, ...asynq.ListOption) ([]*asynq.TaskInfo, error) - }{ - {"pending", a.inspector.ListPendingTasks}, - {"scheduled", a.inspector.ListScheduledTasks}, - {"retry", a.inspector.ListRetryTasks}, - {"active", a.inspector.ListActiveTasks}, + matcher := func(taskType string, payload []byte) bool { + return matchesKnowledge(taskType, payload, knowledgeID) } for _, queue := range queuesScanned { - for _, l := range listers { - if a.queueStateHasMatch(ctx, queue, knowledgeID, l.state, l.list) { + for _, state := range a.cancellableTaskStates() { + if a.queueStateHasMatch(ctx, queue, state.name, state.list, matcher) { return true, nil } } @@ -685,14 +711,249 @@ func (a *asynqTaskInspector) WorkerServerStats( return stats, true, nil } +type taskMatcher func(taskType string, payload []byte) bool + +type cancellableTaskState struct { + name string + list func(string, ...asynq.ListOption) ([]*asynq.TaskInfo, error) + active bool +} + +func (a *asynqTaskInspector) cancellableTaskStates() []cancellableTaskState { + return []cancellableTaskState{ + {name: "pending", list: a.inspector.ListPendingTasks}, + {name: "scheduled", list: a.inspector.ListScheduledTasks}, + {name: "retry", list: a.inspector.ListRetryTasks}, + {name: "active", list: a.inspector.ListActiveTasks, active: true}, + } +} + +// cancelMatchingTasks applies the same matcher to every live queue state. +// Archived tasks are deliberately retained as operator-visible history. +func (a *asynqTaskInspector) cancelMatchingTasks(ctx context.Context, matcher taskMatcher) (int, int) { + deleted := 0 + cancelled := 0 + + // Drain queued states first. Active cancellation is asynchronous: when a + // handler returns context.Canceled, asynq normally moves it to retry, so a + // later settle phase must remove that transitioned record as well. + for _, queue := range queuesScanned { + for _, state := range a.cancellableTaskStates() { + if state.active { + continue + } + deleted += a.processQueueStateMatches( + ctx, queue, state, true, matcher, "delete", + func(task *asynq.TaskInfo) error { return a.inspector.DeleteTask(queue, task.ID) }, + ) + } + } + + // Snapshot all matching active IDs before publishing cancellation. If we + // mutate the active set while paging, exited workers shift later pages and + // can make us skip an entire page. + activeTasks := make([]queueTask, 0) + for _, queue := range queuesScanned { + for _, state := range a.cancellableTaskStates() { + if !state.active { + continue + } + for _, task := range a.snapshotQueueStateMatches(ctx, queue, state, matcher) { + if err := a.inspector.CancelProcessing(task.ID); err != nil { + logger.Warnf(ctx, "[TaskInspector] cancel active type=%s id=%s: %v", task.Type, task.ID, err) + continue + } + cancelled++ + activeTasks = append(activeTasks, queueTask{queue: queue, id: task.ID}) + } + } + } + deleted += a.deleteCancelledTransitions(ctx, activeTasks) + + // Catch tasks that entered pending/scheduled/retry while cancellation was + // settling, plus downstream work emitted immediately before a handler saw + // its cancellation. + for _, queue := range queuesScanned { + for _, state := range a.cancellableTaskStates() { + if state.active { + continue + } + deleted += a.processQueueStateMatches( + ctx, queue, state, true, matcher, "delete", + func(task *asynq.TaskInfo) error { return a.inspector.DeleteTask(queue, task.ID) }, + ) + } + } + return deleted, cancelled +} + +type queueTask struct { + queue string + id string +} + +// snapshotQueueStateMatches returns a read-only snapshot before its caller +// mutates the state. TaskInfo values are only used for immutable task metadata. +func (a *asynqTaskInspector) snapshotQueueStateMatches( + ctx context.Context, + queue string, + state cancellableTaskState, + matcher taskMatcher, +) []*asynq.TaskInfo { + var matches []*asynq.TaskInfo + for page := 1; ; page++ { + if ctx.Err() != nil { + return matches + } + tasks, err := state.list(queue, asynq.PageSize(listPageSize), asynq.Page(page)) + if err != nil { + if !errors.Is(err, asynq.ErrQueueNotFound) { + logger.Warnf(ctx, "[TaskInspector] snapshot %s queue=%s page=%d: %v", state.name, queue, page, err) + } + return matches + } + for _, task := range tasks { + if matcher(task.Type, task.Payload) { + matches = append(matches, task) + } + } + if len(tasks) < listPageSize { + return matches + } + } +} + +// deleteCancelledTransitions waits briefly for signalled active tasks to +// leave active state. A normal context.Canceled result is retried by asynq; +// delete that transitioned record before it can become an orphan. Archived +// and completed tasks remain operator-visible history. +func (a *asynqTaskInspector) deleteCancelledTransitions(ctx context.Context, tasks []queueTask) int { + if len(tasks) == 0 { + return 0 + } + deadline := time.Now().Add(activeCancellationSettleTimeout) + pending := append([]queueTask(nil), tasks...) + deleted := 0 + for len(pending) > 0 { + if ctx.Err() != nil { + return deleted + } + next := pending[:0] + for _, ref := range pending { + info, err := a.inspector.GetTaskInfo(ref.queue, ref.id) + if errors.Is(err, asynq.ErrTaskNotFound) || errors.Is(err, asynq.ErrQueueNotFound) { + continue + } + if err != nil { + logger.Warnf(ctx, "[TaskInspector] inspect cancelled task queue=%s id=%s: %v", ref.queue, ref.id, err) + next = append(next, ref) + continue + } + switch info.State { + case asynq.TaskStatePending, asynq.TaskStateScheduled, asynq.TaskStateRetry: + if err := a.inspector.DeleteTask(ref.queue, ref.id); err != nil { + if !errors.Is(err, asynq.ErrTaskNotFound) { + logger.Warnf(ctx, "[TaskInspector] delete cancelled transition queue=%s id=%s: %v", ref.queue, ref.id, err) + next = append(next, ref) + } + continue + } + deleted++ + case asynq.TaskStateActive: + next = append(next, ref) + } + } + pending = next + if len(pending) == 0 || !time.Now().Before(deadline) { + return deleted + } + wait := activeCancellationPollInterval + if remaining := time.Until(deadline); remaining < wait { + wait = remaining + } + timer := time.NewTimer(wait) + select { + case <-ctx.Done(): + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + return deleted + case <-timer.C: + } + } + return deleted +} + +// processQueueStateMatches pages through one queue state and applies action to +// matching tasks. Deleting a task shifts later entries toward the current +// page, so delete states rescan that page after any successful mutation. +func (a *asynqTaskInspector) processQueueStateMatches( + ctx context.Context, + queue string, + state cancellableTaskState, + rescanAfterSuccess bool, + matcher taskMatcher, + actionName string, + action func(*asynq.TaskInfo) error, +) int { + processed := 0 + page := 1 + passes := 0 + for passes < maxQueueMutationPasses { + passes++ + if ctx.Err() != nil { + return processed + } + tasks, err := state.list(queue, asynq.PageSize(listPageSize), asynq.Page(page)) + if err != nil { + if !errors.Is(err, asynq.ErrQueueNotFound) { + logger.Warnf(ctx, "[TaskInspector] list %s queue=%s page=%d: %v", state.name, queue, page, err) + } + return processed + } + if len(tasks) == 0 { + return processed + } + + processedOnPage := 0 + for _, task := range tasks { + if !matcher(task.Type, task.Payload) { + continue + } + if err := action(task); err != nil { + logger.Warnf(ctx, "[TaskInspector] %s %s type=%s id=%s: %v", actionName, state.name, task.Type, task.ID, err) + continue + } + processed++ + processedOnPage++ + } + if rescanAfterSuccess && processedOnPage > 0 { + continue + } + if len(tasks) < listPageSize { + return processed + } + page++ + } + logger.Warnf(ctx, + "[TaskInspector] stopped %s queue=%s after %d mutation passes", + state.name, queue, maxQueueMutationPasses, + ) + return processed +} + // queueStateHasMatch pages through one (queue, state) list looking for a -// task that references knowledgeID. Mirrors the delete* scanners but is -// strictly read-only and returns early on the first hit. A backend error -// is logged and treated as "no match" (false); the caller's fail-safe -// then errs toward recovering the row rather than preserving it forever. +// matching task. It is strictly read-only and returns on the first hit. A +// backend error is logged and treated as "no match" (false). func (a *asynqTaskInspector) queueStateHasMatch( - ctx context.Context, queue, knowledgeID, state string, + ctx context.Context, + queue string, + state string, list func(string, ...asynq.ListOption) ([]*asynq.TaskInfo, error), + matcher taskMatcher, ) bool { page := 1 for { @@ -706,8 +967,8 @@ func (a *asynqTaskInspector) queueStateHasMatch( if len(tasks) == 0 { return false } - for _, t := range tasks { - if matchesKnowledge(t.Type, t.Payload, knowledgeID) { + for _, task := range tasks { + if matcher(task.Type, task.Payload) { return true } } @@ -731,133 +992,55 @@ func matchesKnowledge(taskType string, payload []byte, knowledgeID string) bool return probe.KnowledgeID == knowledgeID } -func (a *asynqTaskInspector) deletePendingMatches(ctx context.Context, queue, knowledgeID string) int { - deleted := 0 - page := 1 - for { - tasks, err := a.inspector.ListPendingTasks(queue, asynq.PageSize(listPageSize), asynq.Page(page)) - if err != nil { - if !errors.Is(err, asynq.ErrQueueNotFound) { - logger.Warnf(ctx, "[TaskInspector] list pending queue=%s page=%d: %v", queue, page, err) - } - return deleted - } - if len(tasks) == 0 { - return deleted - } - for _, t := range tasks { - if !matchesKnowledge(t.Type, t.Payload, knowledgeID) { - continue - } - if err := a.inspector.DeleteTask(queue, t.ID); err != nil { - logger.Warnf(ctx, "[TaskInspector] delete pending type=%s id=%s: %v", t.Type, t.ID, err) - continue - } - deleted++ - } - if len(tasks) < listPageSize { - return deleted - } - page++ +// matchesKnowledgeBase identifies work made obsolete by deleting a knowledge +// base. In addition to direct KB fields, clone/move payloads carry semantic KB +// references under task-specific field names. knowledgeIDs catches tasks whose +// payload does not carry a KB ID at all. +func matchesKnowledgeBase( + taskType string, + payload []byte, + knowledgeBaseID string, + knowledgeIDs map[string]struct{}, + dataSourceIDs map[string]struct{}, +) bool { + // These cleanup tasks carry snapshots specifically so they can still run + // after the KB row has been soft-deleted. Removing them leaks resources. + if taskType == types.TypeKBDelete || taskType == types.TypeIndexDelete { + return false + } + var probe runtimeTaskPayloadProbe + if err := json.Unmarshal(payload, &probe); err != nil { + return false } -} -func (a *asynqTaskInspector) deleteScheduledMatches(ctx context.Context, queue, knowledgeID string) int { - deleted := 0 - page := 1 - for { - tasks, err := a.inspector.ListScheduledTasks(queue, asynq.PageSize(listPageSize), asynq.Page(page)) - if err != nil { - if !errors.Is(err, asynq.ErrQueueNotFound) { - logger.Warnf(ctx, "[TaskInspector] list scheduled queue=%s page=%d: %v", queue, page, err) + if knowledgeBaseID != "" { + if probe.KnowledgeBaseID == knowledgeBaseID || probe.KBID == knowledgeBaseID { + return true + } + switch taskType { + case types.TypeKBClone: + if probe.SourceID == knowledgeBaseID || probe.TargetID == knowledgeBaseID { + return true } - return deleted - } - if len(tasks) == 0 { - return deleted - } - for _, t := range tasks { - if !matchesKnowledge(t.Type, t.Payload, knowledgeID) { - continue + case types.TypeKnowledgeMove: + if probe.SourceKBID == knowledgeBaseID || probe.TargetKBID == knowledgeBaseID { + return true } - if err := a.inspector.DeleteTask(queue, t.ID); err != nil { - logger.Warnf(ctx, "[TaskInspector] delete scheduled type=%s id=%s: %v", t.Type, t.ID, err) - continue - } - deleted++ } - if len(tasks) < listPageSize { - return deleted - } - page++ } -} -func (a *asynqTaskInspector) deleteRetryMatches(ctx context.Context, queue, knowledgeID string) int { - deleted := 0 - page := 1 - for { - tasks, err := a.inspector.ListRetryTasks(queue, asynq.PageSize(listPageSize), asynq.Page(page)) - if err != nil { - if !errors.Is(err, asynq.ErrQueueNotFound) { - logger.Warnf(ctx, "[TaskInspector] list retry queue=%s page=%d: %v", queue, page, err) - } - return deleted - } - if len(tasks) == 0 { - return deleted - } - for _, t := range tasks { - if !matchesKnowledge(t.Type, t.Payload, knowledgeID) { - continue - } - if err := a.inspector.DeleteTask(queue, t.ID); err != nil { - logger.Warnf(ctx, "[TaskInspector] delete retry type=%s id=%s: %v", t.Type, t.ID, err) - continue - } - deleted++ - } - if len(tasks) < listPageSize { - return deleted - } - page++ + if _, ok := knowledgeIDs[probe.KnowledgeID]; ok && probe.KnowledgeID != "" { + return true } -} - -// cancelActiveMatches signals active workers to abort via -// Inspector.CancelProcessing. The worker's ctx becomes Done() so the -// next blocking call (or our checkpoint reads) bails. The DB-level -// abort flag (parse_status=cancelled) remains the durable signal — -// this is a latency optimization, not the correctness mechanism. -func (a *asynqTaskInspector) cancelActiveMatches(ctx context.Context, queue, knowledgeID string) int { - cancelled := 0 - page := 1 - for { - tasks, err := a.inspector.ListActiveTasks(queue, asynq.PageSize(listPageSize), asynq.Page(page)) - if err != nil { - if !errors.Is(err, asynq.ErrQueueNotFound) { - logger.Warnf(ctx, "[TaskInspector] list active queue=%s page=%d: %v", queue, page, err) - } - return cancelled + for _, knowledgeID := range probe.KnowledgeIDs { + if _, ok := knowledgeIDs[knowledgeID]; ok && knowledgeID != "" { + return true } - if len(tasks) == 0 { - return cancelled - } - for _, t := range tasks { - if !matchesKnowledge(t.Type, t.Payload, knowledgeID) { - continue - } - if err := a.inspector.CancelProcessing(t.ID); err != nil { - logger.Warnf(ctx, "[TaskInspector] cancel active type=%s id=%s: %v", t.Type, t.ID, err) - continue - } - cancelled++ - } - if len(tasks) < listPageSize { - return cancelled - } - page++ } + if _, ok := dataSourceIDs[probe.DataSourceID]; ok && probe.DataSourceID != "" { + return true + } + return false } // noopTaskInspector is the Lite-mode (no Redis) inspector. Inline diff --git a/internal/router/task_inspector_kb_cancel_test.go b/internal/router/task_inspector_kb_cancel_test.go new file mode 100644 index 00000000..58d92381 --- /dev/null +++ b/internal/router/task_inspector_kb_cancel_test.go @@ -0,0 +1,348 @@ +package router + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/alicebob/miniredis/v2" + "github.com/hibiken/asynq" + "github.com/redis/go-redis/v9" +) + +func TestMatchesKnowledgeBase(t *testing.T) { + knowledgeIDs := map[string]struct{}{ + "knowledge-1": {}, + "knowledge-2": {}, + } + dataSourceIDs := map[string]struct{}{"datasource-1": {}} + tests := []struct { + name string + taskType string + payload string + want bool + }{ + {name: "knowledge base id", taskType: types.TypeDocumentProcess, payload: `{"knowledge_base_id":"kb-1"}`, want: true}, + {name: "legacy kb id", taskType: types.TypeFAQImport, payload: `{"kb_id":"kb-1"}`, want: true}, + {name: "clone source", taskType: types.TypeKBClone, payload: `{"source_id":"kb-1"}`, want: true}, + {name: "clone target", taskType: types.TypeKBClone, payload: `{"target_id":"kb-1"}`, want: true}, + {name: "source id is task specific", taskType: types.TypeDocumentProcess, payload: `{"source_id":"kb-1"}`}, + {name: "move source", taskType: types.TypeKnowledgeMove, payload: `{"source_kb_id":"kb-1"}`, want: true}, + {name: "move target", taskType: types.TypeKnowledgeMove, payload: `{"target_kb_id":"kb-1"}`, want: true}, + {name: "move fields are task specific", taskType: types.TypeDocumentProcess, payload: `{"source_kb_id":"kb-1"}`}, + {name: "single knowledge", taskType: types.TypeChunkExtract, payload: `{"knowledge_id":"knowledge-1"}`, want: true}, + { + name: "knowledge collection", + taskType: types.TypeKnowledgeListReparse, + payload: `{"knowledge_ids":["other","knowledge-2"]}`, + want: true, + }, + { + name: "unrelated", + taskType: types.TypeDocumentProcess, + payload: `{"knowledge_base_id":"other","knowledge_id":"other"}`, + }, + {name: "malformed payload", taskType: types.TypeDocumentProcess, payload: `{`}, + {name: "preserve kb delete by kb", taskType: types.TypeKBDelete, payload: `{"knowledge_base_id":"kb-1"}`}, + {name: "preserve kb delete by knowledge", taskType: types.TypeKBDelete, payload: `{"knowledge_id":"knowledge-1"}`}, + {name: "preserve index delete by kb", taskType: types.TypeIndexDelete, payload: `{"knowledge_base_id":"kb-1"}`}, + { + name: "preserve index delete by knowledge", taskType: types.TypeIndexDelete, + payload: `{"knowledge_id":"knowledge-1"}`, + }, + { + name: "data source sync", taskType: types.TypeDataSourceSync, + payload: `{"data_source_id":"datasource-1"}`, want: true, + }, + { + name: "other data source sync", taskType: types.TypeDataSourceSync, + payload: `{"data_source_id":"datasource-other"}`, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got := matchesKnowledgeBase( + test.taskType, []byte(test.payload), "kb-1", knowledgeIDs, dataSourceIDs, + ) + if got != test.want { + t.Fatalf("matchesKnowledgeBase() = %v, want %v", got, test.want) + } + }) + } +} + +func TestMatchesKnowledgePreservesPerKnowledgeAllowList(t *testing.T) { + if !matchesKnowledge(types.TypeDocumentProcess, []byte(`{"knowledge_id":"knowledge-1"}`), "knowledge-1") { + t.Fatal("document task should remain cancellable by knowledge ID") + } + if matchesKnowledge(types.TypeKBClone, []byte(`{"knowledge_id":"knowledge-1"}`), "knowledge-1") { + t.Fatal("KB clone must not become cancellable through the per-knowledge API") + } +} + +func TestCancelTasksForKnowledgeBaseRescansMutatedPages(t *testing.T) { + server := miniredis.RunT(t) + redisClient := redis.NewClient(&redis.Options{Addr: server.Addr()}) + asynqClient := asynq.NewClientFromRedisClient(redisClient) + t.Cleanup(func() { + _ = asynqClient.Close() + _ = redisClient.Close() + }) + inspector := &asynqTaskInspector{ + inspector: asynq.NewInspectorFromRedisClient(redisClient), + redis: redisClient, + } + + const matchingPending = 205 + for i := 0; i < matchingPending; i++ { + enqueueTask(t, asynqClient, types.TypeDocumentProcess, + fmt.Sprintf(`{"knowledge_base_id":"kb-delete","knowledge_id":"knowledge-%d"}`, i), + fmt.Sprintf("matching-%03d", i), + ) + } + for i := 0; i < 5; i++ { + enqueueTask(t, asynqClient, types.TypeDocumentProcess, + `{"knowledge_base_id":"kb-keep","knowledge_id":"keep"}`, + fmt.Sprintf("survivor-%03d", i), + ) + } + enqueueTask(t, asynqClient, types.TypeKBDelete, + `{"knowledge_base_id":"kb-delete"}`, "kb-delete-cleanup", + ) + enqueueTask(t, asynqClient, types.TypeIndexDelete, + `{"knowledge_base_id":"kb-delete"}`, "index-delete-cleanup", + ) + enqueueTask(t, asynqClient, types.TypeDataSourceSync, + `{"data_source_id":"datasource-delete"}`, "datasource-sync", + ) + + scheduleTask(t, asynqClient, types.TypeFAQImport, + `{"kb_id":"kb-delete"}`, "scheduled-kb-match", + ) + scheduleTask(t, asynqClient, types.TypeKnowledgeListReparse, + `{"knowledge_ids":["knowledge-associated"]}`, "scheduled-knowledge-match", + ) + scheduleTask(t, asynqClient, types.TypeFAQImport, + `{"kb_id":"kb-keep"}`, "scheduled-survivor", + ) + + deleted, cancelled, err := inspector.CancelTasksForKnowledgeBase( + context.Background(), "kb-delete", []string{"knowledge-associated"}, []string{"datasource-delete"}, + ) + if err != nil { + t.Fatalf("cancel tasks: %v", err) + } + if want := matchingPending + 3; deleted != want { + t.Fatalf("deleted = %d, want %d", deleted, want) + } + if cancelled != 0 { + t.Fatalf("cancelled active = %d, want 0", cancelled) + } + + pending, err := inspector.inspector.ListPendingTasks( + types.QueueDefault, asynq.PageSize(listPageSize), asynq.Page(1), + ) + if err != nil { + t.Fatalf("list pending: %v", err) + } + if len(pending) != 7 { + t.Fatalf("pending survivors = %d, want 7", len(pending)) + } + if !hasTaskID(pending, "kb-delete-cleanup") { + t.Fatal("kb:delete cleanup task was removed") + } + if !hasTaskID(pending, "index-delete-cleanup") { + t.Fatal("index:delete cleanup task was removed") + } + + scheduled, err := inspector.inspector.ListScheduledTasks( + types.QueueDefault, asynq.PageSize(listPageSize), asynq.Page(1), + ) + if err != nil { + t.Fatalf("list scheduled: %v", err) + } + if len(scheduled) != 1 || scheduled[0].ID != "scheduled-survivor" { + t.Fatalf("scheduled survivors = %v, want scheduled-survivor", taskIDs(scheduled)) + } +} + +func TestCancelTasksForKnowledgeBaseRemovesCancelledActiveRetry(t *testing.T) { + server := miniredis.RunT(t) + redisClient := redis.NewClient(&redis.Options{Addr: server.Addr()}) + asynqClient := asynq.NewClientFromRedisClient(redisClient) + t.Cleanup(func() { + _ = asynqClient.Close() + _ = redisClient.Close() + }) + + inspector := &asynqTaskInspector{ + inspector: asynq.NewInspectorFromRedisClient(redisClient), + redis: redisClient, + } + worker := asynq.NewServerFromRedisClient(redisClient, asynq.Config{ + Concurrency: 1, + Queues: map[string]int{types.QueueDefault: 1}, + TaskCheckInterval: 10 * time.Millisecond, + DelayedTaskCheckInterval: time.Hour, + RetryDelayFunc: func(_ int, _ error, _ *asynq.Task) time.Duration { + return time.Hour + }, + ShutdownTimeout: time.Second, + LogLevel: asynq.FatalLevel, + }) + + handlerStarted := make(chan struct{}) + handlerReturned := make(chan struct{}) + mux := asynq.NewServeMux() + mux.HandleFunc(types.TypeDocumentProcess, func(ctx context.Context, _ *asynq.Task) error { + close(handlerStarted) + <-ctx.Done() + close(handlerReturned) + return ctx.Err() + }) + + if err := worker.Start(mux); err != nil { + t.Fatalf("start asynq worker: %v", err) + } + t.Cleanup(worker.Shutdown) + waitForAsynqCancellationSubscriber(t, redisClient) + + const taskID = "active-kb-match" + if _, err := asynqClient.Enqueue( + asynq.NewTask(types.TypeDocumentProcess, []byte(`{"knowledge_base_id":"kb-delete"}`)), + asynq.Queue(types.QueueDefault), + asynq.TaskID(taskID), + asynq.MaxRetry(3), + ); err != nil { + t.Fatalf("enqueue active task: %v", err) + } + + select { + case <-handlerStarted: + case <-time.After(5 * time.Second): + t.Fatal("handler did not start") + } + waitForTaskState(t, inspector.inspector, taskID, asynq.TaskStateActive) + + deleted, cancelled, err := inspector.CancelTasksForKnowledgeBase( + context.Background(), "kb-delete", nil, nil, + ) + if err != nil { + t.Fatalf("cancel active task: %v", err) + } + if cancelled != 1 { + t.Fatalf("cancelled active = %d, want 1", cancelled) + } + if deleted != 1 { + t.Fatalf("deleted after active cancellation = %d, want 1", deleted) + } + + select { + case <-handlerReturned: + case <-time.After(5 * time.Second): + t.Fatal("cancelled handler did not return") + } + waitForTaskToLeaveLiveStates(t, inspector.inspector, taskID) +} + +func enqueueTask(t *testing.T, client *asynq.Client, taskType, payload, taskID string) { + t.Helper() + if _, err := client.Enqueue( + asynq.NewTask(taskType, []byte(payload)), + asynq.Queue(types.QueueDefault), + asynq.TaskID(taskID), + ); err != nil { + t.Fatalf("enqueue %s: %v", taskID, err) + } +} + +func scheduleTask(t *testing.T, client *asynq.Client, taskType, payload, taskID string) { + t.Helper() + if _, err := client.Enqueue( + asynq.NewTask(taskType, []byte(payload)), + asynq.Queue(types.QueueDefault), + asynq.TaskID(taskID), + asynq.ProcessAt(time.Now().Add(time.Hour)), + ); err != nil { + t.Fatalf("schedule %s: %v", taskID, err) + } +} + +func hasTaskID(tasks []*asynq.TaskInfo, taskID string) bool { + for _, task := range tasks { + if task.ID == taskID { + return true + } + } + return false +} + +func taskIDs(tasks []*asynq.TaskInfo) []string { + ids := make([]string, 0, len(tasks)) + for _, task := range tasks { + ids = append(ids, task.ID) + } + return ids +} + +func waitForTaskState(t *testing.T, inspector *asynq.Inspector, taskID string, want asynq.TaskState) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + var lastState asynq.TaskState + for time.Now().Before(deadline) { + task, err := inspector.GetTaskInfo(types.QueueDefault, taskID) + if err == nil { + lastState = task.State + if task.State == want { + return + } + } else if !errors.Is(err, asynq.ErrTaskNotFound) { + t.Fatalf("get task %s: %v", taskID, err) + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("task %s state = %v, want %v", taskID, lastState, want) +} + +func waitForAsynqCancellationSubscriber(t *testing.T, redisClient *redis.Client) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + subscribers, err := redisClient.PubSubNumSub(context.Background(), "asynq:cancel").Result() + if err != nil { + t.Fatalf("query cancellation subscribers: %v", err) + } + if subscribers["asynq:cancel"] > 0 { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatal("asynq cancellation subscriber did not start") +} + +func waitForTaskToLeaveLiveStates(t *testing.T, inspector *asynq.Inspector, taskID string) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + var lastState asynq.TaskState + for time.Now().Before(deadline) { + task, err := inspector.GetTaskInfo(types.QueueDefault, taskID) + if errors.Is(err, asynq.ErrTaskNotFound) { + return + } + if err != nil { + t.Fatalf("get task %s: %v", taskID, err) + } + lastState = task.State + switch task.State { + case asynq.TaskStatePending, asynq.TaskStateScheduled, asynq.TaskStateRetry, asynq.TaskStateActive: + time.Sleep(10 * time.Millisecond) + default: + return + } + } + t.Fatalf("task %s remained in live state %v", taskID, lastState) +} diff --git a/internal/types/interfaces/task_inspector.go b/internal/types/interfaces/task_inspector.go index 222100d4..a71b2a6e 100644 --- a/internal/types/interfaces/task_inspector.go +++ b/internal/types/interfaces/task_inspector.go @@ -64,6 +64,24 @@ type TaskInspector interface { WorkerServerStats(ctx context.Context) (stats []types.WorkerServerStat, supported bool, err error) } +// KnowledgeBaseTaskCanceller is the optional queue-cleanup capability used +// when a knowledge base is deleted. It is separate from TaskInspector so +// lightweight test doubles and queue backends that cannot inspect tasks do +// not need to implement knowledge-base-wide scanning. +type KnowledgeBaseTaskCanceller interface { + // CancelTasksForKnowledgeBase removes pending/scheduled/retry tasks + // associated with kbID and signals matching active workers to stop. + // knowledgeIDs covers batch tasks whose payload references documents but + // does not carry the parent knowledge-base ID. dataSourceIDs covers sync + // tasks whose payload only identifies a data source. + CancelTasksForKnowledgeBase( + ctx context.Context, + kbID string, + knowledgeIDs []string, + dataSourceIDs []string, + ) (deleted int, cancelled int, err error) +} + // RuntimeTaskInspector is the optional operator surface implemented by queue // backends that retain inspectable task state. It is separate from // TaskInspector so Lite mode and light-weight tests do not need to implement diff --git a/internal/types/interfaces/task_queue.go b/internal/types/interfaces/task_queue.go index 22b56074..349e9c51 100644 --- a/internal/types/interfaces/task_queue.go +++ b/internal/types/interfaces/task_queue.go @@ -83,6 +83,23 @@ type TaskPendingOpsRepository interface { DeleteByDedupKey(ctx context.Context, taskType, scope, scopeID, dedupKey, op string) error } +// TaskPendingOpsScopeCleaner is an optional extension for callers that need +// to discard every durable pending operation owned by a deleted scope. It is +// intentionally separate from TaskPendingOpsRepository so alternate queue +// implementations and existing test doubles are not forced to implement a +// lifecycle-only operation. +type TaskPendingOpsScopeCleaner interface { + DeleteByScope(ctx context.Context, scope, scopeID string) error +} + +// TaskPendingOpsKnowledgeBaseGuard atomically persists a KB-scoped operation +// only while its knowledge base is still active. Implementations must +// serialize the active-KB check with soft deletion so a detached worker cannot +// recreate durable work after the deletion scrub has finished. +type TaskPendingOpsKnowledgeBaseGuard interface { + EnqueueIfKnowledgeBaseActive(ctx context.Context, op *types.TaskPendingOp) (accepted bool, err error) +} + // TaskDeadLetterRepository persists rows for the generic task dead-letter // archive (`task_dead_letters`). Two writers exist: the asynq // dead-letter middleware (one row per archived asynq task), and the diff --git a/internal/types/task.go b/internal/types/task.go index fdcd916b..823465a7 100644 --- a/internal/types/task.go +++ b/internal/types/task.go @@ -396,6 +396,7 @@ type KBDeletePayload struct { TracingContext TenantID uint64 `json:"tenant_id"` KnowledgeBaseID string `json:"knowledge_base_id"` + DataSourceIDs []string `json:"data_source_ids,omitempty"` EffectiveEngines []RetrieverEngineParams `json:"effective_engines"` // VectorStoreID is the bound store snapshot taken at enqueue time (before // soft-delete) so the async worker can resolve the right store. nil means