fix(tasks): dequeue work for deleted knowledge bases

This commit is contained in:
zhangzherui
2026-07-23 04:51:40 +08:00
committed by lyingbug
parent 7fde37eeeb
commit 37de0afad0
14 changed files with 1575 additions and 194 deletions
+62 -1
View File
@@ -10,6 +10,7 @@ import (
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
// taskPendingOpsRepository implements interfaces.TaskPendingOpsRepository.
@@ -26,6 +27,13 @@ func NewTaskPendingOpsRepository(db *gorm.DB) interfaces.TaskPendingOpsRepositor
// Scope/ScopeID/Op (Payload optional). ID, FailCount default to zero;
// EnqueuedAt is filled with the DB-side default if left zero.
func (r *taskPendingOpsRepository) Enqueue(ctx context.Context, op *types.TaskPendingOp) error {
if err := preparePendingOp(op); err != nil {
return err
}
return r.db.WithContext(ctx).Create(op).Error
}
func preparePendingOp(op *types.TaskPendingOp) error {
if op == nil {
return errors.New("task pending ops: nil op")
}
@@ -41,7 +49,48 @@ func (r *taskPendingOpsRepository) Enqueue(ctx context.Context, op *types.TaskPe
// driver-level default handling.
op.Payload = []byte("{}")
}
return r.db.WithContext(ctx).Create(op).Error
return nil
}
// EnqueueIfKnowledgeBaseActive prevents detached wiki cleanup from writing new
// durable work after a KB was soft-deleted. On Postgres the share lock
// serializes this check+insert transaction against the row update performed by
// soft deletion: whichever operation acquires the row first determines the
// order, and the deletion path's subsequent scope scrub removes any insert
// that committed before it.
func (r *taskPendingOpsRepository) EnqueueIfKnowledgeBaseActive(
ctx context.Context,
op *types.TaskPendingOp,
) (bool, error) {
if err := preparePendingOp(op); err != nil {
return false, err
}
if op.Scope != types.TaskScopeKnowledgeBase {
return false, errors.New("task pending ops: guarded enqueue requires knowledge_base scope")
}
accepted := false
err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
query := tx.Model(&types.KnowledgeBase{}).
Select("id").
Where("id = ? AND tenant_id = ?", op.ScopeID, op.TenantID)
dialector := tx.Dialector
if dialector.Name() == "postgres" {
query = query.Clauses(clause.Locking{Strength: "SHARE"})
}
var kb types.KnowledgeBase
if err := query.Take(&kb).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil
}
return err
}
if err := tx.Create(op).Error; err != nil {
return err
}
accepted = true
return nil
})
return accepted, err
}
// PeekBatch returns up to `limit` rows for the (task_type, scope, scope_id)
@@ -228,6 +277,18 @@ func (r *taskPendingOpsRepository) DeleteByIDs(ctx context.Context, ids []int64)
Delete(&types.TaskPendingOp{}).Error
}
// DeleteByScope removes every pending operation owned by a scope, regardless
// of task type. Both scope fields are required so a malformed lifecycle call
// can never turn into an unbounded queue deletion.
func (r *taskPendingOpsRepository) DeleteByScope(ctx context.Context, scope, scopeID string) error {
if scope == "" || scopeID == "" {
return errors.New("task pending ops: scope and scope_id are required")
}
return r.db.WithContext(ctx).
Where("scope = ? AND scope_id = ?", scope, scopeID).
Delete(&types.TaskPendingOp{}).Error
}
// IncrFailCount atomically bumps fail_count for one row and returns the
// new value. We use UPDATE ... RETURNING so the read+write happens in
// one round trip and races between concurrent IncrFailCount callers
@@ -7,6 +7,7 @@ import (
"time"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/driver/sqlite"
@@ -182,6 +183,110 @@ func TestTaskPendingOps_DeleteByIDs_RemovesOnlyTargets(t *testing.T) {
assert.Equal(t, "b", got[0].DedupKey)
}
func TestTaskPendingOps_DeleteByScope_RemovesAllTaskTypesAndIsolatesScopes(t *testing.T) {
db := setupTaskQueueTestDB(t)
repo := NewTaskPendingOpsRepository(db)
cleaner, ok := repo.(interfaces.TaskPendingOpsScopeCleaner)
require.True(t, ok)
ctx := context.Background()
// The deleted KB can have durable work in more than one wiki queue.
require.NoError(t, repo.Enqueue(ctx,
makePendingOp(types.TypeWikiIngest, types.TaskScopeKnowledgeBase, "kb-delete", "ingest", "k1", nil)))
require.NoError(t, repo.Enqueue(ctx,
makePendingOp(types.TypeWikiFinalize, types.TaskScopeKnowledgeBase, "kb-delete", "finalize", "k2", nil)))
// Rows in another KB or another scope must survive even when their IDs or
// task types overlap with the deleted KB.
require.NoError(t, repo.Enqueue(ctx,
makePendingOp(types.TypeWikiIngest, types.TaskScopeKnowledgeBase, "kb-keep", "ingest", "k3", nil)))
require.NoError(t, repo.Enqueue(ctx,
makePendingOp(types.TypeWikiFinalize, types.TaskScopeKnowledge, "kb-delete", "finalize", "k4", nil)))
require.NoError(t, cleaner.DeleteByScope(ctx, types.TaskScopeKnowledgeBase, "kb-delete"))
var remaining []*types.TaskPendingOp
require.NoError(t, db.Order("id ASC").Find(&remaining).Error)
require.Len(t, remaining, 2)
assert.Equal(t, "kb-keep", remaining[0].ScopeID)
assert.Equal(t, types.TaskScopeKnowledge, remaining[1].Scope)
assert.Equal(t, "kb-delete", remaining[1].ScopeID)
}
func TestTaskPendingOps_DeleteByScope_RejectsMissingScope(t *testing.T) {
db := setupTaskQueueTestDB(t)
repo := NewTaskPendingOpsRepository(db)
cleaner, ok := repo.(interfaces.TaskPendingOpsScopeCleaner)
require.True(t, ok)
ctx := context.Background()
require.NoError(t, repo.Enqueue(ctx,
makePendingOp(types.TypeWikiIngest, types.TaskScopeKnowledgeBase, "kb", "ingest", "k1", nil)))
assert.Error(t, cleaner.DeleteByScope(ctx, "", "kb"))
assert.Error(t, cleaner.DeleteByScope(ctx, types.TaskScopeKnowledgeBase, ""))
var count int64
require.NoError(t, db.Model(&types.TaskPendingOp{}).Count(&count).Error)
assert.Equal(t, int64(1), count)
}
func TestTaskPendingOps_EnqueueIfKnowledgeBaseActive(t *testing.T) {
db := setupTaskQueueTestDB(t)
require.NoError(t, db.Exec(`CREATE TABLE knowledge_bases (
id VARCHAR(64) PRIMARY KEY,
tenant_id INTEGER NOT NULL,
deleted_at DATETIME
)`).Error)
require.NoError(t, db.Exec(
"INSERT INTO knowledge_bases (id, tenant_id, deleted_at) VALUES (?, ?, NULL), (?, ?, ?)",
"kb-active", 1, "kb-deleted", 1, time.Now(),
).Error)
repo := NewTaskPendingOpsRepository(db)
guard, ok := repo.(interfaces.TaskPendingOpsKnowledgeBaseGuard)
require.True(t, ok)
ctx := context.Background()
accepted, err := guard.EnqueueIfKnowledgeBaseActive(ctx,
makePendingOp(types.TypeWikiIngest, types.TaskScopeKnowledgeBase, "kb-active", "ingest", "active", nil))
require.NoError(t, err)
assert.True(t, accepted)
for _, tc := range []struct {
name string
op *types.TaskPendingOp
}{
{
name: "soft deleted",
op: makePendingOp(
types.TypeWikiIngest, types.TaskScopeKnowledgeBase, "kb-deleted", "ingest", "deleted", nil,
),
},
{
name: "missing",
op: makePendingOp(
types.TypeWikiIngest, types.TaskScopeKnowledgeBase, "kb-missing", "ingest", "missing", nil,
),
},
{name: "tenant mismatch", op: &types.TaskPendingOp{
TenantID: 2, TaskType: types.TypeWikiIngest, Scope: types.TaskScopeKnowledgeBase,
ScopeID: "kb-active", Op: "ingest", DedupKey: "wrong-tenant",
}},
} {
t.Run(tc.name, func(t *testing.T) {
accepted, err := guard.EnqueueIfKnowledgeBaseActive(ctx, tc.op)
require.NoError(t, err)
assert.False(t, accepted)
})
}
var rows []*types.TaskPendingOp
require.NoError(t, db.Find(&rows).Error)
require.Len(t, rows, 1)
assert.Equal(t, "active", rows[0].DedupKey)
}
// TestTaskPendingOps_IncrFailCount_ReturnsNewValueAndPersists exercises
// the UPDATE...RETURNING flow. Successive bumps should observe monotonic
// counts.
+85 -9
View File
@@ -25,6 +25,8 @@ import (
// ErrInvalidTenantID represents an error for invalid tenant ID
var ErrInvalidTenantID = errors.New("invalid tenant ID")
const kbTaskCleanupTimeout = 5 * time.Second
// knowledgeBaseService implements the knowledge base service interface
type knowledgeBaseService struct {
repo interfaces.KnowledgeBaseRepository
@@ -40,6 +42,8 @@ type knowledgeBaseService struct {
storageResolver interfaces.StorageBackendResolver
graphEngine interfaces.RetrieveGraphRepository
asynqClient interfaces.TaskEnqueuer
taskInspector interfaces.TaskInspector
taskPendingRepo interfaces.TaskPendingOpsRepository
dsRepo interfaces.DataSourceRepository
syncLogRepo interfaces.SyncLogRepository
dsScheduler *datasource.Scheduler
@@ -60,6 +64,8 @@ func NewKnowledgeBaseService(repo interfaces.KnowledgeBaseRepository,
storageResolver interfaces.StorageBackendResolver,
graphEngine interfaces.RetrieveGraphRepository,
asynqClient interfaces.TaskEnqueuer,
taskInspector interfaces.TaskInspector,
taskPendingRepo interfaces.TaskPendingOpsRepository,
dsRepo interfaces.DataSourceRepository,
syncLogRepo interfaces.SyncLogRepository,
dsScheduler *datasource.Scheduler,
@@ -79,6 +85,8 @@ func NewKnowledgeBaseService(repo interfaces.KnowledgeBaseRepository,
storageResolver: storageResolver,
graphEngine: graphEngine,
asynqClient: asynqClient,
taskInspector: taskInspector,
taskPendingRepo: taskPendingRepo,
dsRepo: dsRepo,
syncLogRepo: syncLogRepo,
dsScheduler: dsScheduler,
@@ -720,6 +728,11 @@ func (s *knowledgeBaseService) DeleteKnowledgeBase(ctx context.Context, id strin
recordKBActivity(ctx, s.audit, tenantID, id, types.AuditActionKBDeleted,
"knowledge_base", id, types.AuditOutcomeSuccess, map[string]any{"name": deletedName})
// Stop both ephemeral queue work and durable wiki operations that target
// the now-deleted KB. ProcessKBDelete repeats this with document IDs and
// performs one final scrub after heavy cleanup to close enqueue races.
s.cleanupTasksForKnowledgeBase(ctx, id, nil, nil)
// Step 1b: Remove all organization shares for this KB so org settings no longer show them
if s.shareRepo != nil {
if delErr := s.shareRepo.DeleteByKnowledgeBaseID(ctx, id); delErr != nil {
@@ -729,12 +742,16 @@ func (s *knowledgeBaseService) DeleteKnowledgeBase(ctx context.Context, id strin
// Step 1c: Stop and soft-delete all data sources bound to this KB so cron
// schedules and in-flight sync logs do not keep running against a deleted KB.
s.deleteDataSourcesForKnowledgeBase(ctx, id)
dataSourceIDs := s.deleteDataSourcesForKnowledgeBase(ctx, id)
if len(dataSourceIDs) > 0 {
s.cancelTasksForKnowledgeBase(ctx, id, nil, dataSourceIDs)
}
// Step 2: Enqueue async task for heavy cleanup operations
payload := types.KBDeletePayload{
TenantID: tenantID,
KnowledgeBaseID: id,
DataSourceIDs: dataSourceIDs,
EffectiveEngines: tenantInfo.GetEffectiveEngines(),
VectorStoreID: vectorStoreIDSnapshot, // snapshot taken before soft-delete
}
@@ -772,9 +789,18 @@ func (s *knowledgeBaseService) ProcessKBDelete(ctx context.Context, t *asynq.Tas
tenantID := payload.TenantID
kbID := payload.KnowledgeBaseID
var knowledgeIDs []string
// Set tenant context for downstream services
ctx = context.WithValue(ctx, types.TenantIDContextKey, tenantID)
defer func() {
// Workers may enqueue downstream work while the delete task performs
// heavy storage cleanup. A detached, bounded final scrub runs on every
// return path, including retryable failures and cancellation.
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), kbTaskCleanupTimeout)
defer cancel()
s.cleanupTasksForKnowledgeBase(cleanupCtx, kbID, knowledgeIDs, payload.DataSourceIDs)
}()
logger.Infof(ctx, "Processing KB delete task for knowledge base: %s", kbID)
@@ -788,14 +814,18 @@ func (s *knowledgeBaseService) ProcessKBDelete(ctx context.Context, t *asynq.Tas
return err
}
logger.Infof(ctx, "Found %d knowledge entries to delete", len(knowledgeList))
knowledgeIDs = make([]string, 0, len(knowledgeList))
for _, knowledge := range knowledgeList {
knowledgeIDs = append(knowledgeIDs, knowledge.ID)
}
// Repeat the best-effort queue scrub with document IDs. Some batch tasks
// only carry knowledge_id(s), and active work from the first pass may have
// enqueued another downstream task before cancellation reached it.
s.cleanupTasksForKnowledgeBase(ctx, kbID, knowledgeIDs, payload.DataSourceIDs)
// Step 2: Delete all knowledge entries and their resources
if len(knowledgeList) > 0 {
knowledgeIDs := make([]string, 0, len(knowledgeList))
for _, knowledge := range knowledgeList {
knowledgeIDs = append(knowledgeIDs, knowledge.ID)
}
logger.Infof(ctx, "Deleting all knowledge entries and their resources")
// Delete embeddings from vector store.
@@ -911,23 +941,68 @@ func (s *knowledgeBaseService) ProcessKBDelete(ctx context.Context, t *asynq.Tas
return nil
}
// cancelTasksForKnowledgeBase removes queue work for a deleted KB when the
// configured task backend supports knowledge-base-wide inspection. Queue
// cleanup is an optimization: the soft-deleted database row remains the
// durable source of truth, so backend failures must not fail KB deletion.
func (s *knowledgeBaseService) cancelTasksForKnowledgeBase(
ctx context.Context,
kbID string,
knowledgeIDs []string,
dataSourceIDs []string,
) {
canceller, ok := s.taskInspector.(interfaces.KnowledgeBaseTaskCanceller)
if !ok || kbID == "" {
return
}
if _, _, err := canceller.CancelTasksForKnowledgeBase(ctx, kbID, knowledgeIDs, dataSourceIDs); err != nil {
logger.Warnf(ctx, "Failed to cancel queued tasks for deleted KB %s: %v", kbID, err)
}
}
// cleanupTasksForKnowledgeBase removes both asynq records and durable wiki
// operations. The latter must be cleared as well or startup recovery can
// recreate Redis triggers for a KB that no longer exists.
func (s *knowledgeBaseService) cleanupTasksForKnowledgeBase(
ctx context.Context,
kbID string,
knowledgeIDs []string,
dataSourceIDs []string,
) {
if kbID == "" {
return
}
cleaner, ok := s.taskPendingRepo.(interfaces.TaskPendingOpsScopeCleaner)
if ok {
// Clear durable work before scanning Redis. A large or degraded queue
// must not consume the caller's entire deadline and starve the database
// fence that prevents startup recovery from reviving this KB.
if err := cleaner.DeleteByScope(ctx, types.TaskScopeKnowledgeBase, kbID); err != nil {
logger.Warnf(ctx, "Failed to clear durable tasks for deleted KB %s: %v", kbID, err)
}
}
s.cancelTasksForKnowledgeBase(ctx, kbID, knowledgeIDs, dataSourceIDs)
}
// deleteDataSourcesForKnowledgeBase mirrors DataSourceService.DeleteDataSource for
// every data source attached to the KB. Errors on individual sources are logged
// but do not fail KB deletion — the KB record is already soft-deleted.
func (s *knowledgeBaseService) deleteDataSourcesForKnowledgeBase(ctx context.Context, kbID string) {
func (s *knowledgeBaseService) deleteDataSourcesForKnowledgeBase(ctx context.Context, kbID string) []string {
if s.dsRepo == nil {
return
return nil
}
dataSources, err := s.dsRepo.FindByKnowledgeBase(ctx, kbID)
if err != nil {
logger.Warnf(ctx, "Failed to list data sources for deleted KB %s: %v", kbID, err)
return
return nil
}
dataSourceIDs := make([]string, 0, len(dataSources))
for _, ds := range dataSources {
if ds == nil || ds.ID == "" {
continue
}
dataSourceIDs = append(dataSourceIDs, ds.ID)
if err := s.dsRepo.Delete(ctx, ds.ID); err != nil {
logger.Warnf(ctx, "Failed to delete data source %s for KB %s: %v", ds.ID, kbID, err)
continue
@@ -942,6 +1017,7 @@ func (s *knowledgeBaseService) deleteDataSourcesForKnowledgeBase(ctx context.Con
}
logger.Infof(ctx, "Data source deleted with knowledge base: ds=%s kb=%s", ds.ID, kbID)
}
return dataSourceIDs
}
// SetEmbeddingModel sets the embedding model for a knowledge base
@@ -0,0 +1,293 @@
package service
import (
"context"
"encoding/json"
"errors"
"testing"
"github.com/Tencent/WeKnora/internal/models/embedding"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/hibiken/asynq"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type kbTaskCancelCall struct {
kbID string
knowledgeIDs []string
dataSourceIDs []string
}
type recordingKBTaskInspector struct {
repo *kbDeleteKBRepo
calls []kbTaskCancelCall
cancelErr error
sawSoftDeletedRecord bool
}
func (r *recordingKBTaskInspector) CancelTasksForKnowledge(
context.Context,
string,
) (int, int, error) {
return 0, 0, nil
}
func (r *recordingKBTaskInspector) HasQueuedTasksForKnowledge(context.Context, string) (bool, error) {
return false, nil
}
func (r *recordingKBTaskInspector) QueueStats(context.Context) ([]types.QueueStat, bool, error) {
return nil, true, nil
}
func (r *recordingKBTaskInspector) WorkerServerStats(context.Context) ([]types.WorkerServerStat, bool, error) {
return nil, true, nil
}
func (r *recordingKBTaskInspector) CancelTasksForKnowledgeBase(
_ context.Context,
kbID string,
knowledgeIDs []string,
dataSourceIDs []string,
) (int, int, error) {
r.calls = append(r.calls, kbTaskCancelCall{
kbID: kbID,
knowledgeIDs: append([]string(nil), knowledgeIDs...),
dataSourceIDs: append([]string(nil), dataSourceIDs...),
})
if r.repo != nil && r.repo.deletedID == kbID {
r.sawSoftDeletedRecord = true
}
return 0, 0, r.cancelErr
}
var (
_ interfaces.TaskInspector = (*recordingKBTaskInspector)(nil)
_ interfaces.KnowledgeBaseTaskCanceller = (*recordingKBTaskInspector)(nil)
)
type recordingKBDeleteEnqueuer struct {
calls int
task *asynq.Task
}
type recordingKBPendingRepo struct {
interfaces.TaskPendingOpsRepository
scopeIDs []string
deleteErr error
}
func (r *recordingKBPendingRepo) DeleteByScope(_ context.Context, scope, scopeID string) error {
if scope == types.TaskScopeKnowledgeBase {
r.scopeIDs = append(r.scopeIDs, scopeID)
}
return r.deleteErr
}
func (r *recordingKBDeleteEnqueuer) Enqueue(
task *asynq.Task,
_ ...asynq.Option,
) (*asynq.TaskInfo, error) {
r.calls++
r.task = task
return &asynq.TaskInfo{ID: "kb-delete-task"}, nil
}
func TestDeleteKnowledgeBaseForwardsDataSourceTaskScope(t *testing.T) {
const kbID = "kb-with-datasource"
kbRepo := &kbDeleteKBRepo{fakeKBRepo: *newFakeKBRepo()}
kbRepo.rows[kbID] = &types.KnowledgeBase{ID: kbID, TenantID: 1, Name: "test"}
inspector := &recordingKBTaskInspector{repo: kbRepo}
enqueuer := &recordingKBDeleteEnqueuer{}
dsRepo := newKBDeleteDSRepo(kbID, &types.DataSource{ID: "datasource-1", KnowledgeBaseID: kbID})
svc := &knowledgeBaseService{
repo: kbRepo,
asynqClient: enqueuer,
taskInspector: inspector,
dsRepo: dsRepo,
}
err := svc.DeleteKnowledgeBase(ctxWithTenantStorage(1, "local"), kbID)
require.NoError(t, err)
require.Len(t, inspector.calls, 2)
assert.Empty(t, inspector.calls[0].dataSourceIDs)
assert.Equal(t, []string{"datasource-1"}, inspector.calls[1].dataSourceIDs)
require.NotNil(t, enqueuer.task)
var payload types.KBDeletePayload
require.NoError(t, json.Unmarshal(enqueuer.task.Payload(), &payload))
assert.Equal(t, []string{"datasource-1"}, payload.DataSourceIDs)
}
func TestDeleteKnowledgeBaseCancelsQueuedTasksBestEffort(t *testing.T) {
tests := []struct {
name string
cancelErr error
pendingErr error
}{
{name: "success"},
{name: "inspector failure", cancelErr: errors.New("redis unavailable")},
{name: "durable queue failure", pendingErr: errors.New("database unavailable")},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
const kbID = "kb-task-cleanup"
kbRepo := &kbDeleteKBRepo{fakeKBRepo: *newFakeKBRepo()}
kbRepo.rows[kbID] = &types.KnowledgeBase{ID: kbID, TenantID: 1, Name: "test"}
inspector := &recordingKBTaskInspector{repo: kbRepo, cancelErr: tt.cancelErr}
pendingRepo := &recordingKBPendingRepo{deleteErr: tt.pendingErr}
enqueuer := &recordingKBDeleteEnqueuer{}
svc := &knowledgeBaseService{
repo: kbRepo,
asynqClient: enqueuer,
taskInspector: inspector,
taskPendingRepo: pendingRepo,
}
err := svc.DeleteKnowledgeBase(ctxWithTenantStorage(1, "local"), kbID)
require.NoError(t, err)
require.Len(t, inspector.calls, 1)
assert.Equal(t, kbID, inspector.calls[0].kbID)
assert.Empty(t, inspector.calls[0].knowledgeIDs)
assert.True(t, inspector.sawSoftDeletedRecord)
assert.Equal(t, []string{kbID}, pendingRepo.scopeIDs)
assert.Equal(t, 1, enqueuer.calls)
})
}
}
type emptyKBKnowledgeRepo struct {
interfaces.KnowledgeRepository
}
func (emptyKBKnowledgeRepo) ListKnowledgeByKnowledgeBaseID(
context.Context,
uint64,
string,
) ([]*types.Knowledge, error) {
return nil, nil
}
func TestProcessKBDeleteRepeatsQueueCleanup(t *testing.T) {
inspector := &recordingKBTaskInspector{}
pendingRepo := &recordingKBPendingRepo{}
svc := &knowledgeBaseService{
kgRepo: emptyKBKnowledgeRepo{},
taskInspector: inspector,
taskPendingRepo: pendingRepo,
}
payload, err := json.Marshal(types.KBDeletePayload{TenantID: 1, KnowledgeBaseID: "kb-race"})
require.NoError(t, err)
err = svc.ProcessKBDelete(context.Background(), asynq.NewTask(types.TypeKBDelete, payload))
require.NoError(t, err)
require.Len(t, inspector.calls, 2)
for _, call := range inspector.calls {
assert.Equal(t, "kb-race", call.kbID)
assert.Empty(t, call.knowledgeIDs)
}
assert.Equal(t, []string{"kb-race", "kb-race"}, pendingRepo.scopeIDs)
}
type populatedKBKnowledgeRepo struct {
interfaces.KnowledgeRepository
items []*types.Knowledge
}
func (r populatedKBKnowledgeRepo) ListKnowledgeByKnowledgeBaseID(
context.Context,
uint64,
string,
) ([]*types.Knowledge, error) {
return r.items, nil
}
func (populatedKBKnowledgeRepo) DeleteKnowledgeList(context.Context, uint64, []string) error {
return nil
}
type kbCleanupChunkRepo struct {
interfaces.ChunkRepository
}
func (kbCleanupChunkRepo) ListImageInfoByKnowledgeIDs(
context.Context,
uint64,
[]string,
) ([]interfaces.ChunkImageInfo, error) {
return nil, nil
}
func (kbCleanupChunkRepo) DeleteChunksByKnowledgeID(context.Context, uint64, string) error {
return nil
}
type kbCleanupModelService struct {
interfaces.ModelService
}
func (kbCleanupModelService) GetEmbeddingModel(context.Context, string) (embedding.Embedder, error) {
return kbCleanupEmbedder{}, nil
}
type kbCleanupEmbedder struct{}
func (kbCleanupEmbedder) Embed(context.Context, string) ([]float32, error) { return nil, nil }
func (kbCleanupEmbedder) BatchEmbed(context.Context, []string) ([][]float32, error) {
return nil, nil
}
func (kbCleanupEmbedder) GetModelName() string { return "test" }
func (kbCleanupEmbedder) GetDimensions() int { return 1 }
func (kbCleanupEmbedder) GetModelID() string { return "test" }
func (kbCleanupEmbedder) BatchEmbedWithPool(
context.Context,
embedding.Embedder,
[]string,
) ([][]float32, error) {
return nil, nil
}
func TestProcessKBDeleteCollectsKnowledgeIDsForEveryScrub(t *testing.T) {
inspector := &recordingKBTaskInspector{}
svc := &knowledgeBaseService{
kgRepo: populatedKBKnowledgeRepo{items: []*types.Knowledge{
{ID: "knowledge-1", KnowledgeBaseID: "kb-1", EmbeddingModelID: "model-1"},
{ID: "knowledge-2", KnowledgeBaseID: "kb-1", EmbeddingModelID: "model-1"},
}},
chunkRepo: kbCleanupChunkRepo{},
modelService: kbCleanupModelService{},
taskInspector: inspector,
}
payload, err := json.Marshal(types.KBDeletePayload{TenantID: 1, KnowledgeBaseID: "kb-1"})
require.NoError(t, err)
err = svc.ProcessKBDelete(context.Background(), asynq.NewTask(types.TypeKBDelete, payload))
require.NoError(t, err)
require.Len(t, inspector.calls, 2)
for _, call := range inspector.calls {
assert.Equal(t, []string{"knowledge-1", "knowledge-2"}, call.knowledgeIDs)
}
}
func TestCancelTasksForKnowledgeBaseForwardsKnowledgeIDs(t *testing.T) {
inspector := &recordingKBTaskInspector{}
svc := &knowledgeBaseService{taskInspector: inspector}
svc.cancelTasksForKnowledgeBase(
context.Background(),
"kb-1",
[]string{"knowledge-1", "knowledge-2"},
[]string{"datasource-1"},
)
require.Len(t, inspector.calls, 1)
assert.Equal(t, "kb-1", inspector.calls[0].kbID)
assert.Equal(t, []string{"knowledge-1", "knowledge-2"}, inspector.calls[0].knowledgeIDs)
assert.Equal(t, []string{"datasource-1"}, inspector.calls[0].dataSourceIDs)
}
@@ -0,0 +1,175 @@
package service
import (
"context"
"encoding/json"
"errors"
"testing"
apprepo "github.com/Tencent/WeKnora/internal/application/repository"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/hibiken/asynq"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type wikiKBGuardPendingRepo struct {
interfaces.TaskPendingOpsRepository
accepted bool
guardErr error
guardedOps []*types.TaskPendingOp
deleteErr error
deletedKBs []string
rows []*types.TaskPendingOp
}
func (r *wikiKBGuardPendingRepo) EnqueueIfKnowledgeBaseActive(
_ context.Context,
op *types.TaskPendingOp,
) (bool, error) {
r.guardedOps = append(r.guardedOps, op)
return r.accepted, r.guardErr
}
func (r *wikiKBGuardPendingRepo) DeleteByScope(_ context.Context, scope, scopeID string) error {
if scope == types.TaskScopeKnowledgeBase {
r.deletedKBs = append(r.deletedKBs, scopeID)
}
return r.deleteErr
}
func (r *wikiKBGuardPendingRepo) PeekBatch(
context.Context,
string,
string,
string,
int,
) ([]*types.TaskPendingOp, error) {
return r.rows, nil
}
type wikiGuardTaskQueue struct {
interfaces.TaskEnqueuer
tasks []*asynq.Task
}
func (q *wikiGuardTaskQueue) Enqueue(task *asynq.Task, _ ...asynq.Option) (*asynq.TaskInfo, error) {
q.tasks = append(q.tasks, task)
return &asynq.TaskInfo{ID: "guard-test", Type: task.Type()}, nil
}
type wikiGuardKBService struct {
interfaces.KnowledgeBaseService
kb *types.KnowledgeBase
err error
}
func (s *wikiGuardKBService) GetKnowledgeBaseByIDOnly(
context.Context,
string,
) (*types.KnowledgeBase, error) {
return s.kb, s.err
}
func TestEnqueueWikiWorkSkipsDeletedKnowledgeBase(t *testing.T) {
tests := []struct {
name string
run func(context.Context, interfaces.TaskEnqueuer, interfaces.TaskPendingOpsRepository)
}{
{
name: "ingest",
run: func(ctx context.Context, task interfaces.TaskEnqueuer, repo interfaces.TaskPendingOpsRepository) {
EnqueueWikiIngest(ctx, task, repo, 7, "kb-deleted", "knowledge-1")
},
},
{
name: "retract",
run: func(ctx context.Context, task interfaces.TaskEnqueuer, repo interfaces.TaskPendingOpsRepository) {
EnqueueWikiRetract(ctx, task, repo, WikiRetractPayload{
TenantID: 7, KnowledgeBaseID: "kb-deleted", KnowledgeID: "knowledge-1",
})
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
repo := &wikiKBGuardPendingRepo{accepted: false}
queue := &wikiGuardTaskQueue{}
test.run(context.Background(), queue, repo)
require.Len(t, repo.guardedOps, 1)
assert.Equal(t, "kb-deleted", repo.guardedOps[0].ScopeID)
assert.Empty(t, queue.tasks)
})
}
}
func TestEnqueueWikiFinalizeOnlySchedulesAcceptedRows(t *testing.T) {
for _, accepted := range []bool{false, true} {
t.Run(map[bool]string{false: "deleted", true: "active"}[accepted], func(t *testing.T) {
repo := &wikiKBGuardPendingRepo{accepted: accepted}
queue := &wikiGuardTaskQueue{}
svc := &wikiIngestService{pendingRepo: repo, task: queue}
svc.enqueueFinalize(
context.Background(),
WikiIngestPayload{TenantID: 7, KnowledgeBaseID: "kb-1"},
[]string{"slug-1"},
map[string]string{"slug-1": "Title"},
[]wikiFinalizeChange{{Action: wikiFinalizeAdded, DocTitle: "Document"}},
[]string{"folder-1"},
)
require.Len(t, repo.guardedOps, 3)
if accepted {
require.Len(t, queue.tasks, 1)
assert.Equal(t, types.TypeWikiFinalize, queue.tasks[0].Type())
} else {
assert.Empty(t, queue.tasks)
}
})
}
}
func TestWikiHandlersDrainDeletedKnowledgeBaseQueue(t *testing.T) {
payload, err := json.Marshal(WikiIngestPayload{TenantID: 7, KnowledgeBaseID: "kb-deleted"})
require.NoError(t, err)
for _, taskType := range []string{types.TypeWikiIngest, types.TypeWikiFinalize} {
t.Run(taskType, func(t *testing.T) {
repo := &wikiKBGuardPendingRepo{
rows: []*types.TaskPendingOp{{ID: 1, ScopeID: "kb-deleted"}},
}
svc := &wikiIngestService{
kbService: &wikiGuardKBService{err: apprepo.ErrKnowledgeBaseNotFound},
pendingRepo: repo,
}
err := svc.Handle(context.Background(), asynq.NewTask(taskType, payload))
require.NoError(t, err)
assert.Equal(t, []string{"kb-deleted"}, repo.deletedKBs)
})
}
}
func TestWikiDeletedKnowledgeBaseCleanupFailureRetries(t *testing.T) {
payload, err := json.Marshal(WikiIngestPayload{TenantID: 7, KnowledgeBaseID: "kb-deleted"})
require.NoError(t, err)
wantErr := errors.New("cleanup failed")
repo := &wikiKBGuardPendingRepo{deleteErr: wantErr}
svc := &wikiIngestService{
kbService: &wikiGuardKBService{err: apprepo.ErrKnowledgeBaseNotFound},
pendingRepo: repo,
}
err = svc.ProcessWikiIngest(
context.Background(),
asynq.NewTask(types.TypeWikiIngest, payload),
)
require.ErrorIs(t, err, wantErr)
assert.Equal(t, []string{"kb-deleted"}, repo.deletedKBs)
}
+81 -35
View File
@@ -426,6 +426,23 @@ func (s *wikiIngestService) beginWikiSubspan(ctx context.Context, knowledgeID st
// Lite mode (no Redis) still works as long as Postgres is reachable —
// the queue lives in PG, only the active-batch lock is Redis-only and
// has a process-local fallback (liteLocks) inside the worker.
func enqueueWikiPendingOp(
ctx context.Context,
pendingRepo interfaces.TaskPendingOpsRepository,
op *types.TaskPendingOp,
) (bool, error) {
if pendingRepo == nil {
return true, nil
}
if guard, ok := pendingRepo.(interfaces.TaskPendingOpsKnowledgeBaseGuard); ok {
return guard.EnqueueIfKnowledgeBaseActive(ctx, op)
}
if err := pendingRepo.Enqueue(ctx, op); err != nil {
return false, err
}
return true, nil
}
func EnqueueWikiIngest(
ctx context.Context,
task interfaces.TaskEnqueuer,
@@ -450,20 +467,22 @@ func EnqueueWikiIngest(
logger.Warnf(ctx, "wiki ingest: failed to marshal pending op for %s: %v", knowledgeID, err)
return
}
if pendingRepo != nil {
if err := pendingRepo.Enqueue(ctx, &types.TaskPendingOp{
TenantID: tenantID,
TaskType: wikiTaskType,
Scope: wikiTaskScope,
ScopeID: kbID,
Op: WikiOpIngest,
DedupKey: knowledgeID,
Payload: payloadBytes,
}); err != nil {
logger.Warnf(ctx, "wiki ingest: failed to enqueue pending op for %s: %v", knowledgeID, err)
// Fall through and still schedule the trigger task — the
// next upload (or the next retry pass) will catch the gap.
}
accepted, err := enqueueWikiPendingOp(ctx, pendingRepo, &types.TaskPendingOp{
TenantID: tenantID,
TaskType: wikiTaskType,
Scope: wikiTaskScope,
ScopeID: kbID,
Op: WikiOpIngest,
DedupKey: knowledgeID,
Payload: payloadBytes,
})
if err != nil {
logger.Warnf(ctx, "wiki ingest: failed to enqueue pending op for %s: %v", knowledgeID, err)
return
}
if !accepted {
logger.Infof(ctx, "wiki ingest: skip enqueue for deleted KB %s", kbID)
return
}
trigger := WikiIngestPayload{
@@ -512,18 +531,22 @@ func EnqueueWikiRetract(
logger.Warnf(ctx, "wiki retract: failed to marshal pending op: %v", err)
return
}
if pendingRepo != nil {
if err := pendingRepo.Enqueue(ctx, &types.TaskPendingOp{
TenantID: payload.TenantID,
TaskType: wikiTaskType,
Scope: wikiTaskScope,
ScopeID: payload.KnowledgeBaseID,
Op: WikiOpRetract,
DedupKey: payload.KnowledgeID,
Payload: payloadBytes,
}); err != nil {
logger.Warnf(ctx, "wiki retract: failed to enqueue pending op: %v", err)
}
accepted, err := enqueueWikiPendingOp(ctx, pendingRepo, &types.TaskPendingOp{
TenantID: payload.TenantID,
TaskType: wikiTaskType,
Scope: wikiTaskScope,
ScopeID: payload.KnowledgeBaseID,
Op: WikiOpRetract,
DedupKey: payload.KnowledgeID,
Payload: payloadBytes,
})
if err != nil {
logger.Warnf(ctx, "wiki retract: failed to enqueue pending op: %v", err)
return
}
if !accepted {
logger.Infof(ctx, "wiki retract: skip enqueue for deleted KB %s", payload.KnowledgeBaseID)
return
}
trigger := WikiIngestPayload{
@@ -564,6 +587,25 @@ func wikiIngestCleanupContext(ctx context.Context) (context.Context, context.Can
return context.WithTimeout(context.WithoutCancel(ctx), wikiIngestCleanupTimeout)
}
func (s *wikiIngestService) clearDeletedKnowledgeBasePendingOps(ctx context.Context, kbID string) error {
cleaner, ok := s.pendingRepo.(interfaces.TaskPendingOpsScopeCleaner)
if !ok || kbID == "" {
return nil
}
cleanupCtx, cancel := wikiIngestCleanupContext(ctx)
defer cancel()
return cleaner.DeleteByScope(cleanupCtx, types.TaskScopeKnowledgeBase, kbID)
}
func (s *wikiIngestService) enqueueFinalizeRow(ctx context.Context, op *types.TaskPendingOp) bool {
accepted, err := enqueueWikiPendingOp(ctx, s.pendingRepo, op)
if err != nil {
logger.Warnf(ctx, "wiki finalize: enqueue %s row failed: %v", op.Op, err)
return false
}
return accepted
}
// enqueueFinalize persists this batch's KB-global convergence work into the
// finalize lane of task_pending_ops and schedules a debounced trigger. One
// "slug" row per affected page (carrying its fresh title when this batch wrote
@@ -580,13 +622,14 @@ func (s *wikiIngestService) enqueueFinalize(
if s.pendingRepo == nil {
return
}
acceptedAny := false
for _, slug := range affectedSlugs {
row := wikiFinalizeRow{Slug: slug, Title: freshTitleBySlug[slug]}
b, err := json.Marshal(row)
if err != nil {
continue
}
if err := s.pendingRepo.Enqueue(ctx, &types.TaskPendingOp{
if s.enqueueFinalizeRow(ctx, &types.TaskPendingOp{
TenantID: payload.TenantID,
TaskType: wikiFinalizeTaskType,
Scope: wikiTaskScope,
@@ -594,8 +637,8 @@ func (s *wikiIngestService) enqueueFinalize(
Op: wikiFinalizeOpSlug,
DedupKey: slug,
Payload: b,
}); err != nil {
logger.Warnf(ctx, "wiki finalize: enqueue slug row for %s failed: %v", slug, err)
}) {
acceptedAny = true
}
}
for i := range changes {
@@ -604,7 +647,7 @@ func (s *wikiIngestService) enqueueFinalize(
if err != nil {
continue
}
if err := s.pendingRepo.Enqueue(ctx, &types.TaskPendingOp{
if s.enqueueFinalizeRow(ctx, &types.TaskPendingOp{
TenantID: payload.TenantID,
TaskType: wikiFinalizeTaskType,
Scope: wikiTaskScope,
@@ -612,14 +655,14 @@ func (s *wikiIngestService) enqueueFinalize(
Op: wikiFinalizeOpChange,
DedupKey: "",
Payload: b,
}); err != nil {
logger.Warnf(ctx, "wiki finalize: enqueue change row failed: %v", err)
}) {
acceptedAny = true
}
}
if len(folderIDs) > 0 {
row := wikiFinalizeRow{FolderIDs: uniqueWikiFolderIDs(folderIDs)}
if b, err := json.Marshal(row); err == nil {
if err := s.pendingRepo.Enqueue(ctx, &types.TaskPendingOp{
if s.enqueueFinalizeRow(ctx, &types.TaskPendingOp{
TenantID: payload.TenantID,
TaskType: wikiFinalizeTaskType,
Scope: wikiTaskScope,
@@ -627,11 +670,14 @@ func (s *wikiIngestService) enqueueFinalize(
Op: wikiFinalizeOpFolderPrune,
DedupKey: "",
Payload: b,
}); err != nil {
logger.Warnf(ctx, "wiki finalize: enqueue folder prune row failed: %v", err)
}) {
acceptedAny = true
}
}
}
if !acceptedAny {
return
}
s.scheduleFinalize(ctx, payload)
}
@@ -11,6 +11,7 @@ import (
"unicode/utf8"
"github.com/Tencent/WeKnora/internal/agent"
apprepo "github.com/Tencent/WeKnora/internal/application/repository"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
@@ -280,6 +281,13 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
}
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
if errors.Is(err, apprepo.ErrKnowledgeBaseNotFound) || (err == nil && kb == nil) {
exitStatus = "kb_deleted"
if cleanupErr := s.clearDeletedKnowledgeBasePendingOps(ctx, payload.KnowledgeBaseID); cleanupErr != nil {
return fmt.Errorf("wiki ingest: clear deleted KB queue: %w", cleanupErr)
}
return nil
}
if err != nil {
exitStatus = "get_kb_failed"
return fmt.Errorf("wiki ingest: get KB: %w", err)
@@ -1006,6 +1014,12 @@ func (s *wikiIngestService) ProcessWikiFinalize(ctx context.Context, t *asynq.Ta
}
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
if errors.Is(err, apprepo.ErrKnowledgeBaseNotFound) || (err == nil && kb == nil) {
if cleanupErr := s.clearDeletedKnowledgeBasePendingOps(ctx, payload.KnowledgeBaseID); cleanupErr != nil {
return fmt.Errorf("wiki finalize: clear deleted KB queue: %w", cleanupErr)
}
return nil
}
if err != nil {
return fmt.Errorf("wiki finalize: get KB: %w", err)
}
@@ -34,12 +34,34 @@ func recoverPendingWikiTasks(db *gorm.DB, task interfaces.TaskEnqueuer) {
return
}
ctx := context.Background()
const activeKnowledgeBase = `EXISTS (
SELECT 1 FROM knowledge_bases kb
WHERE kb.id = task_pending_ops.scope_id
AND kb.tenant_id = task_pending_ops.tenant_id
AND kb.deleted_at IS NULL
)`
wikiTaskTypes := []string{types.TypeWikiIngest, types.TypeWikiFinalize}
// Durable rows for a deleted/missing KB must not recreate ephemeral
// triggers at startup. Fail closed if this cleanup cannot be verified.
cleanup := db.WithContext(ctx).
Where("scope = ? AND task_type IN ?", types.TaskScopeKnowledgeBase, wikiTaskTypes).
Where("NOT " + activeKnowledgeBase).
Delete(&types.TaskPendingOp{})
if cleanup.Error != nil {
logger.Warnf(ctx, "[WikiRecovery] failed to clear deleted KB queues: %v", cleanup.Error)
return
}
if cleanup.RowsAffected > 0 {
logger.Infof(ctx, "[WikiRecovery] removed %d pending row(s) for deleted knowledge bases", cleanup.RowsAffected)
}
var scopes []pendingWikiScope
if err := db.WithContext(ctx).
Model(&types.TaskPendingOp{}).
Distinct("tenant_id", "task_type", "scope_id").
Where("scope = ? AND task_type IN ?", types.TaskScopeKnowledgeBase,
[]string{types.TypeWikiIngest, types.TypeWikiFinalize}).
Where("scope = ? AND task_type IN ?", types.TaskScopeKnowledgeBase, wikiTaskTypes).
Where(activeKnowledgeBase).
Find(&scopes).Error; err != nil {
logger.Warnf(ctx, "[WikiRecovery] failed to list pending queues: %v", err)
return
@@ -78,6 +78,14 @@ CREATE TABLE IF NOT EXISTS task_pending_ops (
);
`
const resetPendingKnowledgeBasesDDL = `
CREATE TABLE IF NOT EXISTS knowledge_bases (
id VARCHAR(64) PRIMARY KEY,
tenant_id INTEGER NOT NULL DEFAULT 0,
deleted_at DATETIME
);
`
func setupResetPendingDB(t *testing.T) *gorm.DB {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
@@ -86,6 +94,7 @@ func setupResetPendingDB(t *testing.T) *gorm.DB {
require.NoError(t, db.Exec(resetPendingSyncLogDDL).Error)
require.NoError(t, db.Exec(resetPendingSpansDDL).Error)
require.NoError(t, db.Exec(resetPendingOpsDDL).Error)
require.NoError(t, db.Exec(resetPendingKnowledgeBasesDDL).Error)
return db
}
@@ -295,6 +304,11 @@ func (r *recordingTaskEnqueuer) Enqueue(task *asynq.Task, _ ...asynq.Option) (*a
func TestRecoverPendingWikiTasks_RecreatesOneTriggerPerLaneAndKB(t *testing.T) {
db := setupResetPendingDB(t)
require.NoError(t, db.Exec(
`INSERT INTO knowledge_bases (id, tenant_id, deleted_at)
VALUES (?, ?, NULL), (?, ?, NULL), (?, ?, ?)`,
"kb-a", 7, "kb-b", 8, "kb-deleted", 9, time.Now(),
).Error)
rows := []struct {
tenantID uint64
taskType string
@@ -305,6 +319,8 @@ func TestRecoverPendingWikiTasks_RecreatesOneTriggerPerLaneAndKB(t *testing.T) {
{7, types.TypeWikiIngest, "kb-a", "k-2"}, // same lane: one trigger
{7, types.TypeWikiFinalize, "kb-a", "slug-a"},
{8, types.TypeWikiIngest, "kb-b", "k-3"},
{9, types.TypeWikiIngest, "kb-deleted", "k-deleted"},
{10, types.TypeWikiFinalize, "kb-missing", "k-missing"},
}
for _, row := range rows {
require.NoError(t, db.Exec(
@@ -328,4 +344,10 @@ func TestRecoverPendingWikiTasks_RecreatesOneTriggerPerLaneAndKB(t *testing.T) {
assert.Equal(t, uint64(7), seen[types.TypeWikiIngest+":kb-a"].TenantID)
assert.Equal(t, uint64(7), seen[types.TypeWikiFinalize+":kb-a"].TenantID)
assert.Equal(t, uint64(8), seen[types.TypeWikiIngest+":kb-b"].TenantID)
var orphaned int64
require.NoError(t, db.Model(&types.TaskPendingOp{}).
Where("scope_id IN ?", []string{"kb-deleted", "kb-missing"}).
Count(&orphaned).Error)
assert.Zero(t, orphaned)
}
+330 -147
View File
@@ -96,7 +96,12 @@ var taskTypesForKnowledgeCancel = map[string]struct{}{
// listPageSize caps each Redis LIST call. Asynq pages tasks, so we
// loop until a short page comes back. 100 matches asynq's default.
const listPageSize = 100
const (
listPageSize = 100
maxQueueMutationPasses = 1000
activeCancellationSettleTimeout = time.Second
activeCancellationPollInterval = 25 * time.Millisecond
)
// CancelTasksForKnowledge removes queued tasks whose payload references
// the given knowledge_id and signals active workers running such tasks
@@ -107,18 +112,9 @@ func (a *asynqTaskInspector) CancelTasksForKnowledge(
if a == nil || a.inspector == nil || knowledgeID == "" {
return 0, 0, nil
}
deleted := 0
cancelled := 0
for _, queue := range queuesScanned {
// Pending / Scheduled / Retry can all be deleted by task ID.
// Archived tasks are NOT touched: dead-letter rows are
// already final and should remain visible to operators.
deleted += a.deletePendingMatches(ctx, queue, knowledgeID)
deleted += a.deleteScheduledMatches(ctx, queue, knowledgeID)
deleted += a.deleteRetryMatches(ctx, queue, knowledgeID)
cancelled += a.cancelActiveMatches(ctx, queue, knowledgeID)
}
deleted, cancelled := a.cancelMatchingTasks(ctx, func(taskType string, payload []byte) bool {
return matchesKnowledge(taskType, payload, knowledgeID)
})
logger.Infof(ctx,
"[TaskInspector] knowledge=%s cancel summary: deleted_from_queue=%d active_cancel_signaled=%d",
@@ -127,6 +123,42 @@ func (a *asynqTaskInspector) CancelTasksForKnowledge(
return deleted, cancelled, nil
}
// CancelTasksForKnowledgeBase removes tasks that still reference a knowledge
// base (or one of its knowledges) after the knowledge base has been deleted.
// The kb:delete task is deliberately excluded because it performs the durable
// storage cleanup and must remain queued.
func (a *asynqTaskInspector) CancelTasksForKnowledgeBase(
ctx context.Context, knowledgeBaseID string, knowledgeIDs []string, dataSourceIDs []string,
) (int, int, error) {
if a == nil || a.inspector == nil {
return 0, 0, nil
}
knowledgeIDSet := make(map[string]struct{}, len(knowledgeIDs))
for _, knowledgeID := range knowledgeIDs {
if knowledgeID != "" {
knowledgeIDSet[knowledgeID] = struct{}{}
}
}
dataSourceIDSet := make(map[string]struct{}, len(dataSourceIDs))
for _, dataSourceID := range dataSourceIDs {
if dataSourceID != "" {
dataSourceIDSet[dataSourceID] = struct{}{}
}
}
if knowledgeBaseID == "" && len(knowledgeIDSet) == 0 && len(dataSourceIDSet) == 0 {
return 0, 0, nil
}
deleted, cancelled := a.cancelMatchingTasks(ctx, func(taskType string, payload []byte) bool {
return matchesKnowledgeBase(taskType, payload, knowledgeBaseID, knowledgeIDSet, dataSourceIDSet)
})
logger.Infof(ctx,
"[TaskInspector] knowledge_base=%s cancel summary: deleted_from_queue=%d active_cancel_signaled=%d",
knowledgeBaseID, deleted, cancelled,
)
return deleted, cancelled, nil
}
// HasQueuedTasksForKnowledge reports whether any pending / scheduled /
// retry / active task referencing knowledgeID still lives in the queue.
// Read-only counterpart of CancelTasksForKnowledge — the housekeeping
@@ -138,18 +170,12 @@ func (a *asynqTaskInspector) HasQueuedTasksForKnowledge(
if a == nil || a.inspector == nil || knowledgeID == "" {
return false, nil
}
listers := []struct {
state string
list func(string, ...asynq.ListOption) ([]*asynq.TaskInfo, error)
}{
{"pending", a.inspector.ListPendingTasks},
{"scheduled", a.inspector.ListScheduledTasks},
{"retry", a.inspector.ListRetryTasks},
{"active", a.inspector.ListActiveTasks},
matcher := func(taskType string, payload []byte) bool {
return matchesKnowledge(taskType, payload, knowledgeID)
}
for _, queue := range queuesScanned {
for _, l := range listers {
if a.queueStateHasMatch(ctx, queue, knowledgeID, l.state, l.list) {
for _, state := range a.cancellableTaskStates() {
if a.queueStateHasMatch(ctx, queue, state.name, state.list, matcher) {
return true, nil
}
}
@@ -685,14 +711,249 @@ func (a *asynqTaskInspector) WorkerServerStats(
return stats, true, nil
}
type taskMatcher func(taskType string, payload []byte) bool
type cancellableTaskState struct {
name string
list func(string, ...asynq.ListOption) ([]*asynq.TaskInfo, error)
active bool
}
func (a *asynqTaskInspector) cancellableTaskStates() []cancellableTaskState {
return []cancellableTaskState{
{name: "pending", list: a.inspector.ListPendingTasks},
{name: "scheduled", list: a.inspector.ListScheduledTasks},
{name: "retry", list: a.inspector.ListRetryTasks},
{name: "active", list: a.inspector.ListActiveTasks, active: true},
}
}
// cancelMatchingTasks applies the same matcher to every live queue state.
// Archived tasks are deliberately retained as operator-visible history.
func (a *asynqTaskInspector) cancelMatchingTasks(ctx context.Context, matcher taskMatcher) (int, int) {
deleted := 0
cancelled := 0
// Drain queued states first. Active cancellation is asynchronous: when a
// handler returns context.Canceled, asynq normally moves it to retry, so a
// later settle phase must remove that transitioned record as well.
for _, queue := range queuesScanned {
for _, state := range a.cancellableTaskStates() {
if state.active {
continue
}
deleted += a.processQueueStateMatches(
ctx, queue, state, true, matcher, "delete",
func(task *asynq.TaskInfo) error { return a.inspector.DeleteTask(queue, task.ID) },
)
}
}
// Snapshot all matching active IDs before publishing cancellation. If we
// mutate the active set while paging, exited workers shift later pages and
// can make us skip an entire page.
activeTasks := make([]queueTask, 0)
for _, queue := range queuesScanned {
for _, state := range a.cancellableTaskStates() {
if !state.active {
continue
}
for _, task := range a.snapshotQueueStateMatches(ctx, queue, state, matcher) {
if err := a.inspector.CancelProcessing(task.ID); err != nil {
logger.Warnf(ctx, "[TaskInspector] cancel active type=%s id=%s: %v", task.Type, task.ID, err)
continue
}
cancelled++
activeTasks = append(activeTasks, queueTask{queue: queue, id: task.ID})
}
}
}
deleted += a.deleteCancelledTransitions(ctx, activeTasks)
// Catch tasks that entered pending/scheduled/retry while cancellation was
// settling, plus downstream work emitted immediately before a handler saw
// its cancellation.
for _, queue := range queuesScanned {
for _, state := range a.cancellableTaskStates() {
if state.active {
continue
}
deleted += a.processQueueStateMatches(
ctx, queue, state, true, matcher, "delete",
func(task *asynq.TaskInfo) error { return a.inspector.DeleteTask(queue, task.ID) },
)
}
}
return deleted, cancelled
}
type queueTask struct {
queue string
id string
}
// snapshotQueueStateMatches returns a read-only snapshot before its caller
// mutates the state. TaskInfo values are only used for immutable task metadata.
func (a *asynqTaskInspector) snapshotQueueStateMatches(
ctx context.Context,
queue string,
state cancellableTaskState,
matcher taskMatcher,
) []*asynq.TaskInfo {
var matches []*asynq.TaskInfo
for page := 1; ; page++ {
if ctx.Err() != nil {
return matches
}
tasks, err := state.list(queue, asynq.PageSize(listPageSize), asynq.Page(page))
if err != nil {
if !errors.Is(err, asynq.ErrQueueNotFound) {
logger.Warnf(ctx, "[TaskInspector] snapshot %s queue=%s page=%d: %v", state.name, queue, page, err)
}
return matches
}
for _, task := range tasks {
if matcher(task.Type, task.Payload) {
matches = append(matches, task)
}
}
if len(tasks) < listPageSize {
return matches
}
}
}
// deleteCancelledTransitions waits briefly for signalled active tasks to
// leave active state. A normal context.Canceled result is retried by asynq;
// delete that transitioned record before it can become an orphan. Archived
// and completed tasks remain operator-visible history.
func (a *asynqTaskInspector) deleteCancelledTransitions(ctx context.Context, tasks []queueTask) int {
if len(tasks) == 0 {
return 0
}
deadline := time.Now().Add(activeCancellationSettleTimeout)
pending := append([]queueTask(nil), tasks...)
deleted := 0
for len(pending) > 0 {
if ctx.Err() != nil {
return deleted
}
next := pending[:0]
for _, ref := range pending {
info, err := a.inspector.GetTaskInfo(ref.queue, ref.id)
if errors.Is(err, asynq.ErrTaskNotFound) || errors.Is(err, asynq.ErrQueueNotFound) {
continue
}
if err != nil {
logger.Warnf(ctx, "[TaskInspector] inspect cancelled task queue=%s id=%s: %v", ref.queue, ref.id, err)
next = append(next, ref)
continue
}
switch info.State {
case asynq.TaskStatePending, asynq.TaskStateScheduled, asynq.TaskStateRetry:
if err := a.inspector.DeleteTask(ref.queue, ref.id); err != nil {
if !errors.Is(err, asynq.ErrTaskNotFound) {
logger.Warnf(ctx, "[TaskInspector] delete cancelled transition queue=%s id=%s: %v", ref.queue, ref.id, err)
next = append(next, ref)
}
continue
}
deleted++
case asynq.TaskStateActive:
next = append(next, ref)
}
}
pending = next
if len(pending) == 0 || !time.Now().Before(deadline) {
return deleted
}
wait := activeCancellationPollInterval
if remaining := time.Until(deadline); remaining < wait {
wait = remaining
}
timer := time.NewTimer(wait)
select {
case <-ctx.Done():
if !timer.Stop() {
select {
case <-timer.C:
default:
}
}
return deleted
case <-timer.C:
}
}
return deleted
}
// processQueueStateMatches pages through one queue state and applies action to
// matching tasks. Deleting a task shifts later entries toward the current
// page, so delete states rescan that page after any successful mutation.
func (a *asynqTaskInspector) processQueueStateMatches(
ctx context.Context,
queue string,
state cancellableTaskState,
rescanAfterSuccess bool,
matcher taskMatcher,
actionName string,
action func(*asynq.TaskInfo) error,
) int {
processed := 0
page := 1
passes := 0
for passes < maxQueueMutationPasses {
passes++
if ctx.Err() != nil {
return processed
}
tasks, err := state.list(queue, asynq.PageSize(listPageSize), asynq.Page(page))
if err != nil {
if !errors.Is(err, asynq.ErrQueueNotFound) {
logger.Warnf(ctx, "[TaskInspector] list %s queue=%s page=%d: %v", state.name, queue, page, err)
}
return processed
}
if len(tasks) == 0 {
return processed
}
processedOnPage := 0
for _, task := range tasks {
if !matcher(task.Type, task.Payload) {
continue
}
if err := action(task); err != nil {
logger.Warnf(ctx, "[TaskInspector] %s %s type=%s id=%s: %v", actionName, state.name, task.Type, task.ID, err)
continue
}
processed++
processedOnPage++
}
if rescanAfterSuccess && processedOnPage > 0 {
continue
}
if len(tasks) < listPageSize {
return processed
}
page++
}
logger.Warnf(ctx,
"[TaskInspector] stopped %s queue=%s after %d mutation passes",
state.name, queue, maxQueueMutationPasses,
)
return processed
}
// queueStateHasMatch pages through one (queue, state) list looking for a
// task that references knowledgeID. Mirrors the delete* scanners but is
// strictly read-only and returns early on the first hit. A backend error
// is logged and treated as "no match" (false); the caller's fail-safe
// then errs toward recovering the row rather than preserving it forever.
// matching task. It is strictly read-only and returns on the first hit. A
// backend error is logged and treated as "no match" (false).
func (a *asynqTaskInspector) queueStateHasMatch(
ctx context.Context, queue, knowledgeID, state string,
ctx context.Context,
queue string,
state string,
list func(string, ...asynq.ListOption) ([]*asynq.TaskInfo, error),
matcher taskMatcher,
) bool {
page := 1
for {
@@ -706,8 +967,8 @@ func (a *asynqTaskInspector) queueStateHasMatch(
if len(tasks) == 0 {
return false
}
for _, t := range tasks {
if matchesKnowledge(t.Type, t.Payload, knowledgeID) {
for _, task := range tasks {
if matcher(task.Type, task.Payload) {
return true
}
}
@@ -731,133 +992,55 @@ func matchesKnowledge(taskType string, payload []byte, knowledgeID string) bool
return probe.KnowledgeID == knowledgeID
}
func (a *asynqTaskInspector) deletePendingMatches(ctx context.Context, queue, knowledgeID string) int {
deleted := 0
page := 1
for {
tasks, err := a.inspector.ListPendingTasks(queue, asynq.PageSize(listPageSize), asynq.Page(page))
if err != nil {
if !errors.Is(err, asynq.ErrQueueNotFound) {
logger.Warnf(ctx, "[TaskInspector] list pending queue=%s page=%d: %v", queue, page, err)
}
return deleted
}
if len(tasks) == 0 {
return deleted
}
for _, t := range tasks {
if !matchesKnowledge(t.Type, t.Payload, knowledgeID) {
continue
}
if err := a.inspector.DeleteTask(queue, t.ID); err != nil {
logger.Warnf(ctx, "[TaskInspector] delete pending type=%s id=%s: %v", t.Type, t.ID, err)
continue
}
deleted++
}
if len(tasks) < listPageSize {
return deleted
}
page++
// matchesKnowledgeBase identifies work made obsolete by deleting a knowledge
// base. In addition to direct KB fields, clone/move payloads carry semantic KB
// references under task-specific field names. knowledgeIDs catches tasks whose
// payload does not carry a KB ID at all.
func matchesKnowledgeBase(
taskType string,
payload []byte,
knowledgeBaseID string,
knowledgeIDs map[string]struct{},
dataSourceIDs map[string]struct{},
) bool {
// These cleanup tasks carry snapshots specifically so they can still run
// after the KB row has been soft-deleted. Removing them leaks resources.
if taskType == types.TypeKBDelete || taskType == types.TypeIndexDelete {
return false
}
var probe runtimeTaskPayloadProbe
if err := json.Unmarshal(payload, &probe); err != nil {
return false
}
}
func (a *asynqTaskInspector) deleteScheduledMatches(ctx context.Context, queue, knowledgeID string) int {
deleted := 0
page := 1
for {
tasks, err := a.inspector.ListScheduledTasks(queue, asynq.PageSize(listPageSize), asynq.Page(page))
if err != nil {
if !errors.Is(err, asynq.ErrQueueNotFound) {
logger.Warnf(ctx, "[TaskInspector] list scheduled queue=%s page=%d: %v", queue, page, err)
if knowledgeBaseID != "" {
if probe.KnowledgeBaseID == knowledgeBaseID || probe.KBID == knowledgeBaseID {
return true
}
switch taskType {
case types.TypeKBClone:
if probe.SourceID == knowledgeBaseID || probe.TargetID == knowledgeBaseID {
return true
}
return deleted
}
if len(tasks) == 0 {
return deleted
}
for _, t := range tasks {
if !matchesKnowledge(t.Type, t.Payload, knowledgeID) {
continue
case types.TypeKnowledgeMove:
if probe.SourceKBID == knowledgeBaseID || probe.TargetKBID == knowledgeBaseID {
return true
}
if err := a.inspector.DeleteTask(queue, t.ID); err != nil {
logger.Warnf(ctx, "[TaskInspector] delete scheduled type=%s id=%s: %v", t.Type, t.ID, err)
continue
}
deleted++
}
if len(tasks) < listPageSize {
return deleted
}
page++
}
}
func (a *asynqTaskInspector) deleteRetryMatches(ctx context.Context, queue, knowledgeID string) int {
deleted := 0
page := 1
for {
tasks, err := a.inspector.ListRetryTasks(queue, asynq.PageSize(listPageSize), asynq.Page(page))
if err != nil {
if !errors.Is(err, asynq.ErrQueueNotFound) {
logger.Warnf(ctx, "[TaskInspector] list retry queue=%s page=%d: %v", queue, page, err)
}
return deleted
}
if len(tasks) == 0 {
return deleted
}
for _, t := range tasks {
if !matchesKnowledge(t.Type, t.Payload, knowledgeID) {
continue
}
if err := a.inspector.DeleteTask(queue, t.ID); err != nil {
logger.Warnf(ctx, "[TaskInspector] delete retry type=%s id=%s: %v", t.Type, t.ID, err)
continue
}
deleted++
}
if len(tasks) < listPageSize {
return deleted
}
page++
if _, ok := knowledgeIDs[probe.KnowledgeID]; ok && probe.KnowledgeID != "" {
return true
}
}
// cancelActiveMatches signals active workers to abort via
// Inspector.CancelProcessing. The worker's ctx becomes Done() so the
// next blocking call (or our checkpoint reads) bails. The DB-level
// abort flag (parse_status=cancelled) remains the durable signal —
// this is a latency optimization, not the correctness mechanism.
func (a *asynqTaskInspector) cancelActiveMatches(ctx context.Context, queue, knowledgeID string) int {
cancelled := 0
page := 1
for {
tasks, err := a.inspector.ListActiveTasks(queue, asynq.PageSize(listPageSize), asynq.Page(page))
if err != nil {
if !errors.Is(err, asynq.ErrQueueNotFound) {
logger.Warnf(ctx, "[TaskInspector] list active queue=%s page=%d: %v", queue, page, err)
}
return cancelled
for _, knowledgeID := range probe.KnowledgeIDs {
if _, ok := knowledgeIDs[knowledgeID]; ok && knowledgeID != "" {
return true
}
if len(tasks) == 0 {
return cancelled
}
for _, t := range tasks {
if !matchesKnowledge(t.Type, t.Payload, knowledgeID) {
continue
}
if err := a.inspector.CancelProcessing(t.ID); err != nil {
logger.Warnf(ctx, "[TaskInspector] cancel active type=%s id=%s: %v", t.Type, t.ID, err)
continue
}
cancelled++
}
if len(tasks) < listPageSize {
return cancelled
}
page++
}
if _, ok := dataSourceIDs[probe.DataSourceID]; ok && probe.DataSourceID != "" {
return true
}
return false
}
// noopTaskInspector is the Lite-mode (no Redis) inspector. Inline
@@ -0,0 +1,348 @@
package router
import (
"context"
"errors"
"fmt"
"testing"
"time"
"github.com/Tencent/WeKnora/internal/types"
"github.com/alicebob/miniredis/v2"
"github.com/hibiken/asynq"
"github.com/redis/go-redis/v9"
)
func TestMatchesKnowledgeBase(t *testing.T) {
knowledgeIDs := map[string]struct{}{
"knowledge-1": {},
"knowledge-2": {},
}
dataSourceIDs := map[string]struct{}{"datasource-1": {}}
tests := []struct {
name string
taskType string
payload string
want bool
}{
{name: "knowledge base id", taskType: types.TypeDocumentProcess, payload: `{"knowledge_base_id":"kb-1"}`, want: true},
{name: "legacy kb id", taskType: types.TypeFAQImport, payload: `{"kb_id":"kb-1"}`, want: true},
{name: "clone source", taskType: types.TypeKBClone, payload: `{"source_id":"kb-1"}`, want: true},
{name: "clone target", taskType: types.TypeKBClone, payload: `{"target_id":"kb-1"}`, want: true},
{name: "source id is task specific", taskType: types.TypeDocumentProcess, payload: `{"source_id":"kb-1"}`},
{name: "move source", taskType: types.TypeKnowledgeMove, payload: `{"source_kb_id":"kb-1"}`, want: true},
{name: "move target", taskType: types.TypeKnowledgeMove, payload: `{"target_kb_id":"kb-1"}`, want: true},
{name: "move fields are task specific", taskType: types.TypeDocumentProcess, payload: `{"source_kb_id":"kb-1"}`},
{name: "single knowledge", taskType: types.TypeChunkExtract, payload: `{"knowledge_id":"knowledge-1"}`, want: true},
{
name: "knowledge collection",
taskType: types.TypeKnowledgeListReparse,
payload: `{"knowledge_ids":["other","knowledge-2"]}`,
want: true,
},
{
name: "unrelated",
taskType: types.TypeDocumentProcess,
payload: `{"knowledge_base_id":"other","knowledge_id":"other"}`,
},
{name: "malformed payload", taskType: types.TypeDocumentProcess, payload: `{`},
{name: "preserve kb delete by kb", taskType: types.TypeKBDelete, payload: `{"knowledge_base_id":"kb-1"}`},
{name: "preserve kb delete by knowledge", taskType: types.TypeKBDelete, payload: `{"knowledge_id":"knowledge-1"}`},
{name: "preserve index delete by kb", taskType: types.TypeIndexDelete, payload: `{"knowledge_base_id":"kb-1"}`},
{
name: "preserve index delete by knowledge", taskType: types.TypeIndexDelete,
payload: `{"knowledge_id":"knowledge-1"}`,
},
{
name: "data source sync", taskType: types.TypeDataSourceSync,
payload: `{"data_source_id":"datasource-1"}`, want: true,
},
{
name: "other data source sync", taskType: types.TypeDataSourceSync,
payload: `{"data_source_id":"datasource-other"}`,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
got := matchesKnowledgeBase(
test.taskType, []byte(test.payload), "kb-1", knowledgeIDs, dataSourceIDs,
)
if got != test.want {
t.Fatalf("matchesKnowledgeBase() = %v, want %v", got, test.want)
}
})
}
}
func TestMatchesKnowledgePreservesPerKnowledgeAllowList(t *testing.T) {
if !matchesKnowledge(types.TypeDocumentProcess, []byte(`{"knowledge_id":"knowledge-1"}`), "knowledge-1") {
t.Fatal("document task should remain cancellable by knowledge ID")
}
if matchesKnowledge(types.TypeKBClone, []byte(`{"knowledge_id":"knowledge-1"}`), "knowledge-1") {
t.Fatal("KB clone must not become cancellable through the per-knowledge API")
}
}
func TestCancelTasksForKnowledgeBaseRescansMutatedPages(t *testing.T) {
server := miniredis.RunT(t)
redisClient := redis.NewClient(&redis.Options{Addr: server.Addr()})
asynqClient := asynq.NewClientFromRedisClient(redisClient)
t.Cleanup(func() {
_ = asynqClient.Close()
_ = redisClient.Close()
})
inspector := &asynqTaskInspector{
inspector: asynq.NewInspectorFromRedisClient(redisClient),
redis: redisClient,
}
const matchingPending = 205
for i := 0; i < matchingPending; i++ {
enqueueTask(t, asynqClient, types.TypeDocumentProcess,
fmt.Sprintf(`{"knowledge_base_id":"kb-delete","knowledge_id":"knowledge-%d"}`, i),
fmt.Sprintf("matching-%03d", i),
)
}
for i := 0; i < 5; i++ {
enqueueTask(t, asynqClient, types.TypeDocumentProcess,
`{"knowledge_base_id":"kb-keep","knowledge_id":"keep"}`,
fmt.Sprintf("survivor-%03d", i),
)
}
enqueueTask(t, asynqClient, types.TypeKBDelete,
`{"knowledge_base_id":"kb-delete"}`, "kb-delete-cleanup",
)
enqueueTask(t, asynqClient, types.TypeIndexDelete,
`{"knowledge_base_id":"kb-delete"}`, "index-delete-cleanup",
)
enqueueTask(t, asynqClient, types.TypeDataSourceSync,
`{"data_source_id":"datasource-delete"}`, "datasource-sync",
)
scheduleTask(t, asynqClient, types.TypeFAQImport,
`{"kb_id":"kb-delete"}`, "scheduled-kb-match",
)
scheduleTask(t, asynqClient, types.TypeKnowledgeListReparse,
`{"knowledge_ids":["knowledge-associated"]}`, "scheduled-knowledge-match",
)
scheduleTask(t, asynqClient, types.TypeFAQImport,
`{"kb_id":"kb-keep"}`, "scheduled-survivor",
)
deleted, cancelled, err := inspector.CancelTasksForKnowledgeBase(
context.Background(), "kb-delete", []string{"knowledge-associated"}, []string{"datasource-delete"},
)
if err != nil {
t.Fatalf("cancel tasks: %v", err)
}
if want := matchingPending + 3; deleted != want {
t.Fatalf("deleted = %d, want %d", deleted, want)
}
if cancelled != 0 {
t.Fatalf("cancelled active = %d, want 0", cancelled)
}
pending, err := inspector.inspector.ListPendingTasks(
types.QueueDefault, asynq.PageSize(listPageSize), asynq.Page(1),
)
if err != nil {
t.Fatalf("list pending: %v", err)
}
if len(pending) != 7 {
t.Fatalf("pending survivors = %d, want 7", len(pending))
}
if !hasTaskID(pending, "kb-delete-cleanup") {
t.Fatal("kb:delete cleanup task was removed")
}
if !hasTaskID(pending, "index-delete-cleanup") {
t.Fatal("index:delete cleanup task was removed")
}
scheduled, err := inspector.inspector.ListScheduledTasks(
types.QueueDefault, asynq.PageSize(listPageSize), asynq.Page(1),
)
if err != nil {
t.Fatalf("list scheduled: %v", err)
}
if len(scheduled) != 1 || scheduled[0].ID != "scheduled-survivor" {
t.Fatalf("scheduled survivors = %v, want scheduled-survivor", taskIDs(scheduled))
}
}
func TestCancelTasksForKnowledgeBaseRemovesCancelledActiveRetry(t *testing.T) {
server := miniredis.RunT(t)
redisClient := redis.NewClient(&redis.Options{Addr: server.Addr()})
asynqClient := asynq.NewClientFromRedisClient(redisClient)
t.Cleanup(func() {
_ = asynqClient.Close()
_ = redisClient.Close()
})
inspector := &asynqTaskInspector{
inspector: asynq.NewInspectorFromRedisClient(redisClient),
redis: redisClient,
}
worker := asynq.NewServerFromRedisClient(redisClient, asynq.Config{
Concurrency: 1,
Queues: map[string]int{types.QueueDefault: 1},
TaskCheckInterval: 10 * time.Millisecond,
DelayedTaskCheckInterval: time.Hour,
RetryDelayFunc: func(_ int, _ error, _ *asynq.Task) time.Duration {
return time.Hour
},
ShutdownTimeout: time.Second,
LogLevel: asynq.FatalLevel,
})
handlerStarted := make(chan struct{})
handlerReturned := make(chan struct{})
mux := asynq.NewServeMux()
mux.HandleFunc(types.TypeDocumentProcess, func(ctx context.Context, _ *asynq.Task) error {
close(handlerStarted)
<-ctx.Done()
close(handlerReturned)
return ctx.Err()
})
if err := worker.Start(mux); err != nil {
t.Fatalf("start asynq worker: %v", err)
}
t.Cleanup(worker.Shutdown)
waitForAsynqCancellationSubscriber(t, redisClient)
const taskID = "active-kb-match"
if _, err := asynqClient.Enqueue(
asynq.NewTask(types.TypeDocumentProcess, []byte(`{"knowledge_base_id":"kb-delete"}`)),
asynq.Queue(types.QueueDefault),
asynq.TaskID(taskID),
asynq.MaxRetry(3),
); err != nil {
t.Fatalf("enqueue active task: %v", err)
}
select {
case <-handlerStarted:
case <-time.After(5 * time.Second):
t.Fatal("handler did not start")
}
waitForTaskState(t, inspector.inspector, taskID, asynq.TaskStateActive)
deleted, cancelled, err := inspector.CancelTasksForKnowledgeBase(
context.Background(), "kb-delete", nil, nil,
)
if err != nil {
t.Fatalf("cancel active task: %v", err)
}
if cancelled != 1 {
t.Fatalf("cancelled active = %d, want 1", cancelled)
}
if deleted != 1 {
t.Fatalf("deleted after active cancellation = %d, want 1", deleted)
}
select {
case <-handlerReturned:
case <-time.After(5 * time.Second):
t.Fatal("cancelled handler did not return")
}
waitForTaskToLeaveLiveStates(t, inspector.inspector, taskID)
}
func enqueueTask(t *testing.T, client *asynq.Client, taskType, payload, taskID string) {
t.Helper()
if _, err := client.Enqueue(
asynq.NewTask(taskType, []byte(payload)),
asynq.Queue(types.QueueDefault),
asynq.TaskID(taskID),
); err != nil {
t.Fatalf("enqueue %s: %v", taskID, err)
}
}
func scheduleTask(t *testing.T, client *asynq.Client, taskType, payload, taskID string) {
t.Helper()
if _, err := client.Enqueue(
asynq.NewTask(taskType, []byte(payload)),
asynq.Queue(types.QueueDefault),
asynq.TaskID(taskID),
asynq.ProcessAt(time.Now().Add(time.Hour)),
); err != nil {
t.Fatalf("schedule %s: %v", taskID, err)
}
}
func hasTaskID(tasks []*asynq.TaskInfo, taskID string) bool {
for _, task := range tasks {
if task.ID == taskID {
return true
}
}
return false
}
func taskIDs(tasks []*asynq.TaskInfo) []string {
ids := make([]string, 0, len(tasks))
for _, task := range tasks {
ids = append(ids, task.ID)
}
return ids
}
func waitForTaskState(t *testing.T, inspector *asynq.Inspector, taskID string, want asynq.TaskState) {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
var lastState asynq.TaskState
for time.Now().Before(deadline) {
task, err := inspector.GetTaskInfo(types.QueueDefault, taskID)
if err == nil {
lastState = task.State
if task.State == want {
return
}
} else if !errors.Is(err, asynq.ErrTaskNotFound) {
t.Fatalf("get task %s: %v", taskID, err)
}
time.Sleep(10 * time.Millisecond)
}
t.Fatalf("task %s state = %v, want %v", taskID, lastState, want)
}
func waitForAsynqCancellationSubscriber(t *testing.T, redisClient *redis.Client) {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
subscribers, err := redisClient.PubSubNumSub(context.Background(), "asynq:cancel").Result()
if err != nil {
t.Fatalf("query cancellation subscribers: %v", err)
}
if subscribers["asynq:cancel"] > 0 {
return
}
time.Sleep(10 * time.Millisecond)
}
t.Fatal("asynq cancellation subscriber did not start")
}
func waitForTaskToLeaveLiveStates(t *testing.T, inspector *asynq.Inspector, taskID string) {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
var lastState asynq.TaskState
for time.Now().Before(deadline) {
task, err := inspector.GetTaskInfo(types.QueueDefault, taskID)
if errors.Is(err, asynq.ErrTaskNotFound) {
return
}
if err != nil {
t.Fatalf("get task %s: %v", taskID, err)
}
lastState = task.State
switch task.State {
case asynq.TaskStatePending, asynq.TaskStateScheduled, asynq.TaskStateRetry, asynq.TaskStateActive:
time.Sleep(10 * time.Millisecond)
default:
return
}
}
t.Fatalf("task %s remained in live state %v", taskID, lastState)
}
@@ -64,6 +64,24 @@ type TaskInspector interface {
WorkerServerStats(ctx context.Context) (stats []types.WorkerServerStat, supported bool, err error)
}
// KnowledgeBaseTaskCanceller is the optional queue-cleanup capability used
// when a knowledge base is deleted. It is separate from TaskInspector so
// lightweight test doubles and queue backends that cannot inspect tasks do
// not need to implement knowledge-base-wide scanning.
type KnowledgeBaseTaskCanceller interface {
// CancelTasksForKnowledgeBase removes pending/scheduled/retry tasks
// associated with kbID and signals matching active workers to stop.
// knowledgeIDs covers batch tasks whose payload references documents but
// does not carry the parent knowledge-base ID. dataSourceIDs covers sync
// tasks whose payload only identifies a data source.
CancelTasksForKnowledgeBase(
ctx context.Context,
kbID string,
knowledgeIDs []string,
dataSourceIDs []string,
) (deleted int, cancelled int, err error)
}
// RuntimeTaskInspector is the optional operator surface implemented by queue
// backends that retain inspectable task state. It is separate from
// TaskInspector so Lite mode and light-weight tests do not need to implement
+17
View File
@@ -83,6 +83,23 @@ type TaskPendingOpsRepository interface {
DeleteByDedupKey(ctx context.Context, taskType, scope, scopeID, dedupKey, op string) error
}
// TaskPendingOpsScopeCleaner is an optional extension for callers that need
// to discard every durable pending operation owned by a deleted scope. It is
// intentionally separate from TaskPendingOpsRepository so alternate queue
// implementations and existing test doubles are not forced to implement a
// lifecycle-only operation.
type TaskPendingOpsScopeCleaner interface {
DeleteByScope(ctx context.Context, scope, scopeID string) error
}
// TaskPendingOpsKnowledgeBaseGuard atomically persists a KB-scoped operation
// only while its knowledge base is still active. Implementations must
// serialize the active-KB check with soft deletion so a detached worker cannot
// recreate durable work after the deletion scrub has finished.
type TaskPendingOpsKnowledgeBaseGuard interface {
EnqueueIfKnowledgeBaseActive(ctx context.Context, op *types.TaskPendingOp) (accepted bool, err error)
}
// TaskDeadLetterRepository persists rows for the generic task dead-letter
// archive (`task_dead_letters`). Two writers exist: the asynq
// dead-letter middleware (one row per archived asynq task), and the
+1
View File
@@ -396,6 +396,7 @@ type KBDeletePayload struct {
TracingContext
TenantID uint64 `json:"tenant_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
DataSourceIDs []string `json:"data_source_ids,omitempty"`
EffectiveEngines []RetrieverEngineParams `json:"effective_engines"`
// VectorStoreID is the bound store snapshot taken at enqueue time (before
// soft-delete) so the async worker can resolve the right store. nil means