fix(tasks): dequeue work for deleted knowledge bases
This commit is contained in:
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
// taskPendingOpsRepository implements interfaces.TaskPendingOpsRepository.
|
||||
@@ -26,6 +27,13 @@ func NewTaskPendingOpsRepository(db *gorm.DB) interfaces.TaskPendingOpsRepositor
|
||||
// Scope/ScopeID/Op (Payload optional). ID, FailCount default to zero;
|
||||
// EnqueuedAt is filled with the DB-side default if left zero.
|
||||
func (r *taskPendingOpsRepository) Enqueue(ctx context.Context, op *types.TaskPendingOp) error {
|
||||
if err := preparePendingOp(op); err != nil {
|
||||
return err
|
||||
}
|
||||
return r.db.WithContext(ctx).Create(op).Error
|
||||
}
|
||||
|
||||
func preparePendingOp(op *types.TaskPendingOp) error {
|
||||
if op == nil {
|
||||
return errors.New("task pending ops: nil op")
|
||||
}
|
||||
@@ -41,7 +49,48 @@ func (r *taskPendingOpsRepository) Enqueue(ctx context.Context, op *types.TaskPe
|
||||
// driver-level default handling.
|
||||
op.Payload = []byte("{}")
|
||||
}
|
||||
return r.db.WithContext(ctx).Create(op).Error
|
||||
return nil
|
||||
}
|
||||
|
||||
// EnqueueIfKnowledgeBaseActive prevents detached wiki cleanup from writing new
|
||||
// durable work after a KB was soft-deleted. On Postgres the share lock
|
||||
// serializes this check+insert transaction against the row update performed by
|
||||
// soft deletion: whichever operation acquires the row first determines the
|
||||
// order, and the deletion path's subsequent scope scrub removes any insert
|
||||
// that committed before it.
|
||||
func (r *taskPendingOpsRepository) EnqueueIfKnowledgeBaseActive(
|
||||
ctx context.Context,
|
||||
op *types.TaskPendingOp,
|
||||
) (bool, error) {
|
||||
if err := preparePendingOp(op); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if op.Scope != types.TaskScopeKnowledgeBase {
|
||||
return false, errors.New("task pending ops: guarded enqueue requires knowledge_base scope")
|
||||
}
|
||||
accepted := false
|
||||
err := r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
query := tx.Model(&types.KnowledgeBase{}).
|
||||
Select("id").
|
||||
Where("id = ? AND tenant_id = ?", op.ScopeID, op.TenantID)
|
||||
dialector := tx.Dialector
|
||||
if dialector.Name() == "postgres" {
|
||||
query = query.Clauses(clause.Locking{Strength: "SHARE"})
|
||||
}
|
||||
var kb types.KnowledgeBase
|
||||
if err := query.Take(&kb).Error; err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if err := tx.Create(op).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
accepted = true
|
||||
return nil
|
||||
})
|
||||
return accepted, err
|
||||
}
|
||||
|
||||
// PeekBatch returns up to `limit` rows for the (task_type, scope, scope_id)
|
||||
@@ -228,6 +277,18 @@ func (r *taskPendingOpsRepository) DeleteByIDs(ctx context.Context, ids []int64)
|
||||
Delete(&types.TaskPendingOp{}).Error
|
||||
}
|
||||
|
||||
// DeleteByScope removes every pending operation owned by a scope, regardless
|
||||
// of task type. Both scope fields are required so a malformed lifecycle call
|
||||
// can never turn into an unbounded queue deletion.
|
||||
func (r *taskPendingOpsRepository) DeleteByScope(ctx context.Context, scope, scopeID string) error {
|
||||
if scope == "" || scopeID == "" {
|
||||
return errors.New("task pending ops: scope and scope_id are required")
|
||||
}
|
||||
return r.db.WithContext(ctx).
|
||||
Where("scope = ? AND scope_id = ?", scope, scopeID).
|
||||
Delete(&types.TaskPendingOp{}).Error
|
||||
}
|
||||
|
||||
// IncrFailCount atomically bumps fail_count for one row and returns the
|
||||
// new value. We use UPDATE ... RETURNING so the read+write happens in
|
||||
// one round trip and races between concurrent IncrFailCount callers
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/driver/sqlite"
|
||||
@@ -182,6 +183,110 @@ func TestTaskPendingOps_DeleteByIDs_RemovesOnlyTargets(t *testing.T) {
|
||||
assert.Equal(t, "b", got[0].DedupKey)
|
||||
}
|
||||
|
||||
func TestTaskPendingOps_DeleteByScope_RemovesAllTaskTypesAndIsolatesScopes(t *testing.T) {
|
||||
db := setupTaskQueueTestDB(t)
|
||||
repo := NewTaskPendingOpsRepository(db)
|
||||
cleaner, ok := repo.(interfaces.TaskPendingOpsScopeCleaner)
|
||||
require.True(t, ok)
|
||||
ctx := context.Background()
|
||||
|
||||
// The deleted KB can have durable work in more than one wiki queue.
|
||||
require.NoError(t, repo.Enqueue(ctx,
|
||||
makePendingOp(types.TypeWikiIngest, types.TaskScopeKnowledgeBase, "kb-delete", "ingest", "k1", nil)))
|
||||
require.NoError(t, repo.Enqueue(ctx,
|
||||
makePendingOp(types.TypeWikiFinalize, types.TaskScopeKnowledgeBase, "kb-delete", "finalize", "k2", nil)))
|
||||
|
||||
// Rows in another KB or another scope must survive even when their IDs or
|
||||
// task types overlap with the deleted KB.
|
||||
require.NoError(t, repo.Enqueue(ctx,
|
||||
makePendingOp(types.TypeWikiIngest, types.TaskScopeKnowledgeBase, "kb-keep", "ingest", "k3", nil)))
|
||||
require.NoError(t, repo.Enqueue(ctx,
|
||||
makePendingOp(types.TypeWikiFinalize, types.TaskScopeKnowledge, "kb-delete", "finalize", "k4", nil)))
|
||||
|
||||
require.NoError(t, cleaner.DeleteByScope(ctx, types.TaskScopeKnowledgeBase, "kb-delete"))
|
||||
|
||||
var remaining []*types.TaskPendingOp
|
||||
require.NoError(t, db.Order("id ASC").Find(&remaining).Error)
|
||||
require.Len(t, remaining, 2)
|
||||
assert.Equal(t, "kb-keep", remaining[0].ScopeID)
|
||||
assert.Equal(t, types.TaskScopeKnowledge, remaining[1].Scope)
|
||||
assert.Equal(t, "kb-delete", remaining[1].ScopeID)
|
||||
}
|
||||
|
||||
func TestTaskPendingOps_DeleteByScope_RejectsMissingScope(t *testing.T) {
|
||||
db := setupTaskQueueTestDB(t)
|
||||
repo := NewTaskPendingOpsRepository(db)
|
||||
cleaner, ok := repo.(interfaces.TaskPendingOpsScopeCleaner)
|
||||
require.True(t, ok)
|
||||
ctx := context.Background()
|
||||
|
||||
require.NoError(t, repo.Enqueue(ctx,
|
||||
makePendingOp(types.TypeWikiIngest, types.TaskScopeKnowledgeBase, "kb", "ingest", "k1", nil)))
|
||||
|
||||
assert.Error(t, cleaner.DeleteByScope(ctx, "", "kb"))
|
||||
assert.Error(t, cleaner.DeleteByScope(ctx, types.TaskScopeKnowledgeBase, ""))
|
||||
|
||||
var count int64
|
||||
require.NoError(t, db.Model(&types.TaskPendingOp{}).Count(&count).Error)
|
||||
assert.Equal(t, int64(1), count)
|
||||
}
|
||||
|
||||
func TestTaskPendingOps_EnqueueIfKnowledgeBaseActive(t *testing.T) {
|
||||
db := setupTaskQueueTestDB(t)
|
||||
require.NoError(t, db.Exec(`CREATE TABLE knowledge_bases (
|
||||
id VARCHAR(64) PRIMARY KEY,
|
||||
tenant_id INTEGER NOT NULL,
|
||||
deleted_at DATETIME
|
||||
)`).Error)
|
||||
require.NoError(t, db.Exec(
|
||||
"INSERT INTO knowledge_bases (id, tenant_id, deleted_at) VALUES (?, ?, NULL), (?, ?, ?)",
|
||||
"kb-active", 1, "kb-deleted", 1, time.Now(),
|
||||
).Error)
|
||||
|
||||
repo := NewTaskPendingOpsRepository(db)
|
||||
guard, ok := repo.(interfaces.TaskPendingOpsKnowledgeBaseGuard)
|
||||
require.True(t, ok)
|
||||
ctx := context.Background()
|
||||
|
||||
accepted, err := guard.EnqueueIfKnowledgeBaseActive(ctx,
|
||||
makePendingOp(types.TypeWikiIngest, types.TaskScopeKnowledgeBase, "kb-active", "ingest", "active", nil))
|
||||
require.NoError(t, err)
|
||||
assert.True(t, accepted)
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
op *types.TaskPendingOp
|
||||
}{
|
||||
{
|
||||
name: "soft deleted",
|
||||
op: makePendingOp(
|
||||
types.TypeWikiIngest, types.TaskScopeKnowledgeBase, "kb-deleted", "ingest", "deleted", nil,
|
||||
),
|
||||
},
|
||||
{
|
||||
name: "missing",
|
||||
op: makePendingOp(
|
||||
types.TypeWikiIngest, types.TaskScopeKnowledgeBase, "kb-missing", "ingest", "missing", nil,
|
||||
),
|
||||
},
|
||||
{name: "tenant mismatch", op: &types.TaskPendingOp{
|
||||
TenantID: 2, TaskType: types.TypeWikiIngest, Scope: types.TaskScopeKnowledgeBase,
|
||||
ScopeID: "kb-active", Op: "ingest", DedupKey: "wrong-tenant",
|
||||
}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
accepted, err := guard.EnqueueIfKnowledgeBaseActive(ctx, tc.op)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, accepted)
|
||||
})
|
||||
}
|
||||
|
||||
var rows []*types.TaskPendingOp
|
||||
require.NoError(t, db.Find(&rows).Error)
|
||||
require.Len(t, rows, 1)
|
||||
assert.Equal(t, "active", rows[0].DedupKey)
|
||||
}
|
||||
|
||||
// TestTaskPendingOps_IncrFailCount_ReturnsNewValueAndPersists exercises
|
||||
// the UPDATE...RETURNING flow. Successive bumps should observe monotonic
|
||||
// counts.
|
||||
|
||||
@@ -25,6 +25,8 @@ import (
|
||||
// ErrInvalidTenantID represents an error for invalid tenant ID
|
||||
var ErrInvalidTenantID = errors.New("invalid tenant ID")
|
||||
|
||||
const kbTaskCleanupTimeout = 5 * time.Second
|
||||
|
||||
// knowledgeBaseService implements the knowledge base service interface
|
||||
type knowledgeBaseService struct {
|
||||
repo interfaces.KnowledgeBaseRepository
|
||||
@@ -40,6 +42,8 @@ type knowledgeBaseService struct {
|
||||
storageResolver interfaces.StorageBackendResolver
|
||||
graphEngine interfaces.RetrieveGraphRepository
|
||||
asynqClient interfaces.TaskEnqueuer
|
||||
taskInspector interfaces.TaskInspector
|
||||
taskPendingRepo interfaces.TaskPendingOpsRepository
|
||||
dsRepo interfaces.DataSourceRepository
|
||||
syncLogRepo interfaces.SyncLogRepository
|
||||
dsScheduler *datasource.Scheduler
|
||||
@@ -60,6 +64,8 @@ func NewKnowledgeBaseService(repo interfaces.KnowledgeBaseRepository,
|
||||
storageResolver interfaces.StorageBackendResolver,
|
||||
graphEngine interfaces.RetrieveGraphRepository,
|
||||
asynqClient interfaces.TaskEnqueuer,
|
||||
taskInspector interfaces.TaskInspector,
|
||||
taskPendingRepo interfaces.TaskPendingOpsRepository,
|
||||
dsRepo interfaces.DataSourceRepository,
|
||||
syncLogRepo interfaces.SyncLogRepository,
|
||||
dsScheduler *datasource.Scheduler,
|
||||
@@ -79,6 +85,8 @@ func NewKnowledgeBaseService(repo interfaces.KnowledgeBaseRepository,
|
||||
storageResolver: storageResolver,
|
||||
graphEngine: graphEngine,
|
||||
asynqClient: asynqClient,
|
||||
taskInspector: taskInspector,
|
||||
taskPendingRepo: taskPendingRepo,
|
||||
dsRepo: dsRepo,
|
||||
syncLogRepo: syncLogRepo,
|
||||
dsScheduler: dsScheduler,
|
||||
@@ -720,6 +728,11 @@ func (s *knowledgeBaseService) DeleteKnowledgeBase(ctx context.Context, id strin
|
||||
recordKBActivity(ctx, s.audit, tenantID, id, types.AuditActionKBDeleted,
|
||||
"knowledge_base", id, types.AuditOutcomeSuccess, map[string]any{"name": deletedName})
|
||||
|
||||
// Stop both ephemeral queue work and durable wiki operations that target
|
||||
// the now-deleted KB. ProcessKBDelete repeats this with document IDs and
|
||||
// performs one final scrub after heavy cleanup to close enqueue races.
|
||||
s.cleanupTasksForKnowledgeBase(ctx, id, nil, nil)
|
||||
|
||||
// Step 1b: Remove all organization shares for this KB so org settings no longer show them
|
||||
if s.shareRepo != nil {
|
||||
if delErr := s.shareRepo.DeleteByKnowledgeBaseID(ctx, id); delErr != nil {
|
||||
@@ -729,12 +742,16 @@ func (s *knowledgeBaseService) DeleteKnowledgeBase(ctx context.Context, id strin
|
||||
|
||||
// Step 1c: Stop and soft-delete all data sources bound to this KB so cron
|
||||
// schedules and in-flight sync logs do not keep running against a deleted KB.
|
||||
s.deleteDataSourcesForKnowledgeBase(ctx, id)
|
||||
dataSourceIDs := s.deleteDataSourcesForKnowledgeBase(ctx, id)
|
||||
if len(dataSourceIDs) > 0 {
|
||||
s.cancelTasksForKnowledgeBase(ctx, id, nil, dataSourceIDs)
|
||||
}
|
||||
|
||||
// Step 2: Enqueue async task for heavy cleanup operations
|
||||
payload := types.KBDeletePayload{
|
||||
TenantID: tenantID,
|
||||
KnowledgeBaseID: id,
|
||||
DataSourceIDs: dataSourceIDs,
|
||||
EffectiveEngines: tenantInfo.GetEffectiveEngines(),
|
||||
VectorStoreID: vectorStoreIDSnapshot, // snapshot taken before soft-delete
|
||||
}
|
||||
@@ -772,9 +789,18 @@ func (s *knowledgeBaseService) ProcessKBDelete(ctx context.Context, t *asynq.Tas
|
||||
|
||||
tenantID := payload.TenantID
|
||||
kbID := payload.KnowledgeBaseID
|
||||
var knowledgeIDs []string
|
||||
|
||||
// Set tenant context for downstream services
|
||||
ctx = context.WithValue(ctx, types.TenantIDContextKey, tenantID)
|
||||
defer func() {
|
||||
// Workers may enqueue downstream work while the delete task performs
|
||||
// heavy storage cleanup. A detached, bounded final scrub runs on every
|
||||
// return path, including retryable failures and cancellation.
|
||||
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), kbTaskCleanupTimeout)
|
||||
defer cancel()
|
||||
s.cleanupTasksForKnowledgeBase(cleanupCtx, kbID, knowledgeIDs, payload.DataSourceIDs)
|
||||
}()
|
||||
|
||||
logger.Infof(ctx, "Processing KB delete task for knowledge base: %s", kbID)
|
||||
|
||||
@@ -788,14 +814,18 @@ func (s *knowledgeBaseService) ProcessKBDelete(ctx context.Context, t *asynq.Tas
|
||||
return err
|
||||
}
|
||||
logger.Infof(ctx, "Found %d knowledge entries to delete", len(knowledgeList))
|
||||
knowledgeIDs = make([]string, 0, len(knowledgeList))
|
||||
for _, knowledge := range knowledgeList {
|
||||
knowledgeIDs = append(knowledgeIDs, knowledge.ID)
|
||||
}
|
||||
|
||||
// Repeat the best-effort queue scrub with document IDs. Some batch tasks
|
||||
// only carry knowledge_id(s), and active work from the first pass may have
|
||||
// enqueued another downstream task before cancellation reached it.
|
||||
s.cleanupTasksForKnowledgeBase(ctx, kbID, knowledgeIDs, payload.DataSourceIDs)
|
||||
|
||||
// Step 2: Delete all knowledge entries and their resources
|
||||
if len(knowledgeList) > 0 {
|
||||
knowledgeIDs := make([]string, 0, len(knowledgeList))
|
||||
for _, knowledge := range knowledgeList {
|
||||
knowledgeIDs = append(knowledgeIDs, knowledge.ID)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Deleting all knowledge entries and their resources")
|
||||
|
||||
// Delete embeddings from vector store.
|
||||
@@ -911,23 +941,68 @@ func (s *knowledgeBaseService) ProcessKBDelete(ctx context.Context, t *asynq.Tas
|
||||
return nil
|
||||
}
|
||||
|
||||
// cancelTasksForKnowledgeBase removes queue work for a deleted KB when the
|
||||
// configured task backend supports knowledge-base-wide inspection. Queue
|
||||
// cleanup is an optimization: the soft-deleted database row remains the
|
||||
// durable source of truth, so backend failures must not fail KB deletion.
|
||||
func (s *knowledgeBaseService) cancelTasksForKnowledgeBase(
|
||||
ctx context.Context,
|
||||
kbID string,
|
||||
knowledgeIDs []string,
|
||||
dataSourceIDs []string,
|
||||
) {
|
||||
canceller, ok := s.taskInspector.(interfaces.KnowledgeBaseTaskCanceller)
|
||||
if !ok || kbID == "" {
|
||||
return
|
||||
}
|
||||
if _, _, err := canceller.CancelTasksForKnowledgeBase(ctx, kbID, knowledgeIDs, dataSourceIDs); err != nil {
|
||||
logger.Warnf(ctx, "Failed to cancel queued tasks for deleted KB %s: %v", kbID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// cleanupTasksForKnowledgeBase removes both asynq records and durable wiki
|
||||
// operations. The latter must be cleared as well or startup recovery can
|
||||
// recreate Redis triggers for a KB that no longer exists.
|
||||
func (s *knowledgeBaseService) cleanupTasksForKnowledgeBase(
|
||||
ctx context.Context,
|
||||
kbID string,
|
||||
knowledgeIDs []string,
|
||||
dataSourceIDs []string,
|
||||
) {
|
||||
if kbID == "" {
|
||||
return
|
||||
}
|
||||
cleaner, ok := s.taskPendingRepo.(interfaces.TaskPendingOpsScopeCleaner)
|
||||
if ok {
|
||||
// Clear durable work before scanning Redis. A large or degraded queue
|
||||
// must not consume the caller's entire deadline and starve the database
|
||||
// fence that prevents startup recovery from reviving this KB.
|
||||
if err := cleaner.DeleteByScope(ctx, types.TaskScopeKnowledgeBase, kbID); err != nil {
|
||||
logger.Warnf(ctx, "Failed to clear durable tasks for deleted KB %s: %v", kbID, err)
|
||||
}
|
||||
}
|
||||
s.cancelTasksForKnowledgeBase(ctx, kbID, knowledgeIDs, dataSourceIDs)
|
||||
}
|
||||
|
||||
// deleteDataSourcesForKnowledgeBase mirrors DataSourceService.DeleteDataSource for
|
||||
// every data source attached to the KB. Errors on individual sources are logged
|
||||
// but do not fail KB deletion — the KB record is already soft-deleted.
|
||||
func (s *knowledgeBaseService) deleteDataSourcesForKnowledgeBase(ctx context.Context, kbID string) {
|
||||
func (s *knowledgeBaseService) deleteDataSourcesForKnowledgeBase(ctx context.Context, kbID string) []string {
|
||||
if s.dsRepo == nil {
|
||||
return
|
||||
return nil
|
||||
}
|
||||
|
||||
dataSources, err := s.dsRepo.FindByKnowledgeBase(ctx, kbID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "Failed to list data sources for deleted KB %s: %v", kbID, err)
|
||||
return
|
||||
return nil
|
||||
}
|
||||
dataSourceIDs := make([]string, 0, len(dataSources))
|
||||
for _, ds := range dataSources {
|
||||
if ds == nil || ds.ID == "" {
|
||||
continue
|
||||
}
|
||||
dataSourceIDs = append(dataSourceIDs, ds.ID)
|
||||
if err := s.dsRepo.Delete(ctx, ds.ID); err != nil {
|
||||
logger.Warnf(ctx, "Failed to delete data source %s for KB %s: %v", ds.ID, kbID, err)
|
||||
continue
|
||||
@@ -942,6 +1017,7 @@ func (s *knowledgeBaseService) deleteDataSourcesForKnowledgeBase(ctx context.Con
|
||||
}
|
||||
logger.Infof(ctx, "Data source deleted with knowledge base: ds=%s kb=%s", ds.ID, kbID)
|
||||
}
|
||||
return dataSourceIDs
|
||||
}
|
||||
|
||||
// SetEmbeddingModel sets the embedding model for a knowledge base
|
||||
|
||||
@@ -0,0 +1,293 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/embedding"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type kbTaskCancelCall struct {
|
||||
kbID string
|
||||
knowledgeIDs []string
|
||||
dataSourceIDs []string
|
||||
}
|
||||
|
||||
type recordingKBTaskInspector struct {
|
||||
repo *kbDeleteKBRepo
|
||||
calls []kbTaskCancelCall
|
||||
cancelErr error
|
||||
sawSoftDeletedRecord bool
|
||||
}
|
||||
|
||||
func (r *recordingKBTaskInspector) CancelTasksForKnowledge(
|
||||
context.Context,
|
||||
string,
|
||||
) (int, int, error) {
|
||||
return 0, 0, nil
|
||||
}
|
||||
|
||||
func (r *recordingKBTaskInspector) HasQueuedTasksForKnowledge(context.Context, string) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (r *recordingKBTaskInspector) QueueStats(context.Context) ([]types.QueueStat, bool, error) {
|
||||
return nil, true, nil
|
||||
}
|
||||
|
||||
func (r *recordingKBTaskInspector) WorkerServerStats(context.Context) ([]types.WorkerServerStat, bool, error) {
|
||||
return nil, true, nil
|
||||
}
|
||||
|
||||
func (r *recordingKBTaskInspector) CancelTasksForKnowledgeBase(
|
||||
_ context.Context,
|
||||
kbID string,
|
||||
knowledgeIDs []string,
|
||||
dataSourceIDs []string,
|
||||
) (int, int, error) {
|
||||
r.calls = append(r.calls, kbTaskCancelCall{
|
||||
kbID: kbID,
|
||||
knowledgeIDs: append([]string(nil), knowledgeIDs...),
|
||||
dataSourceIDs: append([]string(nil), dataSourceIDs...),
|
||||
})
|
||||
if r.repo != nil && r.repo.deletedID == kbID {
|
||||
r.sawSoftDeletedRecord = true
|
||||
}
|
||||
return 0, 0, r.cancelErr
|
||||
}
|
||||
|
||||
var (
|
||||
_ interfaces.TaskInspector = (*recordingKBTaskInspector)(nil)
|
||||
_ interfaces.KnowledgeBaseTaskCanceller = (*recordingKBTaskInspector)(nil)
|
||||
)
|
||||
|
||||
type recordingKBDeleteEnqueuer struct {
|
||||
calls int
|
||||
task *asynq.Task
|
||||
}
|
||||
|
||||
type recordingKBPendingRepo struct {
|
||||
interfaces.TaskPendingOpsRepository
|
||||
scopeIDs []string
|
||||
deleteErr error
|
||||
}
|
||||
|
||||
func (r *recordingKBPendingRepo) DeleteByScope(_ context.Context, scope, scopeID string) error {
|
||||
if scope == types.TaskScopeKnowledgeBase {
|
||||
r.scopeIDs = append(r.scopeIDs, scopeID)
|
||||
}
|
||||
return r.deleteErr
|
||||
}
|
||||
|
||||
func (r *recordingKBDeleteEnqueuer) Enqueue(
|
||||
task *asynq.Task,
|
||||
_ ...asynq.Option,
|
||||
) (*asynq.TaskInfo, error) {
|
||||
r.calls++
|
||||
r.task = task
|
||||
return &asynq.TaskInfo{ID: "kb-delete-task"}, nil
|
||||
}
|
||||
|
||||
func TestDeleteKnowledgeBaseForwardsDataSourceTaskScope(t *testing.T) {
|
||||
const kbID = "kb-with-datasource"
|
||||
kbRepo := &kbDeleteKBRepo{fakeKBRepo: *newFakeKBRepo()}
|
||||
kbRepo.rows[kbID] = &types.KnowledgeBase{ID: kbID, TenantID: 1, Name: "test"}
|
||||
inspector := &recordingKBTaskInspector{repo: kbRepo}
|
||||
enqueuer := &recordingKBDeleteEnqueuer{}
|
||||
dsRepo := newKBDeleteDSRepo(kbID, &types.DataSource{ID: "datasource-1", KnowledgeBaseID: kbID})
|
||||
svc := &knowledgeBaseService{
|
||||
repo: kbRepo,
|
||||
asynqClient: enqueuer,
|
||||
taskInspector: inspector,
|
||||
dsRepo: dsRepo,
|
||||
}
|
||||
|
||||
err := svc.DeleteKnowledgeBase(ctxWithTenantStorage(1, "local"), kbID)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Len(t, inspector.calls, 2)
|
||||
assert.Empty(t, inspector.calls[0].dataSourceIDs)
|
||||
assert.Equal(t, []string{"datasource-1"}, inspector.calls[1].dataSourceIDs)
|
||||
require.NotNil(t, enqueuer.task)
|
||||
var payload types.KBDeletePayload
|
||||
require.NoError(t, json.Unmarshal(enqueuer.task.Payload(), &payload))
|
||||
assert.Equal(t, []string{"datasource-1"}, payload.DataSourceIDs)
|
||||
}
|
||||
|
||||
func TestDeleteKnowledgeBaseCancelsQueuedTasksBestEffort(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
cancelErr error
|
||||
pendingErr error
|
||||
}{
|
||||
{name: "success"},
|
||||
{name: "inspector failure", cancelErr: errors.New("redis unavailable")},
|
||||
{name: "durable queue failure", pendingErr: errors.New("database unavailable")},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
const kbID = "kb-task-cleanup"
|
||||
kbRepo := &kbDeleteKBRepo{fakeKBRepo: *newFakeKBRepo()}
|
||||
kbRepo.rows[kbID] = &types.KnowledgeBase{ID: kbID, TenantID: 1, Name: "test"}
|
||||
inspector := &recordingKBTaskInspector{repo: kbRepo, cancelErr: tt.cancelErr}
|
||||
pendingRepo := &recordingKBPendingRepo{deleteErr: tt.pendingErr}
|
||||
enqueuer := &recordingKBDeleteEnqueuer{}
|
||||
svc := &knowledgeBaseService{
|
||||
repo: kbRepo,
|
||||
asynqClient: enqueuer,
|
||||
taskInspector: inspector,
|
||||
taskPendingRepo: pendingRepo,
|
||||
}
|
||||
|
||||
err := svc.DeleteKnowledgeBase(ctxWithTenantStorage(1, "local"), kbID)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Len(t, inspector.calls, 1)
|
||||
assert.Equal(t, kbID, inspector.calls[0].kbID)
|
||||
assert.Empty(t, inspector.calls[0].knowledgeIDs)
|
||||
assert.True(t, inspector.sawSoftDeletedRecord)
|
||||
assert.Equal(t, []string{kbID}, pendingRepo.scopeIDs)
|
||||
assert.Equal(t, 1, enqueuer.calls)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type emptyKBKnowledgeRepo struct {
|
||||
interfaces.KnowledgeRepository
|
||||
}
|
||||
|
||||
func (emptyKBKnowledgeRepo) ListKnowledgeByKnowledgeBaseID(
|
||||
context.Context,
|
||||
uint64,
|
||||
string,
|
||||
) ([]*types.Knowledge, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func TestProcessKBDeleteRepeatsQueueCleanup(t *testing.T) {
|
||||
inspector := &recordingKBTaskInspector{}
|
||||
pendingRepo := &recordingKBPendingRepo{}
|
||||
svc := &knowledgeBaseService{
|
||||
kgRepo: emptyKBKnowledgeRepo{},
|
||||
taskInspector: inspector,
|
||||
taskPendingRepo: pendingRepo,
|
||||
}
|
||||
payload, err := json.Marshal(types.KBDeletePayload{TenantID: 1, KnowledgeBaseID: "kb-race"})
|
||||
require.NoError(t, err)
|
||||
|
||||
err = svc.ProcessKBDelete(context.Background(), asynq.NewTask(types.TypeKBDelete, payload))
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Len(t, inspector.calls, 2)
|
||||
for _, call := range inspector.calls {
|
||||
assert.Equal(t, "kb-race", call.kbID)
|
||||
assert.Empty(t, call.knowledgeIDs)
|
||||
}
|
||||
assert.Equal(t, []string{"kb-race", "kb-race"}, pendingRepo.scopeIDs)
|
||||
}
|
||||
|
||||
type populatedKBKnowledgeRepo struct {
|
||||
interfaces.KnowledgeRepository
|
||||
items []*types.Knowledge
|
||||
}
|
||||
|
||||
func (r populatedKBKnowledgeRepo) ListKnowledgeByKnowledgeBaseID(
|
||||
context.Context,
|
||||
uint64,
|
||||
string,
|
||||
) ([]*types.Knowledge, error) {
|
||||
return r.items, nil
|
||||
}
|
||||
|
||||
func (populatedKBKnowledgeRepo) DeleteKnowledgeList(context.Context, uint64, []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type kbCleanupChunkRepo struct {
|
||||
interfaces.ChunkRepository
|
||||
}
|
||||
|
||||
func (kbCleanupChunkRepo) ListImageInfoByKnowledgeIDs(
|
||||
context.Context,
|
||||
uint64,
|
||||
[]string,
|
||||
) ([]interfaces.ChunkImageInfo, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (kbCleanupChunkRepo) DeleteChunksByKnowledgeID(context.Context, uint64, string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type kbCleanupModelService struct {
|
||||
interfaces.ModelService
|
||||
}
|
||||
|
||||
func (kbCleanupModelService) GetEmbeddingModel(context.Context, string) (embedding.Embedder, error) {
|
||||
return kbCleanupEmbedder{}, nil
|
||||
}
|
||||
|
||||
type kbCleanupEmbedder struct{}
|
||||
|
||||
func (kbCleanupEmbedder) Embed(context.Context, string) ([]float32, error) { return nil, nil }
|
||||
func (kbCleanupEmbedder) BatchEmbed(context.Context, []string) ([][]float32, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (kbCleanupEmbedder) GetModelName() string { return "test" }
|
||||
func (kbCleanupEmbedder) GetDimensions() int { return 1 }
|
||||
func (kbCleanupEmbedder) GetModelID() string { return "test" }
|
||||
func (kbCleanupEmbedder) BatchEmbedWithPool(
|
||||
context.Context,
|
||||
embedding.Embedder,
|
||||
[]string,
|
||||
) ([][]float32, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func TestProcessKBDeleteCollectsKnowledgeIDsForEveryScrub(t *testing.T) {
|
||||
inspector := &recordingKBTaskInspector{}
|
||||
svc := &knowledgeBaseService{
|
||||
kgRepo: populatedKBKnowledgeRepo{items: []*types.Knowledge{
|
||||
{ID: "knowledge-1", KnowledgeBaseID: "kb-1", EmbeddingModelID: "model-1"},
|
||||
{ID: "knowledge-2", KnowledgeBaseID: "kb-1", EmbeddingModelID: "model-1"},
|
||||
}},
|
||||
chunkRepo: kbCleanupChunkRepo{},
|
||||
modelService: kbCleanupModelService{},
|
||||
taskInspector: inspector,
|
||||
}
|
||||
payload, err := json.Marshal(types.KBDeletePayload{TenantID: 1, KnowledgeBaseID: "kb-1"})
|
||||
require.NoError(t, err)
|
||||
|
||||
err = svc.ProcessKBDelete(context.Background(), asynq.NewTask(types.TypeKBDelete, payload))
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Len(t, inspector.calls, 2)
|
||||
for _, call := range inspector.calls {
|
||||
assert.Equal(t, []string{"knowledge-1", "knowledge-2"}, call.knowledgeIDs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCancelTasksForKnowledgeBaseForwardsKnowledgeIDs(t *testing.T) {
|
||||
inspector := &recordingKBTaskInspector{}
|
||||
svc := &knowledgeBaseService{taskInspector: inspector}
|
||||
|
||||
svc.cancelTasksForKnowledgeBase(
|
||||
context.Background(),
|
||||
"kb-1",
|
||||
[]string{"knowledge-1", "knowledge-2"},
|
||||
[]string{"datasource-1"},
|
||||
)
|
||||
|
||||
require.Len(t, inspector.calls, 1)
|
||||
assert.Equal(t, "kb-1", inspector.calls[0].kbID)
|
||||
assert.Equal(t, []string{"knowledge-1", "knowledge-2"}, inspector.calls[0].knowledgeIDs)
|
||||
assert.Equal(t, []string{"datasource-1"}, inspector.calls[0].dataSourceIDs)
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
apprepo "github.com/Tencent/WeKnora/internal/application/repository"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type wikiKBGuardPendingRepo struct {
|
||||
interfaces.TaskPendingOpsRepository
|
||||
accepted bool
|
||||
guardErr error
|
||||
guardedOps []*types.TaskPendingOp
|
||||
deleteErr error
|
||||
deletedKBs []string
|
||||
rows []*types.TaskPendingOp
|
||||
}
|
||||
|
||||
func (r *wikiKBGuardPendingRepo) EnqueueIfKnowledgeBaseActive(
|
||||
_ context.Context,
|
||||
op *types.TaskPendingOp,
|
||||
) (bool, error) {
|
||||
r.guardedOps = append(r.guardedOps, op)
|
||||
return r.accepted, r.guardErr
|
||||
}
|
||||
|
||||
func (r *wikiKBGuardPendingRepo) DeleteByScope(_ context.Context, scope, scopeID string) error {
|
||||
if scope == types.TaskScopeKnowledgeBase {
|
||||
r.deletedKBs = append(r.deletedKBs, scopeID)
|
||||
}
|
||||
return r.deleteErr
|
||||
}
|
||||
|
||||
func (r *wikiKBGuardPendingRepo) PeekBatch(
|
||||
context.Context,
|
||||
string,
|
||||
string,
|
||||
string,
|
||||
int,
|
||||
) ([]*types.TaskPendingOp, error) {
|
||||
return r.rows, nil
|
||||
}
|
||||
|
||||
type wikiGuardTaskQueue struct {
|
||||
interfaces.TaskEnqueuer
|
||||
tasks []*asynq.Task
|
||||
}
|
||||
|
||||
func (q *wikiGuardTaskQueue) Enqueue(task *asynq.Task, _ ...asynq.Option) (*asynq.TaskInfo, error) {
|
||||
q.tasks = append(q.tasks, task)
|
||||
return &asynq.TaskInfo{ID: "guard-test", Type: task.Type()}, nil
|
||||
}
|
||||
|
||||
type wikiGuardKBService struct {
|
||||
interfaces.KnowledgeBaseService
|
||||
kb *types.KnowledgeBase
|
||||
err error
|
||||
}
|
||||
|
||||
func (s *wikiGuardKBService) GetKnowledgeBaseByIDOnly(
|
||||
context.Context,
|
||||
string,
|
||||
) (*types.KnowledgeBase, error) {
|
||||
return s.kb, s.err
|
||||
}
|
||||
|
||||
func TestEnqueueWikiWorkSkipsDeletedKnowledgeBase(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
run func(context.Context, interfaces.TaskEnqueuer, interfaces.TaskPendingOpsRepository)
|
||||
}{
|
||||
{
|
||||
name: "ingest",
|
||||
run: func(ctx context.Context, task interfaces.TaskEnqueuer, repo interfaces.TaskPendingOpsRepository) {
|
||||
EnqueueWikiIngest(ctx, task, repo, 7, "kb-deleted", "knowledge-1")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "retract",
|
||||
run: func(ctx context.Context, task interfaces.TaskEnqueuer, repo interfaces.TaskPendingOpsRepository) {
|
||||
EnqueueWikiRetract(ctx, task, repo, WikiRetractPayload{
|
||||
TenantID: 7, KnowledgeBaseID: "kb-deleted", KnowledgeID: "knowledge-1",
|
||||
})
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
repo := &wikiKBGuardPendingRepo{accepted: false}
|
||||
queue := &wikiGuardTaskQueue{}
|
||||
test.run(context.Background(), queue, repo)
|
||||
|
||||
require.Len(t, repo.guardedOps, 1)
|
||||
assert.Equal(t, "kb-deleted", repo.guardedOps[0].ScopeID)
|
||||
assert.Empty(t, queue.tasks)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnqueueWikiFinalizeOnlySchedulesAcceptedRows(t *testing.T) {
|
||||
for _, accepted := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "deleted", true: "active"}[accepted], func(t *testing.T) {
|
||||
repo := &wikiKBGuardPendingRepo{accepted: accepted}
|
||||
queue := &wikiGuardTaskQueue{}
|
||||
svc := &wikiIngestService{pendingRepo: repo, task: queue}
|
||||
|
||||
svc.enqueueFinalize(
|
||||
context.Background(),
|
||||
WikiIngestPayload{TenantID: 7, KnowledgeBaseID: "kb-1"},
|
||||
[]string{"slug-1"},
|
||||
map[string]string{"slug-1": "Title"},
|
||||
[]wikiFinalizeChange{{Action: wikiFinalizeAdded, DocTitle: "Document"}},
|
||||
[]string{"folder-1"},
|
||||
)
|
||||
|
||||
require.Len(t, repo.guardedOps, 3)
|
||||
if accepted {
|
||||
require.Len(t, queue.tasks, 1)
|
||||
assert.Equal(t, types.TypeWikiFinalize, queue.tasks[0].Type())
|
||||
} else {
|
||||
assert.Empty(t, queue.tasks)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWikiHandlersDrainDeletedKnowledgeBaseQueue(t *testing.T) {
|
||||
payload, err := json.Marshal(WikiIngestPayload{TenantID: 7, KnowledgeBaseID: "kb-deleted"})
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, taskType := range []string{types.TypeWikiIngest, types.TypeWikiFinalize} {
|
||||
t.Run(taskType, func(t *testing.T) {
|
||||
repo := &wikiKBGuardPendingRepo{
|
||||
rows: []*types.TaskPendingOp{{ID: 1, ScopeID: "kb-deleted"}},
|
||||
}
|
||||
svc := &wikiIngestService{
|
||||
kbService: &wikiGuardKBService{err: apprepo.ErrKnowledgeBaseNotFound},
|
||||
pendingRepo: repo,
|
||||
}
|
||||
|
||||
err := svc.Handle(context.Background(), asynq.NewTask(taskType, payload))
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"kb-deleted"}, repo.deletedKBs)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWikiDeletedKnowledgeBaseCleanupFailureRetries(t *testing.T) {
|
||||
payload, err := json.Marshal(WikiIngestPayload{TenantID: 7, KnowledgeBaseID: "kb-deleted"})
|
||||
require.NoError(t, err)
|
||||
wantErr := errors.New("cleanup failed")
|
||||
repo := &wikiKBGuardPendingRepo{deleteErr: wantErr}
|
||||
svc := &wikiIngestService{
|
||||
kbService: &wikiGuardKBService{err: apprepo.ErrKnowledgeBaseNotFound},
|
||||
pendingRepo: repo,
|
||||
}
|
||||
|
||||
err = svc.ProcessWikiIngest(
|
||||
context.Background(),
|
||||
asynq.NewTask(types.TypeWikiIngest, payload),
|
||||
)
|
||||
|
||||
require.ErrorIs(t, err, wantErr)
|
||||
assert.Equal(t, []string{"kb-deleted"}, repo.deletedKBs)
|
||||
}
|
||||
@@ -426,6 +426,23 @@ func (s *wikiIngestService) beginWikiSubspan(ctx context.Context, knowledgeID st
|
||||
// Lite mode (no Redis) still works as long as Postgres is reachable —
|
||||
// the queue lives in PG, only the active-batch lock is Redis-only and
|
||||
// has a process-local fallback (liteLocks) inside the worker.
|
||||
func enqueueWikiPendingOp(
|
||||
ctx context.Context,
|
||||
pendingRepo interfaces.TaskPendingOpsRepository,
|
||||
op *types.TaskPendingOp,
|
||||
) (bool, error) {
|
||||
if pendingRepo == nil {
|
||||
return true, nil
|
||||
}
|
||||
if guard, ok := pendingRepo.(interfaces.TaskPendingOpsKnowledgeBaseGuard); ok {
|
||||
return guard.EnqueueIfKnowledgeBaseActive(ctx, op)
|
||||
}
|
||||
if err := pendingRepo.Enqueue(ctx, op); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func EnqueueWikiIngest(
|
||||
ctx context.Context,
|
||||
task interfaces.TaskEnqueuer,
|
||||
@@ -450,20 +467,22 @@ func EnqueueWikiIngest(
|
||||
logger.Warnf(ctx, "wiki ingest: failed to marshal pending op for %s: %v", knowledgeID, err)
|
||||
return
|
||||
}
|
||||
if pendingRepo != nil {
|
||||
if err := pendingRepo.Enqueue(ctx, &types.TaskPendingOp{
|
||||
TenantID: tenantID,
|
||||
TaskType: wikiTaskType,
|
||||
Scope: wikiTaskScope,
|
||||
ScopeID: kbID,
|
||||
Op: WikiOpIngest,
|
||||
DedupKey: knowledgeID,
|
||||
Payload: payloadBytes,
|
||||
}); err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: failed to enqueue pending op for %s: %v", knowledgeID, err)
|
||||
// Fall through and still schedule the trigger task — the
|
||||
// next upload (or the next retry pass) will catch the gap.
|
||||
}
|
||||
accepted, err := enqueueWikiPendingOp(ctx, pendingRepo, &types.TaskPendingOp{
|
||||
TenantID: tenantID,
|
||||
TaskType: wikiTaskType,
|
||||
Scope: wikiTaskScope,
|
||||
ScopeID: kbID,
|
||||
Op: WikiOpIngest,
|
||||
DedupKey: knowledgeID,
|
||||
Payload: payloadBytes,
|
||||
})
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: failed to enqueue pending op for %s: %v", knowledgeID, err)
|
||||
return
|
||||
}
|
||||
if !accepted {
|
||||
logger.Infof(ctx, "wiki ingest: skip enqueue for deleted KB %s", kbID)
|
||||
return
|
||||
}
|
||||
|
||||
trigger := WikiIngestPayload{
|
||||
@@ -512,18 +531,22 @@ func EnqueueWikiRetract(
|
||||
logger.Warnf(ctx, "wiki retract: failed to marshal pending op: %v", err)
|
||||
return
|
||||
}
|
||||
if pendingRepo != nil {
|
||||
if err := pendingRepo.Enqueue(ctx, &types.TaskPendingOp{
|
||||
TenantID: payload.TenantID,
|
||||
TaskType: wikiTaskType,
|
||||
Scope: wikiTaskScope,
|
||||
ScopeID: payload.KnowledgeBaseID,
|
||||
Op: WikiOpRetract,
|
||||
DedupKey: payload.KnowledgeID,
|
||||
Payload: payloadBytes,
|
||||
}); err != nil {
|
||||
logger.Warnf(ctx, "wiki retract: failed to enqueue pending op: %v", err)
|
||||
}
|
||||
accepted, err := enqueueWikiPendingOp(ctx, pendingRepo, &types.TaskPendingOp{
|
||||
TenantID: payload.TenantID,
|
||||
TaskType: wikiTaskType,
|
||||
Scope: wikiTaskScope,
|
||||
ScopeID: payload.KnowledgeBaseID,
|
||||
Op: WikiOpRetract,
|
||||
DedupKey: payload.KnowledgeID,
|
||||
Payload: payloadBytes,
|
||||
})
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "wiki retract: failed to enqueue pending op: %v", err)
|
||||
return
|
||||
}
|
||||
if !accepted {
|
||||
logger.Infof(ctx, "wiki retract: skip enqueue for deleted KB %s", payload.KnowledgeBaseID)
|
||||
return
|
||||
}
|
||||
|
||||
trigger := WikiIngestPayload{
|
||||
@@ -564,6 +587,25 @@ func wikiIngestCleanupContext(ctx context.Context) (context.Context, context.Can
|
||||
return context.WithTimeout(context.WithoutCancel(ctx), wikiIngestCleanupTimeout)
|
||||
}
|
||||
|
||||
func (s *wikiIngestService) clearDeletedKnowledgeBasePendingOps(ctx context.Context, kbID string) error {
|
||||
cleaner, ok := s.pendingRepo.(interfaces.TaskPendingOpsScopeCleaner)
|
||||
if !ok || kbID == "" {
|
||||
return nil
|
||||
}
|
||||
cleanupCtx, cancel := wikiIngestCleanupContext(ctx)
|
||||
defer cancel()
|
||||
return cleaner.DeleteByScope(cleanupCtx, types.TaskScopeKnowledgeBase, kbID)
|
||||
}
|
||||
|
||||
func (s *wikiIngestService) enqueueFinalizeRow(ctx context.Context, op *types.TaskPendingOp) bool {
|
||||
accepted, err := enqueueWikiPendingOp(ctx, s.pendingRepo, op)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "wiki finalize: enqueue %s row failed: %v", op.Op, err)
|
||||
return false
|
||||
}
|
||||
return accepted
|
||||
}
|
||||
|
||||
// enqueueFinalize persists this batch's KB-global convergence work into the
|
||||
// finalize lane of task_pending_ops and schedules a debounced trigger. One
|
||||
// "slug" row per affected page (carrying its fresh title when this batch wrote
|
||||
@@ -580,13 +622,14 @@ func (s *wikiIngestService) enqueueFinalize(
|
||||
if s.pendingRepo == nil {
|
||||
return
|
||||
}
|
||||
acceptedAny := false
|
||||
for _, slug := range affectedSlugs {
|
||||
row := wikiFinalizeRow{Slug: slug, Title: freshTitleBySlug[slug]}
|
||||
b, err := json.Marshal(row)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if err := s.pendingRepo.Enqueue(ctx, &types.TaskPendingOp{
|
||||
if s.enqueueFinalizeRow(ctx, &types.TaskPendingOp{
|
||||
TenantID: payload.TenantID,
|
||||
TaskType: wikiFinalizeTaskType,
|
||||
Scope: wikiTaskScope,
|
||||
@@ -594,8 +637,8 @@ func (s *wikiIngestService) enqueueFinalize(
|
||||
Op: wikiFinalizeOpSlug,
|
||||
DedupKey: slug,
|
||||
Payload: b,
|
||||
}); err != nil {
|
||||
logger.Warnf(ctx, "wiki finalize: enqueue slug row for %s failed: %v", slug, err)
|
||||
}) {
|
||||
acceptedAny = true
|
||||
}
|
||||
}
|
||||
for i := range changes {
|
||||
@@ -604,7 +647,7 @@ func (s *wikiIngestService) enqueueFinalize(
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if err := s.pendingRepo.Enqueue(ctx, &types.TaskPendingOp{
|
||||
if s.enqueueFinalizeRow(ctx, &types.TaskPendingOp{
|
||||
TenantID: payload.TenantID,
|
||||
TaskType: wikiFinalizeTaskType,
|
||||
Scope: wikiTaskScope,
|
||||
@@ -612,14 +655,14 @@ func (s *wikiIngestService) enqueueFinalize(
|
||||
Op: wikiFinalizeOpChange,
|
||||
DedupKey: "",
|
||||
Payload: b,
|
||||
}); err != nil {
|
||||
logger.Warnf(ctx, "wiki finalize: enqueue change row failed: %v", err)
|
||||
}) {
|
||||
acceptedAny = true
|
||||
}
|
||||
}
|
||||
if len(folderIDs) > 0 {
|
||||
row := wikiFinalizeRow{FolderIDs: uniqueWikiFolderIDs(folderIDs)}
|
||||
if b, err := json.Marshal(row); err == nil {
|
||||
if err := s.pendingRepo.Enqueue(ctx, &types.TaskPendingOp{
|
||||
if s.enqueueFinalizeRow(ctx, &types.TaskPendingOp{
|
||||
TenantID: payload.TenantID,
|
||||
TaskType: wikiFinalizeTaskType,
|
||||
Scope: wikiTaskScope,
|
||||
@@ -627,11 +670,14 @@ func (s *wikiIngestService) enqueueFinalize(
|
||||
Op: wikiFinalizeOpFolderPrune,
|
||||
DedupKey: "",
|
||||
Payload: b,
|
||||
}); err != nil {
|
||||
logger.Warnf(ctx, "wiki finalize: enqueue folder prune row failed: %v", err)
|
||||
}) {
|
||||
acceptedAny = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !acceptedAny {
|
||||
return
|
||||
}
|
||||
s.scheduleFinalize(ctx, payload)
|
||||
}
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/agent"
|
||||
apprepo "github.com/Tencent/WeKnora/internal/application/repository"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/models/chat"
|
||||
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
|
||||
@@ -280,6 +281,13 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
}
|
||||
|
||||
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
|
||||
if errors.Is(err, apprepo.ErrKnowledgeBaseNotFound) || (err == nil && kb == nil) {
|
||||
exitStatus = "kb_deleted"
|
||||
if cleanupErr := s.clearDeletedKnowledgeBasePendingOps(ctx, payload.KnowledgeBaseID); cleanupErr != nil {
|
||||
return fmt.Errorf("wiki ingest: clear deleted KB queue: %w", cleanupErr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
exitStatus = "get_kb_failed"
|
||||
return fmt.Errorf("wiki ingest: get KB: %w", err)
|
||||
@@ -1006,6 +1014,12 @@ func (s *wikiIngestService) ProcessWikiFinalize(ctx context.Context, t *asynq.Ta
|
||||
}
|
||||
|
||||
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, payload.KnowledgeBaseID)
|
||||
if errors.Is(err, apprepo.ErrKnowledgeBaseNotFound) || (err == nil && kb == nil) {
|
||||
if cleanupErr := s.clearDeletedKnowledgeBasePendingOps(ctx, payload.KnowledgeBaseID); cleanupErr != nil {
|
||||
return fmt.Errorf("wiki finalize: clear deleted KB queue: %w", cleanupErr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("wiki finalize: get KB: %w", err)
|
||||
}
|
||||
|
||||
@@ -34,12 +34,34 @@ func recoverPendingWikiTasks(db *gorm.DB, task interfaces.TaskEnqueuer) {
|
||||
return
|
||||
}
|
||||
ctx := context.Background()
|
||||
const activeKnowledgeBase = `EXISTS (
|
||||
SELECT 1 FROM knowledge_bases kb
|
||||
WHERE kb.id = task_pending_ops.scope_id
|
||||
AND kb.tenant_id = task_pending_ops.tenant_id
|
||||
AND kb.deleted_at IS NULL
|
||||
)`
|
||||
wikiTaskTypes := []string{types.TypeWikiIngest, types.TypeWikiFinalize}
|
||||
|
||||
// Durable rows for a deleted/missing KB must not recreate ephemeral
|
||||
// triggers at startup. Fail closed if this cleanup cannot be verified.
|
||||
cleanup := db.WithContext(ctx).
|
||||
Where("scope = ? AND task_type IN ?", types.TaskScopeKnowledgeBase, wikiTaskTypes).
|
||||
Where("NOT " + activeKnowledgeBase).
|
||||
Delete(&types.TaskPendingOp{})
|
||||
if cleanup.Error != nil {
|
||||
logger.Warnf(ctx, "[WikiRecovery] failed to clear deleted KB queues: %v", cleanup.Error)
|
||||
return
|
||||
}
|
||||
if cleanup.RowsAffected > 0 {
|
||||
logger.Infof(ctx, "[WikiRecovery] removed %d pending row(s) for deleted knowledge bases", cleanup.RowsAffected)
|
||||
}
|
||||
|
||||
var scopes []pendingWikiScope
|
||||
if err := db.WithContext(ctx).
|
||||
Model(&types.TaskPendingOp{}).
|
||||
Distinct("tenant_id", "task_type", "scope_id").
|
||||
Where("scope = ? AND task_type IN ?", types.TaskScopeKnowledgeBase,
|
||||
[]string{types.TypeWikiIngest, types.TypeWikiFinalize}).
|
||||
Where("scope = ? AND task_type IN ?", types.TaskScopeKnowledgeBase, wikiTaskTypes).
|
||||
Where(activeKnowledgeBase).
|
||||
Find(&scopes).Error; err != nil {
|
||||
logger.Warnf(ctx, "[WikiRecovery] failed to list pending queues: %v", err)
|
||||
return
|
||||
|
||||
@@ -78,6 +78,14 @@ CREATE TABLE IF NOT EXISTS task_pending_ops (
|
||||
);
|
||||
`
|
||||
|
||||
const resetPendingKnowledgeBasesDDL = `
|
||||
CREATE TABLE IF NOT EXISTS knowledge_bases (
|
||||
id VARCHAR(64) PRIMARY KEY,
|
||||
tenant_id INTEGER NOT NULL DEFAULT 0,
|
||||
deleted_at DATETIME
|
||||
);
|
||||
`
|
||||
|
||||
func setupResetPendingDB(t *testing.T) *gorm.DB {
|
||||
t.Helper()
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
@@ -86,6 +94,7 @@ func setupResetPendingDB(t *testing.T) *gorm.DB {
|
||||
require.NoError(t, db.Exec(resetPendingSyncLogDDL).Error)
|
||||
require.NoError(t, db.Exec(resetPendingSpansDDL).Error)
|
||||
require.NoError(t, db.Exec(resetPendingOpsDDL).Error)
|
||||
require.NoError(t, db.Exec(resetPendingKnowledgeBasesDDL).Error)
|
||||
return db
|
||||
}
|
||||
|
||||
@@ -295,6 +304,11 @@ func (r *recordingTaskEnqueuer) Enqueue(task *asynq.Task, _ ...asynq.Option) (*a
|
||||
|
||||
func TestRecoverPendingWikiTasks_RecreatesOneTriggerPerLaneAndKB(t *testing.T) {
|
||||
db := setupResetPendingDB(t)
|
||||
require.NoError(t, db.Exec(
|
||||
`INSERT INTO knowledge_bases (id, tenant_id, deleted_at)
|
||||
VALUES (?, ?, NULL), (?, ?, NULL), (?, ?, ?)`,
|
||||
"kb-a", 7, "kb-b", 8, "kb-deleted", 9, time.Now(),
|
||||
).Error)
|
||||
rows := []struct {
|
||||
tenantID uint64
|
||||
taskType string
|
||||
@@ -305,6 +319,8 @@ func TestRecoverPendingWikiTasks_RecreatesOneTriggerPerLaneAndKB(t *testing.T) {
|
||||
{7, types.TypeWikiIngest, "kb-a", "k-2"}, // same lane: one trigger
|
||||
{7, types.TypeWikiFinalize, "kb-a", "slug-a"},
|
||||
{8, types.TypeWikiIngest, "kb-b", "k-3"},
|
||||
{9, types.TypeWikiIngest, "kb-deleted", "k-deleted"},
|
||||
{10, types.TypeWikiFinalize, "kb-missing", "k-missing"},
|
||||
}
|
||||
for _, row := range rows {
|
||||
require.NoError(t, db.Exec(
|
||||
@@ -328,4 +344,10 @@ func TestRecoverPendingWikiTasks_RecreatesOneTriggerPerLaneAndKB(t *testing.T) {
|
||||
assert.Equal(t, uint64(7), seen[types.TypeWikiIngest+":kb-a"].TenantID)
|
||||
assert.Equal(t, uint64(7), seen[types.TypeWikiFinalize+":kb-a"].TenantID)
|
||||
assert.Equal(t, uint64(8), seen[types.TypeWikiIngest+":kb-b"].TenantID)
|
||||
|
||||
var orphaned int64
|
||||
require.NoError(t, db.Model(&types.TaskPendingOp{}).
|
||||
Where("scope_id IN ?", []string{"kb-deleted", "kb-missing"}).
|
||||
Count(&orphaned).Error)
|
||||
assert.Zero(t, orphaned)
|
||||
}
|
||||
|
||||
+330
-147
@@ -96,7 +96,12 @@ var taskTypesForKnowledgeCancel = map[string]struct{}{
|
||||
|
||||
// listPageSize caps each Redis LIST call. Asynq pages tasks, so we
|
||||
// loop until a short page comes back. 100 matches asynq's default.
|
||||
const listPageSize = 100
|
||||
const (
|
||||
listPageSize = 100
|
||||
maxQueueMutationPasses = 1000
|
||||
activeCancellationSettleTimeout = time.Second
|
||||
activeCancellationPollInterval = 25 * time.Millisecond
|
||||
)
|
||||
|
||||
// CancelTasksForKnowledge removes queued tasks whose payload references
|
||||
// the given knowledge_id and signals active workers running such tasks
|
||||
@@ -107,18 +112,9 @@ func (a *asynqTaskInspector) CancelTasksForKnowledge(
|
||||
if a == nil || a.inspector == nil || knowledgeID == "" {
|
||||
return 0, 0, nil
|
||||
}
|
||||
deleted := 0
|
||||
cancelled := 0
|
||||
|
||||
for _, queue := range queuesScanned {
|
||||
// Pending / Scheduled / Retry can all be deleted by task ID.
|
||||
// Archived tasks are NOT touched: dead-letter rows are
|
||||
// already final and should remain visible to operators.
|
||||
deleted += a.deletePendingMatches(ctx, queue, knowledgeID)
|
||||
deleted += a.deleteScheduledMatches(ctx, queue, knowledgeID)
|
||||
deleted += a.deleteRetryMatches(ctx, queue, knowledgeID)
|
||||
cancelled += a.cancelActiveMatches(ctx, queue, knowledgeID)
|
||||
}
|
||||
deleted, cancelled := a.cancelMatchingTasks(ctx, func(taskType string, payload []byte) bool {
|
||||
return matchesKnowledge(taskType, payload, knowledgeID)
|
||||
})
|
||||
|
||||
logger.Infof(ctx,
|
||||
"[TaskInspector] knowledge=%s cancel summary: deleted_from_queue=%d active_cancel_signaled=%d",
|
||||
@@ -127,6 +123,42 @@ func (a *asynqTaskInspector) CancelTasksForKnowledge(
|
||||
return deleted, cancelled, nil
|
||||
}
|
||||
|
||||
// CancelTasksForKnowledgeBase removes tasks that still reference a knowledge
|
||||
// base (or one of its knowledges) after the knowledge base has been deleted.
|
||||
// The kb:delete task is deliberately excluded because it performs the durable
|
||||
// storage cleanup and must remain queued.
|
||||
func (a *asynqTaskInspector) CancelTasksForKnowledgeBase(
|
||||
ctx context.Context, knowledgeBaseID string, knowledgeIDs []string, dataSourceIDs []string,
|
||||
) (int, int, error) {
|
||||
if a == nil || a.inspector == nil {
|
||||
return 0, 0, nil
|
||||
}
|
||||
knowledgeIDSet := make(map[string]struct{}, len(knowledgeIDs))
|
||||
for _, knowledgeID := range knowledgeIDs {
|
||||
if knowledgeID != "" {
|
||||
knowledgeIDSet[knowledgeID] = struct{}{}
|
||||
}
|
||||
}
|
||||
dataSourceIDSet := make(map[string]struct{}, len(dataSourceIDs))
|
||||
for _, dataSourceID := range dataSourceIDs {
|
||||
if dataSourceID != "" {
|
||||
dataSourceIDSet[dataSourceID] = struct{}{}
|
||||
}
|
||||
}
|
||||
if knowledgeBaseID == "" && len(knowledgeIDSet) == 0 && len(dataSourceIDSet) == 0 {
|
||||
return 0, 0, nil
|
||||
}
|
||||
|
||||
deleted, cancelled := a.cancelMatchingTasks(ctx, func(taskType string, payload []byte) bool {
|
||||
return matchesKnowledgeBase(taskType, payload, knowledgeBaseID, knowledgeIDSet, dataSourceIDSet)
|
||||
})
|
||||
logger.Infof(ctx,
|
||||
"[TaskInspector] knowledge_base=%s cancel summary: deleted_from_queue=%d active_cancel_signaled=%d",
|
||||
knowledgeBaseID, deleted, cancelled,
|
||||
)
|
||||
return deleted, cancelled, nil
|
||||
}
|
||||
|
||||
// HasQueuedTasksForKnowledge reports whether any pending / scheduled /
|
||||
// retry / active task referencing knowledgeID still lives in the queue.
|
||||
// Read-only counterpart of CancelTasksForKnowledge — the housekeeping
|
||||
@@ -138,18 +170,12 @@ func (a *asynqTaskInspector) HasQueuedTasksForKnowledge(
|
||||
if a == nil || a.inspector == nil || knowledgeID == "" {
|
||||
return false, nil
|
||||
}
|
||||
listers := []struct {
|
||||
state string
|
||||
list func(string, ...asynq.ListOption) ([]*asynq.TaskInfo, error)
|
||||
}{
|
||||
{"pending", a.inspector.ListPendingTasks},
|
||||
{"scheduled", a.inspector.ListScheduledTasks},
|
||||
{"retry", a.inspector.ListRetryTasks},
|
||||
{"active", a.inspector.ListActiveTasks},
|
||||
matcher := func(taskType string, payload []byte) bool {
|
||||
return matchesKnowledge(taskType, payload, knowledgeID)
|
||||
}
|
||||
for _, queue := range queuesScanned {
|
||||
for _, l := range listers {
|
||||
if a.queueStateHasMatch(ctx, queue, knowledgeID, l.state, l.list) {
|
||||
for _, state := range a.cancellableTaskStates() {
|
||||
if a.queueStateHasMatch(ctx, queue, state.name, state.list, matcher) {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
@@ -685,14 +711,249 @@ func (a *asynqTaskInspector) WorkerServerStats(
|
||||
return stats, true, nil
|
||||
}
|
||||
|
||||
type taskMatcher func(taskType string, payload []byte) bool
|
||||
|
||||
type cancellableTaskState struct {
|
||||
name string
|
||||
list func(string, ...asynq.ListOption) ([]*asynq.TaskInfo, error)
|
||||
active bool
|
||||
}
|
||||
|
||||
func (a *asynqTaskInspector) cancellableTaskStates() []cancellableTaskState {
|
||||
return []cancellableTaskState{
|
||||
{name: "pending", list: a.inspector.ListPendingTasks},
|
||||
{name: "scheduled", list: a.inspector.ListScheduledTasks},
|
||||
{name: "retry", list: a.inspector.ListRetryTasks},
|
||||
{name: "active", list: a.inspector.ListActiveTasks, active: true},
|
||||
}
|
||||
}
|
||||
|
||||
// cancelMatchingTasks applies the same matcher to every live queue state.
|
||||
// Archived tasks are deliberately retained as operator-visible history.
|
||||
func (a *asynqTaskInspector) cancelMatchingTasks(ctx context.Context, matcher taskMatcher) (int, int) {
|
||||
deleted := 0
|
||||
cancelled := 0
|
||||
|
||||
// Drain queued states first. Active cancellation is asynchronous: when a
|
||||
// handler returns context.Canceled, asynq normally moves it to retry, so a
|
||||
// later settle phase must remove that transitioned record as well.
|
||||
for _, queue := range queuesScanned {
|
||||
for _, state := range a.cancellableTaskStates() {
|
||||
if state.active {
|
||||
continue
|
||||
}
|
||||
deleted += a.processQueueStateMatches(
|
||||
ctx, queue, state, true, matcher, "delete",
|
||||
func(task *asynq.TaskInfo) error { return a.inspector.DeleteTask(queue, task.ID) },
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Snapshot all matching active IDs before publishing cancellation. If we
|
||||
// mutate the active set while paging, exited workers shift later pages and
|
||||
// can make us skip an entire page.
|
||||
activeTasks := make([]queueTask, 0)
|
||||
for _, queue := range queuesScanned {
|
||||
for _, state := range a.cancellableTaskStates() {
|
||||
if !state.active {
|
||||
continue
|
||||
}
|
||||
for _, task := range a.snapshotQueueStateMatches(ctx, queue, state, matcher) {
|
||||
if err := a.inspector.CancelProcessing(task.ID); err != nil {
|
||||
logger.Warnf(ctx, "[TaskInspector] cancel active type=%s id=%s: %v", task.Type, task.ID, err)
|
||||
continue
|
||||
}
|
||||
cancelled++
|
||||
activeTasks = append(activeTasks, queueTask{queue: queue, id: task.ID})
|
||||
}
|
||||
}
|
||||
}
|
||||
deleted += a.deleteCancelledTransitions(ctx, activeTasks)
|
||||
|
||||
// Catch tasks that entered pending/scheduled/retry while cancellation was
|
||||
// settling, plus downstream work emitted immediately before a handler saw
|
||||
// its cancellation.
|
||||
for _, queue := range queuesScanned {
|
||||
for _, state := range a.cancellableTaskStates() {
|
||||
if state.active {
|
||||
continue
|
||||
}
|
||||
deleted += a.processQueueStateMatches(
|
||||
ctx, queue, state, true, matcher, "delete",
|
||||
func(task *asynq.TaskInfo) error { return a.inspector.DeleteTask(queue, task.ID) },
|
||||
)
|
||||
}
|
||||
}
|
||||
return deleted, cancelled
|
||||
}
|
||||
|
||||
type queueTask struct {
|
||||
queue string
|
||||
id string
|
||||
}
|
||||
|
||||
// snapshotQueueStateMatches returns a read-only snapshot before its caller
|
||||
// mutates the state. TaskInfo values are only used for immutable task metadata.
|
||||
func (a *asynqTaskInspector) snapshotQueueStateMatches(
|
||||
ctx context.Context,
|
||||
queue string,
|
||||
state cancellableTaskState,
|
||||
matcher taskMatcher,
|
||||
) []*asynq.TaskInfo {
|
||||
var matches []*asynq.TaskInfo
|
||||
for page := 1; ; page++ {
|
||||
if ctx.Err() != nil {
|
||||
return matches
|
||||
}
|
||||
tasks, err := state.list(queue, asynq.PageSize(listPageSize), asynq.Page(page))
|
||||
if err != nil {
|
||||
if !errors.Is(err, asynq.ErrQueueNotFound) {
|
||||
logger.Warnf(ctx, "[TaskInspector] snapshot %s queue=%s page=%d: %v", state.name, queue, page, err)
|
||||
}
|
||||
return matches
|
||||
}
|
||||
for _, task := range tasks {
|
||||
if matcher(task.Type, task.Payload) {
|
||||
matches = append(matches, task)
|
||||
}
|
||||
}
|
||||
if len(tasks) < listPageSize {
|
||||
return matches
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// deleteCancelledTransitions waits briefly for signalled active tasks to
|
||||
// leave active state. A normal context.Canceled result is retried by asynq;
|
||||
// delete that transitioned record before it can become an orphan. Archived
|
||||
// and completed tasks remain operator-visible history.
|
||||
func (a *asynqTaskInspector) deleteCancelledTransitions(ctx context.Context, tasks []queueTask) int {
|
||||
if len(tasks) == 0 {
|
||||
return 0
|
||||
}
|
||||
deadline := time.Now().Add(activeCancellationSettleTimeout)
|
||||
pending := append([]queueTask(nil), tasks...)
|
||||
deleted := 0
|
||||
for len(pending) > 0 {
|
||||
if ctx.Err() != nil {
|
||||
return deleted
|
||||
}
|
||||
next := pending[:0]
|
||||
for _, ref := range pending {
|
||||
info, err := a.inspector.GetTaskInfo(ref.queue, ref.id)
|
||||
if errors.Is(err, asynq.ErrTaskNotFound) || errors.Is(err, asynq.ErrQueueNotFound) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[TaskInspector] inspect cancelled task queue=%s id=%s: %v", ref.queue, ref.id, err)
|
||||
next = append(next, ref)
|
||||
continue
|
||||
}
|
||||
switch info.State {
|
||||
case asynq.TaskStatePending, asynq.TaskStateScheduled, asynq.TaskStateRetry:
|
||||
if err := a.inspector.DeleteTask(ref.queue, ref.id); err != nil {
|
||||
if !errors.Is(err, asynq.ErrTaskNotFound) {
|
||||
logger.Warnf(ctx, "[TaskInspector] delete cancelled transition queue=%s id=%s: %v", ref.queue, ref.id, err)
|
||||
next = append(next, ref)
|
||||
}
|
||||
continue
|
||||
}
|
||||
deleted++
|
||||
case asynq.TaskStateActive:
|
||||
next = append(next, ref)
|
||||
}
|
||||
}
|
||||
pending = next
|
||||
if len(pending) == 0 || !time.Now().Before(deadline) {
|
||||
return deleted
|
||||
}
|
||||
wait := activeCancellationPollInterval
|
||||
if remaining := time.Until(deadline); remaining < wait {
|
||||
wait = remaining
|
||||
}
|
||||
timer := time.NewTimer(wait)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
if !timer.Stop() {
|
||||
select {
|
||||
case <-timer.C:
|
||||
default:
|
||||
}
|
||||
}
|
||||
return deleted
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
return deleted
|
||||
}
|
||||
|
||||
// processQueueStateMatches pages through one queue state and applies action to
|
||||
// matching tasks. Deleting a task shifts later entries toward the current
|
||||
// page, so delete states rescan that page after any successful mutation.
|
||||
func (a *asynqTaskInspector) processQueueStateMatches(
|
||||
ctx context.Context,
|
||||
queue string,
|
||||
state cancellableTaskState,
|
||||
rescanAfterSuccess bool,
|
||||
matcher taskMatcher,
|
||||
actionName string,
|
||||
action func(*asynq.TaskInfo) error,
|
||||
) int {
|
||||
processed := 0
|
||||
page := 1
|
||||
passes := 0
|
||||
for passes < maxQueueMutationPasses {
|
||||
passes++
|
||||
if ctx.Err() != nil {
|
||||
return processed
|
||||
}
|
||||
tasks, err := state.list(queue, asynq.PageSize(listPageSize), asynq.Page(page))
|
||||
if err != nil {
|
||||
if !errors.Is(err, asynq.ErrQueueNotFound) {
|
||||
logger.Warnf(ctx, "[TaskInspector] list %s queue=%s page=%d: %v", state.name, queue, page, err)
|
||||
}
|
||||
return processed
|
||||
}
|
||||
if len(tasks) == 0 {
|
||||
return processed
|
||||
}
|
||||
|
||||
processedOnPage := 0
|
||||
for _, task := range tasks {
|
||||
if !matcher(task.Type, task.Payload) {
|
||||
continue
|
||||
}
|
||||
if err := action(task); err != nil {
|
||||
logger.Warnf(ctx, "[TaskInspector] %s %s type=%s id=%s: %v", actionName, state.name, task.Type, task.ID, err)
|
||||
continue
|
||||
}
|
||||
processed++
|
||||
processedOnPage++
|
||||
}
|
||||
if rescanAfterSuccess && processedOnPage > 0 {
|
||||
continue
|
||||
}
|
||||
if len(tasks) < listPageSize {
|
||||
return processed
|
||||
}
|
||||
page++
|
||||
}
|
||||
logger.Warnf(ctx,
|
||||
"[TaskInspector] stopped %s queue=%s after %d mutation passes",
|
||||
state.name, queue, maxQueueMutationPasses,
|
||||
)
|
||||
return processed
|
||||
}
|
||||
|
||||
// queueStateHasMatch pages through one (queue, state) list looking for a
|
||||
// task that references knowledgeID. Mirrors the delete* scanners but is
|
||||
// strictly read-only and returns early on the first hit. A backend error
|
||||
// is logged and treated as "no match" (false); the caller's fail-safe
|
||||
// then errs toward recovering the row rather than preserving it forever.
|
||||
// matching task. It is strictly read-only and returns on the first hit. A
|
||||
// backend error is logged and treated as "no match" (false).
|
||||
func (a *asynqTaskInspector) queueStateHasMatch(
|
||||
ctx context.Context, queue, knowledgeID, state string,
|
||||
ctx context.Context,
|
||||
queue string,
|
||||
state string,
|
||||
list func(string, ...asynq.ListOption) ([]*asynq.TaskInfo, error),
|
||||
matcher taskMatcher,
|
||||
) bool {
|
||||
page := 1
|
||||
for {
|
||||
@@ -706,8 +967,8 @@ func (a *asynqTaskInspector) queueStateHasMatch(
|
||||
if len(tasks) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, t := range tasks {
|
||||
if matchesKnowledge(t.Type, t.Payload, knowledgeID) {
|
||||
for _, task := range tasks {
|
||||
if matcher(task.Type, task.Payload) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -731,133 +992,55 @@ func matchesKnowledge(taskType string, payload []byte, knowledgeID string) bool
|
||||
return probe.KnowledgeID == knowledgeID
|
||||
}
|
||||
|
||||
func (a *asynqTaskInspector) deletePendingMatches(ctx context.Context, queue, knowledgeID string) int {
|
||||
deleted := 0
|
||||
page := 1
|
||||
for {
|
||||
tasks, err := a.inspector.ListPendingTasks(queue, asynq.PageSize(listPageSize), asynq.Page(page))
|
||||
if err != nil {
|
||||
if !errors.Is(err, asynq.ErrQueueNotFound) {
|
||||
logger.Warnf(ctx, "[TaskInspector] list pending queue=%s page=%d: %v", queue, page, err)
|
||||
}
|
||||
return deleted
|
||||
}
|
||||
if len(tasks) == 0 {
|
||||
return deleted
|
||||
}
|
||||
for _, t := range tasks {
|
||||
if !matchesKnowledge(t.Type, t.Payload, knowledgeID) {
|
||||
continue
|
||||
}
|
||||
if err := a.inspector.DeleteTask(queue, t.ID); err != nil {
|
||||
logger.Warnf(ctx, "[TaskInspector] delete pending type=%s id=%s: %v", t.Type, t.ID, err)
|
||||
continue
|
||||
}
|
||||
deleted++
|
||||
}
|
||||
if len(tasks) < listPageSize {
|
||||
return deleted
|
||||
}
|
||||
page++
|
||||
// matchesKnowledgeBase identifies work made obsolete by deleting a knowledge
|
||||
// base. In addition to direct KB fields, clone/move payloads carry semantic KB
|
||||
// references under task-specific field names. knowledgeIDs catches tasks whose
|
||||
// payload does not carry a KB ID at all.
|
||||
func matchesKnowledgeBase(
|
||||
taskType string,
|
||||
payload []byte,
|
||||
knowledgeBaseID string,
|
||||
knowledgeIDs map[string]struct{},
|
||||
dataSourceIDs map[string]struct{},
|
||||
) bool {
|
||||
// These cleanup tasks carry snapshots specifically so they can still run
|
||||
// after the KB row has been soft-deleted. Removing them leaks resources.
|
||||
if taskType == types.TypeKBDelete || taskType == types.TypeIndexDelete {
|
||||
return false
|
||||
}
|
||||
var probe runtimeTaskPayloadProbe
|
||||
if err := json.Unmarshal(payload, &probe); err != nil {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (a *asynqTaskInspector) deleteScheduledMatches(ctx context.Context, queue, knowledgeID string) int {
|
||||
deleted := 0
|
||||
page := 1
|
||||
for {
|
||||
tasks, err := a.inspector.ListScheduledTasks(queue, asynq.PageSize(listPageSize), asynq.Page(page))
|
||||
if err != nil {
|
||||
if !errors.Is(err, asynq.ErrQueueNotFound) {
|
||||
logger.Warnf(ctx, "[TaskInspector] list scheduled queue=%s page=%d: %v", queue, page, err)
|
||||
if knowledgeBaseID != "" {
|
||||
if probe.KnowledgeBaseID == knowledgeBaseID || probe.KBID == knowledgeBaseID {
|
||||
return true
|
||||
}
|
||||
switch taskType {
|
||||
case types.TypeKBClone:
|
||||
if probe.SourceID == knowledgeBaseID || probe.TargetID == knowledgeBaseID {
|
||||
return true
|
||||
}
|
||||
return deleted
|
||||
}
|
||||
if len(tasks) == 0 {
|
||||
return deleted
|
||||
}
|
||||
for _, t := range tasks {
|
||||
if !matchesKnowledge(t.Type, t.Payload, knowledgeID) {
|
||||
continue
|
||||
case types.TypeKnowledgeMove:
|
||||
if probe.SourceKBID == knowledgeBaseID || probe.TargetKBID == knowledgeBaseID {
|
||||
return true
|
||||
}
|
||||
if err := a.inspector.DeleteTask(queue, t.ID); err != nil {
|
||||
logger.Warnf(ctx, "[TaskInspector] delete scheduled type=%s id=%s: %v", t.Type, t.ID, err)
|
||||
continue
|
||||
}
|
||||
deleted++
|
||||
}
|
||||
if len(tasks) < listPageSize {
|
||||
return deleted
|
||||
}
|
||||
page++
|
||||
}
|
||||
}
|
||||
|
||||
func (a *asynqTaskInspector) deleteRetryMatches(ctx context.Context, queue, knowledgeID string) int {
|
||||
deleted := 0
|
||||
page := 1
|
||||
for {
|
||||
tasks, err := a.inspector.ListRetryTasks(queue, asynq.PageSize(listPageSize), asynq.Page(page))
|
||||
if err != nil {
|
||||
if !errors.Is(err, asynq.ErrQueueNotFound) {
|
||||
logger.Warnf(ctx, "[TaskInspector] list retry queue=%s page=%d: %v", queue, page, err)
|
||||
}
|
||||
return deleted
|
||||
}
|
||||
if len(tasks) == 0 {
|
||||
return deleted
|
||||
}
|
||||
for _, t := range tasks {
|
||||
if !matchesKnowledge(t.Type, t.Payload, knowledgeID) {
|
||||
continue
|
||||
}
|
||||
if err := a.inspector.DeleteTask(queue, t.ID); err != nil {
|
||||
logger.Warnf(ctx, "[TaskInspector] delete retry type=%s id=%s: %v", t.Type, t.ID, err)
|
||||
continue
|
||||
}
|
||||
deleted++
|
||||
}
|
||||
if len(tasks) < listPageSize {
|
||||
return deleted
|
||||
}
|
||||
page++
|
||||
if _, ok := knowledgeIDs[probe.KnowledgeID]; ok && probe.KnowledgeID != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// cancelActiveMatches signals active workers to abort via
|
||||
// Inspector.CancelProcessing. The worker's ctx becomes Done() so the
|
||||
// next blocking call (or our checkpoint reads) bails. The DB-level
|
||||
// abort flag (parse_status=cancelled) remains the durable signal —
|
||||
// this is a latency optimization, not the correctness mechanism.
|
||||
func (a *asynqTaskInspector) cancelActiveMatches(ctx context.Context, queue, knowledgeID string) int {
|
||||
cancelled := 0
|
||||
page := 1
|
||||
for {
|
||||
tasks, err := a.inspector.ListActiveTasks(queue, asynq.PageSize(listPageSize), asynq.Page(page))
|
||||
if err != nil {
|
||||
if !errors.Is(err, asynq.ErrQueueNotFound) {
|
||||
logger.Warnf(ctx, "[TaskInspector] list active queue=%s page=%d: %v", queue, page, err)
|
||||
}
|
||||
return cancelled
|
||||
for _, knowledgeID := range probe.KnowledgeIDs {
|
||||
if _, ok := knowledgeIDs[knowledgeID]; ok && knowledgeID != "" {
|
||||
return true
|
||||
}
|
||||
if len(tasks) == 0 {
|
||||
return cancelled
|
||||
}
|
||||
for _, t := range tasks {
|
||||
if !matchesKnowledge(t.Type, t.Payload, knowledgeID) {
|
||||
continue
|
||||
}
|
||||
if err := a.inspector.CancelProcessing(t.ID); err != nil {
|
||||
logger.Warnf(ctx, "[TaskInspector] cancel active type=%s id=%s: %v", t.Type, t.ID, err)
|
||||
continue
|
||||
}
|
||||
cancelled++
|
||||
}
|
||||
if len(tasks) < listPageSize {
|
||||
return cancelled
|
||||
}
|
||||
page++
|
||||
}
|
||||
if _, ok := dataSourceIDs[probe.DataSourceID]; ok && probe.DataSourceID != "" {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// noopTaskInspector is the Lite-mode (no Redis) inspector. Inline
|
||||
|
||||
@@ -0,0 +1,348 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/hibiken/asynq"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
func TestMatchesKnowledgeBase(t *testing.T) {
|
||||
knowledgeIDs := map[string]struct{}{
|
||||
"knowledge-1": {},
|
||||
"knowledge-2": {},
|
||||
}
|
||||
dataSourceIDs := map[string]struct{}{"datasource-1": {}}
|
||||
tests := []struct {
|
||||
name string
|
||||
taskType string
|
||||
payload string
|
||||
want bool
|
||||
}{
|
||||
{name: "knowledge base id", taskType: types.TypeDocumentProcess, payload: `{"knowledge_base_id":"kb-1"}`, want: true},
|
||||
{name: "legacy kb id", taskType: types.TypeFAQImport, payload: `{"kb_id":"kb-1"}`, want: true},
|
||||
{name: "clone source", taskType: types.TypeKBClone, payload: `{"source_id":"kb-1"}`, want: true},
|
||||
{name: "clone target", taskType: types.TypeKBClone, payload: `{"target_id":"kb-1"}`, want: true},
|
||||
{name: "source id is task specific", taskType: types.TypeDocumentProcess, payload: `{"source_id":"kb-1"}`},
|
||||
{name: "move source", taskType: types.TypeKnowledgeMove, payload: `{"source_kb_id":"kb-1"}`, want: true},
|
||||
{name: "move target", taskType: types.TypeKnowledgeMove, payload: `{"target_kb_id":"kb-1"}`, want: true},
|
||||
{name: "move fields are task specific", taskType: types.TypeDocumentProcess, payload: `{"source_kb_id":"kb-1"}`},
|
||||
{name: "single knowledge", taskType: types.TypeChunkExtract, payload: `{"knowledge_id":"knowledge-1"}`, want: true},
|
||||
{
|
||||
name: "knowledge collection",
|
||||
taskType: types.TypeKnowledgeListReparse,
|
||||
payload: `{"knowledge_ids":["other","knowledge-2"]}`,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "unrelated",
|
||||
taskType: types.TypeDocumentProcess,
|
||||
payload: `{"knowledge_base_id":"other","knowledge_id":"other"}`,
|
||||
},
|
||||
{name: "malformed payload", taskType: types.TypeDocumentProcess, payload: `{`},
|
||||
{name: "preserve kb delete by kb", taskType: types.TypeKBDelete, payload: `{"knowledge_base_id":"kb-1"}`},
|
||||
{name: "preserve kb delete by knowledge", taskType: types.TypeKBDelete, payload: `{"knowledge_id":"knowledge-1"}`},
|
||||
{name: "preserve index delete by kb", taskType: types.TypeIndexDelete, payload: `{"knowledge_base_id":"kb-1"}`},
|
||||
{
|
||||
name: "preserve index delete by knowledge", taskType: types.TypeIndexDelete,
|
||||
payload: `{"knowledge_id":"knowledge-1"}`,
|
||||
},
|
||||
{
|
||||
name: "data source sync", taskType: types.TypeDataSourceSync,
|
||||
payload: `{"data_source_id":"datasource-1"}`, want: true,
|
||||
},
|
||||
{
|
||||
name: "other data source sync", taskType: types.TypeDataSourceSync,
|
||||
payload: `{"data_source_id":"datasource-other"}`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
got := matchesKnowledgeBase(
|
||||
test.taskType, []byte(test.payload), "kb-1", knowledgeIDs, dataSourceIDs,
|
||||
)
|
||||
if got != test.want {
|
||||
t.Fatalf("matchesKnowledgeBase() = %v, want %v", got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchesKnowledgePreservesPerKnowledgeAllowList(t *testing.T) {
|
||||
if !matchesKnowledge(types.TypeDocumentProcess, []byte(`{"knowledge_id":"knowledge-1"}`), "knowledge-1") {
|
||||
t.Fatal("document task should remain cancellable by knowledge ID")
|
||||
}
|
||||
if matchesKnowledge(types.TypeKBClone, []byte(`{"knowledge_id":"knowledge-1"}`), "knowledge-1") {
|
||||
t.Fatal("KB clone must not become cancellable through the per-knowledge API")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCancelTasksForKnowledgeBaseRescansMutatedPages(t *testing.T) {
|
||||
server := miniredis.RunT(t)
|
||||
redisClient := redis.NewClient(&redis.Options{Addr: server.Addr()})
|
||||
asynqClient := asynq.NewClientFromRedisClient(redisClient)
|
||||
t.Cleanup(func() {
|
||||
_ = asynqClient.Close()
|
||||
_ = redisClient.Close()
|
||||
})
|
||||
inspector := &asynqTaskInspector{
|
||||
inspector: asynq.NewInspectorFromRedisClient(redisClient),
|
||||
redis: redisClient,
|
||||
}
|
||||
|
||||
const matchingPending = 205
|
||||
for i := 0; i < matchingPending; i++ {
|
||||
enqueueTask(t, asynqClient, types.TypeDocumentProcess,
|
||||
fmt.Sprintf(`{"knowledge_base_id":"kb-delete","knowledge_id":"knowledge-%d"}`, i),
|
||||
fmt.Sprintf("matching-%03d", i),
|
||||
)
|
||||
}
|
||||
for i := 0; i < 5; i++ {
|
||||
enqueueTask(t, asynqClient, types.TypeDocumentProcess,
|
||||
`{"knowledge_base_id":"kb-keep","knowledge_id":"keep"}`,
|
||||
fmt.Sprintf("survivor-%03d", i),
|
||||
)
|
||||
}
|
||||
enqueueTask(t, asynqClient, types.TypeKBDelete,
|
||||
`{"knowledge_base_id":"kb-delete"}`, "kb-delete-cleanup",
|
||||
)
|
||||
enqueueTask(t, asynqClient, types.TypeIndexDelete,
|
||||
`{"knowledge_base_id":"kb-delete"}`, "index-delete-cleanup",
|
||||
)
|
||||
enqueueTask(t, asynqClient, types.TypeDataSourceSync,
|
||||
`{"data_source_id":"datasource-delete"}`, "datasource-sync",
|
||||
)
|
||||
|
||||
scheduleTask(t, asynqClient, types.TypeFAQImport,
|
||||
`{"kb_id":"kb-delete"}`, "scheduled-kb-match",
|
||||
)
|
||||
scheduleTask(t, asynqClient, types.TypeKnowledgeListReparse,
|
||||
`{"knowledge_ids":["knowledge-associated"]}`, "scheduled-knowledge-match",
|
||||
)
|
||||
scheduleTask(t, asynqClient, types.TypeFAQImport,
|
||||
`{"kb_id":"kb-keep"}`, "scheduled-survivor",
|
||||
)
|
||||
|
||||
deleted, cancelled, err := inspector.CancelTasksForKnowledgeBase(
|
||||
context.Background(), "kb-delete", []string{"knowledge-associated"}, []string{"datasource-delete"},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("cancel tasks: %v", err)
|
||||
}
|
||||
if want := matchingPending + 3; deleted != want {
|
||||
t.Fatalf("deleted = %d, want %d", deleted, want)
|
||||
}
|
||||
if cancelled != 0 {
|
||||
t.Fatalf("cancelled active = %d, want 0", cancelled)
|
||||
}
|
||||
|
||||
pending, err := inspector.inspector.ListPendingTasks(
|
||||
types.QueueDefault, asynq.PageSize(listPageSize), asynq.Page(1),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("list pending: %v", err)
|
||||
}
|
||||
if len(pending) != 7 {
|
||||
t.Fatalf("pending survivors = %d, want 7", len(pending))
|
||||
}
|
||||
if !hasTaskID(pending, "kb-delete-cleanup") {
|
||||
t.Fatal("kb:delete cleanup task was removed")
|
||||
}
|
||||
if !hasTaskID(pending, "index-delete-cleanup") {
|
||||
t.Fatal("index:delete cleanup task was removed")
|
||||
}
|
||||
|
||||
scheduled, err := inspector.inspector.ListScheduledTasks(
|
||||
types.QueueDefault, asynq.PageSize(listPageSize), asynq.Page(1),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("list scheduled: %v", err)
|
||||
}
|
||||
if len(scheduled) != 1 || scheduled[0].ID != "scheduled-survivor" {
|
||||
t.Fatalf("scheduled survivors = %v, want scheduled-survivor", taskIDs(scheduled))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCancelTasksForKnowledgeBaseRemovesCancelledActiveRetry(t *testing.T) {
|
||||
server := miniredis.RunT(t)
|
||||
redisClient := redis.NewClient(&redis.Options{Addr: server.Addr()})
|
||||
asynqClient := asynq.NewClientFromRedisClient(redisClient)
|
||||
t.Cleanup(func() {
|
||||
_ = asynqClient.Close()
|
||||
_ = redisClient.Close()
|
||||
})
|
||||
|
||||
inspector := &asynqTaskInspector{
|
||||
inspector: asynq.NewInspectorFromRedisClient(redisClient),
|
||||
redis: redisClient,
|
||||
}
|
||||
worker := asynq.NewServerFromRedisClient(redisClient, asynq.Config{
|
||||
Concurrency: 1,
|
||||
Queues: map[string]int{types.QueueDefault: 1},
|
||||
TaskCheckInterval: 10 * time.Millisecond,
|
||||
DelayedTaskCheckInterval: time.Hour,
|
||||
RetryDelayFunc: func(_ int, _ error, _ *asynq.Task) time.Duration {
|
||||
return time.Hour
|
||||
},
|
||||
ShutdownTimeout: time.Second,
|
||||
LogLevel: asynq.FatalLevel,
|
||||
})
|
||||
|
||||
handlerStarted := make(chan struct{})
|
||||
handlerReturned := make(chan struct{})
|
||||
mux := asynq.NewServeMux()
|
||||
mux.HandleFunc(types.TypeDocumentProcess, func(ctx context.Context, _ *asynq.Task) error {
|
||||
close(handlerStarted)
|
||||
<-ctx.Done()
|
||||
close(handlerReturned)
|
||||
return ctx.Err()
|
||||
})
|
||||
|
||||
if err := worker.Start(mux); err != nil {
|
||||
t.Fatalf("start asynq worker: %v", err)
|
||||
}
|
||||
t.Cleanup(worker.Shutdown)
|
||||
waitForAsynqCancellationSubscriber(t, redisClient)
|
||||
|
||||
const taskID = "active-kb-match"
|
||||
if _, err := asynqClient.Enqueue(
|
||||
asynq.NewTask(types.TypeDocumentProcess, []byte(`{"knowledge_base_id":"kb-delete"}`)),
|
||||
asynq.Queue(types.QueueDefault),
|
||||
asynq.TaskID(taskID),
|
||||
asynq.MaxRetry(3),
|
||||
); err != nil {
|
||||
t.Fatalf("enqueue active task: %v", err)
|
||||
}
|
||||
|
||||
select {
|
||||
case <-handlerStarted:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("handler did not start")
|
||||
}
|
||||
waitForTaskState(t, inspector.inspector, taskID, asynq.TaskStateActive)
|
||||
|
||||
deleted, cancelled, err := inspector.CancelTasksForKnowledgeBase(
|
||||
context.Background(), "kb-delete", nil, nil,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("cancel active task: %v", err)
|
||||
}
|
||||
if cancelled != 1 {
|
||||
t.Fatalf("cancelled active = %d, want 1", cancelled)
|
||||
}
|
||||
if deleted != 1 {
|
||||
t.Fatalf("deleted after active cancellation = %d, want 1", deleted)
|
||||
}
|
||||
|
||||
select {
|
||||
case <-handlerReturned:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("cancelled handler did not return")
|
||||
}
|
||||
waitForTaskToLeaveLiveStates(t, inspector.inspector, taskID)
|
||||
}
|
||||
|
||||
func enqueueTask(t *testing.T, client *asynq.Client, taskType, payload, taskID string) {
|
||||
t.Helper()
|
||||
if _, err := client.Enqueue(
|
||||
asynq.NewTask(taskType, []byte(payload)),
|
||||
asynq.Queue(types.QueueDefault),
|
||||
asynq.TaskID(taskID),
|
||||
); err != nil {
|
||||
t.Fatalf("enqueue %s: %v", taskID, err)
|
||||
}
|
||||
}
|
||||
|
||||
func scheduleTask(t *testing.T, client *asynq.Client, taskType, payload, taskID string) {
|
||||
t.Helper()
|
||||
if _, err := client.Enqueue(
|
||||
asynq.NewTask(taskType, []byte(payload)),
|
||||
asynq.Queue(types.QueueDefault),
|
||||
asynq.TaskID(taskID),
|
||||
asynq.ProcessAt(time.Now().Add(time.Hour)),
|
||||
); err != nil {
|
||||
t.Fatalf("schedule %s: %v", taskID, err)
|
||||
}
|
||||
}
|
||||
|
||||
func hasTaskID(tasks []*asynq.TaskInfo, taskID string) bool {
|
||||
for _, task := range tasks {
|
||||
if task.ID == taskID {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func taskIDs(tasks []*asynq.TaskInfo) []string {
|
||||
ids := make([]string, 0, len(tasks))
|
||||
for _, task := range tasks {
|
||||
ids = append(ids, task.ID)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func waitForTaskState(t *testing.T, inspector *asynq.Inspector, taskID string, want asynq.TaskState) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
var lastState asynq.TaskState
|
||||
for time.Now().Before(deadline) {
|
||||
task, err := inspector.GetTaskInfo(types.QueueDefault, taskID)
|
||||
if err == nil {
|
||||
lastState = task.State
|
||||
if task.State == want {
|
||||
return
|
||||
}
|
||||
} else if !errors.Is(err, asynq.ErrTaskNotFound) {
|
||||
t.Fatalf("get task %s: %v", taskID, err)
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("task %s state = %v, want %v", taskID, lastState, want)
|
||||
}
|
||||
|
||||
func waitForAsynqCancellationSubscriber(t *testing.T, redisClient *redis.Client) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
subscribers, err := redisClient.PubSubNumSub(context.Background(), "asynq:cancel").Result()
|
||||
if err != nil {
|
||||
t.Fatalf("query cancellation subscribers: %v", err)
|
||||
}
|
||||
if subscribers["asynq:cancel"] > 0 {
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("asynq cancellation subscriber did not start")
|
||||
}
|
||||
|
||||
func waitForTaskToLeaveLiveStates(t *testing.T, inspector *asynq.Inspector, taskID string) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
var lastState asynq.TaskState
|
||||
for time.Now().Before(deadline) {
|
||||
task, err := inspector.GetTaskInfo(types.QueueDefault, taskID)
|
||||
if errors.Is(err, asynq.ErrTaskNotFound) {
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("get task %s: %v", taskID, err)
|
||||
}
|
||||
lastState = task.State
|
||||
switch task.State {
|
||||
case asynq.TaskStatePending, asynq.TaskStateScheduled, asynq.TaskStateRetry, asynq.TaskStateActive:
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
default:
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatalf("task %s remained in live state %v", taskID, lastState)
|
||||
}
|
||||
@@ -64,6 +64,24 @@ type TaskInspector interface {
|
||||
WorkerServerStats(ctx context.Context) (stats []types.WorkerServerStat, supported bool, err error)
|
||||
}
|
||||
|
||||
// KnowledgeBaseTaskCanceller is the optional queue-cleanup capability used
|
||||
// when a knowledge base is deleted. It is separate from TaskInspector so
|
||||
// lightweight test doubles and queue backends that cannot inspect tasks do
|
||||
// not need to implement knowledge-base-wide scanning.
|
||||
type KnowledgeBaseTaskCanceller interface {
|
||||
// CancelTasksForKnowledgeBase removes pending/scheduled/retry tasks
|
||||
// associated with kbID and signals matching active workers to stop.
|
||||
// knowledgeIDs covers batch tasks whose payload references documents but
|
||||
// does not carry the parent knowledge-base ID. dataSourceIDs covers sync
|
||||
// tasks whose payload only identifies a data source.
|
||||
CancelTasksForKnowledgeBase(
|
||||
ctx context.Context,
|
||||
kbID string,
|
||||
knowledgeIDs []string,
|
||||
dataSourceIDs []string,
|
||||
) (deleted int, cancelled int, err error)
|
||||
}
|
||||
|
||||
// RuntimeTaskInspector is the optional operator surface implemented by queue
|
||||
// backends that retain inspectable task state. It is separate from
|
||||
// TaskInspector so Lite mode and light-weight tests do not need to implement
|
||||
|
||||
@@ -83,6 +83,23 @@ type TaskPendingOpsRepository interface {
|
||||
DeleteByDedupKey(ctx context.Context, taskType, scope, scopeID, dedupKey, op string) error
|
||||
}
|
||||
|
||||
// TaskPendingOpsScopeCleaner is an optional extension for callers that need
|
||||
// to discard every durable pending operation owned by a deleted scope. It is
|
||||
// intentionally separate from TaskPendingOpsRepository so alternate queue
|
||||
// implementations and existing test doubles are not forced to implement a
|
||||
// lifecycle-only operation.
|
||||
type TaskPendingOpsScopeCleaner interface {
|
||||
DeleteByScope(ctx context.Context, scope, scopeID string) error
|
||||
}
|
||||
|
||||
// TaskPendingOpsKnowledgeBaseGuard atomically persists a KB-scoped operation
|
||||
// only while its knowledge base is still active. Implementations must
|
||||
// serialize the active-KB check with soft deletion so a detached worker cannot
|
||||
// recreate durable work after the deletion scrub has finished.
|
||||
type TaskPendingOpsKnowledgeBaseGuard interface {
|
||||
EnqueueIfKnowledgeBaseActive(ctx context.Context, op *types.TaskPendingOp) (accepted bool, err error)
|
||||
}
|
||||
|
||||
// TaskDeadLetterRepository persists rows for the generic task dead-letter
|
||||
// archive (`task_dead_letters`). Two writers exist: the asynq
|
||||
// dead-letter middleware (one row per archived asynq task), and the
|
||||
|
||||
@@ -396,6 +396,7 @@ type KBDeletePayload struct {
|
||||
TracingContext
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||||
DataSourceIDs []string `json:"data_source_ids,omitempty"`
|
||||
EffectiveEngines []RetrieverEngineParams `json:"effective_engines"`
|
||||
// VectorStoreID is the bound store snapshot taken at enqueue time (before
|
||||
// soft-delete) so the async worker can resolve the right store. nil means
|
||||
|
||||
Reference in New Issue
Block a user