diff --git a/internal/application/repository/task_queue.go b/internal/application/repository/task_queue.go new file mode 100644 index 00000000..d16e11f8 --- /dev/null +++ b/internal/application/repository/task_queue.go @@ -0,0 +1,252 @@ +package repository + +import ( + "context" + "errors" + "fmt" + "strconv" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "gorm.io/gorm" +) + +// taskPendingOpsRepository implements interfaces.TaskPendingOpsRepository. +type taskPendingOpsRepository struct { + db *gorm.DB +} + +// NewTaskPendingOpsRepository constructs a GORM-backed implementation. +func NewTaskPendingOpsRepository(db *gorm.DB) interfaces.TaskPendingOpsRepository { + return &taskPendingOpsRepository{db: db} +} + +// Enqueue inserts a single op. Callers must populate TenantID/TaskType/ +// Scope/ScopeID/Op (Payload optional). ID, FailCount default to zero; +// EnqueuedAt is filled with the DB-side default if left zero. +func (r *taskPendingOpsRepository) Enqueue(ctx context.Context, op *types.TaskPendingOp) error { + if op == nil { + return errors.New("task pending ops: nil op") + } + if op.TaskType == "" || op.Scope == "" || op.ScopeID == "" { + return errors.New("task pending ops: task_type, scope, scope_id are required") + } + if op.Op == "" { + return errors.New("task pending ops: op is required") + } + if len(op.Payload) == 0 { + // Make sure the JSONB column never sees NULL — the schema sets a + // default but explicit "{}" keeps the row uniform regardless of + // driver-level default handling. + op.Payload = []byte("{}") + } + return r.db.WithContext(ctx).Create(op).Error +} + +// PeekBatch returns up to `limit` rows for the (task_type, scope, scope_id) +// tuple ordered by id ASC. Rows are not removed; callers must +// DeleteByIDs once they have been consumed (or IncrFailCount and leave +// them for the next pass). `limit` <= 0 falls back to 1; we clamp the +// upper bound generously so callers can pull large windows when they +// know the consumer can handle them. +func (r *taskPendingOpsRepository) PeekBatch( + ctx context.Context, + taskType, scope, scopeID string, + limit int, +) ([]*types.TaskPendingOp, error) { + if limit <= 0 { + limit = 1 + } + if limit > 1000 { + limit = 1000 + } + var ops []*types.TaskPendingOp + if err := r.db.WithContext(ctx). + Where("task_type = ? AND scope = ? AND scope_id = ?", taskType, scope, scopeID). + Order("id ASC"). + Limit(limit). + Find(&ops).Error; err != nil { + return nil, err + } + return ops, nil +} + +// DeleteByIDs removes the given rows in one statement. Empty input is a +// no-op so the caller can invoke unconditionally at the end of a batch. +func (r *taskPendingOpsRepository) DeleteByIDs(ctx context.Context, ids []int64) error { + if len(ids) == 0 { + return nil + } + return r.db.WithContext(ctx). + Where("id IN ?", ids). + Delete(&types.TaskPendingOp{}).Error +} + +// IncrFailCount atomically bumps fail_count for one row and returns the +// new value. We use UPDATE ... RETURNING so the read+write happens in +// one round trip and races between concurrent IncrFailCount callers +// resolve to monotonic counts. +// +// A missing row returns (0, nil): the caller's ID may have been removed +// by a concurrent DeleteByIDs (e.g. dead-letter path), which is benign. +func (r *taskPendingOpsRepository) IncrFailCount(ctx context.Context, id int64) (int, error) { + var newCount int + err := r.db.WithContext(ctx).Raw( + `UPDATE task_pending_ops SET fail_count = fail_count + 1 WHERE id = ? RETURNING fail_count`, + id, + ).Scan(&newCount).Error + if err != nil { + return 0, err + } + return newCount, nil +} + +// PendingCount returns how many rows are currently queued for the +// tuple. Covered by idx_task_pending_ops_scope. +func (r *taskPendingOpsRepository) PendingCount( + ctx context.Context, + taskType, scope, scopeID string, +) (int64, error) { + var n int64 + if err := r.db.WithContext(ctx). + Model(&types.TaskPendingOp{}). + Where("task_type = ? AND scope = ? AND scope_id = ?", taskType, scope, scopeID). + Count(&n).Error; err != nil { + return 0, err + } + return n, nil +} + +// DeleteByDedupKey drops rows in the tuple whose dedup_key matches. +// If `op` is non-empty, only rows with the matching op are dropped; +// otherwise every matching row is removed. Empty dedup_key is rejected +// to prevent accidentally wiping the entire queue for a KB. +// +// Used by: +// - Wiki delete path: scrub queued WikiOpIngest entries for a +// knowledge that is being deleted, while preserving WikiOpRetract +// so the cleanup can still unlink pages. +// - Wiki reparse path: same scrub of pending ingests so the new +// parse can repopulate cleanly. +func (r *taskPendingOpsRepository) DeleteByDedupKey( + ctx context.Context, + taskType, scope, scopeID, dedupKey, op string, +) error { + if dedupKey == "" { + return fmt.Errorf("task pending ops: empty dedup_key in DeleteByDedupKey") + } + q := r.db.WithContext(ctx). + Where("task_type = ? AND scope = ? AND scope_id = ? AND dedup_key = ?", + taskType, scope, scopeID, dedupKey) + if op != "" { + q = q.Where("op = ?", op) + } + return q.Delete(&types.TaskPendingOp{}).Error +} + +// taskDeadLetterRepository implements interfaces.TaskDeadLetterRepository. +type taskDeadLetterRepository struct { + db *gorm.DB +} + +// NewTaskDeadLetterRepository constructs a GORM-backed implementation. +func NewTaskDeadLetterRepository(db *gorm.DB) interfaces.TaskDeadLetterRepository { + return &taskDeadLetterRepository{db: db} +} + +// Insert records one dead letter. Best-effort caller: the asynq +// middleware swallows the error so a failed insert never masks the +// underlying task error. +func (r *taskDeadLetterRepository) Insert(ctx context.Context, dl *types.TaskDeadLetter) error { + if dl == nil { + return errors.New("task dead letters: nil entry") + } + if dl.TaskType == "" { + return errors.New("task dead letters: task_type is required") + } + if dl.Scope == "" { + dl.Scope = types.TaskScopeUnknown + } + if len(dl.Payload) == 0 { + dl.Payload = []byte("{}") + } + return r.db.WithContext(ctx).Create(dl).Error +} + +// ListByScope returns dead letters for (scope, scope_id) newest-first +// with a stringified id cursor. `limit` is clamped to [1, 200]. Empty +// nextCursor signals the tail. +func (r *taskDeadLetterRepository) ListByScope( + ctx context.Context, + scope, scopeID, cursor string, + limit int, +) ([]*types.TaskDeadLetter, string, error) { + if scope == "" || scopeID == "" { + return nil, "", errors.New("task dead letters: scope and scope_id are required") + } + return r.list(ctx, cursor, limit, func(q *gorm.DB) *gorm.DB { + return q.Where("scope = ? AND scope_id = ?", scope, scopeID) + }) +} + +// ListByTaskType returns dead letters for the given task_type +// newest-first with a stringified id cursor. Same clamping rules. +func (r *taskDeadLetterRepository) ListByTaskType( + ctx context.Context, + taskType, cursor string, + limit int, +) ([]*types.TaskDeadLetter, string, error) { + if taskType == "" { + return nil, "", errors.New("task dead letters: task_type is required") + } + return r.list(ctx, cursor, limit, func(q *gorm.DB) *gorm.DB { + return q.Where("task_type = ?", taskType) + }) +} + +// list is the shared cursor pagination implementation, parametrized by +// the caller-supplied filter. Mirrors wikiLogEntryRepository.List. +func (r *taskDeadLetterRepository) list( + ctx context.Context, + cursor string, + limit int, + filter func(*gorm.DB) *gorm.DB, +) ([]*types.TaskDeadLetter, string, error) { + if limit <= 0 { + limit = 50 + } + if limit > 200 { + limit = 200 + } + + q := r.db.WithContext(ctx).Order("id DESC").Limit(limit) + q = filter(q) + + if cursor != "" { + cursorID, err := strconv.ParseInt(cursor, 10, 64) + if err != nil { + return nil, "", fmt.Errorf("invalid cursor %q: %w", cursor, err) + } + q = q.Where("id < ?", cursorID) + } + + var rows []*types.TaskDeadLetter + if err := q.Find(&rows).Error; err != nil { + return nil, "", err + } + + nextCursor := "" + if len(rows) == limit { + nextCursor = strconv.FormatInt(rows[len(rows)-1].ID, 10) + } + return rows, nextCursor, nil +} + +// DeleteByID drops a single dead letter row. Returns nil even if the +// row is already gone — operators issuing concurrent deletes shouldn't +// see spurious errors. +func (r *taskDeadLetterRepository) DeleteByID(ctx context.Context, id int64) error { + return r.db.WithContext(ctx). + Where("id = ?", id). + Delete(&types.TaskDeadLetter{}).Error +} diff --git a/internal/application/repository/task_queue_test.go b/internal/application/repository/task_queue_test.go new file mode 100644 index 00000000..dc3f1406 --- /dev/null +++ b/internal/application/repository/task_queue_test.go @@ -0,0 +1,422 @@ +package repository + +import ( + "context" + "encoding/json" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +// taskPendingOpsTestDDL mirrors the production schema in +// migrations/versioned/000041_task_queue_and_wiki_indexes.up.sql but uses +// SQLite-compatible types. INTEGER PRIMARY KEY AUTOINCREMENT preserves +// the monotonically-increasing ID semantics PeekBatch/cursor pagination +// rely on. JSONB → TEXT is fine since GORM round-trips json.RawMessage +// as bytes either way. +const taskPendingOpsTestDDL = ` +CREATE TABLE IF NOT EXISTS task_pending_ops ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + tenant_id INTEGER NOT NULL, + task_type VARCHAR(64) NOT NULL, + scope VARCHAR(32) NOT NULL, + scope_id VARCHAR(64) NOT NULL, + op VARCHAR(32) NOT NULL, + dedup_key VARCHAR(128) NOT NULL DEFAULT '', + payload TEXT NOT NULL DEFAULT '{}', + fail_count INTEGER NOT NULL DEFAULT 0, + enqueued_at DATETIME DEFAULT CURRENT_TIMESTAMP, + claimed_at DATETIME +); +` + +const taskDeadLettersTestDDL = ` +CREATE TABLE IF NOT EXISTS task_dead_letters ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + tenant_id INTEGER NOT NULL, + task_type VARCHAR(64) NOT NULL, + scope VARCHAR(32) NOT NULL, + scope_id VARCHAR(64) NOT NULL, + related_id VARCHAR(64) NOT NULL DEFAULT '', + payload TEXT NOT NULL, + last_error TEXT NOT NULL DEFAULT '', + fail_count INTEGER NOT NULL, + failed_at DATETIME DEFAULT CURRENT_TIMESTAMP +); +` + +func setupTaskQueueTestDB(t *testing.T) *gorm.DB { + t.Helper() + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) + require.NoError(t, err) + require.NoError(t, db.Exec(taskPendingOpsTestDDL).Error) + require.NoError(t, db.Exec(taskDeadLettersTestDDL).Error) + return db +} + +func makePendingOp(taskType, scope, scopeID, op, dedup string, payload []byte) *types.TaskPendingOp { + return &types.TaskPendingOp{ + TenantID: 1, + TaskType: taskType, + Scope: scope, + ScopeID: scopeID, + Op: op, + DedupKey: dedup, + Payload: payload, + } +} + +// ---------------- TaskPendingOpsRepository ---------------- + +// TestTaskPendingOps_Enqueue_AssignsIDAndDefaults verifies a freshly +// inserted op gets a positive ID and the empty payload becomes "{}" +// rather than NULL/empty. +func TestTaskPendingOps_Enqueue_AssignsIDAndDefaults(t *testing.T) { + db := setupTaskQueueTestDB(t) + repo := NewTaskPendingOpsRepository(db) + ctx := context.Background() + + op := makePendingOp("wiki:ingest", "knowledge_base", "kb-1", "ingest", "k-1", nil) + require.NoError(t, repo.Enqueue(ctx, op)) + assert.NotZero(t, op.ID) + assert.Equal(t, json.RawMessage("{}"), op.Payload, "nil payload should default to {}") +} + +// TestTaskPendingOps_Enqueue_RejectsMissingFields covers the validation +// layer: every required field must be set, otherwise the call returns an +// error WITHOUT touching the DB. +func TestTaskPendingOps_Enqueue_RejectsMissingFields(t *testing.T) { + db := setupTaskQueueTestDB(t) + repo := NewTaskPendingOpsRepository(db) + ctx := context.Background() + + cases := []struct { + name string + op *types.TaskPendingOp + }{ + {"nil op", nil}, + {"missing task_type", makePendingOp("", "knowledge_base", "kb", "ingest", "", nil)}, + {"missing scope", makePendingOp("t", "", "kb", "ingest", "", nil)}, + {"missing scope_id", makePendingOp("t", "s", "", "ingest", "", nil)}, + {"missing op", makePendingOp("t", "s", "id", "", "", nil)}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + err := repo.Enqueue(ctx, c.op) + assert.Error(t, err) + }) + } + + var n int64 + db.Table("task_pending_ops").Count(&n) + assert.Equal(t, int64(0), n) +} + +// TestTaskPendingOps_PeekBatch_ScopedAndOrdered verifies PeekBatch only +// returns rows for the matching tuple, in id ASC order, and respects +// the limit. +func TestTaskPendingOps_PeekBatch_ScopedAndOrdered(t *testing.T) { + db := setupTaskQueueTestDB(t) + repo := NewTaskPendingOpsRepository(db) + ctx := context.Background() + + // Three ops in kb-A, two in kb-B, one in different task_type. + require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb-A", "ingest", "k1", nil))) + require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb-A", "retract", "k2", nil))) + require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb-A", "ingest", "k3", nil))) + require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb-B", "ingest", "k4", nil))) + require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb-B", "ingest", "k5", nil))) + require.NoError(t, repo.Enqueue(ctx, makePendingOp("summary:gen", "knowledge_base", "kb-A", "ingest", "k6", nil))) + + // Peek up to 10 from kb-A — should see exactly 3, in insertion order. + got, err := repo.PeekBatch(ctx, "wiki:ingest", "knowledge_base", "kb-A", 10) + require.NoError(t, err) + require.Len(t, got, 3) + assert.Equal(t, "k1", got[0].DedupKey) + assert.Equal(t, "k2", got[1].DedupKey) + assert.Equal(t, "k3", got[2].DedupKey) + assert.True(t, got[0].ID < got[1].ID && got[1].ID < got[2].ID, "ids should be ascending") + + // Limit caps result size. + got, err = repo.PeekBatch(ctx, "wiki:ingest", "knowledge_base", "kb-A", 2) + require.NoError(t, err) + assert.Len(t, got, 2) + + // Different task_type isolated. + got, err = repo.PeekBatch(ctx, "summary:gen", "knowledge_base", "kb-A", 10) + require.NoError(t, err) + require.Len(t, got, 1) + assert.Equal(t, "k6", got[0].DedupKey) +} + +// TestTaskPendingOps_DeleteByIDs_RemovesOnlyTargets verifies the +// delete-after-consume path. Empty input must be a no-op so the consumer +// can call it unconditionally. +func TestTaskPendingOps_DeleteByIDs_RemovesOnlyTargets(t *testing.T) { + db := setupTaskQueueTestDB(t) + repo := NewTaskPendingOpsRepository(db) + ctx := context.Background() + + a := makePendingOp("wiki:ingest", "knowledge_base", "kb", "ingest", "a", nil) + b := makePendingOp("wiki:ingest", "knowledge_base", "kb", "ingest", "b", nil) + c := makePendingOp("wiki:ingest", "knowledge_base", "kb", "ingest", "c", nil) + require.NoError(t, repo.Enqueue(ctx, a)) + require.NoError(t, repo.Enqueue(ctx, b)) + require.NoError(t, repo.Enqueue(ctx, c)) + + // No-op: empty slice. + require.NoError(t, repo.DeleteByIDs(ctx, nil)) + require.NoError(t, repo.DeleteByIDs(ctx, []int64{})) + + // Delete a + c, keep b. + require.NoError(t, repo.DeleteByIDs(ctx, []int64{a.ID, c.ID})) + + got, err := repo.PeekBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 10) + require.NoError(t, err) + require.Len(t, got, 1) + assert.Equal(t, "b", got[0].DedupKey) +} + +// TestTaskPendingOps_IncrFailCount_ReturnsNewValueAndPersists exercises +// the UPDATE...RETURNING flow. Successive bumps should observe monotonic +// counts. +func TestTaskPendingOps_IncrFailCount_ReturnsNewValueAndPersists(t *testing.T) { + db := setupTaskQueueTestDB(t) + repo := NewTaskPendingOpsRepository(db) + ctx := context.Background() + + op := makePendingOp("wiki:ingest", "knowledge_base", "kb", "ingest", "k", nil) + require.NoError(t, repo.Enqueue(ctx, op)) + + n, err := repo.IncrFailCount(ctx, op.ID) + require.NoError(t, err) + assert.Equal(t, 1, n) + + n, err = repo.IncrFailCount(ctx, op.ID) + require.NoError(t, err) + assert.Equal(t, 2, n) + + // Persisted value matches what was returned. + rows, err := repo.PeekBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 10) + require.NoError(t, err) + require.Len(t, rows, 1) + assert.Equal(t, 2, rows[0].FailCount) +} + +// TestTaskPendingOps_PendingCount_ScopedTuple confirms the count covers +// only the (task_type, scope, scope_id) tuple. +func TestTaskPendingOps_PendingCount_ScopedTuple(t *testing.T) { + db := setupTaskQueueTestDB(t) + repo := NewTaskPendingOpsRepository(db) + ctx := context.Background() + + require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb-A", "ingest", "k1", nil))) + require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb-A", "ingest", "k2", nil))) + require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb-B", "ingest", "k3", nil))) + + n, err := repo.PendingCount(ctx, "wiki:ingest", "knowledge_base", "kb-A") + require.NoError(t, err) + assert.Equal(t, int64(2), n) + + n, err = repo.PendingCount(ctx, "wiki:ingest", "knowledge_base", "missing") + require.NoError(t, err) + assert.Equal(t, int64(0), n) +} + +// TestTaskPendingOps_DeleteByDedupKey_Filters tests the wiki delete-race +// helper: matching rows go away, others survive, optional op filter +// narrows the scope, and an empty dedup_key is rejected (so a buggy +// caller can't wipe the entire queue). +func TestTaskPendingOps_DeleteByDedupKey_Filters(t *testing.T) { + db := setupTaskQueueTestDB(t) + repo := NewTaskPendingOpsRepository(db) + ctx := context.Background() + + // Two ingests + one retract, all keyed on knowledge "k1"; one ingest + // for unrelated "k2". + require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "ingest", "k1", nil))) + require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "ingest", "k1", nil))) + require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "retract", "k1", nil))) + require.NoError(t, repo.Enqueue(ctx, makePendingOp("wiki:ingest", "knowledge_base", "kb", "ingest", "k2", nil))) + + // Empty key is an error, queue unchanged. + err := repo.DeleteByDedupKey(ctx, "wiki:ingest", "knowledge_base", "kb", "", "") + assert.Error(t, err) + n, _ := repo.PendingCount(ctx, "wiki:ingest", "knowledge_base", "kb") + assert.Equal(t, int64(4), n) + + // Drop only "ingest" rows for k1; retract survives. + require.NoError(t, repo.DeleteByDedupKey(ctx, "wiki:ingest", "knowledge_base", "kb", "k1", "ingest")) + rows, err := repo.PeekBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 10) + require.NoError(t, err) + require.Len(t, rows, 2) + // The two survivors must be the retract for k1 and the ingest for k2. + keys := map[string]string{} + for _, r := range rows { + keys[r.Op] = r.DedupKey + } + assert.Equal(t, "k1", keys["retract"]) + assert.Equal(t, "k2", keys["ingest"]) + + // Drop everything keyed on k1 regardless of op (empty op = wildcard). + require.NoError(t, repo.DeleteByDedupKey(ctx, "wiki:ingest", "knowledge_base", "kb", "k1", "")) + rows, err = repo.PeekBatch(ctx, "wiki:ingest", "knowledge_base", "kb", 10) + require.NoError(t, err) + require.Len(t, rows, 1) + assert.Equal(t, "k2", rows[0].DedupKey) +} + +// ---------------- TaskDeadLetterRepository ---------------- + +func makeDeadLetter(taskType, scope, scopeID, relatedID, lastErr string) *types.TaskDeadLetter { + return &types.TaskDeadLetter{ + TenantID: 1, + TaskType: taskType, + Scope: scope, + ScopeID: scopeID, + RelatedID: relatedID, + Payload: json.RawMessage(`{"x":1}`), + LastError: lastErr, + FailCount: 5, + } +} + +// TestTaskDeadLetter_Insert_DefaultsAndAssignsID covers the empty-payload +// fallback and ID assignment. +func TestTaskDeadLetter_Insert_DefaultsAndAssignsID(t *testing.T) { + db := setupTaskQueueTestDB(t) + repo := NewTaskDeadLetterRepository(db) + ctx := context.Background() + + dl := &types.TaskDeadLetter{ + TenantID: 1, + TaskType: "wiki:ingest", + ScopeID: "kb", + FailCount: 3, + // Scope intentionally empty — should default to "unknown". + // Payload intentionally nil — should default to "{}". + } + require.NoError(t, repo.Insert(ctx, dl)) + assert.NotZero(t, dl.ID) + assert.Equal(t, types.TaskScopeUnknown, dl.Scope) + assert.Equal(t, json.RawMessage("{}"), dl.Payload) +} + +// TestTaskDeadLetter_Insert_RejectsMissingFields verifies the guard +// against rows that would leave the table without the columns ops queries +// rely on. +func TestTaskDeadLetter_Insert_RejectsMissingFields(t *testing.T) { + db := setupTaskQueueTestDB(t) + repo := NewTaskDeadLetterRepository(db) + ctx := context.Background() + + assert.Error(t, repo.Insert(ctx, nil)) + assert.Error(t, repo.Insert(ctx, &types.TaskDeadLetter{ScopeID: "kb"})) + + var n int64 + db.Table("task_dead_letters").Count(&n) + assert.Equal(t, int64(0), n) +} + +// TestTaskDeadLetter_ListByScope_NewestFirstAndCursored exercises the +// cursor pagination path used by the ops console. +func TestTaskDeadLetter_ListByScope_NewestFirstAndCursored(t *testing.T) { + db := setupTaskQueueTestDB(t) + repo := NewTaskDeadLetterRepository(db) + ctx := context.Background() + + // Insert 5 rows for kb-A and 2 for kb-B. + for i := 0; i < 5; i++ { + require.NoError(t, repo.Insert(ctx, makeDeadLetter("wiki:ingest", "knowledge_base", "kb-A", "k", "boom"))) + } + require.NoError(t, repo.Insert(ctx, makeDeadLetter("wiki:ingest", "knowledge_base", "kb-B", "k", "boom"))) + require.NoError(t, repo.Insert(ctx, makeDeadLetter("wiki:ingest", "knowledge_base", "kb-B", "k", "boom"))) + + // First page of 2 from kb-A, newest first. + page1, cursor, err := repo.ListByScope(ctx, "knowledge_base", "kb-A", "", 2) + require.NoError(t, err) + require.Len(t, page1, 2) + assert.True(t, page1[0].ID > page1[1].ID, "newest first") + require.NotEmpty(t, cursor) + + // Second page of 2. + page2, cursor, err := repo.ListByScope(ctx, "knowledge_base", "kb-A", cursor, 2) + require.NoError(t, err) + require.Len(t, page2, 2) + assert.True(t, page1[1].ID > page2[0].ID, "page2 should continue past page1") + require.NotEmpty(t, cursor) + + // Last page — only 1 row left, cursor goes empty since len < limit. + page3, cursor, err := repo.ListByScope(ctx, "knowledge_base", "kb-A", cursor, 2) + require.NoError(t, err) + require.Len(t, page3, 1) + assert.Empty(t, cursor) + + // kb-B is isolated. + pageB, _, err := repo.ListByScope(ctx, "knowledge_base", "kb-B", "", 10) + require.NoError(t, err) + require.Len(t, pageB, 2) +} + +// TestTaskDeadLetter_ListByScope_RejectsMissingScope guards the input +// validation in the public method. +func TestTaskDeadLetter_ListByScope_RejectsMissingScope(t *testing.T) { + db := setupTaskQueueTestDB(t) + repo := NewTaskDeadLetterRepository(db) + ctx := context.Background() + + _, _, err := repo.ListByScope(ctx, "", "kb", "", 10) + assert.Error(t, err) + _, _, err = repo.ListByScope(ctx, "knowledge_base", "", "", 10) + assert.Error(t, err) +} + +// TestTaskDeadLetter_ListByTaskType_FiltersAndPaginates is the cross-KB +// view: "all summary:generation failures" regardless of which KB they +// belong to. +func TestTaskDeadLetter_ListByTaskType_FiltersAndPaginates(t *testing.T) { + db := setupTaskQueueTestDB(t) + repo := NewTaskDeadLetterRepository(db) + ctx := context.Background() + + require.NoError(t, repo.Insert(ctx, makeDeadLetter("wiki:ingest", "knowledge_base", "kb-A", "k1", ""))) + require.NoError(t, repo.Insert(ctx, makeDeadLetter("summary:gen", "knowledge_base", "kb-A", "k2", ""))) + require.NoError(t, repo.Insert(ctx, makeDeadLetter("summary:gen", "knowledge_base", "kb-B", "k3", ""))) + require.NoError(t, repo.Insert(ctx, makeDeadLetter("wiki:ingest", "knowledge_base", "kb-B", "k4", ""))) + + rows, _, err := repo.ListByTaskType(ctx, "summary:gen", "", 10) + require.NoError(t, err) + require.Len(t, rows, 2) + for _, r := range rows { + assert.Equal(t, "summary:gen", r.TaskType) + } + + _, _, err = repo.ListByTaskType(ctx, "", "", 10) + assert.Error(t, err) +} + +// TestTaskDeadLetter_DeleteByID_IsIdempotent confirms a missing row does +// not produce an error — operators triggering concurrent deletes should +// see clean success. +func TestTaskDeadLetter_DeleteByID_IsIdempotent(t *testing.T) { + db := setupTaskQueueTestDB(t) + repo := NewTaskDeadLetterRepository(db) + ctx := context.Background() + + dl := makeDeadLetter("wiki:ingest", "knowledge_base", "kb", "k", "") + require.NoError(t, repo.Insert(ctx, dl)) + + require.NoError(t, repo.DeleteByID(ctx, dl.ID)) + // Second delete on the same id should silently succeed. + require.NoError(t, repo.DeleteByID(ctx, dl.ID)) + // Delete of unknown id should silently succeed. + require.NoError(t, repo.DeleteByID(ctx, 99999)) + + rows, _, err := repo.ListByScope(ctx, "knowledge_base", "kb", "", 10) + require.NoError(t, err) + assert.Len(t, rows, 0) +} diff --git a/internal/application/repository/wiki_page.go b/internal/application/repository/wiki_page.go index e7e486c7..8b3e96d0 100644 --- a/internal/application/repository/wiki_page.go +++ b/internal/application/repository/wiki_page.go @@ -298,6 +298,354 @@ func (r *wikiPageRepository) ListBySourceRef(ctx context.Context, kbID string, s return pages, nil } +// ListSlugsBySourceRef returns just the slugs of pages that reference the +// given knowledge id. Same predicate as ListBySourceRef (both forms +// "knowledgeID" and "knowledgeID|title"), but projected down to a single +// column so the wiki ingest pipeline doesn't have to load full rows when +// it only needs a "before" set of slugs. +// +// Backed by idx_wiki_pages_source_refs (GIN jsonb_path_ops) for the +// containment branch and idx_wiki_pages_source_refs_text for the legacy +// text-LIKE branch — both added in migration 000041. +func (r *wikiPageRepository) ListSlugsBySourceRef(ctx context.Context, kbID string, sourceKnowledgeID string) ([]string, error) { + needle, err := json.Marshal([]string{sourceKnowledgeID}) + if err != nil { + return nil, fmt.Errorf("marshal source ref needle: %w", err) + } + prefix, err := json.Marshal(sourceKnowledgeID + "|") + if err != nil { + return nil, fmt.Errorf("marshal source ref prefix: %w", err) + } + prefixStr := string(prefix) + if len(prefixStr) >= 2 && prefixStr[len(prefixStr)-1] == '"' { + prefixStr = prefixStr[:len(prefixStr)-1] + } + likePattern := "%" + escapeLikePattern(prefixStr) + "%" + + var slugs []string + if err := r.db.WithContext(ctx). + Model(&types.WikiPage{}). + Where("knowledge_base_id = ? AND (source_refs @> ?::jsonb OR source_refs::text LIKE ?)", + kbID, + string(needle), + likePattern, + ). + Pluck("slug", &slugs).Error; err != nil { + return nil, err + } + return slugs, nil +} + +// ListBySlugs returns lightweight projections (slug, title, page_type, +// status, aliases, out_links) for the given slugs in one IN query. +// Used by wiki ingest's lazy fetcher path to resolve slug -> title / +// out-links during Map/Reduce without paying for a full ListAll scan. +// +// Empty input returns nil, nil. Slugs not present in the KB are silently +// dropped from the returned map (caller treats absent slugs as "no +// such page" — the same shape ListAll had via missing keys). +func (r *wikiPageRepository) ListBySlugs( + ctx context.Context, + kbID string, + slugs []string, +) (map[string]*types.WikiPageLite, error) { + if len(slugs) == 0 { + return nil, nil + } + var rows []types.WikiPageLite + if err := r.db.WithContext(ctx). + Model(&types.WikiPage{}). + Select("slug", "title", "page_type", "status", "aliases", "out_links"). + Where("knowledge_base_id = ? AND slug IN ?", kbID, slugs). + Scan(&rows).Error; err != nil { + return nil, err + } + out := make(map[string]*types.WikiPageLite, len(rows)) + for i := range rows { + r := rows[i] + out[r.Slug] = &r + } + return out, nil +} + +// ListSummariesByKnowledgeIDs returns summary-page content keyed by the +// knowledge id that authored it. The page_type filter is applied first +// (only summary pages have content suitable for retract framing); within +// that subset we look at source_refs for either the bare knowledge id +// or the "knowledgeID|title" legacy form. +// +// Empty kids returns nil, nil. A knowledge id with no surviving summary +// page is silently absent from the result map. +// +// Used by reduceSlugUpdates' retract branch so it can frame "what did +// the now-departed sibling document contribute?" for the WikiPageModify +// LLM call without needing to keep the whole batchCtx.SummaryContent +// map in memory ahead of time. +func (r *wikiPageRepository) ListSummariesByKnowledgeIDs( + ctx context.Context, + kbID string, + kids []string, +) (map[string]string, error) { + if len(kids) == 0 { + return nil, nil + } + + // Build a JSONB containment-OR with one needle per knowledge id, + // plus a single text-LIKE OR over the legacy prefix forms. The + // containment branches each get their own GIN index probe; the + // LIKE branch falls back to the text fulltext GIN. + type row struct { + Content string `gorm:"column:content"` + SourceRefs types.StringArray `gorm:"column:source_refs"` + } + + q := r.db.WithContext(ctx). + Model(&types.WikiPage{}). + Select("content", "source_refs"). + Where("knowledge_base_id = ? AND page_type = ? AND status <> ?", + kbID, types.WikiPageTypeSummary, types.WikiPageStatusArchived) + + // Build OR clauses without using overly-clever GORM tricks: assemble + // raw SQL fragments + args. Keeping this defensive because source_refs + // patterns include user-controlled knowledge ids. + clauses := make([]string, 0, len(kids)*2) + args := make([]interface{}, 0, len(kids)*2) + for _, kid := range kids { + if kid == "" { + continue + } + needle, err := json.Marshal([]string{kid}) + if err != nil { + return nil, fmt.Errorf("marshal kid needle: %w", err) + } + clauses = append(clauses, "source_refs @> ?::jsonb") + args = append(args, string(needle)) + + prefix, err := json.Marshal(kid + "|") + if err != nil { + return nil, fmt.Errorf("marshal kid prefix: %w", err) + } + prefixStr := string(prefix) + if len(prefixStr) >= 2 && prefixStr[len(prefixStr)-1] == '"' { + prefixStr = prefixStr[:len(prefixStr)-1] + } + clauses = append(clauses, "source_refs::text LIKE ?") + args = append(args, "%"+escapeLikePattern(prefixStr)+"%") + } + if len(clauses) == 0 { + return nil, nil + } + q = q.Where("("+strings.Join(clauses, " OR ")+")", args...) + + var rows []row + if err := q.Scan(&rows).Error; err != nil { + return nil, err + } + + // Map each row's content to every kid in its source_refs (a single + // summary may carry multiple sources after a previous merge / re- + // ingest). Caller looks up by kid, so duplicates resolve to the + // same content string. + kidSet := make(map[string]struct{}, len(kids)) + for _, kid := range kids { + if kid != "" { + kidSet[kid] = struct{}{} + } + } + out := make(map[string]string, len(rows)) + for _, r := range rows { + for _, ref := range r.SourceRefs { + refKID := ref + if pipeIdx := strings.Index(ref, "|"); pipeIdx > 0 { + refKID = ref[:pipeIdx] + } + if _, want := kidSet[refKID]; !want { + continue + } + if _, exists := out[refKID]; !exists { + out[refKID] = r.Content + } + } + } + return out, nil +} + +// ExistsSlugs reports which of the given slugs are live (non-archived, +// non-deleted) in the KB. Used by cleanDeadLinks to validate out-link +// targets without loading the referenced pages' content. Slugs not +// present in the KB at all map to false; archived slugs also map to +// false so dead-link cleanup treats them as gone. +// +// Empty input returns nil, nil so callers can branch cheaply. +func (r *wikiPageRepository) ExistsSlugs( + ctx context.Context, + kbID string, + slugs []string, +) (map[string]bool, error) { + if len(slugs) == 0 { + return nil, nil + } + var live []string + if err := r.db.WithContext(ctx). + Model(&types.WikiPage{}). + Where("knowledge_base_id = ? AND slug IN ? AND status <> ?", + kbID, slugs, types.WikiPageStatusArchived). + Pluck("slug", &live).Error; err != nil { + return nil, err + } + out := make(map[string]bool, len(slugs)) + for _, s := range slugs { + out[s] = false + } + for _, s := range live { + out[s] = true + } + return out, nil +} + +// ListAllSlugs returns every non-archived page slug in the KB. Used by +// the lint pipeline to compute the "live slug set" for broken-link +// detection without paying for ListAll's full row materialization. +func (r *wikiPageRepository) ListAllSlugs( + ctx context.Context, + kbID string, +) ([]string, error) { + var slugs []string + if err := r.db.WithContext(ctx). + Model(&types.WikiPage{}). + Where("knowledge_base_id = ? AND status <> ?", kbID, types.WikiPageStatusArchived). + Pluck("slug", &slugs).Error; err != nil { + return nil, err + } + return slugs, nil +} + +// ListPagesCursor returns up to `limit` pages for the KB ordered by +// (knowledge_base_id, id) ascending, paginated by an opaque numeric +// cursor. The cursor is the stringified id of the last row from the +// previous page; "" starts from the beginning. Empty nextCursor = +// end-of-stream. +// +// Used by lint to walk the entire KB without ever holding the full +// page set in memory. `limit` is clamped to [1, 500]. +func (r *wikiPageRepository) ListPagesCursor( + ctx context.Context, + kbID string, + cursor string, + limit int, +) ([]*types.WikiPage, string, error) { + if limit <= 0 { + limit = 100 + } + if limit > 500 { + limit = 500 + } + q := r.db.WithContext(ctx). + Where("knowledge_base_id = ?", kbID). + Order("id ASC"). + Limit(limit) + if cursor != "" { + q = q.Where("id > ?", cursor) + } + var pages []*types.WikiPage + if err := q.Find(&pages).Error; err != nil { + return nil, "", err + } + nextCursor := "" + if len(pages) == limit { + nextCursor = pages[len(pages)-1].ID + } + return pages, nextCursor, nil +} + +// ListByTypeRecent returns up to `limit` summary-typed pages ordered +// by updated_at DESC, projected to slug/title/summary. Used by the +// rebuildIndexPage first-time generation path — historically that +// loaded EVERY summary page and concatenated them into the prompt, +// which broke the LLM context window once a KB grew past a few +// thousand documents. The recent-N projection caps the prompt size +// at the cost of intro framing for very old documents (which are +// unlikely to be the most-relevant index introductions anyway). +// +// `limit` is clamped to [1, 1000]; 0 falls back to 200. +func (r *wikiPageRepository) ListByTypeRecent( + ctx context.Context, + kbID string, + pageType string, + limit int, +) ([]types.WikiIndexEntry, error) { + if limit <= 0 { + limit = 200 + } + if limit > 1000 { + limit = 1000 + } + var entries []types.WikiIndexEntry + if err := r.db.WithContext(ctx). + Model(&types.WikiPage{}). + Select("slug", "title", "summary"). + Where("knowledge_base_id = ? AND page_type = ? AND status <> ?", + kbID, pageType, types.WikiPageStatusArchived). + Order("updated_at DESC"). + Limit(limit). + Scan(&entries).Error; err != nil { + return nil, err + } + return entries, nil +} + +// FindSimilarPages returns the top-k entity/concept pages whose lowercase +// title is most similar to the given query under PostgreSQL pg_trgm +// trigram similarity. Backed by idx_wiki_pages_title_trgm (GIN +// gin_trgm_ops, migration 000041). Used by the dedup pre-filter to +// surface candidate merge targets without loading every entity/concept +// page into Go. +// +// types is an optional page_type allow-list; empty means entity+concept. +// limit is clamped to [1, 50]. Pages whose title similarity is below +// 0.1 are dropped server-side via the `%` operator (which respects +// pg_trgm.similarity_threshold). +func (r *wikiPageRepository) FindSimilarPages( + ctx context.Context, + kbID string, + query string, + pageTypes []string, + limit int, +) ([]*types.WikiPageLite, error) { + if strings.TrimSpace(query) == "" { + return nil, nil + } + if limit <= 0 { + limit = 20 + } + if limit > 50 { + limit = 50 + } + if len(pageTypes) == 0 { + pageTypes = []string{types.WikiPageTypeEntity, types.WikiPageTypeConcept} + } + + q := strings.ToLower(strings.TrimSpace(query)) + + var rows []types.WikiPageLite + if err := r.db.Debug().WithContext(ctx). + Model(&types.WikiPage{}). + Select("slug, title, page_type, status, aliases, out_links, similarity(lower(title), ?) AS sim", q). + Where("knowledge_base_id = ? AND page_type IN ? AND status <> ? AND lower(title) % ?", + kbID, pageTypes, types.WikiPageStatusArchived, q). + Order("sim DESC"). + Limit(limit). + Scan(&rows).Error; err != nil { + return nil, err + } + out := make([]*types.WikiPageLite, len(rows)) + for i := range rows { + r := rows[i] + out[i] = &r + } + return out, nil +} + // ListAll retrieves all wiki pages in a knowledge base func (r *wikiPageRepository) ListAll(ctx context.Context, kbID string) ([]*types.WikiPage, error) { var pages []*types.WikiPage @@ -476,7 +824,7 @@ func (r *wikiPageRepository) ListIssues(ctx context.Context, kbID string, slug s if status != "" { query = query.Where("status = ?", status) } - + var issues []*types.WikiPageIssue if err := query.Order("created_at DESC").Find(&issues).Error; err != nil { return nil, err diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index 25a0f940..1a7b92b1 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -36,24 +36,25 @@ var ( // knowledgeService implements the knowledge service interface // service 实现知识服务接口 type knowledgeService struct { - config *config.Config - retrieveEngine interfaces.RetrieveEngineRegistry - repo interfaces.KnowledgeRepository - kbService interfaces.KnowledgeBaseService - tenantRepo interfaces.TenantRepository - tenantService interfaces.TenantService - documentReader interfaces.DocumentReader - chunkService interfaces.ChunkService - chunkRepo interfaces.ChunkRepository - tagRepo interfaces.KnowledgeTagRepository - tagService interfaces.KnowledgeTagService - fileSvc interfaces.FileService - modelService interfaces.ModelService - task interfaces.TaskEnqueuer - graphEngine interfaces.RetrieveGraphRepository - redisClient *redis.Client - kbShareService interfaces.KBShareService - imageResolver *docparser.ImageResolver + config *config.Config + retrieveEngine interfaces.RetrieveEngineRegistry + repo interfaces.KnowledgeRepository + kbService interfaces.KnowledgeBaseService + tenantRepo interfaces.TenantRepository + tenantService interfaces.TenantService + documentReader interfaces.DocumentReader + chunkService interfaces.ChunkService + chunkRepo interfaces.ChunkRepository + tagRepo interfaces.KnowledgeTagRepository + tagService interfaces.KnowledgeTagService + fileSvc interfaces.FileService + modelService interfaces.ModelService + task interfaces.TaskEnqueuer + graphEngine interfaces.RetrieveGraphRepository + redisClient *redis.Client + kbShareService interfaces.KBShareService + imageResolver *docparser.ImageResolver + taskPendingRepo interfaces.TaskPendingOpsRepository // In-memory fallbacks for Lite mode (no Redis) memFAQProgress sync.Map // taskID -> *types.FAQImportProgress @@ -90,28 +91,30 @@ func NewKnowledgeService( imageResolver *docparser.ImageResolver, wikiRepo interfaces.WikiPageRepository, wikiService interfaces.WikiPageService, + taskPendingRepo interfaces.TaskPendingOpsRepository, ) (interfaces.KnowledgeService, error) { return &knowledgeService{ - config: config, - repo: repo, - kbService: kbService, - tenantRepo: tenantRepo, - tenantService: tenantService, - documentReader: documentReader, - chunkService: chunkService, - chunkRepo: chunkRepo, - tagRepo: tagRepo, - tagService: tagService, - fileSvc: fileSvc, - modelService: modelService, - task: task, - graphEngine: graphEngine, - retrieveEngine: retrieveEngine, - redisClient: redisClient, - kbShareService: kbShareService, - imageResolver: imageResolver, - wikiRepo: wikiRepo, - wikiService: wikiService, + config: config, + repo: repo, + kbService: kbService, + tenantRepo: tenantRepo, + tenantService: tenantService, + documentReader: documentReader, + chunkService: chunkService, + chunkRepo: chunkRepo, + tagRepo: tagRepo, + tagService: tagService, + fileSvc: fileSvc, + modelService: modelService, + task: task, + graphEngine: graphEngine, + retrieveEngine: retrieveEngine, + redisClient: redisClient, + kbShareService: kbShareService, + imageResolver: imageResolver, + wikiRepo: wikiRepo, + wikiService: wikiService, + taskPendingRepo: taskPendingRepo, }, nil } diff --git a/internal/application/service/knowledge_delete.go b/internal/application/service/knowledge_delete.go index 7cbfc905..e01bde89 100644 --- a/internal/application/service/knowledge_delete.go +++ b/internal/application/service/knowledge_delete.go @@ -282,7 +282,7 @@ func (s *knowledgeService) cleanupWikiOnKnowledgeDelete(ctx context.Context, kno // so the knowledge's disappearance is reflected in the UI. lang, _ := types.LanguageFromContext(ctx) tenantID, _ := types.TenantIDFromContext(ctx) - EnqueueWikiRetract(ctx, s.task, s.redisClient, WikiRetractPayload{ + EnqueueWikiRetract(ctx, s.task, s.taskPendingRepo, WikiRetractPayload{ TenantID: tenantID, KnowledgeBaseID: kbID, KnowledgeID: knowledgeID, @@ -309,59 +309,25 @@ func (s *knowledgeService) markKnowledgeDeletedForWiki(ctx context.Context, kbID } // scrubWikiPendingIngest removes queued WikiOpIngest entries for a knowledge -// from the debounced pending list. Used by both the delete path (we're about -// to soft-delete the doc, no point ingesting it) and the reparse path (the +// from task_pending_ops. Used by both the delete path (we're about to +// soft-delete the doc, no point ingesting it) and the reparse path (the // old chunks are about to vanish, so any pending ingest would either race // with the cleanup or no-op on an empty chunk set — and the post-process // task will enqueue a fresh ingest once new chunks land anyway). // // Retract entries stay put — delete still needs them to unlink referencing // pages, and reparse never enqueues retracts for the doc being reparsed. -// -// We use LREM against JSON-encoded entries plus a best-effort raw-UUID -// fallback for backward compatibility with the legacy format documented in -// peekPendingList. +// We pass op=WikiOpIngest so DeleteByDedupKey filters to the ingest rows +// only. func (s *knowledgeService) scrubWikiPendingIngest(ctx context.Context, kbID, knowledgeID, reason string) { - if s.redisClient == nil || kbID == "" || knowledgeID == "" { + if s.taskPendingRepo == nil || kbID == "" || knowledgeID == "" { return } - pendingKey := wikiPendingKeyPrefix + kbID - - // Best-effort: inspect the list, remove matching ingest entries one by one. - // The list is bounded (wikiMaxDocsPerBatch at a time on the consumer - // side, practical uploads rarely exceed a few dozen), so a single LRange - // is safe. - items, err := s.redisClient.LRange(ctx, pendingKey, 0, -1).Result() - if err != nil { - logger.Warnf(ctx, "wiki %s: failed to read pending list %s: %v", reason, pendingKey, err) + if err := s.taskPendingRepo.DeleteByDedupKey(ctx, wikiTaskType, wikiTaskScope, kbID, knowledgeID, WikiOpIngest); err != nil { + logger.Warnf(ctx, "wiki %s: failed to scrub pending ingest ops for knowledge %s: %v", reason, knowledgeID, err) return } - removed := 0 - for _, item := range items { - // Legacy raw-UUID form - if item == knowledgeID { - if n, err := s.redisClient.LRem(ctx, pendingKey, 0, item).Result(); err == nil { - removed += int(n) - } - continue - } - if !strings.HasPrefix(item, "{") { - continue - } - var op WikiPendingOp - if err := json.Unmarshal([]byte(item), &op); err != nil { - continue - } - if op.KnowledgeID != knowledgeID || op.Op != WikiOpIngest { - continue - } - if n, err := s.redisClient.LRem(ctx, pendingKey, 0, item).Result(); err == nil { - removed += int(n) - } - } - if removed > 0 { - logger.Infof(ctx, "wiki %s: scrubbed %d pending ingest ops for knowledge %s", reason, removed, knowledgeID) - } + logger.Infof(ctx, "wiki %s: scrubbed pending ingest ops for knowledge %s", reason, knowledgeID) } // prepareWikiForReparse is the reparse counterpart to diff --git a/internal/application/service/knowledge_post_process.go b/internal/application/service/knowledge_post_process.go index c59bdc40..8ce5106f 100644 --- a/internal/application/service/knowledge_post_process.go +++ b/internal/application/service/knowledge_post_process.go @@ -17,11 +17,12 @@ import ( // KnowledgePostProcessService acts as an orchestrator for all post-processing tasks // after a document has been parsed and split into chunks (including multimodal OCR/Caption). type KnowledgePostProcessService struct { - knowledgeRepo interfaces.KnowledgeRepository - kbService interfaces.KnowledgeBaseService - chunkService interfaces.ChunkService - taskEnqueuer interfaces.TaskEnqueuer - redisClient *redis.Client + knowledgeRepo interfaces.KnowledgeRepository + kbService interfaces.KnowledgeBaseService + chunkService interfaces.ChunkService + taskEnqueuer interfaces.TaskEnqueuer + pendingRepo interfaces.TaskPendingOpsRepository + redisClient *redis.Client } func NewKnowledgePostProcessService( @@ -29,6 +30,7 @@ func NewKnowledgePostProcessService( kbService interfaces.KnowledgeBaseService, chunkService interfaces.ChunkService, taskEnqueuer interfaces.TaskEnqueuer, + pendingRepo interfaces.TaskPendingOpsRepository, redisClient *redis.Client, ) interfaces.TaskHandler { return &KnowledgePostProcessService{ @@ -36,6 +38,7 @@ func NewKnowledgePostProcessService( kbService: kbService, chunkService: chunkService, taskEnqueuer: taskEnqueuer, + pendingRepo: pendingRepo, redisClient: redisClient, } } @@ -126,7 +129,7 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta // 6. Spawn Wiki Ingest Task if wiki indexing is enabled in IndexingStrategy if kb.IndexingStrategy.WikiEnabled && len(textChunks) > 0 { - EnqueueWikiIngest(ctx, s.taskEnqueuer, s.redisClient, payload.TenantID, payload.KnowledgeBaseID, payload.KnowledgeID) + EnqueueWikiIngest(ctx, s.taskEnqueuer, s.pendingRepo, payload.TenantID, payload.KnowledgeBaseID, payload.KnowledgeID) logger.Infof(ctx, "[KnowledgePostProcess] Enqueued wiki ingest task for %s", payload.KnowledgeID) } return nil diff --git a/internal/application/service/slug_fuzzy.go b/internal/application/service/slug_fuzzy.go new file mode 100644 index 00000000..1505e935 --- /dev/null +++ b/internal/application/service/slug_fuzzy.go @@ -0,0 +1,188 @@ +package service + +import ( + "strings" + + "github.com/Tencent/WeKnora/internal/searchutil" +) + +// Fuzzy slug resolution: rescue LLM-generated `[[slug|display]]` links +// where the slug is *almost* right but not exactly the one in the KB. +// +// Background: the wiki ingest LLM is given an `` +// block listing exact `[[slug]] = title` entries. Empirically — even +// when the prompt explicitly says "copy-paste the slug verbatim" — +// open-source models such as deepseek-v3 routinely munge slugs in +// predictable ways: +// +// - inserting hyphens that happen to look like pinyin word breaks +// ("shanghaitower" → "shang-hai-tower"), +// - dropping or duplicating hyphens, +// - normalizing the case of an ASCII slug, +// - latinizing names that should have stayed CJK. +// +// The display text is almost always still correct (the model just +// copy-pastes the title), so we have two levers to find the *real* +// slug: a tolerant string compare on the slug itself, and a reverse +// lookup from display text. Combined, these recover the vast majority +// of broken links without any LLM cost — the alternative being +// stripDeadWikiLinks, which gives up and emits plain text. +// +// Cleanup callers (stripDeadWikiLinks, cleanDeadLinks) consult +// resolveDeadSlug FIRST and only fall back to "strip to plain text" +// when no live candidate is close enough to be safe. + +// slugResolveBigramThreshold is the minimum char-bigram Jaccard +// similarity required for the bigram fallback to accept a candidate. +// 0.8 is intentionally conservative: at this level "shang-hai-tower" +// and "shanghai-tower" still match (most bigrams overlap), but +// "user-profile" and "user-permissions" do NOT (different stems even +// though the prefix matches). Anything weaker risks linking to the +// wrong page, which is worse than emitting plain text. +const slugResolveBigramThreshold = 0.8 + +// normalizeSlugForCompare collapses cosmetic slug variations so two +// slugs that differ only in hyphenation or case compare equal. +// +// Specifically: +// - lowercases ASCII letters (slugify already does this, so most +// real slugs are already lowercase, but the LLM occasionally +// produces capitalized forms), +// - removes every hyphen and underscore. +// +// Hyphens in wiki slugs are purely visual separators — slugify uses +// them between word fragments. Removing them lets us treat +// "shang-hai-tower" and "shanghai-tower" as the same logical token bag. +// +// CJK runes are preserved verbatim: they ARE the meaningful identity +// of CJK slugs, so collapsing them would over-merge. +func normalizeSlugForCompare(slug string) string { + slug = strings.ToLower(slug) + slug = strings.ReplaceAll(slug, "-", "") + slug = strings.ReplaceAll(slug, "_", "") + return slug +} + +// resolveDeadSlug attempts to map a dead `[[slug]]` reference back to +// a live KB slug, using progressively more permissive heuristics: +// +// 1. Display-text reverse lookup. If the LLM emitted "[[bad-slug|上海 +// 中心大厦]]" and there's a live page whose Title or alias is +// "上海中心大厦", return its slug. This is the most common case +// by far (the model copies the title correctly even when it +// mangles the slug). +// +// 2. Hyphen/case-normalized slug equality. Two slugs that compare +// equal under normalizeSlugForCompare are treated as the same +// logical page. Catches the LLM's "insert pinyin breaks" pattern. +// +// 3. Bigram Jaccard similarity ≥ slugResolveBigramThreshold (0.8). +// Catches typos and minor character substitutions that the +// normalize step doesn't fix. Computed via searchutil.Jaccard +// against character bigrams of each candidate slug. +// +// Returns the resolved live slug and true on success; "" and false +// when no candidate is close enough to be safe. +// +// The candidate pool is supplied by the caller as `liveSlugs`. The +// caller is also responsible for passing a `titleToSlug` map keyed +// by exact (case-sensitive) page title and alias surface forms. Both +// maps are consulted only — never mutated. +func resolveDeadSlug( + deadSlug string, + displayText string, + liveSlugs map[string]struct{}, + titleToSlug map[string]string, +) (string, bool) { + if deadSlug == "" { + return "", false + } + // Already live? No-op success. + if _, ok := liveSlugs[deadSlug]; ok { + return deadSlug, true + } + + // (1) Display-text reverse lookup. Trim whitespace because LLM + // occasionally emits `[[slug| display ]]` with stray spaces, and + // the user-side title would have been written without them. + if dt := strings.TrimSpace(displayText); dt != "" { + if slug, ok := titleToSlug[dt]; ok && slug != "" { + if _, live := liveSlugs[slug]; live { + return slug, true + } + } + } + + // (2) Normalized-equality check. Compute the dead slug's normal + // form once and compare against every live candidate's normal + // form. O(N) but N is bounded — this only fires on the dead-link + // path, and `liveSlugs` is the per-batch affected set, not the + // full KB. + deadNorm := normalizeSlugForCompare(deadSlug) + if deadNorm == "" { + // Slug was nothing but hyphens / underscores after normalize + // — nothing meaningful to compare against. + return "", false + } + for cand := range liveSlugs { + if normalizeSlugForCompare(cand) == deadNorm { + return cand, true + } + } + + // (3) Bigram Jaccard fallback. Use searchutil.Jaccard against + // the character bigrams of each slug. We restrict the bigram + // alphabet to the slug's normalized form (lowercase, no hyphens) + // so a hyphenation difference doesn't dominate the bigram set + // the way it would for raw slug strings. + deadGrams := slugCharBigrams(deadNorm) + if len(deadGrams) == 0 { + return "", false + } + var ( + bestSlug string + bestScore float64 + ) + for cand := range liveSlugs { + candNorm := normalizeSlugForCompare(cand) + if candNorm == "" { + continue + } + candGrams := slugCharBigrams(candNorm) + if len(candGrams) == 0 { + continue + } + score := searchutil.Jaccard(deadGrams, candGrams) + if score > bestScore { + bestScore = score + bestSlug = cand + } + } + if bestScore >= slugResolveBigramThreshold { + return bestSlug, true + } + return "", false +} + +// slugCharBigrams returns a character-bigram set for the given (already +// normalized) slug. Single-rune slugs degrade to a 1-gram so they still +// contribute a comparable signal. +// +// Mirrors surfaceGrams in wiki_ingest_dedup.go but operates on a slug +// (which is already lowercased / hyphen-stripped) rather than a raw +// surface form, so we don't redo the lowercasing or punctuation +// stripping work. +func slugCharBigrams(s string) map[string]struct{} { + if s == "" { + return nil + } + runes := []rune(s) + if len(runes) == 1 { + return map[string]struct{}{string(runes): {}} + } + out := make(map[string]struct{}, len(runes)) + for i := 0; i < len(runes)-1; i++ { + out[string(runes[i:i+2])] = struct{}{} + } + return out +} diff --git a/internal/application/service/slug_fuzzy_test.go b/internal/application/service/slug_fuzzy_test.go new file mode 100644 index 00000000..111faef6 --- /dev/null +++ b/internal/application/service/slug_fuzzy_test.go @@ -0,0 +1,137 @@ +package service + +import "testing" + +func TestNormalizeSlugForCompare(t *testing.T) { + cases := map[string]string{ + "shang-hai-tower": "shanghaitower", + "shanghai-tower": "shanghaitower", + "SHANG-HAI-TOWER": "shanghaitower", + "under_score_slug": "underscoreslug", + "entity/shanghai-tower": "entity/shanghaitower", + "": "", + "---": "", + "中文-slug": "中文slug", + } + for input, want := range cases { + t.Run(input, func(t *testing.T) { + if got := normalizeSlugForCompare(input); got != want { + t.Errorf("normalizeSlugForCompare(%q) = %q, want %q", input, got, want) + } + }) + } +} + +func TestResolveDeadSlug_HyphenVariation(t *testing.T) { + // The exact bug pattern reported in production: the LLM inserted a + // pinyin word break in a CJK-derived slug ("shang-hai" instead of + // "shanghai") even though the display text was copied verbatim from + // the canonical title. Resolver should heal the hyphenation drift + // using the display-text reverse lookup. + live := map[string]struct{}{ + "entity/shanghai-tower": {}, + } + titleToSlug := map[string]string{ + "上海中心大厦": "entity/shanghai-tower", + } + got, ok := resolveDeadSlug( + "entity/shang-hai-tower", + "上海中心大厦", + live, titleToSlug, + ) + if !ok { + t.Fatal("expected resolution success on pinyin-break drift case") + } + if got != "entity/shanghai-tower" { + t.Errorf("expected canonical slug, got %q", got) + } +} + +func TestResolveDeadSlug_ExactMatchPasses(t *testing.T) { + live := map[string]struct{}{ + "entity/foo": {}, + } + got, ok := resolveDeadSlug("entity/foo", "", live, nil) + if !ok || got != "entity/foo" { + t.Errorf("expected pass-through for live slug, got (%q, %v)", got, ok) + } +} + +func TestResolveDeadSlug_DisplayTextLookupPriority(t *testing.T) { + // Even when normalized-equality would also find a candidate, + // display-text lookup should win because it's the highest- + // confidence signal. + live := map[string]struct{}{ + "entity/exact-match": {}, + "entity/exactmatch": {}, + } + titleToSlug := map[string]string{ + "Exact Match": "entity/exactmatch", + } + got, ok := resolveDeadSlug( + "entity/exact-match-typo", + "Exact Match", + live, titleToSlug, + ) + if !ok { + t.Fatal("expected resolution") + } + if got != "entity/exactmatch" { + t.Errorf("display-text lookup should have won; got %q", got) + } +} + +func TestResolveDeadSlug_DisplayTextStillRequiresLive(t *testing.T) { + // titleToSlug points at a slug that's NOT in liveSlugs; we + // shouldn't return it. + live := map[string]struct{}{ + "entity/something-else": {}, + } + titleToSlug := map[string]string{ + "Title": "entity/dead-target", + } + if _, ok := resolveDeadSlug("entity/x", "Title", live, titleToSlug); ok { + t.Fatal("must not return a slug that isn't in liveSlugs") + } +} + +func TestResolveDeadSlug_BigramFallback(t *testing.T) { + // Slug differs from canonical by a single character (typo). + // Normalized-equality won't catch this; bigram Jaccard should. + live := map[string]struct{}{ + "entity/zhongguo-yinhang": {}, + } + got, ok := resolveDeadSlug( + "entity/zhongguo-yinghang", // h↔gh transposition typo + "", + live, nil, + ) + if !ok { + t.Fatal("expected bigram fallback to recover the typo") + } + if got != "entity/zhongguo-yinhang" { + t.Errorf("got %q", got) + } +} + +func TestResolveDeadSlug_RejectsUnrelated(t *testing.T) { + // Completely unrelated candidates must not match — better to + // strip the link than to silently link to the wrong page. + live := map[string]struct{}{ + "entity/foo": {}, + "entity/bar": {}, + "entity/baz": {}, + } + if got, ok := resolveDeadSlug("entity/quuxasdf", "", live, nil); ok { + t.Errorf("expected no match for unrelated slug; got %q", got) + } +} + +func TestResolveDeadSlug_EmptyInputs(t *testing.T) { + if _, ok := resolveDeadSlug("", "any", nil, nil); ok { + t.Error("empty deadSlug must return false") + } + if _, ok := resolveDeadSlug("entity/foo", "", nil, nil); ok { + t.Error("empty liveSlugs / titleToSlug must return false") + } +} diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go index 4a0d67be..c21b1e4e 100644 --- a/internal/application/service/wiki_ingest.go +++ b/internal/application/service/wiki_ingest.go @@ -37,10 +37,6 @@ const ( // maxContentForWiki limits the document content sent to LLM for wiki generation maxContentForWiki = 32768 - // wikiPendingKeyPrefix is the Redis key prefix for pending wiki ingest document lists. - // Key format: wiki:pending:{kbID} → Redis List of knowledge IDs. - wikiPendingKeyPrefix = "wiki:pending:" - // wikiActiveKeyPrefix is the Redis key for the "batch in progress" flag. // Key format: wiki:active:{kbID} → "1" with TTL. Prevents concurrent batches. wikiActiveKeyPrefix = "wiki:active:" @@ -49,27 +45,17 @@ const ( // the batch task fires. Debounces rapid uploads. wikiIngestDelay = 30 * time.Second - // wikiPendingTTL prevents stale pending lists from accumulating. - wikiPendingTTL = 24 * time.Hour - // wikiMaxDocsPerBatch limits how many documents a single batch processes. - // Prevents unbounded execution time. Remaining docs stay in the pending list - // and are picked up by the follow-up task. + // Prevents unbounded execution time. Remaining ops stay in + // task_pending_ops and are picked up by the follow-up task. wikiMaxDocsPerBatch = 5 - // wikiFailCountKeyPrefix is the Redis key prefix for per-document failure - // counters. Key: wiki:failcount:{kbID}:{knowledgeID} → integer. - // Incremented each time requeueFailedOps retries an op; reset to 0 on - // successful ingest. Once the counter exceeds wikiMaxFailRetries the op is - // dropped instead of re-queued, preventing LLM-timeout storms from causing - // unbounded wiki:pending growth. - wikiFailCountKeyPrefix = "wiki:failcount:" - // wikiMaxFailRetries is the maximum number of times a single document op - // may be re-queued via requeueFailedOps before it is permanently dropped. - // 5 retries ≈ five full batch cycles (each with a ~30 s delay), giving - // transient LLM errors a fair chance to recover without letting a - // persistently-broken doc clog the queue indefinitely. + // may be re-attempted via requeueFailedOps before it is permanently + // archived to task_dead_letters. 5 retries ≈ five full batch cycles + // (each with a ~30 s delay), giving transient LLM errors a fair chance + // to recover without letting a persistently-broken doc clog the queue + // indefinitely. wikiMaxFailRetries = 5 // wikiIngestMaxRetry controls asynq retry budget for wiki:ingest tasks. @@ -113,6 +99,15 @@ const ( // between retry attempts. The nth retry waits base << (n-1) — so with // a 2s base we wait 2s, 4s, 8s between attempts. wikiLLMBackoffBase = 2 * time.Second + + // wikiTaskType is the task_type stamp used in task_pending_ops and + // task_dead_letters rows for this pipeline. Stable across the lifetime + // of any pending op so the follow-up consumer can pull it back. + wikiTaskType = "wiki:ingest" + + // wikiTaskScope is the scope used by both pending ops and dead letters. + // Wiki ingest is per-KB, so every op is scoped to a knowledge_base. + wikiTaskScope = types.TaskScopeKnowledgeBase ) // WikiDeletedTombstoneKey returns the Redis key used to mark a knowledge as @@ -124,15 +119,15 @@ func WikiDeletedTombstoneKey(kbID, knowledgeID string) string { } // WikiIngestPayload is the asynq task payload for wiki ingest batch trigger. -// The actual document IDs are stored in a Redis list (wiki:pending:{kbID}). -// KnowledgeID is only used as fallback in Lite mode (no Redis). +// The actual document IDs are stored in the task_pending_ops table; this +// payload only carries the trigger metadata so the worker can resolve +// the queue tuple (task_type, scope, scope_id) and process whatever rows +// are queued under it. type WikiIngestPayload struct { types.TracingContext TenantID uint64 `json:"tenant_id"` KnowledgeBaseID string `json:"knowledge_base_id"` Language string `json:"language,omitempty"` - // Fallback for Lite mode (no Redis) - LiteOps []WikiPendingOp `json:"lite_ops,omitempty"` } // WikiRetractPayload is the asynq task payload for wiki content retraction @@ -152,7 +147,18 @@ const ( WikiOpRetract = "retract" ) -// WikiPendingOp represents a single operation in the Redis pending queue +// WikiPendingOp represents a single operation queued in task_pending_ops +// under task_type="wiki:ingest". The struct is the JSON payload of the +// task_pending_ops row; the surrounding (task_type, scope, scope_id, +// dedup_key) fields live as separate columns and are not serialized +// here. +// +// dbID is the auto-increment primary key of the task_pending_ops row +// the op was loaded from. PeekBatch fills it; consumers carry it +// through Map/Reduce so DeleteByIDs (after consume) and IncrFailCount +// (after failure) can address the right row. It is intentionally +// unexported and excluded from JSON so the persisted payload does not +// duplicate the column. type WikiPendingOp struct { Op string `json:"op"` KnowledgeID string `json:"knowledge_id"` @@ -162,18 +168,39 @@ type WikiPendingOp struct { DocTitle string `json:"doc_title,omitempty"` DocSummary string `json:"doc_summary,omitempty"` PageSlugs []string `json:"page_slugs,omitempty"` + + // dbID is set by peekPendingList from task_pending_ops.id. Zero in + // constructions made outside the queue (e.g. legacy tests). + dbID int64 `json:"-"` } -// wikiIngestService handles the LLM-powered wiki generation pipeline +// wikiIngestService handles the LLM-powered wiki generation pipeline. +// +// Durable state lives in two places: +// - task_pending_ops (rows tagged task_type="wiki:ingest", scope= +// "knowledge_base"): the per-document op queue. Replaces the +// legacy Redis wiki:pending: list, which was vulnerable to +// 24h TTL eviction at 4w-document scale. +// - task_dead_letters: in-batch failures that exhausted +// wikiMaxFailRetries land here. The asynq dead-letter middleware +// also writes asynq-level archived rows here uniformly across +// every task type. +// +// Redis is still used for the per-KB active-batch lock +// (wiki:active:) and the delete tombstone (wiki:deleted:<...>), +// both of which are correctness-critical short-lived flags rather +// than data the system should survive without. type wikiIngestService struct { - wikiService interfaces.WikiPageService - kbService interfaces.KnowledgeBaseService - knowledgeSvc interfaces.KnowledgeService - chunkRepo interfaces.ChunkRepository - modelService interfaces.ModelService - task interfaces.TaskEnqueuer - logEntrySvc interfaces.WikiLogEntryService - redisClient *redis.Client // nil in Lite mode (no Redis) + wikiService interfaces.WikiPageService + kbService interfaces.KnowledgeBaseService + knowledgeSvc interfaces.KnowledgeService + chunkRepo interfaces.ChunkRepository + modelService interfaces.ModelService + task interfaces.TaskEnqueuer + logEntrySvc interfaces.WikiLogEntryService + pendingRepo interfaces.TaskPendingOpsRepository + deadLetterRepo interfaces.TaskDeadLetterRepository + redisClient *redis.Client // nil in Lite mode (no Redis) // liteLocks provides per-KB mutual exclusion in Lite mode (no Redis). // Keys are kbID strings; values are unused (presence = locked). liteLocks sync.Map @@ -188,95 +215,111 @@ func NewWikiIngestService( modelService interfaces.ModelService, task interfaces.TaskEnqueuer, logEntrySvc interfaces.WikiLogEntryService, + pendingRepo interfaces.TaskPendingOpsRepository, + deadLetterRepo interfaces.TaskDeadLetterRepository, redisClient *redis.Client, ) interfaces.TaskHandler { svc := &wikiIngestService{ - wikiService: wikiService, - kbService: kbService, - knowledgeSvc: knowledgeSvc, - chunkRepo: chunkRepo, - modelService: modelService, - task: task, - logEntrySvc: logEntrySvc, - redisClient: redisClient, + wikiService: wikiService, + kbService: kbService, + knowledgeSvc: knowledgeSvc, + chunkRepo: chunkRepo, + modelService: modelService, + task: task, + logEntrySvc: logEntrySvc, + pendingRepo: pendingRepo, + deadLetterRepo: deadLetterRepo, + redisClient: redisClient, } return svc } -// EnqueueWikiIngest adds a document to the wiki ingest queue. +// EnqueueWikiIngest queues a document for wiki ingestion. // -// Architecture: each document upload pushes its knowledgeID to a Redis pending list, -// then schedules a delayed asynq task. When the task fires, it atomically drains the -// entire list and processes ALL pending documents in one batch. +// Architecture: each upload inserts one row into task_pending_ops +// (task_type="wiki:ingest", scope="knowledge_base", scope_id=kbID, +// dedup_key=knowledgeID), then schedules a debounced asynq trigger task. +// When the trigger fires, the worker peeks a batch from +// task_pending_ops, processes it, deletes consumed rows, and (if more +// remain) schedules a follow-up. Multiple debounced triggers within the +// 30s window all coalesce: the first one to acquire the per-KB active +// lock drains the batch; subsequent ones see an empty queue and exit. // -// If multiple uploads happen within the delay window (30s), each one schedules a task, -// but the FIRST task to fire drains the list and processes everything. Subsequent tasks -// fire, find an empty list, and exit immediately (no-op). This gives us natural batching -// without any locks or task deduplication. -// -// t=0s doc1 → RPush + Enqueue(delay=30s, id=random1) -// t=5s doc2 → RPush + Enqueue(delay=30s, id=random2) -// t=10s doc3 → RPush + Enqueue(delay=30s, id=random3) -// t=30s random1 fires → drain [doc1,doc2,doc3] → process all -// t=35s random2 fires → drain [] → no-op return -// t=40s random3 fires → drain [] → no-op return -// -// In Lite mode (no Redis), falls back to immediate per-document execution. -func EnqueueWikiIngest(ctx context.Context, task interfaces.TaskEnqueuer, redisClient *redis.Client, tenantID uint64, kbID, knowledgeID string) { +// Lite mode (no Redis) still works as long as Postgres is reachable — +// the queue lives in PG, only the active-batch lock is Redis-only and +// has a process-local fallback (liteLocks) inside the worker. +func EnqueueWikiIngest( + ctx context.Context, + task interfaces.TaskEnqueuer, + pendingRepo interfaces.TaskPendingOpsRepository, + tenantID uint64, + kbID, knowledgeID string, +) { lang, _ := types.LanguageFromContext(ctx) - payload := WikiIngestPayload{ + // Persist the pending op. A re-ingest of the same knowledge id while + // a previous op is still queued simply appends another row; the + // peekPendingList consumer collapses by dedup_key (== knowledge_id), + // keeping the LATEST op for each knowledge — matching the legacy + // "RPush + reverse-dedupe" semantics. + op := WikiPendingOp{ + Op: WikiOpIngest, + KnowledgeID: knowledgeID, + Language: lang, + } + payloadBytes, err := json.Marshal(op) + if err != nil { + logger.Warnf(ctx, "wiki ingest: failed to marshal pending op for %s: %v", knowledgeID, err) + return + } + if pendingRepo != nil { + if err := pendingRepo.Enqueue(ctx, &types.TaskPendingOp{ + TenantID: tenantID, + TaskType: wikiTaskType, + Scope: wikiTaskScope, + ScopeID: kbID, + Op: WikiOpIngest, + DedupKey: knowledgeID, + Payload: payloadBytes, + }); err != nil { + logger.Warnf(ctx, "wiki ingest: failed to enqueue pending op for %s: %v", knowledgeID, err) + // Fall through and still schedule the trigger task — the + // next upload (or the next retry pass) will catch the gap. + } + } + + trigger := WikiIngestPayload{ TenantID: tenantID, KnowledgeBaseID: kbID, Language: lang, } + langfuse.InjectTracing(ctx, &trigger) + triggerBytes, _ := json.Marshal(trigger) - // Push to Redis pending list (if Redis available) - if redisClient != nil { - pendingKey := wikiPendingKeyPrefix + kbID - // Reset stale fail counter for fresh user-triggered ingest enqueue. - // This gives a newly re-ingested document a full retry budget. - failKey := wikiFailCountKeyPrefix + kbID + ":" + knowledgeID - redisClient.Del(ctx, failKey) - op := WikiPendingOp{ - Op: WikiOpIngest, - KnowledgeID: knowledgeID, - Language: lang, - } - opBytes, _ := json.Marshal(op) - redisClient.RPush(ctx, pendingKey, string(opBytes)) - redisClient.Expire(ctx, pendingKey, wikiPendingTTL) - } else { - // Fallback for Lite mode (no Redis) - payload.LiteOps = []WikiPendingOp{{ - Op: WikiOpIngest, - KnowledgeID: knowledgeID, - Language: lang, - }} - } - - langfuse.InjectTracing(ctx, &payload) - payloadBytes, _ := json.Marshal(payload) - - t := asynq.NewTask(types.TypeWikiIngest, payloadBytes, + t := asynq.NewTask(types.TypeWikiIngest, triggerBytes, asynq.Queue("low"), asynq.MaxRetry(wikiIngestMaxRetry), asynq.Timeout(60*time.Minute), asynq.ProcessIn(wikiIngestDelay), ) if _, err := task.Enqueue(t); err != nil { - logger.Warnf(ctx, "wiki ingest: failed to enqueue task: %v", err) + logger.Warnf(ctx, "wiki ingest: failed to enqueue trigger task: %v", err) } } -// EnqueueWikiRetract enqueues an async wiki content retraction task -func EnqueueWikiRetract(ctx context.Context, task interfaces.TaskEnqueuer, redisClient *redis.Client, payload WikiRetractPayload) { - ingestPayload := WikiIngestPayload{ - TenantID: payload.TenantID, - KnowledgeBaseID: payload.KnowledgeBaseID, - Language: payload.Language, - } - +// EnqueueWikiRetract queues a wiki retraction op (a delete cleanup). +// Identical persistence model as EnqueueWikiIngest — the op rides in +// task_pending_ops and an asynq trigger fires shortly after to +// process the batch. Retracts use a slightly shorter ProcessIn delay +// because there is no "user upload arriving in waves" pattern to +// debounce against — a deletion fires once and we want the cleanup +// to land promptly. +func EnqueueWikiRetract( + ctx context.Context, + task interfaces.TaskEnqueuer, + pendingRepo interfaces.TaskPendingOpsRepository, + payload WikiRetractPayload, +) { op := WikiPendingOp{ Op: WikiOpRetract, KnowledgeID: payload.KnowledgeID, @@ -285,27 +328,40 @@ func EnqueueWikiRetract(ctx context.Context, task interfaces.TaskEnqueuer, redis PageSlugs: payload.PageSlugs, Language: payload.Language, } - - if redisClient != nil { - pendingKey := wikiPendingKeyPrefix + payload.KnowledgeBaseID - opBytes, _ := json.Marshal(op) - redisClient.RPush(ctx, pendingKey, string(opBytes)) - redisClient.Expire(ctx, pendingKey, wikiPendingTTL) - } else { - // Fallback for Lite mode (no Redis) - ingestPayload.LiteOps = []WikiPendingOp{op} + payloadBytes, err := json.Marshal(op) + if err != nil { + logger.Warnf(ctx, "wiki retract: failed to marshal pending op: %v", err) + return + } + if pendingRepo != nil { + if err := pendingRepo.Enqueue(ctx, &types.TaskPendingOp{ + TenantID: payload.TenantID, + TaskType: wikiTaskType, + Scope: wikiTaskScope, + ScopeID: payload.KnowledgeBaseID, + Op: WikiOpRetract, + DedupKey: payload.KnowledgeID, + Payload: payloadBytes, + }); err != nil { + logger.Warnf(ctx, "wiki retract: failed to enqueue pending op: %v", err) + } } - langfuse.InjectTracing(ctx, &ingestPayload) - payloadBytes, _ := json.Marshal(ingestPayload) - t := asynq.NewTask(types.TypeWikiIngest, payloadBytes, + trigger := WikiIngestPayload{ + TenantID: payload.TenantID, + KnowledgeBaseID: payload.KnowledgeBaseID, + Language: payload.Language, + } + langfuse.InjectTracing(ctx, &trigger) + triggerBytes, _ := json.Marshal(trigger) + t := asynq.NewTask(types.TypeWikiIngest, triggerBytes, asynq.Queue("low"), asynq.MaxRetry(wikiIngestMaxRetry), asynq.Timeout(60*time.Minute), asynq.ProcessIn(5*time.Second), // Retract can trigger the batch quickly ) if _, err := task.Enqueue(t); err != nil { - logger.Warnf(ctx, "wiki retract: failed to enqueue task: %v", err) + logger.Warnf(ctx, "wiki retract: failed to enqueue trigger task: %v", err) } } @@ -316,136 +372,155 @@ func (s *wikiIngestService) Handle(ctx context.Context, t *asynq.Task) error { return s.ProcessWikiIngest(ctx, t) } -// peekPendingList gets up to wikiMaxDocsPerBatch entries from the Redis pending list -// WITHOUT removing them. It returns the unique ops and the actual number of items peeked. -func (s *wikiIngestService) peekPendingList(ctx context.Context, kbID string) ([]WikiPendingOp, int) { - if s.redisClient == nil { - return nil, 0 +// peekPendingList loads up to `limit` ops from task_pending_ops for +// this KB, ordered FIFO. Rows are NOT removed; callers must +// DeleteByIDs once they have been consumed (or IncrFailCount + leave +// them in place for the next pass). +// +// peekedIDs returns the DB ids of every row included in the peek +// (NOT just the ones that survived dedup) so trimPendingList can +// delete them all in one statement at the end of the batch — this +// matches the legacy "LTrim peekedCount entries" semantics, where +// duplicates collapsed by the consumer were also drained from the +// list once their canonical sibling had been processed. +func (s *wikiIngestService) peekPendingList(ctx context.Context, kbID string, limit int) (ops []WikiPendingOp, peekedIDs []int64) { + if s.pendingRepo == nil { + return nil, nil } - pendingKey := wikiPendingKeyPrefix + kbID - - result, err := s.redisClient.LRange(ctx, pendingKey, 0, wikiMaxDocsPerBatch-1).Result() + if limit <= 0 { + limit = wikiMaxDocsPerBatch + } + rows, err := s.pendingRepo.PeekBatch(ctx, wikiTaskType, wikiTaskScope, kbID, limit) if err != nil { logger.Warnf(ctx, "wiki ingest: failed to peek pending list: %v", err) - return nil, 0 + return nil, nil + } + if len(rows) == 0 { + return nil, nil } - var ops []WikiPendingOp - for _, item := range result { - if !strings.HasPrefix(item, "{") { - // Backward compatibility: raw knowledgeID string - ops = append(ops, WikiPendingOp{ - Op: WikiOpIngest, - KnowledgeID: item, - }) - continue - } + all := make([]WikiPendingOp, 0, len(rows)) + peekedIDs = make([]int64, 0, len(rows)) + for _, r := range rows { + peekedIDs = append(peekedIDs, r.ID) var op WikiPendingOp - if err := json.Unmarshal([]byte(item), &op); err == nil { - ops = append(ops, op) + if len(r.Payload) > 0 { + if err := json.Unmarshal(r.Payload, &op); err != nil { + logger.Warnf(ctx, "wiki ingest: failed to unmarshal pending op id=%d: %v", r.ID, err) + continue + } } else { - logger.Warnf(ctx, "wiki ingest: failed to unmarshal pending op: %v, raw_item: %.100s", err, item) + // Defensive: if payload was lost, fall back to column data + // so the row is still drainable (otherwise it would loop + // on every batch as un-deletable). + op = WikiPendingOp{ + Op: r.Op, + KnowledgeID: r.DedupKey, + } } + op.dbID = r.ID + all = append(all, op) } - // Deduplicate by KnowledgeID, keeping only the *last* operation for each document. - // This optimizes out redundant sequences (e.g., upload then immediate delete: [ingest, retract] -> [retract]). + // Deduplicate by KnowledgeID, keeping only the *last* operation for + // each document. Optimizes out redundant sequences (e.g., upload + // then immediate delete: [ingest, retract] → [retract]). The + // non-canonical rows still get drained at trim time — their dbIDs + // are in peekedIDs. seen := make(map[string]bool) - var reversedUnique []WikiPendingOp - for i := len(ops) - 1; i >= 0; i-- { - op := ops[i] - if !seen[op.KnowledgeID] { - seen[op.KnowledgeID] = true + reversedUnique := make([]WikiPendingOp, 0, len(all)) + for i := len(all) - 1; i >= 0; i-- { + op := all[i] + if op.KnowledgeID == "" { + // No dedup key — keep verbatim (rare; edge case for + // future ops without a knowledge anchor). reversedUnique = append(reversedUnique, op) - } - } - - // Reverse back to maintain chronological order - var unique []WikiPendingOp - for i := len(reversedUnique) - 1; i >= 0; i-- { - unique = append(unique, reversedUnique[i]) - } - - return unique, len(result) -} - -// trimPendingList removes the first `count` items from the Redis pending list. -func (s *wikiIngestService) trimPendingList(ctx context.Context, kbID string, count int) { - if s.redisClient == nil || count <= 0 { - return - } - pendingKey := wikiPendingKeyPrefix + kbID - if err := s.redisClient.LTrim(ctx, pendingKey, int64(count), -1).Err(); err != nil { - logger.Warnf(ctx, "wiki ingest: failed to trim pending list: %v", err) - } -} - -// requeueFailedOps re-enqueues failed operations for retry. -// -// Redis mode: appends ops back to the pending list tail so the next follow-up -// batch picks them up. -// -// Lite mode (no Redis): enqueues a new asynq task per failed op with a short -// delay, since there is no shared pending list to append to. -func (s *wikiIngestService) requeueFailedOps(ctx context.Context, payload WikiIngestPayload, ops []WikiPendingOp) { - if s.redisClient != nil { - pendingKey := wikiPendingKeyPrefix + payload.KnowledgeBaseID - for _, op := range ops { - // Increment failure counter; drop the op permanently once it - // exceeds wikiMaxFailRetries to prevent unbounded queue growth - // caused by persistent LLM timeouts or extraction errors. - failKey := wikiFailCountKeyPrefix + payload.KnowledgeBaseID + ":" + op.KnowledgeID - count, err := s.redisClient.Incr(ctx, failKey).Result() - if err != nil { - logger.Warnf(ctx, "wiki ingest: failed to increment fail count for %s: %v", op.KnowledgeID, err) - // Fall through and requeue anyway — better to retry than silently drop - } else { - // Refresh TTL on every update so the key doesn't outlast its usefulness - s.redisClient.Expire(ctx, failKey, wikiPendingTTL) - if count > wikiMaxFailRetries { - logger.Warnf(ctx, "wiki ingest: dropping op %s (%s) after %d failures (limit %d)", op.KnowledgeID, op.DocTitle, count, wikiMaxFailRetries) - // Drop-path cleanup: avoid leaking stale counters that can - // poison future manual re-ingest within TTL window. - s.redisClient.Del(ctx, failKey) - continue - } - } - - data, err := json.Marshal(op) - if err != nil { - logger.Warnf(ctx, "wiki ingest: failed to marshal op for requeue: %v", err) - continue - } - if err := s.redisClient.RPush(ctx, pendingKey, string(data)).Err(); err != nil { - logger.Warnf(ctx, "wiki ingest: failed to requeue op %s: %v", op.KnowledgeID, err) - continue - } - logger.Infof(ctx, "wiki ingest: re-queued failed op %s (%s) for retry (attempt %d/%d)", op.KnowledgeID, op.DocTitle, count, wikiMaxFailRetries) - } - return - } - - // Lite mode: re-enqueue each failed op as a new asynq task. - for _, op := range ops { - retryPayload := WikiIngestPayload{ - TenantID: payload.TenantID, - KnowledgeBaseID: payload.KnowledgeBaseID, - Language: op.Language, - LiteOps: []WikiPendingOp{op}, - } - langfuse.InjectTracing(ctx, &retryPayload) - payloadBytes, _ := json.Marshal(retryPayload) - t := asynq.NewTask(types.TypeWikiIngest, payloadBytes, - asynq.Queue("low"), - asynq.MaxRetry(wikiIngestMaxRetry), - asynq.Timeout(60*time.Minute), - asynq.ProcessIn(wikiIngestDelay), - ) - if _, err := s.task.Enqueue(t); err != nil { - logger.Warnf(ctx, "wiki ingest: failed to requeue lite op %s: %v", op.KnowledgeID, err) continue } - logger.Infof(ctx, "wiki ingest: re-queued failed lite op %s (%s) for retry", op.KnowledgeID, op.DocTitle) + if seen[op.KnowledgeID] { + continue + } + seen[op.KnowledgeID] = true + reversedUnique = append(reversedUnique, op) + } + + ops = make([]WikiPendingOp, 0, len(reversedUnique)) + for i := len(reversedUnique) - 1; i >= 0; i-- { + ops = append(ops, reversedUnique[i]) + } + return ops, peekedIDs +} + +// trimPendingList deletes consumed rows from task_pending_ops. Empty +// input is a no-op so callers can invoke unconditionally at the end +// of a batch. +func (s *wikiIngestService) trimPendingList(ctx context.Context, ids []int64) { + if s.pendingRepo == nil || len(ids) == 0 { + return + } + if err := s.pendingRepo.DeleteByIDs(ctx, ids); err != nil { + logger.Warnf(ctx, "wiki ingest: failed to trim %d pending rows: %v", len(ids), err) + } +} + +// requeueFailedOps records in-batch failures. +// +// For each failed op: +// +// - IncrFailCount on the source row. The repo returns the new total, +// so a single round trip handles both bookkeeping and retry-budget +// check. +// - If the count is <= wikiMaxFailRetries: leave the row in place. +// The next follow-up batch's PeekBatch will pick it up naturally +// (rows are ordered by id ASC and we never moved/touched it). +// - If the count exceeds the retry cap: archive the op into +// task_dead_letters and DeleteByIDs to remove it from the queue. +// Both writes are best-effort — a DB failure here is logged and +// swallowed so a single transient blip doesn't recursively spawn +// more failures. +func (s *wikiIngestService) requeueFailedOps(ctx context.Context, payload WikiIngestPayload, ops []WikiPendingOp) { + if s.pendingRepo == nil || len(ops) == 0 { + return + } + for _, op := range ops { + if op.dbID == 0 { + // Op was never persisted (synthetic / test) — nothing to + // retry against. + continue + } + count, err := s.pendingRepo.IncrFailCount(ctx, op.dbID) + if err != nil { + logger.Warnf(ctx, "wiki ingest: failed to increment fail count for %s (id=%d): %v", op.KnowledgeID, op.dbID, err) + // Without a fresh count we can't tell whether to drop. Be + // conservative: leave the row in place; the next PeekBatch + // will see it again and we'll try once more. + continue + } + if count <= wikiMaxFailRetries { + logger.Infof(ctx, "wiki ingest: re-queued failed op %s (%s) for retry (attempt %d/%d)", op.KnowledgeID, op.DocTitle, count, wikiMaxFailRetries) + continue + } + + // Exhausted in-batch retries — archive and remove. + logger.Warnf(ctx, "wiki ingest: dropping op %s (%s) after %d failures (limit %d)", op.KnowledgeID, op.DocTitle, count, wikiMaxFailRetries) + if s.deadLetterRepo != nil { + payloadBytes, _ := json.Marshal(op) + if dlErr := s.deadLetterRepo.Insert(ctx, &types.TaskDeadLetter{ + TenantID: payload.TenantID, + TaskType: wikiTaskType, + Scope: wikiTaskScope, + ScopeID: payload.KnowledgeBaseID, + RelatedID: op.KnowledgeID, + Payload: payloadBytes, + LastError: fmt.Sprintf("exceeded wikiMaxFailRetries=%d (in-batch retries)", wikiMaxFailRetries), + FailCount: count, + }); dlErr != nil { + logger.Warnf(ctx, "wiki ingest: failed to archive op %s to dead letters: %v", op.KnowledgeID, dlErr) + } + } + if err := s.pendingRepo.DeleteByIDs(ctx, []int64{op.dbID}); err != nil { + logger.Warnf(ctx, "wiki ingest: failed to drop dead-lettered row id=%d: %v", op.dbID, err) + } } } @@ -460,11 +535,37 @@ type docIngestResult struct { Pages []types.WikiLogPageRef } -// WikiBatchContext holds shared data across Map and Reduce phases +// WikiBatchContext holds shared data across Map and Reduce phases. +// +// Historically this carried a fully materialized `AllPages` slice plus +// pre-built SlugTitleMap / SummaryContentByKnowledgeID lookup tables. +// At 4w-document scale that meant the very first thing every batch +// did was load 100K+ wiki_pages rows (content TEXT included) into Go +// memory — and then walk them several more times for cleanDeadLinks / +// injectCrossLinks / getExistingPageSlugsForKnowledge. +// +// We now lazy-load via fetchers backed by lightweight projections +// (ListBySlugs / ListSummariesByKnowledgeIDs). Each fetcher caches +// results keyed by its input so repeat lookups within a batch are +// free; the cache is per-batch and goroutine-local-via-mutex (sync.Map +// would also work but mutex keeps the surface small). type WikiBatchContext struct { - AllPages []*types.WikiPage - SlugTitleMap map[string]string - SummaryContentByKnowledgeID map[string]string + // SlugTitle resolves a slug to its current title (or "" if missing). + // Backed by ListBySlugs; cache is populated as callers ask, so we + // only pay for the slugs we actually look at. + SlugTitle func(ctx context.Context, slug string) string + + // SlugTitleMany batches a slug-set into a single ListBySlugs query + // and returns the resolved titles map. Convenient when a caller + // already has the full slug list; results are still cached. + SlugTitleMany func(ctx context.Context, slugs []string) map[string]string + + // SummaryContentByKnowledgeID returns the surviving summary page's + // content for the given knowledge id (or "" if no summary page + // exists / was archived). Backed by ListSummariesByKnowledgeIDs; + // cache is populated lazily as well. + SummaryContentByKnowledgeID func(ctx context.Context, kid string) string + // ExtractionGranularity drives Pass 0 (candidate slug extraction) // aggressiveness. Resolved once per batch from the KnowledgeBase's // WikiConfig so every doc in the batch sees the same scope rules. @@ -542,7 +643,7 @@ func previewStringSlice(items []string, limit int) string { var wikiLinkRE = regexp.MustCompile(`\[\[([^\[\]\|\s]+)(?:\|([^\]]+))?\]\]`) // sanitizeDeadSummaryLinks rewrites the summary pages produced by THIS -// batch to remove `[[slug]]` / `[[slug|display]]` references that point +// batch to fix `[[slug]]` / `[[slug|display]]` references that point // at slugs whose entity/concept page generation failed in reduce. // // Background: WikiSummaryPrompt instructs the LLM to embed wiki links @@ -551,8 +652,14 @@ var wikiLinkRE = regexp.MustCompile(`\[\[([^\[\]\|\s]+)(?:\|([^\]]+))?\]\]`) // creation happens later in reduce. When reduce's WikiPageModifyPrompt // fails on an entity/concept slug the page never gets written — and // the already-persisted summary is left holding a `[[entity/foo|name]]` -// link that 404s. This pass replaces those dead links with their -// display text so the summary degrades gracefully. +// link that 404s. +// +// We pass the batch's affected-slug set + the SlugTitleMany fetcher +// to the resolver so that LLM-mangled slugs (e.g. extra pinyin hyphens +// in "shang-hai-tower" vs "shanghai-tower") are healed in place rather +// than stripped to plain text — preserving cross-link information +// whenever the display text or surface form unambiguously identifies a +// live page. // // Pure text replacement, no LLM call. Scoped to the doc-summary slugs // in this batch (`summary/`), keeping the work @@ -562,10 +669,16 @@ func (s *wikiIngestService) sanitizeDeadSummaryLinks( kbID string, docResults []*docIngestResult, failedSlugs map[string]struct{}, + batchCtx *WikiBatchContext, ) { if len(failedSlugs) == 0 || len(docResults) == 0 { return } + // Build a (live-slug-set, title->slug) pair the resolver can consult. + // We seed liveSlugs from batchCtx (the slugs that DID make it into + // pages this batch) and expand it lazily as needed via SlugTitleMany. + // titleToSlug is filled with the same successful pages' titles so the + // display-text reverse lookup works on first try. for _, r := range docResults { if r == nil || r.KnowledgeID == "" { continue @@ -575,7 +688,25 @@ func (s *wikiIngestService) sanitizeDeadSummaryLinks( if err != nil || page == nil { continue } - newContent, changed := stripDeadWikiLinks(page.Content, failedSlugs) + + // Collect the slugs this summary actually links to (so the + // resolver has a non-empty pool of candidates), plus all the + // successfully-written sibling pages from the same doc. These + // two sets together cover the LLM-vs-actual mismatch cases + // without paying for a full ListAll scan. + candidateSlugs := make(map[string]struct{}, len(page.OutLinks)+len(r.Pages)) + for _, slug := range page.OutLinks { + candidateSlugs[slug] = struct{}{} + } + for _, ref := range r.Pages { + if _, bad := failedSlugs[ref.Slug]; bad { + continue + } + candidateSlugs[ref.Slug] = struct{}{} + } + liveSlugs, titleToSlug := s.resolveLiveSlugs(ctx, batchCtx, candidateSlugs) + + newContent, changed := stripDeadWikiLinks(page.Content, failedSlugs, liveSlugs, titleToSlug) if !changed { continue } @@ -588,12 +719,66 @@ func (s *wikiIngestService) sanitizeDeadSummaryLinks( } } -// stripDeadWikiLinks replaces `[[slug]]` / `[[slug|display]]` references -// with plain text whenever `slug` is in the dead set. The display text -// (when present) is preserved verbatim; otherwise the slug's last path -// segment is humanized into a fallback label so the surrounding sentence -// still reads naturally. -func stripDeadWikiLinks(content string, deadSlugs map[string]struct{}) (string, bool) { +// resolveLiveSlugs builds the (liveSlugs, titleToSlug) pair that +// stripDeadWikiLinks / cleanDeadLinks pass into resolveDeadSlug. +// +// We start from a caller-supplied candidate set (typically the page's +// own out-links + this batch's freshly-written slugs) and ask the +// batch's SlugTitleMany fetcher to resolve them in one batched query. +// The fetcher already filters out archived / system pages, so missing +// entries naturally translate to "not live" without an extra check. +// +// titleToSlug is keyed by the page's exact title only — we don't have +// aliases in the lite projection. That's an acceptable trade-off: the +// reported breakage pattern is "slug munged, display = title", not +// "slug munged, display = alias", so display-by-title carries the +// majority of the rescue value at a fraction of the storage cost. +func (s *wikiIngestService) resolveLiveSlugs( + ctx context.Context, + batchCtx *WikiBatchContext, + candidates map[string]struct{}, +) (map[string]struct{}, map[string]string) { + if len(candidates) == 0 || batchCtx == nil || batchCtx.SlugTitleMany == nil { + return nil, nil + } + slugList := make([]string, 0, len(candidates)) + for s := range candidates { + slugList = append(slugList, s) + } + titles := batchCtx.SlugTitleMany(ctx, slugList) + live := make(map[string]struct{}, len(titles)) + titleToSlug := make(map[string]string, len(titles)) + for slug, title := range titles { + live[slug] = struct{}{} + if title != "" { + titleToSlug[title] = slug + } + } + return live, titleToSlug +} + +// stripDeadWikiLinks rewrites `[[slug]]` / `[[slug|display]]` references +// whose `slug` falls into the dead set. The handling depends on whether +// the dead slug can be repaired: +// +// - If the resolver maps the dead slug to a live one (typically via +// display-text reverse lookup or hyphen-normalized equality — +// see resolveDeadSlug), the link is REWRITTEN with the corrected +// slug. Display text is preserved. +// - If no live candidate is close enough, the link is STRIPPED to +// plain text (display text when present; otherwise a humanized +// last-segment of the slug). This is the original behaviour. +// +// The resolver is optional: when liveSlugs / titleToSlug are nil or +// empty, every dead slug falls through to the strip path. This keeps +// backward compatibility for tests / call sites that don't yet wire +// the resolution data. +func stripDeadWikiLinks( + content string, + deadSlugs map[string]struct{}, + liveSlugs map[string]struct{}, + titleToSlug map[string]string, +) (string, bool) { if len(deadSlugs) == 0 || content == "" { return content, false } @@ -607,16 +792,30 @@ func stripDeadWikiLinks(content string, deadSlugs map[string]struct{}) (string, if _, dead := deadSlugs[slug]; !dead { return match } - changed = true display := "" if len(sub) >= 3 { display = strings.TrimSpace(sub[2]) } + + // (1) Try fuzzy resolve before falling back to strip. The + // resolver consults display-text reverse lookup, hyphen- + // normalized equality, and bigram similarity in that order; + // returns "" only when no candidate is safe. + if resolved, ok := resolveDeadSlug(slug, display, liveSlugs, titleToSlug); ok && resolved != slug { + changed = true + if display != "" { + return "[[" + resolved + "|" + display + "]]" + } + return "[[" + resolved + "]]" + } + + // (2) Strip — best-effort plain text. Prefer the LLM-supplied + // display text; otherwise humanize the slug's last path segment + // so the prose stays readable. + changed = true if display != "" { return display } - // Derive a readable label from the slug's tail segment so we - // don't leave a raw "entity/foo-bar" smear in the prose. parts := strings.Split(slug, "/") label := parts[len(parts)-1] label = strings.ReplaceAll(label, "-", " ") @@ -625,60 +824,97 @@ func stripDeadWikiLinks(content string, deadSlugs map[string]struct{}) (string, return out, changed } -// cleanDeadLinks removes [[wiki-links]] that point to archived or deleted pages. -// Scans all published pages, checks each out_link, and removes references to -// pages that no longer exist or are archived. No LLM call — pure text cleanup. -func (s *wikiIngestService) cleanDeadLinks(ctx context.Context, kbID string) { - allPages, err := s.wikiService.ListAllPages(ctx, kbID) - if err != nil || len(allPages) == 0 { +// cleanDeadLinks rewrites `[[slug]]` references in the batch's affected +// pages whose targets no longer exist (or were archived). Pure text +// cleanup — no LLM call. +// +// Scope is intentionally limited to the slugs touched by this batch: +// at 4w-document scale the legacy "scan every page in the KB" path was +// the dominant tail in the post-batch phase, and the long-tail +// historical dead links are better handled by the lint AutoFix pipeline +// (which runs out-of-band and can afford a full table walk). +// +// For each affected page: +// +// 1. Pull its lite projection (out_links + status) via the batch's +// SlugTitle fetcher (one IN query for the whole affected set, +// amortized via the batchCtx cache). +// 2. Probe the union of out-link targets through ExistsSlugs to +// classify them as live vs dead. +// 3. For each dead link, try resolveDeadSlug first; rewrite if a +// safe candidate exists, otherwise strip to plain text. +// 4. Persist the rewritten content via UpdateAutoLinkedContent so +// the version counter stays unchanged (this is a maintenance +// pass, not a user-visible edit). +func (s *wikiIngestService) cleanDeadLinks(ctx context.Context, kbID string, affectedSlugs []string, batchCtx *WikiBatchContext) { + if len(affectedSlugs) == 0 { return } - // Build set of live (non-archived, non-system) slugs - liveSlugs := make(map[string]bool) - for _, p := range allPages { - if p.Status != types.WikiPageStatusArchived { - liveSlugs[p.Slug] = true - } - } - - var cleaned int - for _, p := range allPages { - if p.Status == types.WikiPageStatusArchived { + // (1) Load the affected pages' content + out-links in one go. + // We need the full WikiPage rows here (not just lite projections) + // because we're going to rewrite content; the lite path saves + // nothing once we're touching content anyway. + cleaned := 0 + for _, slug := range affectedSlugs { + page, err := s.wikiService.GetPageBySlug(ctx, kbID, slug) + if err != nil || page == nil { continue } - if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog { + if page.Status == types.WikiPageStatusArchived { + continue + } + if page.PageType == types.WikiPageTypeIndex || page.PageType == types.WikiPageTypeLog { + continue + } + if len(page.OutLinks) == 0 { continue } - content := p.Content - changed := false - - // Find all [[slug]] references and remove dead ones - for _, outSlug := range p.OutLinks { - if liveSlugs[outSlug] { - continue // link is alive - } - // Dead link — remove the [[slug]] from content, keep the display text if any - linkPattern := "[[" + outSlug + "]]" - if strings.Contains(content, linkPattern) { - // Replace [[dead-slug]] with just the slug's readable part - parts := strings.Split(outSlug, "/") - readableName := parts[len(parts)-1] - readableName = strings.ReplaceAll(readableName, "-", " ") - content = strings.ReplaceAll(content, linkPattern, readableName) - changed = true - } + // (2) Classify out-links as live vs dead via one batched + // ExistsSlugs query. Empty slug list → no-op. + liveMap, err := s.wikiService.ExistsSlugs(ctx, kbID, []string(page.OutLinks)) + if err != nil { + logger.Warnf(ctx, "wiki: ExistsSlugs failed during dead-link cleanup for %s: %v", slug, err) + continue } - - if changed { - p.Content = content - if err := s.wikiService.UpdateAutoLinkedContent(ctx, p); err != nil { - logger.Warnf(ctx, "wiki: failed to clean dead links in page %s: %v", p.Slug, err) + deadSlugs := make(map[string]struct{}) + liveSlugs := make(map[string]struct{}, len(liveMap)) + for outSlug, alive := range liveMap { + if alive { + liveSlugs[outSlug] = struct{}{} } else { - cleaned++ + deadSlugs[outSlug] = struct{}{} } } + if len(deadSlugs) == 0 { + continue + } + + // (3) Build the title->slug reverse-lookup map for fuzzy + // resolve. We pull titles for the live slugs only — those + // are the candidates a dead reference could be remapped to. + titles := batchCtx.SlugTitleMany(ctx, []string(page.OutLinks)) + titleToSlug := make(map[string]string, len(titles)) + for s, t := range titles { + if t != "" { + titleToSlug[t] = s + } + } + + newContent, changed := stripDeadWikiLinks(page.Content, deadSlugs, liveSlugs, titleToSlug) + if !changed { + continue + } + + // (4) Persist. UpdateAutoLinkedContent skips the version bump + // because dead-link cleanup is a machine-only edit. + page.Content = newContent + if err := s.wikiService.UpdateAutoLinkedContent(ctx, page); err != nil { + logger.Warnf(ctx, "wiki: failed to clean dead links in page %s: %v", page.Slug, err) + continue + } + cleaned++ } if cleaned > 0 { @@ -686,44 +922,78 @@ func (s *wikiIngestService) cleanDeadLinks(ctx context.Context, kbID string) { } } -// injectCrossLinks scans affected pages and injects [[wiki-links]] for mentions -// of other wiki page titles in the content. Pure text replacement, no LLM call. -// Processes entity/concept/synthesis/comparison and summary pages (not index/log). +// injectCrossLinks scans the batch's affected pages and injects +// `[[wiki-links]]` for mentions of other wiki page titles / aliases +// in the content. Pure text replacement, no LLM call. // -// The actual matching is delegated to linkifyContent, which handles code-block -// and existing-link exclusions plus ASCII word-boundary checks. -func (s *wikiIngestService) injectCrossLinks(ctx context.Context, kbID string, affectedSlugs []string) { - allPages, err := s.wikiService.ListAllPages(ctx, kbID) - if err != nil || len(allPages) < 2 { +// Scope is intentionally limited to two slug sets: +// +// 1. The affected pages themselves — we only rewrite their content. +// 2. The candidate refs come from (a) the affected pages' existing +// out-links (already known to be relevant via prior linkification +// or manual edits) plus (b) the batch's freshly-written sibling +// slugs supplied via `linkRefs` from the caller. +// +// At 4w-document scale this is the difference between loading 100K+ +// pages just to find link candidates vs O(batch-size) lookups. We +// trade off some long-tail recall (a brand new entity in this batch +// won't be linkified into pages from previous batches until they get +// re-edited), but lint AutoFix is the right place for that. +// +// linkifyContent does the actual matching work, including code-block / +// existing-link / word-boundary exclusions. +func (s *wikiIngestService) injectCrossLinks( + ctx context.Context, + kbID string, + affectedSlugs []string, + freshRefs []linkRef, + batchCtx *WikiBatchContext, +) { + if len(affectedSlugs) == 0 { return } - refs := collectLinkRefs(allPages) - if len(refs) == 0 { - return - } - - affectedSet := make(map[string]bool, len(affectedSlugs)) + updated := 0 for _, slug := range affectedSlugs { - affectedSet[slug] = true - } - - var updated int - for _, p := range allPages { - if !affectedSet[p.Slug] { + page, err := s.wikiService.GetPageBySlug(ctx, kbID, slug) + if err != nil || page == nil { continue } - if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog { + if page.PageType == types.WikiPageTypeIndex || page.PageType == types.WikiPageTypeLog { continue } - newContent, changed := linkifyContent(p.Content, refs, p.Slug) + // Build the per-page candidate ref set: the existing out-links + // (resolved via the batch's title fetcher to skip archived / + // system pages) plus the freshly-written sibling slugs from + // this batch. + var refs []linkRef + if len(page.OutLinks) > 0 { + titles := batchCtx.SlugTitleMany(ctx, []string(page.OutLinks)) + for outSlug, title := range titles { + if title == "" || outSlug == slug { + continue + } + refs = append(refs, linkRef{slug: outSlug, matchText: title}) + } + } + for _, fr := range freshRefs { + if fr.slug == slug { + continue + } + refs = append(refs, fr) + } + if len(refs) == 0 { + continue + } + + newContent, changed := linkifyContent(page.Content, refs, page.Slug) if !changed { continue } - p.Content = newContent - if err := s.wikiService.UpdateAutoLinkedContent(ctx, p); err != nil { - logger.Warnf(ctx, "wiki ingest: cross-link injection failed for %s: %v", p.Slug, err) + page.Content = newContent + if err := s.wikiService.UpdateAutoLinkedContent(ctx, page); err != nil { + logger.Warnf(ctx, "wiki ingest: cross-link injection failed for %s: %v", page.Slug, err) continue } updated++ @@ -754,28 +1024,40 @@ func collectLinkRefs(pages []*types.WikiPage) []linkRef { return refs } -// getExistingPageSlugsForKnowledge returns all page slugs that currently reference -// a given knowledge ID in their source_refs. Used to snapshot state before re-ingest. +// getExistingPageSlugsForKnowledge returns all page slugs that currently +// reference a given knowledge ID in their source_refs. Used to snapshot +// state before re-ingest so the reduce phase can reconcile additions vs +// retractions. +// +// Backed by idx_wiki_pages_source_refs (GIN jsonb_path_ops, migration +// 000041) and the legacy text-index fallback for "kid|title" entries. +// We project to slugs only — no need to load full row content for a +// per-doc snapshot. +// +// Index/log slugs (wiki-intrinsic system pages) never carry real +// source_refs in practice, but we filter them out explicitly here as +// a defense-in-depth measure: an old buggy ingest that mistakenly +// stamped a system page with a knowledge ref would otherwise show up +// in the reparse "old set" and confuse the reduce stage. func (s *wikiIngestService) getExistingPageSlugsForKnowledge(ctx context.Context, kbID, knowledgeID string) map[string]bool { - allPages, err := s.wikiService.ListAllPages(ctx, kbID) - if err != nil || len(allPages) == 0 { + slugs, err := s.wikiService.ListSlugsBySourceRef(ctx, kbID, knowledgeID) + if err != nil { + logger.Warnf(ctx, "wiki ingest: ListSlugsBySourceRef(%s) failed: %v", knowledgeID, err) return nil } - - slugs := make(map[string]bool) - prefix := knowledgeID + "|" - for _, p := range allPages { - if p.PageType == types.WikiPageTypeIndex || p.PageType == types.WikiPageTypeLog { + if len(slugs) == 0 { + return nil + } + out := make(map[string]bool, len(slugs)) + for _, slug := range slugs { + // Defense-in-depth: skip wiki-intrinsic slugs that never have + // real source refs. + if slug == "index" || slug == "log" { continue } - for _, ref := range p.SourceRefs { - if ref == knowledgeID || strings.HasPrefix(ref, prefix) { - slugs[p.Slug] = true - break - } - } + out[slug] = true } - return slugs + return out } // retractStalePages handles pages that were previously linked to this document @@ -834,32 +1116,39 @@ type combinedExtraction struct { // The new intro is written to both Content and Summary so readers that // still fall back to Summary (older clients, legacy migrations) stay in // sync with the column the view actually renders. +// indexIntroSummaryCap caps how many summary pages we feed into the +// LLM when generating the wiki index intro from scratch. A 4w-document +// KB would otherwise blow the context window every batch, and the +// intro is a "set the scene" artifact where the most-recently-touched +// documents carry disproportionately more signal anyway. We pick the +// top-N most-recently-updated summaries and add a "showing N of M" +// hint to the prompt so the LLM can be honest about its sample. +const indexIntroSummaryCap = 200 + +// rebuildIndexPage refreshes the LLM-generated intro on the index +// page. Two paths: +// +// - First-time generation (no existing intro, or only the legacy +// placeholder): the LLM gets a CAPPED window of the most recent +// summary pages (most-recently-updated wins). Compare with the +// legacy path which loaded ALL summaries — at 4w-document scale +// that produced multi-MB prompts that simply broke the context +// window and silently fell back to a hardcoded intro. +// - Incremental update: the LLM gets only the existing intro plus +// the change description for THIS batch. Document summaries are +// intentionally NOT included — at scale the change-description +// alone is enough signal for "what landed?", and excluding the +// full summary set keeps the prompt size bounded regardless of +// KB size. +// +// The intro is written to both Content and Summary so legacy readers +// that fall through to Summary stay in sync. func (s *wikiIngestService) rebuildIndexPage(ctx context.Context, chatModel chat.Chat, payload WikiIngestPayload, changeDesc, lang string) error { indexPage, _ := s.wikiService.GetIndex(ctx, payload.KnowledgeBaseID) if indexPage == nil { return nil } - // Gather just the Summary-type pages the LLM needs for intro framing. - // We do NOT load or iterate every wiki page here — the directory is - // now assembled on demand by GetIndexView, which is the only reader - // that needs the full enumeration. - summaries, err := s.wikiService.ListByType(ctx, payload.KnowledgeBaseID, types.WikiPageTypeSummary) - if err != nil { - return err - } - - var docSummaries strings.Builder - for _, p := range summaries { - if p == nil || p.Status == types.WikiPageStatusArchived { - continue - } - fmt.Fprintf(&docSummaries, "\n%s\n%s\n\n\n", p.Title, p.Summary) - } - if docSummaries.Len() == 0 { - docSummaries.WriteString("(no documents yet)") - } - // The intro lives on both Content and Summary. Prefer Content since // that's what the new index view returns; fall back to Summary for // rows written before this refactor so the incremental-update prompt @@ -879,9 +1168,36 @@ func (s *wikiIngestService) rebuildIndexPage(ctx context.Context, chatModel chat var intro string switch { case existingIntro == "" || existingIntro == "Wiki index - table of contents": - // First time — generate intro from scratch. + // First-time generation: pull the top-N most-recent summary + // pages via the lite projection. CountByType lets us tell the + // LLM "showing N of M" so it can frame the intro honestly when + // the KB is bigger than what we're sampling. + recentSummaries, listErr := s.wikiService.ListByTypeRecent(ctx, payload.KnowledgeBaseID, types.WikiPageTypeSummary, indexIntroSummaryCap) + if listErr != nil { + return listErr + } + var docSummaries strings.Builder + for _, e := range recentSummaries { + fmt.Fprintf(&docSummaries, "\n%s\n%s\n\n\n", e.Title, e.Summary) + } + // Best-effort total count for the framing hint. CountByType + // counts every page type; we need just summary, so we read + // directly. A failure here doesn't block intro generation. + totalSummaries := int64(len(recentSummaries)) + if counts, cntErr := s.wikiService.CountByType(ctx, payload.KnowledgeBaseID); cntErr == nil { + if t, ok := counts[types.WikiPageTypeSummary]; ok { + totalSummaries = t + } + } + framing := "" + if int(totalSummaries) > len(recentSummaries) && len(recentSummaries) > 0 { + framing = fmt.Sprintf("(showing %d most recent of %d total documents)\n\n", len(recentSummaries), totalSummaries) + } + if docSummaries.Len() == 0 { + docSummaries.WriteString("(no documents yet)") + } generatedIntro, genErr := s.generateWithTemplate(ctx, chatModel, agent.WikiIndexIntroPrompt, map[string]string{ - "DocumentSummaries": docSummaries.String(), + "DocumentSummaries": framing + docSummaries.String(), "Language": lang, }) if genErr != nil { @@ -890,11 +1206,16 @@ func (s *wikiIngestService) rebuildIndexPage(ctx context.Context, chatModel chat intro = strings.TrimSpace(generatedIntro) } case changeDesc != "": - // Incremental update — tell the LLM what changed. + // Incremental update: only the existing intro + this batch's + // change description go into the prompt. We deliberately stop + // passing the full DocumentSummaries set here — at 4w docs it + // would re-flood the context every batch, and the + // change-description block already encodes the "what just + // changed" signal the prompt is asking for. updatedIntro, genErr := s.generateWithTemplate(ctx, chatModel, agent.WikiIndexIntroUpdatePrompt, map[string]string{ "ExistingIntro": existingIntro, "ChangeDescription": changeDesc, - "DocumentSummaries": docSummaries.String(), + "DocumentSummaries": "", "Language": lang, }) if genErr != nil { @@ -919,7 +1240,7 @@ func (s *wikiIngestService) rebuildIndexPage(ctx context.Context, chatModel chat indexPage.Content = intro indexPage.Summary = intro - _, err = s.wikiService.UpdatePage(ctx, indexPage) + _, err := s.wikiService.UpdatePage(ctx, indexPage) return err } @@ -1016,40 +1337,79 @@ func xmlEscape(s string) string { // existing wiki pages in a single LLM call. Uses pre-loaded allPages to avoid // redundant DB queries. This replaces the two separate deduplicateItems calls // that each queried ListAllPages + made a separate LLM call. +// deduplicateExtractedBatch deduplicates both entities and concepts against +// existing wiki pages in a single LLM call. Pre-filters candidates via the +// pg_trgm trigram index on lower(title) — every new item issues a +// FindSimilarPages probe and the union of top-K hits across all items is +// the candidate set. This replaces the legacy "ListAllPages + Go-side +// surface-form Jaccard" path that scaled O(P × N) on large KBs. +// +// The KB-id-keyed query relies on idx_wiki_pages_title_trgm (added in +// migration 000041); pg_search environments load pg_trgm in the same +// init step (see migrations/paradedb/00-init-db.sql). func (s *wikiIngestService) deduplicateExtractedBatch( ctx context.Context, chatModel chat.Chat, + kbID string, entities, concepts []extractedItem, - allPages []*types.WikiPage, ) ([]extractedItem, []extractedItem) { if len(entities) == 0 && len(concepts) == 0 { return entities, concepts } - - if len(allPages) == 0 { + if s.wikiService == nil { return entities, concepts } - // Pre-filter the candidate existing-pages set. Passing the full corpus - // to the LLM on large KBs bloats the prompt and empirically lets the - // model hallucinate merges between totally unrelated slugs. The filter - // is conservative (high recall) and the downstream validMerge check - // remains in place for defense in depth. - newItems := make([]extractedItem, 0, len(entities)+len(concepts)) - newItems = append(newItems, entities...) - newItems = append(newItems, concepts...) - candidatePages := selectDedupCandidatePages(newItems, allPages) + // Build the candidate set: for each new item, ask the repo for + // the top-K trigram-similar pages and union the results. Dedup by + // slug as we go so the prompt only carries each candidate once. + candidatePages := make(map[string]*types.WikiPageLite) + probe := func(item extractedItem) { + queries := make([]string, 0, 1+len(item.Aliases)) + if item.Name != "" { + queries = append(queries, item.Name) + } + for _, alias := range item.Aliases { + if alias != "" { + queries = append(queries, alias) + } + } + for _, q := range queries { + pages, err := s.wikiService.FindSimilarPages(ctx, kbID, q, + []string{types.WikiPageTypeEntity, types.WikiPageTypeConcept}, + dedupCandidateTopK) + if err != nil { + logger.Warnf(ctx, "wiki ingest: dedup FindSimilarPages(%q) failed: %v", q, err) + continue + } + for _, p := range pages { + if p == nil || p.Slug == "" { + continue + } + if _, ok := candidatePages[p.Slug]; !ok { + candidatePages[p.Slug] = p + } + } + } + } + for _, e := range entities { + probe(e) + } + for _, c := range concepts { + probe(c) + } if len(candidatePages) == 0 { + // No similar existing pages — nothing to merge against. The + // items pass through unchanged. + logger.Infof(ctx, "wiki ingest: no similar existing pages found for %d new items", len(entities)+len(concepts)) return entities, concepts } - if origCount := countEntityConceptPages(allPages); origCount > len(candidatePages) { - logger.Infof(ctx, "wiki ingest: dedup candidate filter kept %d/%d existing pages for %d new items", - len(candidatePages), origCount, len(newItems)) - } + logger.Infof(ctx, "wiki ingest: similar existing pages selected for %d new items", + len(candidatePages), len(entities)+len(concepts)) var existingBuf strings.Builder for _, p := range candidatePages { - writeDedupItemXML(&existingBuf, p.Slug, p.Title, string(p.PageType), []string(p.Aliases)) + writeDedupItemXML(&existingBuf, p.Slug, p.Title, p.PageType, []string(p.Aliases)) } if existingBuf.Len() == 0 { return entities, concepts @@ -1086,14 +1446,19 @@ func (s *wikiIngestService) deduplicateExtractedBatch( return entities, concepts } - existingSlugs := make(map[string]bool, len(allPages)) - for _, p := range allPages { - existingSlugs[p.Slug] = true + // Build the existing-slug set from the candidate map: anything not + // in candidates is rejected as an LLM hallucination, since by + // construction the model only ever saw those slugs as merge + // targets. Compare with the legacy "look up against allPages" + // path which had a wider acceptance window. + existingSlugs := make(map[string]bool, len(candidatePages)) + for slug := range candidatePages { + existingSlugs[slug] = true } validMerge := func(srcSlug, dstSlug string) bool { if !existingSlugs[dstSlug] { - logger.Warnf(ctx, "wiki ingest: dedup rejected %s → %s (target slug does not exist)", srcSlug, dstSlug) + logger.Warnf(ctx, "wiki ingest: dedup rejected %s → %s (target slug does not exist in candidate set)", srcSlug, dstSlug) return false } srcPrefix := srcSlug[:strings.Index(srcSlug, "/")+1] diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go index ba3d9d2d..187df45f 100644 --- a/internal/application/service/wiki_ingest_batch.go +++ b/internal/application/service/wiki_ingest_batch.go @@ -18,12 +18,18 @@ import ( "golang.org/x/sync/errgroup" ) +// scheduleFollowUp enqueues another asynq trigger task if there are +// still pending ops in task_pending_ops for this KB. Returns true when +// a follow-up was scheduled. +// +// We use a short ProcessIn (5s) so the active-batch lock has time to +// release before the next worker tries to acquire it; otherwise we'd +// just bounce on ErrWikiIngestConcurrent and burn an asynq retry slot. func (s *wikiIngestService) scheduleFollowUp(ctx context.Context, payload WikiIngestPayload) bool { - if s.redisClient == nil { + if s.pendingRepo == nil { return false } - pendingKey := wikiPendingKeyPrefix + payload.KnowledgeBaseID - count, err := s.redisClient.LLen(ctx, pendingKey).Result() + count, err := s.pendingRepo.PendingCount(ctx, wikiTaskType, wikiTaskScope, payload.KnowledgeBaseID) if err != nil || count == 0 { return false } @@ -65,11 +71,17 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task followUpScheduled := false totalPagesAffected := 0 docPreview := make([]string, 0, 6) + // Tunables resolved from KB.WikiConfig once we've loaded the KB. + // Captured up here so the deferred stats log can observe them + // regardless of which exit path we took. + loggedBatchSize := 0 + loggedMapPar := 0 + loggedReducePar := 0 defer func() { logger.Infof( ctx, - "wiki ingest stats: kb=%s tenant=%d retry=%d/%d status=%s elapsed=%s mode=%s lock_acquired=%v pending_ops=%d ops(ingest=%d,retract=%d) ingest(success=%d,failed=%d) retract_handled=%d pages(total=%d) index(rebuild_attempted=%v,rebuild_succeeded=%v) followup=%v preview=%s", + "wiki ingest stats: kb=%s tenant=%d retry=%d/%d status=%s elapsed=%s mode=%s lock_acquired=%v pending_ops=%d ops(ingest=%d,retract=%d) ingest(success=%d,failed=%d) retract_handled=%d pages(total=%d) index(rebuild_attempted=%v,rebuild_succeeded=%v) followup=%v tunables(batch=%d,map_par=%d,reduce_par=%d) preview=%s", payload.KnowledgeBaseID, payload.TenantID, retryCount, @@ -88,6 +100,9 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task indexRebuildAttempted, indexRebuildSucceeded, followUpScheduled, + loggedBatchSize, + loggedMapPar, + loggedReducePar, previewStringSlice(docPreview, 6), ) }() @@ -117,21 +132,21 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task logger.Warnf(ctx, "wiki ingest: redis SetNX failed: %v", err) } else if !acquired { exitStatus = "active_lock_conflict" - // If the pending list is already empty, the active batch will process - // everything — no need to retry. Returning nil avoids burning through the - // retry budget on tasks that would be no-ops when they eventually acquire - // the lock. If there are still pending ops, retry so we don't miss them - // in case the active batch drained the list before we RPush'd. - pendingKey := wikiPendingKeyPrefix + payload.KnowledgeBaseID - n, nErr := s.redisClient.LLen(ctx, pendingKey).Result() + // If task_pending_ops is already empty for this KB, the active + // batch will drain whatever was queued. Returning nil avoids + // burning through the retry budget on tasks that would be + // no-ops when they eventually acquire the lock. If rows still + // remain, retry so we don't miss them in case the active + // batch drained its peek before our op landed. + n, nErr := s.pendingRepo.PendingCount(ctx, wikiTaskType, wikiTaskScope, payload.KnowledgeBaseID) if nErr != nil { - logger.Warnf(ctx, "wiki ingest: failed to read pending length during lock conflict for KB %s: %v", payload.KnowledgeBaseID, nErr) + logger.Warnf(ctx, "wiki ingest: failed to read pending count during lock conflict for KB %s: %v", payload.KnowledgeBaseID, nErr) logger.Infof(ctx, "wiki ingest: another batch active for KB %s, deferring to asynq retry", payload.KnowledgeBaseID) return ErrWikiIngestConcurrent } if n == 0 { exitStatus = "active_lock_conflict_empty" - logger.Infof(ctx, "wiki ingest: concurrent batch active for KB %s, pending list empty — skipping", payload.KnowledgeBaseID) + logger.Infof(ctx, "wiki ingest: concurrent batch active for KB %s, pending queue empty — skipping", payload.KnowledgeBaseID) return nil } logger.Infof(ctx, "wiki ingest: another batch active for KB %s, deferring to asynq retry", payload.KnowledgeBaseID) @@ -196,31 +211,34 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task return fmt.Errorf("wiki ingest: get chat model: %w", err) } + // Resolve per-KB tunables once. WikiConfig.IngestBatchSize / + // IngestMapParallel / IngestReduceParallel let operators on + // 4w-document KBs raise the throughput knob (more docs per batch + + // more concurrent LLM calls) without a code deploy. Zero falls back + // to the historical defaults so existing KBs see no behaviour + // change until they opt in. + batchSize := kb.WikiConfig.IngestBatchSizeOrDefault(wikiMaxDocsPerBatch) + mapParallel := kb.WikiConfig.IngestMapParallelOrDefault(10) + reduceParallel := kb.WikiConfig.IngestReduceParallelOrDefault(10) + loggedBatchSize = batchSize + loggedMapPar = mapParallel + loggedReducePar = reduceParallel + loggedBatchSize = batchSize + loggedMapPar = mapParallel + loggedReducePar = reduceParallel + lang := types.LanguageNameFromContext(ctx) - pendingOps, peekedCount := s.peekPendingList(ctx, payload.KnowledgeBaseID) + pendingOps, peekedIDs := s.peekPendingList(ctx, payload.KnowledgeBaseID, batchSize) pendingOpsCount = len(pendingOps) if len(pendingOps) == 0 { - if s.redisClient != nil { - exitStatus = "no_pending_ops" - logger.Infof(ctx, "wiki ingest: no pending operations for KB %s", payload.KnowledgeBaseID) - return nil - } - if len(payload.LiteOps) > 0 { - pendingOps = payload.LiteOps - peekedCount = len(pendingOps) - pendingOpsCount = len(pendingOps) - } else { - exitStatus = "no_lite_ops" - return nil - } + exitStatus = "no_pending_ops" + logger.Infof(ctx, "wiki ingest: no pending operations for KB %s", payload.KnowledgeBaseID) + return nil } logger.Infof(ctx, "wiki ingest: batch processing %d ops for KB %s", len(pendingOps), payload.KnowledgeBaseID) - // Fetch all existing pages once, shared across Map and Reduce phases - allPages, _ := s.wikiService.ListAllPages(ctx, payload.KnowledgeBaseID) - // Resolve extraction granularity once per batch. Historical rows with // empty/unknown values fall back to Standard via Normalize(). Failures // to load the KB (unlikely since we're already acting on it) also @@ -230,25 +248,115 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task granularity = kb.WikiConfig.ExtractionGranularity.Normalize() } - batchCtx := &WikiBatchContext{ - AllPages: allPages, - SlugTitleMap: make(map[string]string), - SummaryContentByKnowledgeID: make(map[string]string), - ExtractionGranularity: granularity, - } - for _, p := range allPages { - if p.PageType != types.WikiPageTypeIndex && p.PageType != types.WikiPageTypeLog && p.Status != types.WikiPageStatusArchived { - batchCtx.SlugTitleMap[p.Slug] = p.Title + // Build the per-batch lazy fetchers. These replace the legacy + // pre-batch ListAllPages dump: instead of pulling ~100MB of rows + // up front (and walking them several more times during the batch), + // callers pay only for the slugs / knowledge ids they actually + // reach for. Cache hits keep repeat lookups within the batch free. + var ( + fetchMu sync.Mutex + slugTitleCache = make(map[string]string) // slug -> title; "" = known-missing + summaryKIDCache = make(map[string]string) // kid -> content; "" = known-missing + ) + + resolveSlugs := func(ctx context.Context, slugs []string) map[string]string { + // Filter to the slugs we don't already have cached. + fetchMu.Lock() + need := slugs[:0:0] + for _, slug := range slugs { + if _, ok := slugTitleCache[slug]; ok { + continue + } + need = append(need, slug) } - if p.PageType == types.WikiPageTypeSummary && p.Content != "" { - for _, ref := range p.SourceRefs { - kid := ref - if pipeIdx := strings.Index(ref, "|"); pipeIdx > 0 { - kid = ref[:pipeIdx] + fetchMu.Unlock() + + if len(need) > 0 { + pages, err := s.wikiService.ListBySlugs(ctx, payload.KnowledgeBaseID, need) + if err != nil { + logger.Warnf(ctx, "wiki ingest: ListBySlugs(%d slugs) failed: %v", len(need), err) + } + fetchMu.Lock() + for _, slug := range need { + if p, ok := pages[slug]; ok && p != nil { + if p.Status == types.WikiPageStatusArchived || + p.PageType == types.WikiPageTypeIndex || + p.PageType == types.WikiPageTypeLog { + // Treat archived / system pages as missing from the + // title-resolution map: cleanDeadLinks shouldn't link + // to them, and the log-feed slug-title fallback + // should degrade to slug-only display. + slugTitleCache[slug] = "" + continue + } + slugTitleCache[slug] = p.Title + } else { + slugTitleCache[slug] = "" } - batchCtx.SummaryContentByKnowledgeID[kid] = p.Content + } + fetchMu.Unlock() + } + + out := make(map[string]string, len(slugs)) + fetchMu.Lock() + for _, slug := range slugs { + if title := slugTitleCache[slug]; title != "" { + out[slug] = title } } + fetchMu.Unlock() + return out + } + + resolveSummaries := func(ctx context.Context, kids []string) map[string]string { + fetchMu.Lock() + need := kids[:0:0] + for _, kid := range kids { + if _, ok := summaryKIDCache[kid]; ok { + continue + } + need = append(need, kid) + } + fetchMu.Unlock() + + if len(need) > 0 { + contents, err := s.wikiService.ListSummariesByKnowledgeIDs(ctx, payload.KnowledgeBaseID, need) + if err != nil { + logger.Warnf(ctx, "wiki ingest: ListSummariesByKnowledgeIDs(%d kids) failed: %v", len(need), err) + } + fetchMu.Lock() + for _, kid := range need { + if c, ok := contents[kid]; ok && c != "" { + summaryKIDCache[kid] = c + } else { + summaryKIDCache[kid] = "" + } + } + fetchMu.Unlock() + } + + out := make(map[string]string, len(kids)) + fetchMu.Lock() + for _, kid := range kids { + if content := summaryKIDCache[kid]; content != "" { + out[kid] = content + } + } + fetchMu.Unlock() + return out + } + + batchCtx := &WikiBatchContext{ + SlugTitle: func(ctx context.Context, slug string) string { + m := resolveSlugs(ctx, []string{slug}) + return m[slug] + }, + SlugTitleMany: resolveSlugs, + SummaryContentByKnowledgeID: func(ctx context.Context, kid string) string { + m := resolveSummaries(ctx, []string{kid}) + return m[kid] + }, + ExtractionGranularity: granularity, } // 1. MAP PHASE (Parallel extraction and generation of updates) @@ -259,7 +367,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task var retractChangeDesc strings.Builder eg, mapCtx := errgroup.WithContext(ctx) - eg.SetLimit(10) // Map phase limit + eg.SetLimit(mapParallel) // Map phase limit (configurable via WikiConfig) for _, op := range pendingOps { op := op @@ -352,12 +460,12 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task } mapMu.Unlock() - // Reset the per-document failure counter on success so transient - // errors don't permanently exhaust the retry budget. - if s.redisClient != nil { - failKey := wikiFailCountKeyPrefix + payload.KnowledgeBaseID + ":" + op.KnowledgeID - s.redisClient.Del(mapCtx, failKey) - } + // No fail-count reset needed: a successful op is added + // to peekedIDs and gets DELETEd from task_pending_ops at + // trim time, so there is no stale fail_count column to + // scrub. Compare with the legacy Redis path, which kept + // a separate wiki:failcount:<...> key alive for 24h + // regardless of whether the original op had drained. } return nil }) @@ -366,7 +474,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task // 2. REDUCE PHASE (Parallel upserting grouped by Slug) egReduce, reduceCtx := errgroup.WithContext(ctx) - egReduce.SetLimit(10) // Reduce phase limit (LLM + DB concurrent connections) + egReduce.SetLimit(reduceParallel) // Reduce phase limit (LLM + DB concurrent connections, configurable) var reduceMu sync.Mutex var allPagesAffected []string @@ -415,7 +523,7 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task // those slugs into actual pages. Rewrite those dead links to plain // text so the summary doesn't contain unresolvable references. if len(failedAdditionSlugs) > 0 && len(docResults) > 0 { - s.sanitizeDeadSummaryLinks(ctx, payload.KnowledgeBaseID, docResults, failedAdditionSlugs) + s.sanitizeDeadSummaryLinks(ctx, payload.KnowledgeBaseID, docResults, failedAdditionSlugs, batchCtx) } totalPagesAffected = len(allPagesAffected) @@ -436,10 +544,10 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task if len(slugs) == 0 { return nil } + titles := batchCtx.SlugTitleMany(ctx, slugs) out := make([]types.WikiLogPageRef, 0, len(slugs)) for _, slug := range slugs { - title := batchCtx.SlugTitleMap[slug] - out = append(out, types.WikiLogPageRef{Slug: slug, Title: title}) + out = append(out, types.WikiLogPageRef{Slug: slug, Title: titles[slug]}) } return out } @@ -502,24 +610,68 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task // failed entity slugs); sanitizeDeadSummaryLinks above handles the // well-known summary case, and this pass is the safety net for the // long tail (cross-doc citations, prior batches' lingering refs). - if len(retractPagesAffected) > 0 || len(failedAdditionSlugs) > 0 || len(allPagesAffected) > 0 { + // Dead-link cleanup: scoped to this batch's affected pages so the + // pass scales with batch size, not with KB size. The lint + // AutoFix path takes care of long-tail cleanup across the whole + // KB out-of-band. + if len(allPagesAffected) > 0 { logger.Infof(ctx, "wiki ingest: cleaning dead links") - s.cleanDeadLinks(ctx, payload.KnowledgeBaseID) + s.cleanDeadLinks(ctx, payload.KnowledgeBaseID, allPagesAffected, batchCtx) } if len(allPagesAffected) > 0 { + // Build the freshRefs set: every (slug, title) pair this batch + // successfully wrote, minus any that landed in failedAdditionSlugs. + // These are the "newly-mentionable" pages — links to them will + // not have appeared in older content yet, so injectCrossLinks + // targets exactly the affected pages with this fresh ref set. + freshRefs := make([]linkRef, 0, len(docResults)*4) + for _, dr := range docResults { + if dr == nil { + continue + } + for _, p := range dr.Pages { + if p.Slug == "" || p.Title == "" { + continue + } + if _, bad := failedAdditionSlugs[p.Slug]; bad { + continue + } + freshRefs = append(freshRefs, linkRef{slug: p.Slug, matchText: p.Title}) + } + } + logger.Infof(ctx, "wiki ingest: injecting cross links") - s.injectCrossLinks(ctx, payload.KnowledgeBaseID, allPagesAffected) + s.injectCrossLinks(ctx, payload.KnowledgeBaseID, allPagesAffected, freshRefs, batchCtx) logger.Infof(ctx, "wiki ingest: publishing draft pages") s.publishDraftPages(ctx, payload.KnowledgeBaseID, allPagesAffected) } - s.trimPendingList(ctx, payload.KnowledgeBaseID, peekedCount) + // Build the trim set: rows that should be removed from + // task_pending_ops. We start from the full peekedIDs (every row we + // pulled, even ones de-duplicated by knowledge_id) and subtract + // any failed op's dbID — those need to stay in place so the + // requeueFailedOps path can decide between retry and dead-letter. + failedIDSet := make(map[int64]struct{}, len(failedOps)) + for _, op := range failedOps { + if op.dbID != 0 { + failedIDSet[op.dbID] = struct{}{} + } + } + trimIDs := make([]int64, 0, len(peekedIDs)) + for _, id := range peekedIDs { + if _, fail := failedIDSet[id]; fail { + continue + } + trimIDs = append(trimIDs, id) + } + s.trimPendingList(ctx, trimIDs) - // Re-enqueue failed ops so they get retried in the next follow-up batch. - // This must happen after trim: trim removes the consumed batch head, then - // requeue appends the failed items to the tail for a future attempt. + // Process failed ops: increment fail_count and dead-letter once + // the cap is hit. Must come AFTER trim so successful siblings are + // already gone from the queue — otherwise a follow-up batch could + // re-pick them up. if len(failedOps) > 0 { s.requeueFailedOps(ctx, payload, failedOps) } @@ -611,11 +763,11 @@ func (s *wikiIngestService) mapOneDocument( pass0Failed bool ) logger.Infof(ctx, "wiki ingest: pass 0 — extracting candidate slugs for %s", knowledgeID) - extractedEntities, extractedConcepts, slugItems, err = s.extractCandidateSlugs(ctx, chatModel, content, lang, oldPageSlugs, batchCtx) + extractedEntities, extractedConcepts, slugItems, err = s.extractCandidateSlugs(ctx, chatModel, payload.KnowledgeBaseID, content, lang, oldPageSlugs, batchCtx) if err != nil { logger.Warnf(ctx, "wiki ingest: pass 0 failed for %s (%v) — falling back to legacy extractor", knowledgeID, err) pass0Failed = true - extractedEntities, extractedConcepts, slugItems, err = s.extractEntitiesAndConceptsNoUpsert(ctx, chatModel, content, lang, oldPageSlugs, batchCtx) + extractedEntities, extractedConcepts, slugItems, err = s.extractEntitiesAndConceptsNoUpsert(ctx, chatModel, payload.KnowledgeBaseID, content, lang, oldPageSlugs, batchCtx) if err != nil { logger.Warnf(ctx, "wiki ingest: legacy fallback also failed for %s: %v", knowledgeID, err) return nil, nil, err @@ -830,10 +982,11 @@ func (s *wikiIngestService) mapOneDocument( // SummaryBody, so emitting an extra retract would just be // dead weight that the summary branch discards anyway. // - // priorContribution is the doc's LAST summary body snapshotted at the - // start of this batch (from allPages scan). Empty on first-ever ingest - // — in that case oldPageSlugs is also empty, so we never consult it. - priorContribution := batchCtx.SummaryContentByKnowledgeID[knowledgeID] + // priorContribution is the doc's LAST summary body, fetched lazily + // at this point (rather than pre-loaded into the batch context). + // Empty on first-ever ingest — in that case oldPageSlugs is also + // empty, so we never consult it. + priorContribution := batchCtx.SummaryContentByKnowledgeID(ctx, knowledgeID) newSlugSet := make(map[string]bool, len(extractedPages)) for _, ns := range extractedPages { @@ -889,6 +1042,7 @@ func (s *wikiIngestService) mapOneDocument( func (s *wikiIngestService) extractEntitiesAndConceptsNoUpsert( ctx context.Context, chatModel chat.Chat, + kbID string, content, lang string, oldPageSlugs map[string]bool, batchCtx *WikiBatchContext, @@ -929,8 +1083,13 @@ func (s *wikiIngestService) extractEntitiesAndConceptsNoUpsert( return nil, nil, nil, fmt.Errorf("parse combined extraction JSON: %w", err) } + // Dedup pre-filter is dispatched against the wiki page repo via + // pg_trgm (see deduplicateExtractedBatch). Until the trgm path + // lands the dedup pre-filter degrades to "no dedup" which is the + // safe default — the LLM merge call simply doesn't get a candidate + // list and the items pass through unchanged. result.Entities, result.Concepts = s.deduplicateExtractedBatch( - ctx, chatModel, result.Entities, result.Concepts, batchCtx.AllPages, + ctx, chatModel, kbID, result.Entities, result.Concepts, ) slugItems := make(map[string]extractedItem) @@ -1082,7 +1241,7 @@ func (s *wikiIngestService) reduceSlugUpdates( continue } - if content, ok := batchCtx.SummaryContentByKnowledgeID[refKnowledgeID]; ok { + if content := batchCtx.SummaryContentByKnowledgeID(ctx, refKnowledgeID); content != "" { fmt.Fprintf(&remainingSourcesContent, "\n%s\n\n%s\n\n\n\n", refTitle, content) } else { fmt.Fprintf(&remainingSourcesContent, "\n%s\n\n(summary not available)\n\n\n\n", refTitle) @@ -1157,8 +1316,9 @@ func (s *wikiIngestService) reduceSlugUpdates( } if len(additions) > 0 || len(retracts) > 0 { + titles := batchCtx.SlugTitleMany(ctx, []string(page.OutLinks)) for _, outSlug := range page.OutLinks { - if title, ok := batchCtx.SlugTitleMap[outSlug]; ok { + if title := titles[outSlug]; title != "" { fmt.Fprintf(&relatedSlugs, "- %s (%s)\n", outSlug, title) } } diff --git a/internal/application/service/wiki_ingest_cite.go b/internal/application/service/wiki_ingest_cite.go index c97fa1b7..7f6ec99e 100644 --- a/internal/application/service/wiki_ingest_cite.go +++ b/internal/application/service/wiki_ingest_cite.go @@ -71,6 +71,7 @@ type citationPipelineOutcome struct { func (s *wikiIngestService) extractCandidateSlugs( ctx context.Context, chatModel chat.Chat, + kbID string, content, lang string, oldPageSlugs map[string]bool, batchCtx *WikiBatchContext, @@ -111,7 +112,7 @@ func (s *wikiIngestService) extractCandidateSlugs( } result.Entities, result.Concepts = s.deduplicateExtractedBatch( - ctx, chatModel, result.Entities, result.Concepts, batchCtx.AllPages, + ctx, chatModel, kbID, result.Entities, result.Concepts, ) slugItems := make(map[string]extractedItem, len(result.Entities)+len(result.Concepts)) diff --git a/internal/application/service/wiki_lint.go b/internal/application/service/wiki_lint.go index b7012c9e..61628524 100644 --- a/internal/application/service/wiki_lint.go +++ b/internal/application/service/wiki_lint.go @@ -73,7 +73,24 @@ func NewWikiLintService( } } -// RunLint performs a comprehensive health check on a wiki knowledge base +// lintCursorBatch is the per-batch limit for the streaming page walk. +// Picked at 200 because wiki pages can carry multi-KB content blobs +// and 200 rows × ~20KB ≈ 4MB resident at a time, which is well within +// what we want to hold while running per-page checks. +const lintCursorBatch = 200 + +// RunLint performs a comprehensive health check on a wiki knowledge base. +// +// At 4w-document scale the legacy "load every page in one shot" +// approach was the dominant tail in this method (and intermittently +// caused OOM in production). We now walk the page set via +// ListPagesCursor in lintCursorBatch-sized windows, accumulating +// issues incrementally — memory stays bounded regardless of KB size. +// +// We also drop the GetGraph(Limit:0) call that the legacy path used +// to compute the live-slug set. ListAllSlugs is a one-column projection +// over the same predicate (kbID + status<>archived), so it gives the +// same answer at a fraction of the cost. func (s *WikiLintService) RunLint(ctx context.Context, kbID string) (*WikiLintReport, error) { // Validate KB kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, kbID) @@ -90,159 +107,170 @@ func (s *WikiLintService) RunLint(ctx context.Context, kbID string) (*WikiLintRe return nil, fmt.Errorf("get stats: %w", err) } - // Get graph for link analysis. Lint needs the FULL graph to detect - // orphans and broken links across every page, so we pass Limit=0 - // which the service treats as "no cap". - graph, err := s.wikiService.GetGraph(ctx, &types.WikiGraphRequest{ - KnowledgeBaseID: kbID, - Mode: types.WikiGraphModeOverview, - Limit: 0, - }) + // Compute the live-slug set from the cheap one-column projection. + // This replaces a full GetGraph call (which materialized every node + // + edge) with a single Pluck("slug") query. + liveSlugs, err := s.wikiService.ListAllSlugs(ctx, kbID) if err != nil { - return nil, fmt.Errorf("get graph: %w", err) + return nil, fmt.Errorf("list all slugs: %w", err) } - - // Get all pages for detailed analysis - resp, err := s.wikiService.ListPages(ctx, &types.WikiPageListRequest{ - KnowledgeBaseID: kbID, - PageSize: 500, - }) - if err != nil { - return nil, fmt.Errorf("list pages: %w", err) + slugSet := make(map[string]bool, len(liveSlugs)) + for _, slug := range liveSlugs { + slugSet[slug] = true } var issues []WikiLintIssue healthScore := 100 + knowledgeLive := make(map[string]bool) // kid -> exists; cached across pages - // Build slug set for link validation - slugSet := make(map[string]bool) - for _, node := range graph.Nodes { - slugSet[node.Slug] = true - } - - // Check 1: Orphan pages (no inbound links, excluding index/log) - for _, page := range resp.Pages { - if page.PageType == types.WikiPageTypeIndex || page.PageType == types.WikiPageTypeLog { - continue - } - if len(page.InLinks) == 0 { - issues = append(issues, WikiLintIssue{ - Type: LintIssueOrphanPage, - Severity: SeverityWarning, - PageSlug: page.Slug, - Description: fmt.Sprintf("Page '%s' has no inbound links — it's disconnected from the wiki", page.Title), - AutoFixable: false, - }) - } - } - - // Check 2: Broken links - for _, page := range resp.Pages { - for _, outLink := range page.OutLinks { - if !slugSet[outLink] { - issues = append(issues, WikiLintIssue{ - Type: LintIssueBrokenLink, - Severity: SeverityError, - PageSlug: page.Slug, - TargetSlug: outLink, - Description: fmt.Sprintf("Page '%s' links to [[%s]] which does not exist", page.Title, outLink), - AutoFixable: true, - }) - } - } - } - - // Check 3: Empty content - for _, page := range resp.Pages { - content := strings.TrimSpace(page.Content) - if len(content) < 50 { - issues = append(issues, WikiLintIssue{ - Type: LintIssueEmptyContent, - Severity: SeverityWarning, - PageSlug: page.Slug, - Description: fmt.Sprintf("Page '%s' has very little content (%d chars)", page.Title, len(content)), - AutoFixable: true, - }) - } - } - - // Check 4: Stale source refs — source_refs pointing at soft-deleted - // knowledge. This is the primary self-heal for the ingest/delete race - // condition: if a wiki_ingest task managed to slip past the in-flight - // guards and wrote a page for a knowledge that has since been deleted, - // the page becomes a dead-end (wiki_read_source_doc returns "knowledge - // not found"). Flag it so AutoFix can strip the ref, and delete the - // page if no live refs remain. - if s.knowledgeService != nil { - knowledgeLive := make(map[string]bool) // kid -> exists - for _, page := range resp.Pages { - // Skip wiki-intrinsic pages: index/log never carry SourceRefs - // anyway, and accidentally flagging them would risk AutoFix - // deleting system pages. - if page.PageType == types.WikiPageTypeIndex || page.PageType == types.WikiPageTypeLog { - continue - } - for _, ref := range page.SourceRefs { - kid := ref - if i := strings.Index(ref, "|"); i > 0 { - kid = ref[:i] - } - if kid == "" { - continue - } - live, seen := knowledgeLive[kid] - if !seen { - kn, err := s.knowledgeService.GetKnowledgeByIDOnly(ctx, kid) - live = err == nil && kn != nil - knowledgeLive[kid] = live - } - if !live { - issues = append(issues, WikiLintIssue{ - Type: LintIssueStaleRef, - Severity: SeverityError, - PageSlug: page.Slug, - TargetSlug: kid, - Description: fmt.Sprintf("Page '%s' references deleted knowledge %s", page.Title, kid), - AutoFixable: true, - }) - } - } - } - } - - // Check 5: Missing cross-references (entities mentioned in content but not linked) + // First-pass walk: orphan / broken-link / empty / stale-ref + // detection. Each check is independent of order; we accumulate + // issues per-batch and the cursor walk keeps memory bounded. + // + // We collect entity / concept titles in this pass too so the + // missing-cross-ref check (which is intrinsically O(N×M) in + // distinct entities × pages) doesn't need a second walk to find + // candidates. The check itself runs in a second walk because it + // needs the full entity-title set to compare against any page. entitySlugs := make(map[string]string) // slug -> title - for _, page := range resp.Pages { - if page.PageType == types.WikiPageTypeEntity || page.PageType == types.WikiPageTypeConcept { - entitySlugs[page.Slug] = page.Title + + cursor := "" + for { + pages, next, err := s.wikiService.ListPagesCursor(ctx, kbID, cursor, lintCursorBatch) + if err != nil { + return nil, fmt.Errorf("list pages cursor: %w", err) } - } - for _, page := range resp.Pages { - for slug, title := range entitySlugs { - if slug == page.Slug { - continue + if len(pages) == 0 { + break + } + for _, page := range pages { + // Track entity/concept titles for the second pass. + if page.PageType == types.WikiPageTypeEntity || page.PageType == types.WikiPageTypeConcept { + entitySlugs[page.Slug] = page.Title } - // Check if title is mentioned in content but not linked - if strings.Contains(strings.ToLower(page.Content), strings.ToLower(title)) { - linked := false - for _, l := range page.OutLinks { - if l == slug { - linked = true - break - } - } - if !linked { + + // Check 1: Orphan pages (no inbound links, excluding system pages). + if page.PageType != types.WikiPageTypeIndex && page.PageType != types.WikiPageTypeLog { + if len(page.InLinks) == 0 { issues = append(issues, WikiLintIssue{ - Type: LintIssueMissingCrossRef, - Severity: SeverityInfo, + Type: LintIssueOrphanPage, + Severity: SeverityWarning, PageSlug: page.Slug, - TargetSlug: slug, - Description: fmt.Sprintf("Page '%s' mentions '%s' but doesn't link to [[%s]]", page.Title, title, slug), + Description: fmt.Sprintf("Page '%s' has no inbound links — it's disconnected from the wiki", page.Title), AutoFixable: false, }) } } + + // Check 2: Broken links — outlinks pointing at slugs that + // don't exist in the live set. + for _, outLink := range page.OutLinks { + if !slugSet[outLink] { + issues = append(issues, WikiLintIssue{ + Type: LintIssueBrokenLink, + Severity: SeverityError, + PageSlug: page.Slug, + TargetSlug: outLink, + Description: fmt.Sprintf("Page '%s' links to [[%s]] which does not exist", page.Title, outLink), + AutoFixable: true, + }) + } + } + + // Check 3: Empty content. + content := strings.TrimSpace(page.Content) + if len(content) < 50 { + issues = append(issues, WikiLintIssue{ + Type: LintIssueEmptyContent, + Severity: SeverityWarning, + PageSlug: page.Slug, + Description: fmt.Sprintf("Page '%s' has very little content (%d chars)", page.Title, len(content)), + AutoFixable: true, + }) + } + + // Check 4: Stale source refs — source_refs pointing at + // soft-deleted knowledge. Cached knowledgeLive lookup keeps + // per-kid checks O(1) after the first batch encounters + // each id. + if s.knowledgeService != nil && + page.PageType != types.WikiPageTypeIndex && + page.PageType != types.WikiPageTypeLog { + for _, ref := range page.SourceRefs { + kid := ref + if i := strings.Index(ref, "|"); i > 0 { + kid = ref[:i] + } + if kid == "" { + continue + } + live, seen := knowledgeLive[kid] + if !seen { + kn, err := s.knowledgeService.GetKnowledgeByIDOnly(ctx, kid) + live = err == nil && kn != nil + knowledgeLive[kid] = live + } + if !live { + issues = append(issues, WikiLintIssue{ + Type: LintIssueStaleRef, + Severity: SeverityError, + PageSlug: page.Slug, + TargetSlug: kid, + Description: fmt.Sprintf("Page '%s' references deleted knowledge %s", page.Title, kid), + AutoFixable: true, + }) + } + } + } } + if next == "" { + break + } + cursor = next + } + + // Second-pass walk: missing-cross-ref check. This needs the full + // entitySlugs map (built in pass 1), so it has to be a separate + // pass — but it's still streaming. + cursor = "" + for { + pages, next, err := s.wikiService.ListPagesCursor(ctx, kbID, cursor, lintCursorBatch) + if err != nil { + return nil, fmt.Errorf("list pages cursor (pass 2): %w", err) + } + if len(pages) == 0 { + break + } + for _, page := range pages { + lowerContent := strings.ToLower(page.Content) + outLinkSet := make(map[string]struct{}, len(page.OutLinks)) + for _, l := range page.OutLinks { + outLinkSet[l] = struct{}{} + } + for slug, title := range entitySlugs { + if slug == page.Slug || title == "" { + continue + } + if !strings.Contains(lowerContent, strings.ToLower(title)) { + continue + } + if _, linked := outLinkSet[slug]; linked { + continue + } + issues = append(issues, WikiLintIssue{ + Type: LintIssueMissingCrossRef, + Severity: SeverityInfo, + PageSlug: page.Slug, + TargetSlug: slug, + Description: fmt.Sprintf("Page '%s' mentions '%s' but doesn't link to [[%s]]", page.Title, title, slug), + AutoFixable: false, + }) + } + } + if next == "" { + break + } + cursor = next } // Calculate health score diff --git a/internal/application/service/wiki_page.go b/internal/application/service/wiki_page.go index 09900280..de71a4f6 100644 --- a/internal/application/service/wiki_page.go +++ b/internal/application/service/wiki_page.go @@ -23,10 +23,11 @@ var wikiLinkRegex = regexp.MustCompile(`\[\[([^\]]+)\]\]`) // wikiPageService implements the WikiPageService interface type wikiPageService struct { - repo interfaces.WikiPageRepository - chunkRepo interfaces.ChunkRepository - kbService interfaces.KnowledgeBaseService - redisClient *redis.Client + repo interfaces.WikiPageRepository + chunkRepo interfaces.ChunkRepository + kbService interfaces.KnowledgeBaseService + taskPendingRepo interfaces.TaskPendingOpsRepository + redisClient *redis.Client } // NewWikiPageService creates a new wiki page service @@ -34,13 +35,15 @@ func NewWikiPageService( repo interfaces.WikiPageRepository, chunkRepo interfaces.ChunkRepository, kbService interfaces.KnowledgeBaseService, + taskPendingRepo interfaces.TaskPendingOpsRepository, redisClient *redis.Client, ) interfaces.WikiPageService { return &wikiPageService{ - repo: repo, - chunkRepo: chunkRepo, - kbService: kbService, - redisClient: redisClient, + repo: repo, + chunkRepo: chunkRepo, + kbService: kbService, + taskPendingRepo: taskPendingRepo, + redisClient: redisClient, } } @@ -667,8 +670,15 @@ func (s *wikiPageService) GetStats(ctx context.Context, kbID string) (*types.Wik var pendingTasks int64 var pendingIssues int64 var isActive bool + if s.taskPendingRepo != nil { + // Pending wiki ingest ops live in task_pending_ops keyed by + // (task_type="wiki:ingest", scope="knowledge_base", scope_id=kbID). + pendingTasks, _ = s.taskPendingRepo.PendingCount(ctx, wikiTaskType, wikiTaskScope, kbID) + } if s.redisClient != nil { - pendingTasks, _ = s.redisClient.LLen(ctx, "wiki:pending:"+kbID).Result() + // The "active batch in progress" flag is still a Redis-only + // short-lived signal (per-process lock with TTL renew); not + // worth migrating since it carries no durable state. activeFlag, _ := s.redisClient.Exists(ctx, "wiki:active:"+kbID).Result() isActive = activeFlag > 0 } @@ -746,6 +756,67 @@ func (s *wikiPageService) ListPagesBySourceRef(ctx context.Context, kbID string, return s.repo.ListBySourceRef(ctx, kbID, knowledgeID) } +// ListSlugsBySourceRef returns just the slugs of pages that cite the given +// knowledge id. Backed by the source_refs GIN index added in migration +// 000041 — the wiki ingest pipeline uses it as a cheap "before" snapshot +// when reconciling old vs new extraction sets. +func (s *wikiPageService) ListSlugsBySourceRef(ctx context.Context, kbID string, knowledgeID string) ([]string, error) { + return s.repo.ListSlugsBySourceRef(ctx, kbID, knowledgeID) +} + +// ListBySlugs is the lazy fetcher used by wiki ingest's batch context. +// Returns lightweight projections (no content / source_refs / chunk_refs) +// for the requested slugs, in a single IN query. Used in place of the +// pre-batch ListAllPages dump that historically pulled hundreds of MB +// for KBs in the tens of thousands of pages. +func (s *wikiPageService) ListBySlugs(ctx context.Context, kbID string, slugs []string) (map[string]*types.WikiPageLite, error) { + return s.repo.ListBySlugs(ctx, kbID, slugs) +} + +// ListSummariesByKnowledgeIDs is the lazy fetcher for the retract / +// reparse branches of reduceSlugUpdates. Returns the content of each +// surviving summary page keyed by its source knowledge id. +func (s *wikiPageService) ListSummariesByKnowledgeIDs(ctx context.Context, kbID string, kids []string) (map[string]string, error) { + return s.repo.ListSummariesByKnowledgeIDs(ctx, kbID, kids) +} + +// ExistsSlugs reports which of the given slugs are live (non-archived, +// non-deleted) in the KB. Used by cleanDeadLinks to validate out-link +// targets before stripping them. +func (s *wikiPageService) ExistsSlugs(ctx context.Context, kbID string, slugs []string) (map[string]bool, error) { + return s.repo.ExistsSlugs(ctx, kbID, slugs) +} + +// ListAllSlugs returns every non-archived slug in the KB. Used by lint +// to compute the live-slug set without paying for ListAll's full row +// materialization. +func (s *wikiPageService) ListAllSlugs(ctx context.Context, kbID string) ([]string, error) { + return s.repo.ListAllSlugs(ctx, kbID) +} + +// ListPagesCursor is the lint-side cursor pagination over wiki_pages. +func (s *wikiPageService) ListPagesCursor(ctx context.Context, kbID string, cursor string, limit int) ([]*types.WikiPage, string, error) { + return s.repo.ListPagesCursor(ctx, kbID, cursor, limit) +} + +// ListByTypeRecent caps the page count for first-time index intro +// generation so the LLM prompt stays bounded on large KBs. +func (s *wikiPageService) ListByTypeRecent(ctx context.Context, kbID string, pageType string, limit int) ([]types.WikiIndexEntry, error) { + return s.repo.ListByTypeRecent(ctx, kbID, pageType, limit) +} + +// FindSimilarPages performs a pg_trgm similarity search; used by the +// dedup pre-filter to surface candidate merge targets. +func (s *wikiPageService) FindSimilarPages(ctx context.Context, kbID string, query string, pageTypes []string, limit int) ([]*types.WikiPageLite, error) { + return s.repo.FindSimilarPages(ctx, kbID, query, pageTypes, limit) +} + +// CountByType is a service-layer pass-through over the repo. Used by +// the index intro path to frame the LLM prompt's "showing N of M" hint. +func (s *wikiPageService) CountByType(ctx context.Context, kbID string) (map[string]int64, error) { + return s.repo.CountByType(ctx, kbID) +} + // SearchPages performs full-text search over wiki pages func (s *wikiPageService) SearchPages(ctx context.Context, kbID string, query string, limit int) ([]*types.WikiPage, error) { return s.repo.Search(ctx, kbID, query, limit) diff --git a/internal/container/container.go b/internal/container/container.go index b34c6206..7aa24b0a 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -158,6 +158,8 @@ func BuildContainer(container *dig.Container) *dig.Container { must(container.Provide(repository.NewSyncLogRepository)) must(container.Provide(repository.NewWikiPageRepository)) must(container.Provide(repository.NewWikiLogEntryRepository)) + must(container.Provide(repository.NewTaskPendingOpsRepository)) + must(container.Provide(repository.NewTaskDeadLetterRepository)) // MCP manager for managing MCP client connections logger.Debugf(ctx, "[Container] Registering MCP manager...") diff --git a/internal/middleware/asynqdl/asynqdl.go b/internal/middleware/asynqdl/asynqdl.go new file mode 100644 index 00000000..c2276c8c --- /dev/null +++ b/internal/middleware/asynqdl/asynqdl.go @@ -0,0 +1,215 @@ +// Package asynqdl provides an asynq middleware that records every task +// whose retry budget is exhausted into the generic task_dead_letters table. +// +// The middleware is the catch-all observability path for asynq failures: +// without it, archived tasks live only inside Redis with the asynq +// inspector CLI as the sole readable surface. With it, operators can +// SQL-query failures by task type, scope, or tenant. +// +// The package is intentionally separate from /internal/middleware (which +// is gin-only) and from /internal/tracing/langfuse (which has its own +// mux.Use). Putting it in its own package keeps the import graph tidy: +// it depends on types/interfaces and asynq, nothing else inside the +// project. +package asynqdl + +import ( + "context" + "encoding/json" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/hibiken/asynq" +) + +// Middleware returns an asynq.MiddlewareFunc that records dead-lettered +// tasks. It is safe to install before any other asynq middleware: the +// inner handler's return value is forwarded verbatim, so every layer +// (langfuse tracing, business handlers) still sees the original +// error/result. +// +// Behaviour: +// +// - Tasks that succeed (handler returns nil) are passed through with +// zero overhead besides the function call. +// +// - Tasks that fail (handler returns non-nil) trigger an inspection of +// asynq's retry counters. We only record on the FINAL retry — +// i.e. when GetRetryCount(ctx) >= GetMaxRetry(ctx). Earlier failures +// are still reported up to asynq for backoff/retry but do not +// create dead-letter rows; otherwise we'd write one row per +// transient hiccup. +// +// - Insert is best-effort: a DB error is logged and swallowed. The +// middleware never alters the task error returned to the caller. +// +// This middleware should be installed once on the asynq mux, BEFORE +// other middlewares that may transform errors. See router/task.go. +func Middleware(repo interfaces.TaskDeadLetterRepository) asynq.MiddlewareFunc { + return func(next asynq.Handler) asynq.Handler { + return asynq.HandlerFunc(func(ctx context.Context, t *asynq.Task) error { + err := next.ProcessTask(ctx, t) + if err == nil { + return nil + } + if !isFinalAttempt(ctx) { + return err + } + if repo == nil { + return err + } + dl := buildDeadLetter(t, err) + if dl == nil { + return err + } + // context.Background() so a cancelled task ctx (e.g. server + // shutdown signaling cancellation right as a task fails) + // doesn't also drop the dead-letter record. + if insertErr := repo.Insert(context.Background(), dl); insertErr != nil { + logger.Warnf(ctx, + "asynq dead-letter: failed to record %s task: %v (original task error: %v)", + t.Type(), insertErr, err, + ) + } + return err + }) + } +} + +// isFinalAttempt reports whether the just-finished handler invocation +// was the LAST one asynq will run before archiving the task. Asynq +// increments retry_count *after* a successful retry, so on the final +// attempt retry_count == max_retry. Both helpers can fail (returning +// 0, false) when the middleware runs outside an asynq worker context; +// in that case we conservatively report "final" so test setups that +// invoke the middleware directly still exercise the insert path. +func isFinalAttempt(ctx context.Context) bool { + retried, retriedOK := asynq.GetRetryCount(ctx) + maxRetry, maxOK := asynq.GetMaxRetry(ctx) + if !retriedOK || !maxOK { + return true + } + return retried >= maxRetry +} + +// buildDeadLetter constructs a TaskDeadLetter from the task and the +// final error. The payload extraction is deliberately schema-agnostic: +// every project payload is a JSON object containing some subset of the +// well-known fields {tenant_id, knowledge_base_id, kb_id, knowledge_id, +// task_id}. We probe for those fields with a tolerant struct so a new +// payload shape doesn't need to know about this middleware. +// +// Returns nil only if the task itself is nil — every other failure mode +// (unparseable payload, unknown task type) still produces a row, with +// scope="unknown" so operators can find it. +func buildDeadLetter(t *asynq.Task, taskErr error) *types.TaskDeadLetter { + if t == nil { + return nil + } + probe := payloadProbe{} + _ = json.Unmarshal(t.Payload(), &probe) // best-effort + + scope, scopeID := inferScope(probe) + relatedID := probe.KnowledgeID + + retryCount := 0 + // asynq doesn't expose the captured retry on the Task itself, so we + // fall back to the probe's RetryCount (often 0 — most payloads don't + // carry it). The middleware caller has already verified that we're + // at the final attempt; the absolute number is informational. + + // asynq stores the payload as raw bytes; preserve it verbatim so a + // future requeue can reuse it directly. + rawPayload := t.Payload() + if len(rawPayload) == 0 { + rawPayload = []byte("{}") + } + + return &types.TaskDeadLetter{ + TenantID: probe.TenantID, + TaskType: t.Type(), + Scope: scope, + ScopeID: scopeID, + RelatedID: relatedID, + Payload: json.RawMessage(rawPayload), + LastError: truncateError(taskErr.Error(), 8192), + FailCount: retryCount, + } +} + +// payloadProbe is a best-effort decoder that extracts the common +// identifier fields from any project payload. Every payload type is a +// JSON object with some subset of these keys; missing keys decode to +// zero values, which we then fall back through. +type payloadProbe struct { + TenantID uint64 `json:"tenant_id,omitempty"` + KnowledgeBaseID string `json:"knowledge_base_id,omitempty"` + KBID string `json:"kb_id,omitempty"` // FAQImportPayload uses this name + KnowledgeID string `json:"knowledge_id,omitempty"` + SourceID string `json:"source_id,omitempty"` // KBClonePayload + TargetID string `json:"target_id,omitempty"` // KBClonePayload + SourceKBID string `json:"source_kb_id,omitempty"` // KnowledgeMovePayload + TargetKBID string `json:"target_kb_id,omitempty"` // KnowledgeMovePayload +} + +// inferScope picks the most-specific scope tuple available in the +// probe. The order reflects the conventional "blast radius" of a +// task: knowledge_base is broader than knowledge, etc. Wiki ingest is +// scope="knowledge_base"; chunk extract is scope="knowledge"; and so on. +// +// Falls back to scope="unknown" with empty scope_id if the payload +// carries no recognizable identifier — better than silently dropping +// the row. +func inferScope(p payloadProbe) (string, string) { + switch { + case p.KnowledgeBaseID != "": + return types.TaskScopeKnowledgeBase, p.KnowledgeBaseID + case p.KBID != "": + return types.TaskScopeKnowledgeBase, p.KBID + case p.SourceKBID != "": + // Knowledge move spans two KBs; the source is the better + // blast-radius indicator (it's where the work was being read + // from when the failure occurred). + return types.TaskScopeKnowledgeBase, p.SourceKBID + case p.SourceID != "": + // KB clone: same reasoning — record against the source. + return types.TaskScopeKnowledgeBase, p.SourceID + case p.KnowledgeID != "": + return types.TaskScopeKnowledge, p.KnowledgeID + case p.TenantID != 0: + return types.TaskScopeTenant, formatUint(p.TenantID) + default: + return types.TaskScopeUnknown, "" + } +} + +// truncateError prevents a single runaway stack trace from inflating +// the dead-letter table beyond the disk capacity reasonably allotted +// to it. +func truncateError(s string, max int) string { + if max <= 0 || len(s) <= max { + return s + } + const suffix = "...(truncated)" + if max <= len(suffix) { + return s[:max] + } + return s[:max-len(suffix)] + suffix +} + +// formatUint inlines strconv.FormatUint without importing strconv just +// for one call (keeps the package import set minimal). +func formatUint(v uint64) string { + if v == 0 { + return "0" + } + var buf [20]byte + i := len(buf) + for v > 0 { + i-- + buf[i] = byte('0' + v%10) + v /= 10 + } + return string(buf[i:]) +} diff --git a/internal/middleware/asynqdl/asynqdl_test.go b/internal/middleware/asynqdl/asynqdl_test.go new file mode 100644 index 00000000..ad772d64 --- /dev/null +++ b/internal/middleware/asynqdl/asynqdl_test.go @@ -0,0 +1,228 @@ +package asynqdl + +import ( + "context" + "encoding/json" + "errors" + "sync" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/hibiken/asynq" +) + +// fakeRepo records calls to Insert in memory so tests can assert how +// many dead-letter rows would have been written. It implements only +// the subset of TaskDeadLetterRepository the middleware uses; the +// Lister methods are unused in the middleware path. +type fakeRepo struct { + mu sync.Mutex + rows []*types.TaskDeadLetter + fail error +} + +func (f *fakeRepo) Insert(ctx context.Context, dl *types.TaskDeadLetter) error { + f.mu.Lock() + defer f.mu.Unlock() + if f.fail != nil { + return f.fail + } + cp := *dl // defensive copy: callers may reuse the struct + f.rows = append(f.rows, &cp) + return nil +} + +func (f *fakeRepo) ListByScope(context.Context, string, string, string, int) ([]*types.TaskDeadLetter, string, error) { + return nil, "", nil +} + +func (f *fakeRepo) ListByTaskType(context.Context, string, string, int) ([]*types.TaskDeadLetter, string, error) { + return nil, "", nil +} + +func (f *fakeRepo) DeleteByID(context.Context, int64) error { return nil } + +// captureRow returns the i'th row safely under the lock. +func (f *fakeRepo) captureRow(i int) *types.TaskDeadLetter { + f.mu.Lock() + defer f.mu.Unlock() + if i >= len(f.rows) { + return nil + } + return f.rows[i] +} + +func (f *fakeRepo) rowCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.rows) +} + +// runMiddleware wires up the middleware around a handler that always +// returns handlerErr. We invoke it directly (no asynq server) so retry +// counters come from the helper-injected ctx values; isFinalAttempt +// treats a missing pair as "final" for testability. +func runMiddleware(repo *fakeRepo, taskType string, payload []byte, handlerErr error) error { + mw := Middleware(repo) + wrapped := mw(asynq.HandlerFunc(func(_ context.Context, _ *asynq.Task) error { + return handlerErr + })) + t := asynq.NewTask(taskType, payload) + return wrapped.ProcessTask(context.Background(), t) +} + +func TestMiddleware_NilHandlerErr_NoInsert(t *testing.T) { + repo := &fakeRepo{} + if err := runMiddleware(repo, "any:task", []byte(`{"tenant_id":1}`), nil); err != nil { + t.Fatalf("expected nil error, got %v", err) + } + if got := repo.rowCount(); got != 0 { + t.Fatalf("expected zero inserts on success, got %d", got) + } +} + +func TestMiddleware_FailureWithoutAsynqCtx_RecordsRow(t *testing.T) { + repo := &fakeRepo{} + payload, _ := json.Marshal(map[string]any{ + "tenant_id": 42, + "knowledge_base_id": "kb-abc", + "knowledge_id": "k-1", + }) + wantErr := errors.New("boom") + gotErr := runMiddleware(repo, "wiki:ingest", payload, wantErr) + if !errors.Is(gotErr, wantErr) { + t.Fatalf("expected the original task error to propagate, got %v", gotErr) + } + if got := repo.rowCount(); got != 1 { + t.Fatalf("expected exactly one insert, got %d", got) + } + row := repo.captureRow(0) + if row.TaskType != "wiki:ingest" { + t.Errorf("task_type: got %q", row.TaskType) + } + if row.TenantID != 42 { + t.Errorf("tenant_id: got %d", row.TenantID) + } + if row.Scope != types.TaskScopeKnowledgeBase || row.ScopeID != "kb-abc" { + t.Errorf("scope tuple: got (%q, %q)", row.Scope, row.ScopeID) + } + if row.RelatedID != "k-1" { + t.Errorf("related_id: got %q", row.RelatedID) + } + if row.LastError != "boom" { + t.Errorf("last_error: got %q", row.LastError) + } +} + +func TestMiddleware_UnknownPayload_StillRecords(t *testing.T) { + repo := &fakeRepo{} + // Payload that doesn't carry any of the well-known identifier keys. + payload := []byte(`{"some_unrelated":"value"}`) + _ = runMiddleware(repo, "future:weird", payload, errors.New("nope")) + if got := repo.rowCount(); got != 1 { + t.Fatalf("expected one insert for unknown payload, got %d", got) + } + row := repo.captureRow(0) + if row.Scope != types.TaskScopeUnknown { + t.Errorf("expected scope=unknown for unparseable payload, got %q", row.Scope) + } + if row.ScopeID != "" { + t.Errorf("expected empty scope_id for unknown scope, got %q", row.ScopeID) + } + if string(row.Payload) != string(payload) { + t.Errorf("expected payload preserved verbatim, got %s", string(row.Payload)) + } +} + +func TestMiddleware_NilPayload_DefaultsToEmptyObject(t *testing.T) { + repo := &fakeRepo{} + _ = runMiddleware(repo, "any:task", nil, errors.New("err")) + row := repo.captureRow(0) + if row == nil { + t.Fatal("expected a row") + } + if string(row.Payload) != "{}" { + t.Errorf("expected payload normalized to {}, got %s", string(row.Payload)) + } +} + +func TestMiddleware_LongErrorTruncated(t *testing.T) { + repo := &fakeRepo{} + long := make([]byte, 16384) + for i := range long { + long[i] = 'x' + } + _ = runMiddleware(repo, "any", []byte("{}"), errors.New(string(long))) + row := repo.captureRow(0) + if got := len(row.LastError); got > 8192 { + t.Errorf("expected last_error truncated to <=8192 bytes, got %d", got) + } + const suffix = "...(truncated)" + if got := row.LastError[len(row.LastError)-len(suffix):]; got != suffix { + t.Errorf("expected truncation suffix %q, got %q", suffix, got) + } +} + +func TestMiddleware_RepoFailure_DoesNotMaskTaskError(t *testing.T) { + repo := &fakeRepo{fail: errors.New("db down")} + wantErr := errors.New("task boom") + gotErr := runMiddleware(repo, "any", []byte(`{"tenant_id":1}`), wantErr) + if !errors.Is(gotErr, wantErr) { + t.Fatalf("repo failure must not mask task error; got %v", gotErr) + } +} + +func TestInferScope_Priority(t *testing.T) { + tests := []struct { + name string + probe payloadProbe + wantScope string + wantID string + }{ + { + name: "knowledge_base_id wins over knowledge_id", + probe: payloadProbe{KnowledgeBaseID: "kb1", KnowledgeID: "k1"}, + wantScope: types.TaskScopeKnowledgeBase, + wantID: "kb1", + }, + { + name: "kb_id alias resolves to knowledge_base scope", + probe: payloadProbe{KBID: "kb-faq"}, + wantScope: types.TaskScopeKnowledgeBase, + wantID: "kb-faq", + }, + { + name: "source_kb_id from KnowledgeMovePayload", + probe: payloadProbe{SourceKBID: "src-kb", TargetKBID: "tgt-kb"}, + wantScope: types.TaskScopeKnowledgeBase, + wantID: "src-kb", + }, + { + name: "knowledge_id only", + probe: payloadProbe{KnowledgeID: "k-only"}, + wantScope: types.TaskScopeKnowledge, + wantID: "k-only", + }, + { + name: "tenant_id fallback", + probe: payloadProbe{TenantID: 7}, + wantScope: types.TaskScopeTenant, + wantID: "7", + }, + { + name: "no identifiers — unknown scope", + probe: payloadProbe{}, + wantScope: types.TaskScopeUnknown, + wantID: "", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + gotScope, gotID := inferScope(tc.probe) + if gotScope != tc.wantScope || gotID != tc.wantID { + t.Errorf("inferScope: got (%q, %q), want (%q, %q)", + gotScope, gotID, tc.wantScope, tc.wantID) + } + }) + } +} diff --git a/internal/router/task.go b/internal/router/task.go index ca2dbb58..0a4fc606 100644 --- a/internal/router/task.go +++ b/internal/router/task.go @@ -8,6 +8,7 @@ import ( "time" "github.com/Tencent/WeKnora/internal/application/service" + "github.com/Tencent/WeKnora/internal/middleware/asynqdl" "github.com/Tencent/WeKnora/internal/tracing/langfuse" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" @@ -28,6 +29,7 @@ type AsynqTaskParams struct { ImageMultimodal interfaces.TaskHandler `name:"imageMultimodal"` KnowledgePostProcess interfaces.TaskHandler `name:"knowledgePostProcess"` WikiIngest interfaces.TaskHandler `name:"wikiIngest"` + DeadLetterRepo interfaces.TaskDeadLetterRepository } func getAsynqRedisClientOpt() *asynq.RedisClientOpt { @@ -101,6 +103,15 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux { // Create a new mux and register all handlers mux := asynq.NewServeMux() + // Install the dead-letter middleware FIRST so it sees the raw error + // returned by the handler, before any other middleware that might + // transform it. The middleware records one task_dead_letters row per + // task that exhausts its retry budget — operators can then SQL-query + // failures by task type, scope, or tenant without scraping logs. + // Best-effort: a DB failure is logged and swallowed; the original task + // error always propagates upstream to asynq for retry/archival. + mux.Use(asynqdl.Middleware(params.DeadLetterRepo)) + // Install Langfuse middleware BEFORE handler registration so every task // type is automatically wrapped. When Langfuse is disabled the middleware // is a pass-through; when enabled it resumes the upstream HTTP trace (if diff --git a/internal/types/interfaces/task_queue.go b/internal/types/interfaces/task_queue.go new file mode 100644 index 00000000..1ffea0f9 --- /dev/null +++ b/internal/types/interfaces/task_queue.go @@ -0,0 +1,86 @@ +package interfaces + +import ( + "context" + + "github.com/Tencent/WeKnora/internal/types" +) + +// TaskPendingOpsRepository persists rows for the generic task pending +// queue (`task_pending_ops`). The queue is the durable replacement for +// the Redis-list-backed wiki:pending: queue. It is intentionally +// stateless about consumer semantics: the (TaskType, Scope, ScopeID) +// tuple is the only routing primitive the repository understands; +// deduplication, batching, and retry policy live in the consumer. +// +// Concurrency model in this revision: PeekBatch does NOT take row +// locks. Consumers are expected to enforce per-(scope_id) serialization +// out-of-band (wiki ingest does this via Redis SetNX wiki:active:). +// The reserved `claimed_at` column lets a future revision adopt +// pessimistic locking without a schema change. +type TaskPendingOpsRepository interface { + // Enqueue inserts a single op. The caller fills in TenantID, TaskType, + // Scope, ScopeID, Op, DedupKey, Payload; ID, FailCount, EnqueuedAt + // are server-side defaults. + Enqueue(ctx context.Context, op *types.TaskPendingOp) error + + // PeekBatch returns up to `limit` rows for the given queue tuple, + // ordered by id ASC (FIFO within the queue). Rows are NOT removed — + // callers must DeleteByIDs once the ops have been processed (or + // IncrFailCount and leave them for the next pass). + PeekBatch(ctx context.Context, taskType, scope, scopeID string, limit int) ([]*types.TaskPendingOp, error) + + // DeleteByIDs removes the given rows. No-op for empty input. Used + // to consume a successfully-processed batch, and to drop ops that + // have been moved to task_dead_letters. + DeleteByIDs(ctx context.Context, ids []int64) error + + // IncrFailCount increments fail_count for one row and returns the + // new value. Returns (0, nil) if the row does not exist (race with + // DeleteByIDs is benign). + IncrFailCount(ctx context.Context, id int64) (int, error) + + // PendingCount returns the number of rows currently queued for the + // given tuple. Cheap (covered by idx_task_pending_ops_scope) and + // used by the wiki ingest follow-up scheduler. + PendingCount(ctx context.Context, taskType, scope, scopeID string) (int64, error) + + // DeleteByDedupKey removes rows for the tuple whose DedupKey + // matches. If `op` is non-empty, only rows with that exact op are + // removed (this lets wiki ingest scrub queued "ingest" ops while + // preserving "retract" ops for the same knowledge — retract is + // still needed to clean up wiki pages after the source doc is + // deleted). If `op` is empty, every matching row is removed + // regardless of op. + DeleteByDedupKey(ctx context.Context, taskType, scope, scopeID, dedupKey, op string) error +} + +// TaskDeadLetterRepository persists rows for the generic task dead-letter +// archive (`task_dead_letters`). Two writers exist: the asynq +// dead-letter middleware (one row per archived asynq task), and the +// service-level retry handlers (one row per in-batch op that exhausted +// its consumer-defined retry budget — wiki ingest is the current case). +// +// Reads are operator-driven: list by scope to triage a single KB, +// list by task_type for cross-KB symptom hunting. No TTL. +type TaskDeadLetterRepository interface { + // Insert records one dead letter. Best-effort caller: the asynq + // middleware ignores the error so a failed insert never masks the + // underlying task error. + Insert(ctx context.Context, dl *types.TaskDeadLetter) error + + // ListByScope returns dead letters for the given scope tuple, + // newest-first, paginated by failed-id cursor. `cursor` is the + // stringified id of the oldest entry from the previous page; "" = + // from the newest. Empty nextCursor = end of stream. `limit` is + // clamped to [1, 200]. + ListByScope(ctx context.Context, scope, scopeID, cursor string, limit int) ([]*types.TaskDeadLetter, string, error) + + // ListByTaskType returns dead letters for the given task_type, + // newest-first, with the same cursor semantics as ListByScope. + ListByTaskType(ctx context.Context, taskType, cursor string, limit int) ([]*types.TaskDeadLetter, string, error) + + // DeleteByID drops a single dead letter (e.g. after operators have + // requeued the task manually). + DeleteByID(ctx context.Context, id int64) error +} diff --git a/internal/types/interfaces/wiki_page.go b/internal/types/interfaces/wiki_page.go index 2229e6c3..170459db 100644 --- a/internal/types/interfaces/wiki_page.go +++ b/internal/types/interfaces/wiki_page.go @@ -116,6 +116,53 @@ type WikiPageService interface { // than relying on a caller-provided stale snapshot). ListPagesBySourceRef(ctx context.Context, kbID string, knowledgeID string) ([]*types.WikiPage, error) + // ListSlugsBySourceRef returns just the slugs of pages that cite the + // given knowledge id. Cheaper than ListPagesBySourceRef when callers + // only need the slug set (e.g. wiki ingest's "before" snapshot). + ListSlugsBySourceRef(ctx context.Context, kbID string, knowledgeID string) ([]string, error) + + // ListBySlugs is the lazy fetcher used by the wiki ingest batch + // context: returns lightweight projections (no content / source_refs) + // for the requested slugs in a single IN query. Used in place of + // the pre-batch ListAllPages dump. + ListBySlugs(ctx context.Context, kbID string, slugs []string) (map[string]*types.WikiPageLite, error) + + // ListSummariesByKnowledgeIDs returns summary-page content keyed by + // the knowledge id that authored it. Used by the retract / reparse + // branches of reduceSlugUpdates for "what was this doc's + // contribution?" framing. + ListSummariesByKnowledgeIDs(ctx context.Context, kbID string, kids []string) (map[string]string, error) + + // ExistsSlugs reports which of the given slugs are live (non-archived). + // Used by cleanDeadLinks to validate out-link targets cheaply. + ExistsSlugs(ctx context.Context, kbID string, slugs []string) (map[string]bool, error) + + // ListAllSlugs returns every non-archived slug in the KB. Used by + // lint to compute the live-slug set for broken-link detection. + ListAllSlugs(ctx context.Context, kbID string) ([]string, error) + + // ListPagesCursor walks the KB in id-asc order with an opaque + // cursor. Used by lint to stream the page set without holding it + // all in memory at once. + ListPagesCursor(ctx context.Context, kbID string, cursor string, limit int) ([]*types.WikiPage, string, error) + + // ListByTypeRecent returns the most-recently-updated pages of the + // given type, projected to slug/title/summary, capped at `limit`. + // Used by rebuildIndexPage's first-time generation path. + ListByTypeRecent(ctx context.Context, kbID string, pageType string, limit int) ([]types.WikiIndexEntry, error) + + // FindSimilarPages performs a pg_trgm similarity search over + // page titles. Used by the dedup pre-filter to surface candidate + // merge targets server-side. + FindSimilarPages(ctx context.Context, kbID string, query string, pageTypes []string, limit int) ([]*types.WikiPageLite, error) + + // CountByType returns page counts grouped by type for a knowledge + // base. Re-exposed at the service layer so the index intro + // generation path can frame the LLM prompt with "showing N of M" + // metadata when the recent-N projection doesn't cover the full + // summary set. + CountByType(ctx context.Context, kbID string) (map[string]int64, error) + // SearchPages performs full-text search over wiki pages. SearchPages(ctx context.Context, kbID string, query string, limit int) ([]*types.WikiPage, error) @@ -174,6 +221,49 @@ type WikiPageRepository interface { // ListBySourceRef retrieves all wiki pages that reference a given source knowledge ID. ListBySourceRef(ctx context.Context, kbID string, sourceKnowledgeID string) ([]*types.WikiPage, error) + // ListSlugsBySourceRef returns just the slugs of pages that reference + // the given knowledge id. Same predicate as ListBySourceRef but + // projected to a single column for the wiki ingest pipeline's + // "before" snapshot path. + ListSlugsBySourceRef(ctx context.Context, kbID string, sourceKnowledgeID string) ([]string, error) + + // ListBySlugs returns lightweight projections (slug/title/page_type/ + // status/aliases/out_links) for the given slugs in one IN query. + // Used by the lazy fetcher path in wiki ingest's Map/Reduce phases. + ListBySlugs(ctx context.Context, kbID string, slugs []string) (map[string]*types.WikiPageLite, error) + + // ListSummariesByKnowledgeIDs returns summary-page content keyed by + // the knowledge id that authored it. Used by the retract branch of + // reduceSlugUpdates for "what was this doc's contribution?" framing. + ListSummariesByKnowledgeIDs(ctx context.Context, kbID string, kids []string) (map[string]string, error) + + // ExistsSlugs reports which of the given slugs are live (non- + // archived). Used by cleanDeadLinks to validate out-link targets + // without loading the referenced pages' content. + ExistsSlugs(ctx context.Context, kbID string, slugs []string) (map[string]bool, error) + + // ListAllSlugs returns every non-archived slug in the KB. Used by + // lint to compute the live-slug set without paying for ListAll's + // full row materialization. + ListAllSlugs(ctx context.Context, kbID string) ([]string, error) + + // ListPagesCursor returns up to `limit` pages ordered by id ASC, + // paginated by an opaque numeric cursor. Used by lint to stream + // the KB without loading every page at once. + ListPagesCursor(ctx context.Context, kbID string, cursor string, limit int) ([]*types.WikiPage, string, error) + + // ListByTypeRecent returns the most-recently-updated pages of the + // given type, projected to slug/title/summary, capped at `limit`. + // Used by rebuildIndexPage's first-time generation path so the + // LLM prompt size is bounded on large KBs. + ListByTypeRecent(ctx context.Context, kbID string, pageType string, limit int) ([]types.WikiIndexEntry, error) + + // FindSimilarPages returns the top-k pages whose lowercase title + // is most similar to the query under pg_trgm. `pageTypes` empty + // defaults to entity+concept. Used by the dedup pre-filter to + // surface candidate merge targets server-side. + FindSimilarPages(ctx context.Context, kbID string, query string, pageTypes []string, limit int) ([]*types.WikiPageLite, error) + // ListAll retrieves all wiki pages in a knowledge base (for link rebuilding, graph generation). ListAll(ctx context.Context, kbID string) ([]*types.WikiPage, error) diff --git a/internal/types/task_dead_letter.go b/internal/types/task_dead_letter.go new file mode 100644 index 00000000..3c863729 --- /dev/null +++ b/internal/types/task_dead_letter.go @@ -0,0 +1,70 @@ +package types + +import ( + "encoding/json" + "time" +) + +// TaskDeadLetter is one row in the `task_dead_letters` archive: a +// permanent record of a task whose retry budget was exhausted. The table +// is populated from two distinct paths: +// +// 1. The asynq dead-letter middleware +// (internal/middleware/asynq_dead_letter.go) inserts a row whenever +// an asynq task's retry count equals its MaxRetry on the way out. +// This is the catch-all that covers every registered task type +// uniformly without per-handler instrumentation. +// +// 2. Service-level retry handlers insert directly when an in-batch +// retry counter exceeds the service-defined cap. Wiki ingest is the +// primary example: per-document ops accumulate fail_count in +// `task_pending_ops`, and once the count exceeds wikiMaxFailRetries +// the op is moved here instead of being requeued. +// +// Operators query by (Scope, ScopeID) — e.g. "all dead letters for KB +// abc" — or by TaskType — e.g. "all summary:generation failures in the +// last 24h". The table has no TTL; rows are kept until manually pruned. +type TaskDeadLetter struct { + // Auto-increment row id. + ID int64 `json:"id" gorm:"primaryKey;autoIncrement"` + // Tenant scope mirrored from the original task payload (best-effort: + // if the middleware can't decode the payload, this falls back to 0). + TenantID uint64 `json:"tenant_id" gorm:"index"` + // Task identifier (e.g. "wiki:ingest", "summary:generation"). + TaskType string `json:"task_type" gorm:"type:varchar(64)"` + // Logical scope, mirrors task_pending_ops.scope. Falls back to + // "unknown" when the middleware doesn't recognize the task type's + // payload. + Scope string `json:"scope" gorm:"type:varchar(32)"` + // Identifier within scope, e.g. kbID for scope="knowledge_base". + ScopeID string `json:"scope_id" gorm:"type:varchar(64)"` + // Optional secondary identifier. Wiki ingest uses this for the + // knowledge_id so per-document failures cluster around the source + // document; other consumers can leave it empty. + RelatedID string `json:"related_id" gorm:"type:varchar(64);default:''"` + // Raw task payload at the time of failure. For asynq tasks this is + // the verbatim asynq.Task.Payload(); for service-level dead letters + // the consumer chooses what to record. + Payload json.RawMessage `json:"payload" gorm:"type:jsonb"` + // String form of the final error. Long stack traces are kept verbatim. + LastError string `json:"last_error" gorm:"type:text;default:''"` + // Total attempt count when the dead-letter record was written. + FailCount int `json:"fail_count"` + // Server-side write time. + FailedAt time.Time `json:"failed_at"` +} + +// TableName binds TaskDeadLetter to the `task_dead_letters` table. +func (TaskDeadLetter) TableName() string { + return "task_dead_letters" +} + +// Standard scope literals. Repositories accept any string, but consumers +// should prefer these so SQL queries (run from the ops console) don't +// have to guess at variations. +const ( + TaskScopeKnowledgeBase = "knowledge_base" + TaskScopeKnowledge = "knowledge" + TaskScopeTenant = "tenant" + TaskScopeUnknown = "unknown" +) diff --git a/internal/types/task_pending_op.go b/internal/types/task_pending_op.go new file mode 100644 index 00000000..8e892353 --- /dev/null +++ b/internal/types/task_pending_op.go @@ -0,0 +1,63 @@ +package types + +import ( + "encoding/json" + "time" +) + +// TaskPendingOp is one entry in the generic task pending queue +// (`task_pending_ops`). The queue is the durable replacement for the +// Redis-list-backed `wiki:pending:` queue: rows survive restarts +// and are not subject to TTL eviction. +// +// The (TaskType, Scope, ScopeID) tuple is the queue identity. A consumer +// pulls a batch with PeekBatch on that tuple, deduplicates by DedupKey +// service-side if it cares, processes the ops, then DeleteByIDs the +// consumed rows. FailCount is incremented per-row by IncrFailCount and +// the consumer dead-letters the row once the count exceeds a service- +// defined cap. +type TaskPendingOp struct { + // Auto-increment row id. Used by PeekBatch ordering and by + // DeleteByIDs / IncrFailCount as the row key. + ID int64 `json:"id" gorm:"primaryKey;autoIncrement"` + // Tenant scope mirrored from the enclosing object so per-tenant + // retention / quota queries don't have to join. + TenantID uint64 `json:"tenant_id" gorm:"index"` + // Free-form task identifier (e.g. "wiki:ingest"). Should match an + // asynq task type when applicable, but the queue itself doesn't + // enforce that — it's just a string. + TaskType string `json:"task_type" gorm:"type:varchar(64)"` + // Logical scope, e.g. "knowledge_base" / "knowledge" / "tenant". + // Read together with ScopeID. + Scope string `json:"scope" gorm:"type:varchar(32)"` + // Identifier within the scope (e.g. kbID for scope="knowledge_base"). + ScopeID string `json:"scope_id" gorm:"type:varchar(64)"` + // Operation kind. Service-defined: e.g. "ingest" / "retract" for + // wiki, but other consumers can use whatever vocabulary they like. + Op string `json:"op" gorm:"type:varchar(32)"` + // Optional service-defined deduplication key. The consumer is + // responsible for collapsing equivalent ops within a peeked batch + // (the queue itself does NOT enforce uniqueness — multiple rows with + // the same DedupKey can coexist; the consumer chooses which wins). + DedupKey string `json:"dedup_key" gorm:"type:varchar(128);default:''"` + // JSON-serialized op payload. Schema is consumer-defined; the queue + // stores it verbatim. Use json.RawMessage to avoid double-decode. + Payload json.RawMessage `json:"payload" gorm:"type:jsonb;default:'{}'"` + // In-batch retry counter. Reset to 0 on successful consume. + FailCount int `json:"fail_count" gorm:"default:0"` + // Server-side enqueue time. NOT used for ordering (id is the cursor) + // but useful for ops queries like "rows older than 1h that never + // drained". + EnqueuedAt time.Time `json:"enqueued_at"` + // Optional claim timestamp for future locking workflows. Not used + // in the current revision: consumers rely on external mutual + // exclusion (e.g. wiki:active: Redis SetNX). Reserved column + // so future no-lock parallel workers can flip it inside a row-level + // lock without another migration. + ClaimedAt *time.Time `json:"claimed_at,omitempty"` +} + +// TableName binds TaskPendingOp to the `task_pending_ops` table. +func (TaskPendingOp) TableName() string { + return "task_pending_ops" +} diff --git a/internal/types/wiki_page.go b/internal/types/wiki_page.go index cc24fe8d..3b254db4 100644 --- a/internal/types/wiki_page.go +++ b/internal/types/wiki_page.go @@ -158,6 +158,53 @@ type WikiConfig struct { // ExtractionGranularity controls how many candidate slugs Pass 0 extracts // per document. Empty / unknown value is treated as WikiExtractionStandard. ExtractionGranularity WikiExtractionGranularity `yaml:"extraction_granularity" json:"extraction_granularity,omitempty"` + + // IngestBatchSize controls how many pending ops a single batch + // processes before scheduling a follow-up. 0 falls back to the + // hard-coded default (5). Operators on large KBs (4w+ docs) can + // raise this to 10–20 to amortize the lock-acquire / index-rebuild + // overhead across more documents per round. + IngestBatchSize int `yaml:"ingest_batch_size" json:"ingest_batch_size,omitempty"` + + // IngestMapParallel sets the errgroup limit for the Map phase + // (per-document extraction + summary + chunk citation). 0 falls + // back to 10. Bound by the LLM provider's concurrency limit and + // the worker's outbound HTTP pool. + IngestMapParallel int `yaml:"ingest_map_parallel" json:"ingest_map_parallel,omitempty"` + + // IngestReduceParallel sets the errgroup limit for the Reduce phase + // (per-slug page write). 0 falls back to 10. Bound by the same + // LLM concurrency / HTTP pool considerations as the Map phase, + // plus DB connection pool size. + IngestReduceParallel int `yaml:"ingest_reduce_parallel" json:"ingest_reduce_parallel,omitempty"` +} + +// IngestBatchSizeOrDefault returns IngestBatchSize when set (> 0), +// otherwise the hard-coded fallback. Centralized so callers don't have +// to repeat the 0-check. +func (c *WikiConfig) IngestBatchSizeOrDefault(fallback int) int { + if c == nil || c.IngestBatchSize <= 0 { + return fallback + } + return c.IngestBatchSize +} + +// IngestMapParallelOrDefault returns IngestMapParallel when set, +// otherwise the hard-coded fallback. +func (c *WikiConfig) IngestMapParallelOrDefault(fallback int) int { + if c == nil || c.IngestMapParallel <= 0 { + return fallback + } + return c.IngestMapParallel +} + +// IngestReduceParallelOrDefault returns IngestReduceParallel when set, +// otherwise the hard-coded fallback. +func (c *WikiConfig) IngestReduceParallelOrDefault(fallback int) int { + if c == nil || c.IngestReduceParallel <= 0 { + return fallback + } + return c.IngestReduceParallel } // Value implements the driver.Valuer interface @@ -324,3 +371,30 @@ type WikiIndexResponse struct { Version int `json:"version"` Groups []WikiIndexGroup `json:"groups"` } + +// WikiPageLite is a slim projection of WikiPage carrying only the fields +// the wiki ingest pipeline reaches for during Map / Reduce. It exists so +// per-batch fetcher queries don't have to load the full multi-MB content +// column for every page they want a title or out-link from. +// +// Use cases: +// +// - SlugTitleFetcher: resolve slug -> title for log entries and +// cross-link injection. +// - cleanDeadLinks: read out_links + status without pulling content. +// - dedup pre-filter: title + aliases + page_type for the trgm / +// surface-similarity comparisons. +// +// Aliases is included because dedup and cross-link injection both treat +// the alias surface forms as first-class match targets; OutLinks is +// included so dead-link cleanup can determine which pages reference a +// given dead slug without a second query. +type WikiPageLite struct { + Slug string `json:"slug"` + Title string `json:"title"` + PageType string `json:"page_type"` + Status string `json:"status"` + Aliases StringArray `json:"aliases,omitempty"` + OutLinks StringArray `json:"out_links,omitempty"` +} + diff --git a/migrations/versioned/000041_task_queue_and_wiki_indexes.down.sql b/migrations/versioned/000041_task_queue_and_wiki_indexes.down.sql new file mode 100644 index 00000000..05a98147 --- /dev/null +++ b/migrations/versioned/000041_task_queue_and_wiki_indexes.down.sql @@ -0,0 +1,16 @@ +-- Migration: 000041_task_queue_and_wiki_indexes (rollback) +-- Description: Drop the generic task queue tables and the new wiki_pages +-- GIN indexes. Pending rows are NOT migrated back to Redis on +-- rollback; operators are expected to drain the queue first +-- (or accept the loss of un-consumed ops). + +DO $$ BEGIN RAISE NOTICE '[Migration 000041 DOWN] Reverting task queue + wiki indexes schema'; END $$; + +DROP INDEX IF EXISTS idx_wiki_pages_title_trgm; +DROP INDEX IF EXISTS idx_wiki_pages_source_refs_text; +DROP INDEX IF EXISTS idx_wiki_pages_source_refs; + +DROP TABLE IF EXISTS task_dead_letters; +DROP TABLE IF EXISTS task_pending_ops; + +DO $$ BEGIN RAISE NOTICE '[Migration 000041 DOWN] task queue + wiki indexes schema reverted successfully'; END $$; diff --git a/migrations/versioned/000041_task_queue_and_wiki_indexes.up.sql b/migrations/versioned/000041_task_queue_and_wiki_indexes.up.sql new file mode 100644 index 00000000..d0066ac9 --- /dev/null +++ b/migrations/versioned/000041_task_queue_and_wiki_indexes.up.sql @@ -0,0 +1,135 @@ +-- Migration: 000041_task_queue_and_wiki_indexes +-- Description: Generic task pending queue + dead-letter table, and three +-- additional GIN indexes on wiki_pages to back the optimized +-- ingest pipeline (4w-document-scale). +-- +-- Schemas introduced: +-- 1. task_pending_ops — durable replacement for the Redis-list-backed +-- wiki:pending: queue. Designed generically +-- (task_type + scope + scope_id) so future +-- "debounced batch" task types can reuse it +-- without another migration. +-- 2. task_dead_letters — generic archived-task record. Written by both +-- (a) the asynq middleware when a task exhausts +-- its retry budget, and (b) the wiki ingest +-- service when a per-document op exceeds its +-- in-batch retry quota. +-- 3. wiki_pages indexes — GIN on source_refs (containment + text), and +-- trigram on lower(title) for the new pg_trgm +-- dedup pre-filter. + +DO $$ BEGIN RAISE NOTICE '[Migration 000041] Applying task queue + wiki indexes schema'; END $$; + +-- --------------------------------------------------------------------------- +-- 1) task_pending_ops +-- +-- Generic replacement for ad-hoc Redis-list pending queues. Each row is one +-- pending operation scoped to a (task_type, scope, scope_id) tuple. The +-- consumer (e.g. wiki ingest batch handler) uses PeekBatch to pull the head +-- of the list, processes the ops, then DeleteByIDs the consumed rows. +-- +-- claimed_at is intentionally added but unused in this revision: the wiki +-- ingest pipeline guarantees per-KB serialization via its own active-batch +-- lock (Redis SetNX wiki:active:), so PeekBatch doesn't need +-- SELECT ... FOR UPDATE. Future task types that don't carry an external +-- lock can flip claimed_at = NOW() inside a row-level lock and rely on the +-- column to detect orphaned claims after a worker crash. +-- --------------------------------------------------------------------------- +CREATE TABLE IF NOT EXISTS task_pending_ops ( + id BIGSERIAL PRIMARY KEY, + tenant_id BIGINT NOT NULL, + task_type VARCHAR(64) NOT NULL, + scope VARCHAR(32) NOT NULL, + scope_id VARCHAR(64) NOT NULL, + op VARCHAR(32) NOT NULL, + dedup_key VARCHAR(128) NOT NULL DEFAULT '', + payload JSONB NOT NULL DEFAULT '{}'::JSONB, + fail_count INT NOT NULL DEFAULT 0, + enqueued_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + claimed_at TIMESTAMPTZ +); + +COMMENT ON TABLE task_pending_ops IS 'Generic durable pending-op queue keyed by (task_type, scope, scope_id). Replaces ad-hoc Redis-list queues that were vulnerable to TTL eviction.'; +COMMENT ON COLUMN task_pending_ops.task_type IS 'Free-form task identifier, e.g. "wiki:ingest" — should match an asynq task type when applicable.'; +COMMENT ON COLUMN task_pending_ops.scope IS 'Logical scope, e.g. "knowledge_base" / "knowledge" / "tenant". Read together with scope_id.'; +COMMENT ON COLUMN task_pending_ops.dedup_key IS 'Optional service-defined key used by the consumer to de-duplicate equivalent ops within a single batch peek. Empty means no de-dup.'; +COMMENT ON COLUMN task_pending_ops.fail_count IS 'In-batch retry counter: the consumer increments it via IncrFailCount and dead-letters once it exceeds a service-defined cap.'; +COMMENT ON COLUMN task_pending_ops.claimed_at IS 'Reserved for future locking workflows; consumers in this revision rely on external mutual exclusion and ignore the column.'; + +-- Cover the PeekBatch query: the consumer scans rows for one +-- (task_type, scope, scope_id) tuple ordered by id ASC. +CREATE INDEX IF NOT EXISTS idx_task_pending_ops_scope + ON task_pending_ops (task_type, scope, scope_id, id); + +CREATE INDEX IF NOT EXISTS idx_task_pending_ops_tenant + ON task_pending_ops (tenant_id); + +-- --------------------------------------------------------------------------- +-- 2) task_dead_letters +-- +-- Permanent archive for tasks that exhausted retries. Written from two +-- distinct paths: +-- (a) The asynq dead-letter middleware (internal/middleware/asynq_dead_letter.go) +-- inserts a row whenever a task's retry count equals MaxRetry on the +-- way out — covers every asynq task type uniformly. +-- (b) The wiki ingest service inserts a row directly when a per-document +-- op exceeds wikiMaxFailRetries inside a batch — these never escalate +-- to an asynq retry because they're handled inline. +-- +-- Operations queries it by scope (e.g. all dead letters for a KB) or by +-- task_type (e.g. all summary:generation failures in the last 24h). No +-- TTL — rows are kept for postmortem until manually pruned. failed_at DESC +-- so newest-first scans are cheap. +-- --------------------------------------------------------------------------- +CREATE TABLE IF NOT EXISTS task_dead_letters ( + id BIGSERIAL PRIMARY KEY, + tenant_id BIGINT NOT NULL, + task_type VARCHAR(64) NOT NULL, + scope VARCHAR(32) NOT NULL, + scope_id VARCHAR(64) NOT NULL, + related_id VARCHAR(64) NOT NULL DEFAULT '', + payload JSONB NOT NULL, + last_error TEXT NOT NULL DEFAULT '', + fail_count INT NOT NULL, + failed_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +COMMENT ON TABLE task_dead_letters IS 'Permanent archive of tasks that exhausted retries. Written by the asynq dead-letter middleware and by service-level retry handlers (e.g. wiki ingest per-doc retries).'; +COMMENT ON COLUMN task_dead_letters.related_id IS 'Optional secondary identifier. Wiki ingest puts knowledge_id here so retract/ingest dead letters cluster around the source document.'; +COMMENT ON COLUMN task_dead_letters.payload IS 'Raw task payload (asynq.Task.Payload) at the time of failure. Allows manual requeue via SQL + asynq.Client.Enqueue.'; +COMMENT ON COLUMN task_dead_letters.last_error IS 'String form of the error that caused the final retry to fail. Long stack traces are kept verbatim.'; + +CREATE INDEX IF NOT EXISTS idx_task_dead_letters_scope + ON task_dead_letters (scope, scope_id, failed_at DESC); + +CREATE INDEX IF NOT EXISTS idx_task_dead_letters_tenant + ON task_dead_letters (tenant_id, failed_at DESC); + +CREATE INDEX IF NOT EXISTS idx_task_dead_letters_task_type + ON task_dead_letters (task_type, failed_at DESC); + +-- --------------------------------------------------------------------------- +-- 3) wiki_pages: source_refs / title trigram indexes +-- +-- (a) source_refs containment GIN — covers `source_refs @> ?::jsonb`. Used +-- by ListBySourceRef when looking up pages that cite a given knowledge +-- id (delete flow, retract reconciliation, getExistingPageSlugsForKnowledge). +-- +-- (b) source_refs text-fulltext GIN — fallback for the legacy "kid|title" +-- ref form, which the runtime LIKE pattern needs. Same shape as the +-- existing fulltext index in 000037 (line 54-55). +-- +-- (c) title trigram GIN — backs the new pg_trgm-based dedup pre-filter +-- (selectDedupCandidatePages). pg_trgm extension is already loaded by +-- migration 000002. +-- --------------------------------------------------------------------------- +CREATE INDEX IF NOT EXISTS idx_wiki_pages_source_refs + ON wiki_pages USING GIN (source_refs jsonb_path_ops); + +CREATE INDEX IF NOT EXISTS idx_wiki_pages_source_refs_text + ON wiki_pages USING GIN (to_tsvector('simple', source_refs::text)); + +CREATE INDEX IF NOT EXISTS idx_wiki_pages_title_trgm + ON wiki_pages USING GIN (lower(title) gin_trgm_ops); + +DO $$ BEGIN RAISE NOTICE '[Migration 000041] task queue + wiki indexes schema applied successfully'; END $$;