fix: mark knowledge delete failures after retry exhaustion
This commit is contained in:
@@ -322,6 +322,26 @@ func (r *knowledgeRepository) UpdateKnowledgeColumns(
|
||||
return r.db.WithContext(ctx).Model(&types.Knowledge{}).Where("id = ?", id).Updates(values).Error
|
||||
}
|
||||
|
||||
// UpdateActiveDeletingKnowledgeColumns only touches rows that are still visible
|
||||
// to normal queries and have not moved out of the transient deleting state.
|
||||
func (r *knowledgeRepository) UpdateActiveDeletingKnowledgeColumns(
|
||||
ctx context.Context,
|
||||
id string,
|
||||
values map[string]interface{},
|
||||
) (bool, error) {
|
||||
if len(values) == 0 {
|
||||
return false, nil
|
||||
}
|
||||
result := r.db.WithContext(ctx).
|
||||
Model(&types.Knowledge{}).
|
||||
Where("id = ? AND parse_status = ?", id, types.ParseStatusDeleting).
|
||||
Updates(values)
|
||||
if result.Error != nil {
|
||||
return false, result.Error
|
||||
}
|
||||
return result.RowsAffected > 0, nil
|
||||
}
|
||||
|
||||
// FinalizeSubtask atomically decrements pending_subtasks_count and, when
|
||||
// the counter reaches zero while parse_status is still 'finalizing',
|
||||
// flips the row to 'completed' in the same statement so concurrent
|
||||
|
||||
@@ -91,6 +91,20 @@ func reloadKnowledgeRow(t *testing.T, db *gorm.DB, id string) (status string, co
|
||||
return status, count
|
||||
}
|
||||
|
||||
func insertKnowledgeWithStatus(t *testing.T, db *gorm.DB, status string, deleted bool) string {
|
||||
t.Helper()
|
||||
id := uuid.New().String()
|
||||
deletedAt := interface{}(nil)
|
||||
if deleted {
|
||||
deletedAt = "2026-06-16 12:00:00"
|
||||
}
|
||||
require.NoError(t, db.Exec(`
|
||||
INSERT INTO knowledges (id, tenant_id, knowledge_base_id, type, title, source, parse_status, deleted_at)
|
||||
VALUES (?, 1, ?, 'document', 'delete-test', 'manual', ?, ?)
|
||||
`, id, uuid.New().String(), status, deletedAt).Error)
|
||||
return id
|
||||
}
|
||||
|
||||
// TestFinalizeSubtask_Concurrent_ExactlyOnePromote spawns N goroutines that
|
||||
// each call FinalizeSubtask after SetFinalizing(N), and asserts:
|
||||
// - the counter ends at zero,
|
||||
@@ -268,3 +282,39 @@ func TestUpdateKnowledge_PendingCounterOmittedOnReset(t *testing.T) {
|
||||
_, count = reloadKnowledgeRow(t, db, id)
|
||||
assert.Equal(t, 0, count)
|
||||
}
|
||||
|
||||
func TestUpdateActiveDeletingKnowledgeColumns_GuardsStateAndSoftDelete(t *testing.T) {
|
||||
db := setupKnowledgeTestDB(t)
|
||||
repo := NewKnowledgeRepository(db).(*knowledgeRepository)
|
||||
ctx := context.Background()
|
||||
|
||||
activeDeletingID := insertKnowledgeWithStatus(t, db, types.ParseStatusDeleting, false)
|
||||
activeCompletedID := insertKnowledgeWithStatus(t, db, types.ParseStatusCompleted, false)
|
||||
deletedDeletingID := insertKnowledgeWithStatus(t, db, types.ParseStatusDeleting, true)
|
||||
|
||||
updated, err := repo.UpdateActiveDeletingKnowledgeColumns(ctx, activeDeletingID, map[string]interface{}{
|
||||
"parse_status": types.ParseStatusFailed,
|
||||
"error_message": "delete task exhausted retries",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.True(t, updated)
|
||||
|
||||
updated, err = repo.UpdateActiveDeletingKnowledgeColumns(ctx, activeCompletedID, map[string]interface{}{
|
||||
"parse_status": types.ParseStatusFailed,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.False(t, updated)
|
||||
|
||||
updated, err = repo.UpdateActiveDeletingKnowledgeColumns(ctx, deletedDeletingID, map[string]interface{}{
|
||||
"parse_status": types.ParseStatusFailed,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.False(t, updated)
|
||||
|
||||
status, _ := reloadKnowledgeRow(t, db, activeDeletingID)
|
||||
assert.Equal(t, types.ParseStatusFailed, status)
|
||||
status, _ = reloadKnowledgeRow(t, db, activeCompletedID)
|
||||
assert.Equal(t, types.ParseStatusCompleted, status)
|
||||
status, _ = reloadKnowledgeRow(t, db, deletedDeletingID)
|
||||
assert.Equal(t, types.ParseStatusDeleting, status)
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ package asynqdl
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
@@ -164,6 +165,7 @@ func buildDeadLetter(t *asynq.Task, taskErr error, attempts int) *types.TaskDead
|
||||
Payload: json.RawMessage(rawPayload),
|
||||
LastError: truncateError(taskErr.Error(), 8192),
|
||||
FailCount: attempts,
|
||||
FailedAt: time.Now(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -117,6 +117,9 @@ func TestMiddleware_FailureWithoutAsynqCtx_RecordsRow(t *testing.T) {
|
||||
if row.FailCount != 0 {
|
||||
t.Errorf("fail_count: expected 0 outside worker ctx, got %d", row.FailCount)
|
||||
}
|
||||
if row.FailedAt.IsZero() {
|
||||
t.Error("failed_at: expected non-zero timestamp")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMiddleware_UnknownPayload_StillRecords(t *testing.T) {
|
||||
|
||||
@@ -267,6 +267,10 @@ var taskTypesAffectingKnowledgeStatus = map[string]struct{}{
|
||||
types.TypeManualProcess: {},
|
||||
}
|
||||
|
||||
type deadLetterKnowledgeListDeletePayload struct {
|
||||
KnowledgeIDs []string `json:"knowledge_ids,omitempty"`
|
||||
}
|
||||
|
||||
// newDeadLetterKnowledgeFailer returns the callback wired into the asynq
|
||||
// dead-letter middleware. When a document-related task exhausts its retry
|
||||
// budget, this callback marks the corresponding Knowledge row as failed so
|
||||
@@ -288,6 +292,10 @@ func newDeadLetterKnowledgeFailer(ks interfaces.KnowledgeService, tracker servic
|
||||
if t == nil {
|
||||
return
|
||||
}
|
||||
if t.Type() == types.TypeKnowledgeListDelete {
|
||||
markKnowledgeListDeleteFailed(ctx, repo, t, taskErr)
|
||||
return
|
||||
}
|
||||
if _, ok := taskTypesAffectingKnowledgeStatus[t.Type()]; !ok {
|
||||
return
|
||||
}
|
||||
@@ -320,3 +328,37 @@ func newDeadLetterKnowledgeFailer(ks interfaces.KnowledgeService, tracker servic
|
||||
logger.Infof(ctx, "dead-letter callback: marked knowledge %s as failed (task=%s)", probe.KnowledgeID, t.Type())
|
||||
}
|
||||
}
|
||||
|
||||
func markKnowledgeListDeleteFailed(
|
||||
ctx context.Context,
|
||||
repo interfaces.KnowledgeRepository,
|
||||
t *asynq.Task,
|
||||
taskErr error,
|
||||
) {
|
||||
var payload deadLetterKnowledgeListDeletePayload
|
||||
if err := json.Unmarshal(t.Payload(), &payload); err != nil || len(payload.KnowledgeIDs) == 0 {
|
||||
return
|
||||
}
|
||||
errMsg := "delete task exhausted retries: " + taskErr.Error()
|
||||
if len(errMsg) > 8192 {
|
||||
errMsg = errMsg[:8192]
|
||||
}
|
||||
for _, knowledgeID := range payload.KnowledgeIDs {
|
||||
if knowledgeID == "" {
|
||||
continue
|
||||
}
|
||||
updated, err := repo.UpdateActiveDeletingKnowledgeColumns(ctx, knowledgeID, map[string]interface{}{
|
||||
"parse_status": types.ParseStatusFailed,
|
||||
"error_message": errMsg,
|
||||
})
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "dead-letter callback: failed to mark delete failure for knowledge %s: %v", knowledgeID, err)
|
||||
continue
|
||||
}
|
||||
if !updated {
|
||||
logger.Infof(ctx, "dead-letter callback: skipped marking knowledge %s after delete task exhaustion because it is no longer active deleting", knowledgeID)
|
||||
continue
|
||||
}
|
||||
logger.Infof(ctx, "dead-letter callback: marked knowledge %s as failed after delete task exhausted retries", knowledgeID)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -230,6 +230,9 @@ type KnowledgeRepository interface {
|
||||
// statement so callers that flip several related fields (e.g. parse_status +
|
||||
// error_message) cannot leave the row in a half-updated state.
|
||||
UpdateKnowledgeColumns(ctx context.Context, id string, values map[string]interface{}) error
|
||||
// UpdateActiveDeletingKnowledgeColumns updates an active, non-deleted knowledge row
|
||||
// only when it is still in the transient deleting state.
|
||||
UpdateActiveDeletingKnowledgeColumns(ctx context.Context, id string, values map[string]interface{}) (bool, error)
|
||||
// FinalizeSubtask atomically decrements pending_subtasks_count for the
|
||||
// given knowledge and promotes parse_status from "finalizing" to
|
||||
// "completed" when the count reaches zero. Returns the post-decrement
|
||||
|
||||
Reference in New Issue
Block a user