fix: mark knowledge delete failures after retry exhaustion

This commit is contained in:
langcaiye
2026-06-16 12:08:00 +08:00
committed by lyingbug
parent 6efd2093f5
commit 0c3d45d3cd
6 changed files with 120 additions and 0 deletions
@@ -322,6 +322,26 @@ func (r *knowledgeRepository) UpdateKnowledgeColumns(
return r.db.WithContext(ctx).Model(&types.Knowledge{}).Where("id = ?", id).Updates(values).Error
}
// UpdateActiveDeletingKnowledgeColumns only touches rows that are still visible
// to normal queries and have not moved out of the transient deleting state.
func (r *knowledgeRepository) UpdateActiveDeletingKnowledgeColumns(
ctx context.Context,
id string,
values map[string]interface{},
) (bool, error) {
if len(values) == 0 {
return false, nil
}
result := r.db.WithContext(ctx).
Model(&types.Knowledge{}).
Where("id = ? AND parse_status = ?", id, types.ParseStatusDeleting).
Updates(values)
if result.Error != nil {
return false, result.Error
}
return result.RowsAffected > 0, nil
}
// FinalizeSubtask atomically decrements pending_subtasks_count and, when
// the counter reaches zero while parse_status is still 'finalizing',
// flips the row to 'completed' in the same statement so concurrent
@@ -91,6 +91,20 @@ func reloadKnowledgeRow(t *testing.T, db *gorm.DB, id string) (status string, co
return status, count
}
func insertKnowledgeWithStatus(t *testing.T, db *gorm.DB, status string, deleted bool) string {
t.Helper()
id := uuid.New().String()
deletedAt := interface{}(nil)
if deleted {
deletedAt = "2026-06-16 12:00:00"
}
require.NoError(t, db.Exec(`
INSERT INTO knowledges (id, tenant_id, knowledge_base_id, type, title, source, parse_status, deleted_at)
VALUES (?, 1, ?, 'document', 'delete-test', 'manual', ?, ?)
`, id, uuid.New().String(), status, deletedAt).Error)
return id
}
// TestFinalizeSubtask_Concurrent_ExactlyOnePromote spawns N goroutines that
// each call FinalizeSubtask after SetFinalizing(N), and asserts:
// - the counter ends at zero,
@@ -268,3 +282,39 @@ func TestUpdateKnowledge_PendingCounterOmittedOnReset(t *testing.T) {
_, count = reloadKnowledgeRow(t, db, id)
assert.Equal(t, 0, count)
}
func TestUpdateActiveDeletingKnowledgeColumns_GuardsStateAndSoftDelete(t *testing.T) {
db := setupKnowledgeTestDB(t)
repo := NewKnowledgeRepository(db).(*knowledgeRepository)
ctx := context.Background()
activeDeletingID := insertKnowledgeWithStatus(t, db, types.ParseStatusDeleting, false)
activeCompletedID := insertKnowledgeWithStatus(t, db, types.ParseStatusCompleted, false)
deletedDeletingID := insertKnowledgeWithStatus(t, db, types.ParseStatusDeleting, true)
updated, err := repo.UpdateActiveDeletingKnowledgeColumns(ctx, activeDeletingID, map[string]interface{}{
"parse_status": types.ParseStatusFailed,
"error_message": "delete task exhausted retries",
})
require.NoError(t, err)
assert.True(t, updated)
updated, err = repo.UpdateActiveDeletingKnowledgeColumns(ctx, activeCompletedID, map[string]interface{}{
"parse_status": types.ParseStatusFailed,
})
require.NoError(t, err)
assert.False(t, updated)
updated, err = repo.UpdateActiveDeletingKnowledgeColumns(ctx, deletedDeletingID, map[string]interface{}{
"parse_status": types.ParseStatusFailed,
})
require.NoError(t, err)
assert.False(t, updated)
status, _ := reloadKnowledgeRow(t, db, activeDeletingID)
assert.Equal(t, types.ParseStatusFailed, status)
status, _ = reloadKnowledgeRow(t, db, activeCompletedID)
assert.Equal(t, types.ParseStatusCompleted, status)
status, _ = reloadKnowledgeRow(t, db, deletedDeletingID)
assert.Equal(t, types.ParseStatusDeleting, status)
}
+2
View File
@@ -16,6 +16,7 @@ package asynqdl
import (
"context"
"encoding/json"
"time"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
@@ -164,6 +165,7 @@ func buildDeadLetter(t *asynq.Task, taskErr error, attempts int) *types.TaskDead
Payload: json.RawMessage(rawPayload),
LastError: truncateError(taskErr.Error(), 8192),
FailCount: attempts,
FailedAt: time.Now(),
}
}
@@ -117,6 +117,9 @@ func TestMiddleware_FailureWithoutAsynqCtx_RecordsRow(t *testing.T) {
if row.FailCount != 0 {
t.Errorf("fail_count: expected 0 outside worker ctx, got %d", row.FailCount)
}
if row.FailedAt.IsZero() {
t.Error("failed_at: expected non-zero timestamp")
}
}
func TestMiddleware_UnknownPayload_StillRecords(t *testing.T) {
+42
View File
@@ -267,6 +267,10 @@ var taskTypesAffectingKnowledgeStatus = map[string]struct{}{
types.TypeManualProcess: {},
}
type deadLetterKnowledgeListDeletePayload struct {
KnowledgeIDs []string `json:"knowledge_ids,omitempty"`
}
// newDeadLetterKnowledgeFailer returns the callback wired into the asynq
// dead-letter middleware. When a document-related task exhausts its retry
// budget, this callback marks the corresponding Knowledge row as failed so
@@ -288,6 +292,10 @@ func newDeadLetterKnowledgeFailer(ks interfaces.KnowledgeService, tracker servic
if t == nil {
return
}
if t.Type() == types.TypeKnowledgeListDelete {
markKnowledgeListDeleteFailed(ctx, repo, t, taskErr)
return
}
if _, ok := taskTypesAffectingKnowledgeStatus[t.Type()]; !ok {
return
}
@@ -320,3 +328,37 @@ func newDeadLetterKnowledgeFailer(ks interfaces.KnowledgeService, tracker servic
logger.Infof(ctx, "dead-letter callback: marked knowledge %s as failed (task=%s)", probe.KnowledgeID, t.Type())
}
}
func markKnowledgeListDeleteFailed(
ctx context.Context,
repo interfaces.KnowledgeRepository,
t *asynq.Task,
taskErr error,
) {
var payload deadLetterKnowledgeListDeletePayload
if err := json.Unmarshal(t.Payload(), &payload); err != nil || len(payload.KnowledgeIDs) == 0 {
return
}
errMsg := "delete task exhausted retries: " + taskErr.Error()
if len(errMsg) > 8192 {
errMsg = errMsg[:8192]
}
for _, knowledgeID := range payload.KnowledgeIDs {
if knowledgeID == "" {
continue
}
updated, err := repo.UpdateActiveDeletingKnowledgeColumns(ctx, knowledgeID, map[string]interface{}{
"parse_status": types.ParseStatusFailed,
"error_message": errMsg,
})
if err != nil {
logger.Warnf(ctx, "dead-letter callback: failed to mark delete failure for knowledge %s: %v", knowledgeID, err)
continue
}
if !updated {
logger.Infof(ctx, "dead-letter callback: skipped marking knowledge %s after delete task exhaustion because it is no longer active deleting", knowledgeID)
continue
}
logger.Infof(ctx, "dead-letter callback: marked knowledge %s as failed after delete task exhausted retries", knowledgeID)
}
}
+3
View File
@@ -230,6 +230,9 @@ type KnowledgeRepository interface {
// statement so callers that flip several related fields (e.g. parse_status +
// error_message) cannot leave the row in a half-updated state.
UpdateKnowledgeColumns(ctx context.Context, id string, values map[string]interface{}) error
// UpdateActiveDeletingKnowledgeColumns updates an active, non-deleted knowledge row
// only when it is still in the transient deleting state.
UpdateActiveDeletingKnowledgeColumns(ctx context.Context, id string, values map[string]interface{}) (bool, error)
// FinalizeSubtask atomically decrements pending_subtasks_count for the
// given knowledge and promotes parse_status from "finalizing" to
// "completed" when the count reaches zero. Returns the post-decrement