feat(system): expand runtime queue task inspection and safe actions
Replace failed-only task views with state-aware runtime task listing, operator-safe cancel/run-now/delete actions, orphan detection, and multimodal queue cleanup for deleted knowledge rows.
This commit is contained in:
+174
-62
@@ -19,6 +19,7 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/application/service/file"
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/database"
|
||||
apperrors "github.com/Tencent/WeKnora/internal/errors"
|
||||
"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
modellimiter "github.com/Tencent/WeKnora/internal/models/limiter"
|
||||
@@ -32,6 +33,10 @@ import (
|
||||
"github.com/neo4j/neo4j-go-driver/v6/neo4j"
|
||||
)
|
||||
|
||||
type runtimeKnowledgeCanceller interface {
|
||||
CancelKnowledgeParse(ctx context.Context, knowledgeID string) (*types.Knowledge, error)
|
||||
}
|
||||
|
||||
// SystemHandler handles system-related requests
|
||||
type SystemHandler struct {
|
||||
cfg *config.Config
|
||||
@@ -49,6 +54,10 @@ type SystemHandler struct {
|
||||
// Lite mode), so GetRuntimeQueues can distinguish "no queues in this
|
||||
// deployment" from "queues are empty".
|
||||
taskInspector interfaces.TaskInspector
|
||||
// knowledgeSvc supplies the domain-level cancellation path used by the
|
||||
// runtime task console. It updates business state and tracing before queue
|
||||
// records are removed, unlike a raw Redis deletion.
|
||||
knowledgeSvc runtimeKnowledgeCanceller
|
||||
}
|
||||
|
||||
// NewSystemHandler creates a new system handler
|
||||
@@ -60,6 +69,7 @@ func NewSystemHandler(cfg *config.Config,
|
||||
systemSettingSvc interfaces.SystemSettingService,
|
||||
auditSvc interfaces.AuditLogService,
|
||||
taskInspector interfaces.TaskInspector,
|
||||
knowledgeSvc interfaces.KnowledgeService,
|
||||
) *SystemHandler {
|
||||
return &SystemHandler{
|
||||
cfg: cfg,
|
||||
@@ -70,6 +80,7 @@ func NewSystemHandler(cfg *config.Config,
|
||||
systemSettingSvc: systemSettingSvc,
|
||||
auditSvc: auditSvc,
|
||||
taskInspector: taskInspector,
|
||||
knowledgeSvc: knowledgeSvc,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1593,15 +1604,12 @@ func (h *SystemHandler) GetRuntimeQueues(c *gin.Context) {
|
||||
c.JSON(http.StatusOK, resp)
|
||||
}
|
||||
|
||||
// RuntimeFailedTasksResponse is a page of tasks that exhausted automatic
|
||||
// retries. Available=false is used by Lite mode, where there is no durable
|
||||
// queue backend to inspect or mutate.
|
||||
type RuntimeFailedTasksResponse struct {
|
||||
Available bool `json:"available"`
|
||||
Tasks []types.FailedTaskInfo `json:"tasks"`
|
||||
Page int `json:"page"`
|
||||
PageSize int `json:"page_size"`
|
||||
HasMore bool `json:"has_more"`
|
||||
type RuntimeTasksResponse struct {
|
||||
Available bool `json:"available"`
|
||||
Tasks []types.RuntimeTaskInfo `json:"tasks"`
|
||||
Page int `json:"page"`
|
||||
PageSize int `json:"page_size"`
|
||||
HasMore bool `json:"has_more"`
|
||||
}
|
||||
|
||||
func isKnownRuntimeQueue(name string) bool {
|
||||
@@ -1613,7 +1621,7 @@ func isKnownRuntimeQueue(name string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func runtimeFailedTaskPage(c *gin.Context) (int, int) {
|
||||
func runtimeTaskPage(c *gin.Context) (int, int) {
|
||||
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
|
||||
pageSize, _ := strconv.Atoi(c.DefaultQuery("page_size", "20"))
|
||||
if page < 1 {
|
||||
@@ -1628,40 +1636,37 @@ func runtimeFailedTaskPage(c *gin.Context) (int, int) {
|
||||
return page, pageSize
|
||||
}
|
||||
|
||||
// ListRuntimeFailedTasks godoc
|
||||
// @Summary List tasks that stopped after exhausting automatic retries
|
||||
// @Tags System Admin
|
||||
// @Produce json
|
||||
// @Param queue path string true "Queue name"
|
||||
// @Success 200 {object} RuntimeFailedTasksResponse
|
||||
// @Router /system/admin/runtime/queues/{queue}/failed-tasks [get]
|
||||
func (h *SystemHandler) ListRuntimeFailedTasks(c *gin.Context) {
|
||||
func (h *SystemHandler) listRuntimeTasks(c *gin.Context, state types.RuntimeTaskState) {
|
||||
queue := c.Param("queue")
|
||||
if !isKnownRuntimeQueue(queue) {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Unknown task queue"})
|
||||
return
|
||||
}
|
||||
page, pageSize := runtimeFailedTaskPage(c)
|
||||
inspector, ok := h.taskInspector.(interfaces.FailedTaskInspector)
|
||||
if !state.Valid() {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Unknown task state"})
|
||||
return
|
||||
}
|
||||
page, pageSize := runtimeTaskPage(c)
|
||||
inspector, ok := h.taskInspector.(interfaces.RuntimeTaskInspector)
|
||||
if !ok {
|
||||
c.JSON(http.StatusOK, RuntimeFailedTasksResponse{
|
||||
c.JSON(http.StatusOK, RuntimeTasksResponse{
|
||||
Available: false,
|
||||
Tasks: []types.FailedTaskInfo{},
|
||||
Tasks: []types.RuntimeTaskInfo{},
|
||||
Page: page,
|
||||
PageSize: pageSize,
|
||||
})
|
||||
return
|
||||
}
|
||||
tasks, supported, err := inspector.ListFailedTasks(c.Request.Context(), queue, page, pageSize)
|
||||
tasks, supported, err := inspector.ListRuntimeTasks(c.Request.Context(), queue, state, page, pageSize)
|
||||
if err != nil {
|
||||
logger.Errorf(c.Request.Context(), "list failed queue tasks queue=%s: %v", queue, err)
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to list stopped tasks"})
|
||||
logger.Errorf(c.Request.Context(), "list runtime queue tasks queue=%s state=%s: %v", queue, state, err)
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to list queue tasks"})
|
||||
return
|
||||
}
|
||||
if tasks == nil {
|
||||
tasks = []types.FailedTaskInfo{}
|
||||
tasks = []types.RuntimeTaskInfo{}
|
||||
}
|
||||
c.JSON(http.StatusOK, RuntimeFailedTasksResponse{
|
||||
c.JSON(http.StatusOK, RuntimeTasksResponse{
|
||||
Available: supported,
|
||||
Tasks: tasks,
|
||||
Page: page,
|
||||
@@ -1670,29 +1675,55 @@ func (h *SystemHandler) ListRuntimeFailedTasks(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// ListRuntimeTasks returns one live task-state page for the unified operator
|
||||
// drawer. Raw task payloads are never included.
|
||||
// @Summary List runtime queue tasks by state
|
||||
// @Tags System Admin
|
||||
// @Produce json
|
||||
// @Param queue path string true "Queue name"
|
||||
// @Param state query string true "Task state" Enums(pending,active,scheduled,retry,archived,completed)
|
||||
// @Param page query int false "Page" default(1)
|
||||
// @Param page_size query int false "Page size" default(20)
|
||||
// @Success 200 {object} RuntimeTasksResponse
|
||||
// @Router /system/admin/runtime/queues/{queue}/tasks [get]
|
||||
func (h *SystemHandler) ListRuntimeTasks(c *gin.Context) {
|
||||
h.listRuntimeTasks(c, types.RuntimeTaskState(c.Query("state")))
|
||||
}
|
||||
|
||||
func (h *SystemHandler) emitQueueTaskAudit(
|
||||
ctx context.Context,
|
||||
action types.AuditAction,
|
||||
queue, taskID string,
|
||||
extra map[string]string,
|
||||
) {
|
||||
if h.auditSvc == nil {
|
||||
return
|
||||
}
|
||||
actorID, _ := types.UserIDFromContext(ctx)
|
||||
details, _ := json.Marshal(map[string]string{"queue": queue, "task_id": taskID})
|
||||
detailMap := map[string]string{"queue": queue, "task_id": taskID}
|
||||
for key, value := range extra {
|
||||
detailMap[key] = value
|
||||
}
|
||||
details, _ := json.Marshal(detailMap)
|
||||
targetType := "queue_task"
|
||||
targetID := taskID
|
||||
if targetID == "" {
|
||||
targetType = "task_queue"
|
||||
targetID = queue
|
||||
}
|
||||
_ = h.auditSvc.Log(ctx, &types.AuditLog{
|
||||
TenantID: 0,
|
||||
ActorUserID: actorID,
|
||||
ActorRole: "system_admin",
|
||||
Action: action,
|
||||
TargetType: "queue_task",
|
||||
TargetID: taskID,
|
||||
TargetType: targetType,
|
||||
TargetID: targetID,
|
||||
Outcome: types.AuditOutcomeSuccess,
|
||||
Details: types.JSON(details),
|
||||
})
|
||||
}
|
||||
|
||||
func runtimeFailedTaskParams(c *gin.Context) (string, string, bool) {
|
||||
func runtimeTaskParams(c *gin.Context) (string, string, bool) {
|
||||
queue := c.Param("queue")
|
||||
taskID := c.Param("task_id")
|
||||
if !isKnownRuntimeQueue(queue) || taskID == "" {
|
||||
@@ -1702,56 +1733,137 @@ func runtimeFailedTaskParams(c *gin.Context) (string, string, bool) {
|
||||
return queue, taskID, true
|
||||
}
|
||||
|
||||
// RetryRuntimeFailedTask schedules one stopped task for one immediate manual
|
||||
// run. It does not reset the automatic retry counter.
|
||||
func (h *SystemHandler) RetryRuntimeFailedTask(c *gin.Context) {
|
||||
queue, taskID, ok := runtimeFailedTaskParams(c)
|
||||
func (h *SystemHandler) mutateRuntimeTask(c *gin.Context, action types.RuntimeTaskAction) {
|
||||
queue, taskID, ok := runtimeTaskParams(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
inspector, supported := h.taskInspector.(interfaces.FailedTaskInspector)
|
||||
inspector, supported := h.taskInspector.(interfaces.RuntimeTaskInspector)
|
||||
if !supported {
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "Task queue is unavailable"})
|
||||
return
|
||||
}
|
||||
available, err := inspector.RetryFailedTask(c.Request.Context(), queue, taskID)
|
||||
task, available, err := inspector.GetRuntimeTask(c.Request.Context(), queue, taskID)
|
||||
if !available {
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "Task queue is unavailable"})
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
logger.Errorf(c.Request.Context(), "retry failed queue task queue=%s task=%s: %v", queue, taskID, err)
|
||||
c.JSON(http.StatusConflict, gin.H{"error": "Task is no longer available for retry"})
|
||||
if err != nil || task == nil {
|
||||
c.JSON(http.StatusConflict, gin.H{"error": "Task is no longer available"})
|
||||
return
|
||||
}
|
||||
h.emitQueueTaskAudit(c.Request.Context(), types.AuditActionSystemQueueTaskRetried, queue, taskID)
|
||||
if !task.Allows(action) {
|
||||
c.JSON(http.StatusConflict, gin.H{"error": "Action is not allowed for the current task state"})
|
||||
return
|
||||
}
|
||||
|
||||
var auditAction types.AuditAction
|
||||
auditDetails := map[string]string{
|
||||
"task_type": task.Type,
|
||||
"from_state": string(task.State),
|
||||
"action": string(action),
|
||||
}
|
||||
switch action {
|
||||
case types.RuntimeTaskActionCancel:
|
||||
if h.knowledgeSvc == nil || task.TenantID == 0 || task.KnowledgeID == "" {
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "Business cancellation is unavailable"})
|
||||
return
|
||||
}
|
||||
cancelCtx := context.WithValue(c.Request.Context(), types.TenantIDContextKey, task.TenantID)
|
||||
if _, err = h.knowledgeSvc.CancelKnowledgeParse(cancelCtx, task.KnowledgeID); err != nil {
|
||||
if isKnowledgeGone(err) {
|
||||
if h.purgeOrphanRuntimeTask(c.Request.Context(), inspector, queue, taskID, task.KnowledgeID) {
|
||||
logger.Warnf(c.Request.Context(),
|
||||
"cancel runtime queue task queue=%s task=%s: knowledge gone, purged queue record",
|
||||
queue, taskID)
|
||||
auditDetails["orphan_purge"] = "true"
|
||||
auditAction = types.AuditActionSystemQueueTaskCancelled
|
||||
break
|
||||
}
|
||||
}
|
||||
logger.Errorf(c.Request.Context(), "cancel runtime queue task queue=%s task=%s: %v", queue, taskID, err)
|
||||
c.JSON(http.StatusConflict, gin.H{"error": "Task can no longer be cancelled"})
|
||||
return
|
||||
}
|
||||
auditAction = types.AuditActionSystemQueueTaskCancelled
|
||||
case types.RuntimeTaskActionRunNow:
|
||||
available, err = inspector.RunRuntimeTask(c.Request.Context(), queue, taskID)
|
||||
if !available {
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "Task queue is unavailable"})
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
logger.Errorf(c.Request.Context(), "run queue task now queue=%s task=%s: %v", queue, taskID, err)
|
||||
c.JSON(http.StatusConflict, gin.H{"error": "Task is no longer available to run"})
|
||||
return
|
||||
}
|
||||
auditAction = types.AuditActionSystemQueueTaskRunNow
|
||||
case types.RuntimeTaskActionDelete:
|
||||
available, err = inspector.DeleteRuntimeTask(c.Request.Context(), queue, taskID)
|
||||
if !available {
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "Task queue is unavailable"})
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
logger.Errorf(c.Request.Context(), "delete queue task queue=%s task=%s: %v", queue, taskID, err)
|
||||
c.JSON(http.StatusConflict, gin.H{"error": "Task is no longer available for deletion"})
|
||||
return
|
||||
}
|
||||
auditAction = types.AuditActionSystemQueueTaskDeleted
|
||||
default:
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Unknown task action"})
|
||||
return
|
||||
}
|
||||
h.emitQueueTaskAudit(c.Request.Context(), auditAction, queue, taskID, auditDetails)
|
||||
c.JSON(http.StatusOK, gin.H{"success": true})
|
||||
}
|
||||
|
||||
// DeleteRuntimeFailedTask removes one stopped task from the queue archive. It
|
||||
// does not rerun the task or change its related business object.
|
||||
func (h *SystemHandler) DeleteRuntimeFailedTask(c *gin.Context) {
|
||||
queue, taskID, ok := runtimeFailedTaskParams(c)
|
||||
if !ok {
|
||||
return
|
||||
func isKnowledgeGone(err error) bool {
|
||||
if errors.Is(err, repository.ErrKnowledgeNotFound) {
|
||||
return true
|
||||
}
|
||||
inspector, supported := h.taskInspector.(interfaces.FailedTaskInspector)
|
||||
if !supported {
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "Task queue is unavailable"})
|
||||
return
|
||||
var appErr *apperrors.AppError
|
||||
return errors.As(err, &appErr) && appErr.Code == apperrors.ErrNotFound
|
||||
}
|
||||
|
||||
// purgeOrphanRuntimeTask removes queue records for a task whose knowledge row
|
||||
// is already gone. Best-effort: CancelTasksForKnowledge sweeps every queue
|
||||
// state; ForceDeleteRuntimeTask is the fallback for the specific task ID.
|
||||
func (h *SystemHandler) purgeOrphanRuntimeTask(
|
||||
ctx context.Context,
|
||||
inspector interfaces.RuntimeTaskInspector,
|
||||
queue, taskID, knowledgeID string,
|
||||
) bool {
|
||||
if h.taskInspector != nil && knowledgeID != "" {
|
||||
deleted, _, err := h.taskInspector.CancelTasksForKnowledge(ctx, knowledgeID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "purge orphan tasks for knowledge %s: %v", knowledgeID, err)
|
||||
} else if deleted > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
available, err := inspector.DeleteFailedTask(c.Request.Context(), queue, taskID)
|
||||
if !available {
|
||||
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "Task queue is unavailable"})
|
||||
return
|
||||
supported, err := inspector.ForceDeleteRuntimeTask(ctx, queue, taskID)
|
||||
if !supported || err != nil {
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "force delete orphan task queue=%s task=%s: %v", queue, taskID, err)
|
||||
}
|
||||
return false
|
||||
}
|
||||
if err != nil {
|
||||
logger.Errorf(c.Request.Context(), "delete failed queue task queue=%s task=%s: %v", queue, taskID, err)
|
||||
c.JSON(http.StatusConflict, gin.H{"error": "Task is no longer available for deletion"})
|
||||
return
|
||||
}
|
||||
h.emitQueueTaskAudit(c.Request.Context(), types.AuditActionSystemQueueTaskDeleted, queue, taskID)
|
||||
c.JSON(http.StatusOK, gin.H{"success": true})
|
||||
return true
|
||||
}
|
||||
|
||||
// MutateRuntimeTask executes a backend-advertised action after re-reading the
|
||||
// current task state to close click/race windows.
|
||||
// @Summary Run a safe runtime task action
|
||||
// @Tags System Admin
|
||||
// @Produce json
|
||||
// @Param queue path string true "Queue name"
|
||||
// @Param task_id path string true "Task ID"
|
||||
// @Param action path string true "Action" Enums(cancel,run_now,delete)
|
||||
// @Success 200 {object} map[string]bool
|
||||
// @Router /system/admin/runtime/queues/{queue}/tasks/{task_id}/actions/{action} [post]
|
||||
func (h *SystemHandler) MutateRuntimeTask(c *gin.Context) {
|
||||
h.mutateRuntimeTask(c, types.RuntimeTaskAction(c.Param("action")))
|
||||
}
|
||||
|
||||
func (h *SystemHandler) ListSystemSettings(c *gin.Context) {
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/application/repository"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
@@ -75,21 +76,79 @@ func (runtimeTestInspector) WorkerServerStats(context.Context) ([]types.WorkerSe
|
||||
}, true, nil
|
||||
}
|
||||
|
||||
type runtimeFailedTestInspector struct {
|
||||
type runtimeTaskTestInspector struct {
|
||||
runtimeTestInspector
|
||||
tasks []types.FailedTaskInfo
|
||||
retriedTask string
|
||||
deletedTask string
|
||||
mutatedQueue string
|
||||
tasks []types.RuntimeTaskInfo
|
||||
retriedTask string
|
||||
deletedTask string
|
||||
forceDeleted string
|
||||
cancelKnowledge string
|
||||
cancelDeleted int
|
||||
mutatedQueue string
|
||||
}
|
||||
|
||||
func (r *runtimeFailedTestInspector) ListFailedTasks(
|
||||
context.Context, string, int, int,
|
||||
) ([]types.FailedTaskInfo, bool, error) {
|
||||
func (r *runtimeTaskTestInspector) CancelTasksForKnowledge(
|
||||
_ context.Context, knowledgeID string,
|
||||
) (int, int, error) {
|
||||
r.cancelKnowledge = knowledgeID
|
||||
if r.cancelDeleted > 0 {
|
||||
return r.cancelDeleted, 0, nil
|
||||
}
|
||||
return 0, 0, nil
|
||||
}
|
||||
|
||||
type runtimeKnowledgeCancelTest struct {
|
||||
tenantID uint64
|
||||
knowledgeID string
|
||||
err error
|
||||
}
|
||||
|
||||
func (r *runtimeKnowledgeCancelTest) CancelKnowledgeParse(
|
||||
ctx context.Context, knowledgeID string,
|
||||
) (*types.Knowledge, error) {
|
||||
r.tenantID, _ = ctx.Value(types.TenantIDContextKey).(uint64)
|
||||
r.knowledgeID = knowledgeID
|
||||
if r.err != nil {
|
||||
return nil, r.err
|
||||
}
|
||||
return &types.Knowledge{ID: knowledgeID, TenantID: r.tenantID}, nil
|
||||
}
|
||||
|
||||
func (r *runtimeTaskTestInspector) ListRuntimeTasks(
|
||||
_ context.Context, _ string, state types.RuntimeTaskState, _, _ int,
|
||||
) ([]types.RuntimeTaskInfo, bool, error) {
|
||||
for i := range r.tasks {
|
||||
if r.tasks[i].State == "" {
|
||||
r.tasks[i].State = state
|
||||
}
|
||||
if r.tasks[i].AllowedActions == nil && state == types.RuntimeTaskArchived {
|
||||
r.tasks[i].AllowedActions = []types.RuntimeTaskAction{
|
||||
types.RuntimeTaskActionRunNow,
|
||||
types.RuntimeTaskActionDelete,
|
||||
}
|
||||
}
|
||||
}
|
||||
return r.tasks, true, nil
|
||||
}
|
||||
|
||||
func (r *runtimeFailedTestInspector) RetryFailedTask(
|
||||
func (r *runtimeTaskTestInspector) GetRuntimeTask(
|
||||
_ context.Context, queue, taskID string,
|
||||
) (*types.RuntimeTaskInfo, bool, error) {
|
||||
for i := range r.tasks {
|
||||
if r.tasks[i].ID == taskID {
|
||||
return &r.tasks[i], true, nil
|
||||
}
|
||||
}
|
||||
return &types.RuntimeTaskInfo{
|
||||
ID: taskID, Queue: queue, State: types.RuntimeTaskArchived,
|
||||
AllowedActions: []types.RuntimeTaskAction{
|
||||
types.RuntimeTaskActionRunNow,
|
||||
types.RuntimeTaskActionDelete,
|
||||
},
|
||||
}, true, nil
|
||||
}
|
||||
|
||||
func (r *runtimeTaskTestInspector) RunRuntimeTask(
|
||||
_ context.Context, queue, taskID string,
|
||||
) (bool, error) {
|
||||
r.mutatedQueue = queue
|
||||
@@ -97,7 +156,7 @@ func (r *runtimeFailedTestInspector) RetryFailedTask(
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (r *runtimeFailedTestInspector) DeleteFailedTask(
|
||||
func (r *runtimeTaskTestInspector) DeleteRuntimeTask(
|
||||
_ context.Context, queue, taskID string,
|
||||
) (bool, error) {
|
||||
r.mutatedQueue = queue
|
||||
@@ -105,6 +164,14 @@ func (r *runtimeFailedTestInspector) DeleteFailedTask(
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (r *runtimeTaskTestInspector) ForceDeleteRuntimeTask(
|
||||
_ context.Context, queue, taskID string,
|
||||
) (bool, error) {
|
||||
r.mutatedQueue = queue
|
||||
r.forceDeleted = taskID
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func TestGetRuntimeQueuesReportsIsolatedPoolCapacity(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
handler := &SystemHandler{
|
||||
@@ -186,9 +253,9 @@ func TestGetRuntimeQueuesFallsBackFromInvalidHistoricalConcurrency(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListRuntimeFailedTasksReturnsSafeTaskDetails(t *testing.T) {
|
||||
func TestListRuntimeTasksReturnsSafeTaskDetails(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
inspector := &runtimeFailedTestInspector{tasks: []types.FailedTaskInfo{{
|
||||
inspector := &runtimeTaskTestInspector{tasks: []types.RuntimeTaskInfo{{
|
||||
ID: "task-1",
|
||||
Queue: types.QueueDefault,
|
||||
Type: types.TypeDocumentProcess,
|
||||
@@ -202,13 +269,13 @@ func TestListRuntimeFailedTasksReturnsSafeTaskDetails(t *testing.T) {
|
||||
recorder := httptest.NewRecorder()
|
||||
ctx, _ := gin.CreateTestContext(recorder)
|
||||
ctx.Params = gin.Params{{Key: "queue", Value: types.QueueDefault}}
|
||||
ctx.Request = httptest.NewRequest(http.MethodGet, "/api/v1/system/admin/runtime/queues/default/failed-tasks", nil)
|
||||
ctx.Request = httptest.NewRequest(http.MethodGet, "/api/v1/system/admin/runtime/queues/default/tasks?state=archived", nil)
|
||||
|
||||
handler.ListRuntimeFailedTasks(ctx)
|
||||
handler.ListRuntimeTasks(ctx)
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
var response RuntimeFailedTasksResponse
|
||||
var response RuntimeTasksResponse
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
@@ -220,9 +287,9 @@ func TestListRuntimeFailedTasksReturnsSafeTaskDetails(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeFailedTaskMutationsDelegateToInspector(t *testing.T) {
|
||||
func TestRuntimeTaskMutationsDelegateToInspector(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
inspector := &runtimeFailedTestInspector{}
|
||||
inspector := &runtimeTaskTestInspector{}
|
||||
handler := &SystemHandler{taskInspector: inspector}
|
||||
|
||||
retryRecorder := httptest.NewRecorder()
|
||||
@@ -230,9 +297,10 @@ func TestRuntimeFailedTaskMutationsDelegateToInspector(t *testing.T) {
|
||||
retryCtx.Params = gin.Params{
|
||||
{Key: "queue", Value: types.QueueDefault},
|
||||
{Key: "task_id", Value: "task-1"},
|
||||
{Key: "action", Value: string(types.RuntimeTaskActionRunNow)},
|
||||
}
|
||||
retryCtx.Request = httptest.NewRequest(http.MethodPost, "/retry", nil)
|
||||
handler.RetryRuntimeFailedTask(retryCtx)
|
||||
handler.MutateRuntimeTask(retryCtx)
|
||||
if retryRecorder.Code != http.StatusOK || inspector.retriedTask != "task-1" {
|
||||
t.Fatalf("retry failed: status=%d inspector=%+v", retryRecorder.Code, inspector)
|
||||
}
|
||||
@@ -242,24 +310,94 @@ func TestRuntimeFailedTaskMutationsDelegateToInspector(t *testing.T) {
|
||||
deleteCtx.Params = gin.Params{
|
||||
{Key: "queue", Value: types.QueueDefault},
|
||||
{Key: "task_id", Value: "task-2"},
|
||||
{Key: "action", Value: string(types.RuntimeTaskActionDelete)},
|
||||
}
|
||||
deleteCtx.Request = httptest.NewRequest(http.MethodDelete, "/task-2", nil)
|
||||
handler.DeleteRuntimeFailedTask(deleteCtx)
|
||||
handler.MutateRuntimeTask(deleteCtx)
|
||||
if deleteRecorder.Code != http.StatusOK || inspector.deletedTask != "task-2" {
|
||||
t.Fatalf("delete failed: status=%d inspector=%+v", deleteRecorder.Code, inspector)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListRuntimeFailedTasksRejectsUnknownQueue(t *testing.T) {
|
||||
func TestListRuntimeTasksRejectsUnknownQueue(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
handler := &SystemHandler{taskInspector: &runtimeFailedTestInspector{}}
|
||||
handler := &SystemHandler{taskInspector: &runtimeTaskTestInspector{}}
|
||||
recorder := httptest.NewRecorder()
|
||||
ctx, _ := gin.CreateTestContext(recorder)
|
||||
ctx.Params = gin.Params{{Key: "queue", Value: "unknown"}}
|
||||
ctx.Request = httptest.NewRequest(http.MethodGet, "/failed-tasks", nil)
|
||||
ctx.Request = httptest.NewRequest(http.MethodGet, "/tasks?state=archived", nil)
|
||||
|
||||
handler.ListRuntimeFailedTasks(ctx)
|
||||
handler.ListRuntimeTasks(ctx)
|
||||
if recorder.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want %d", recorder.Code, http.StatusBadRequest)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListRuntimeTasksRejectsUnknownState(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
handler := &SystemHandler{taskInspector: &runtimeTaskTestInspector{}}
|
||||
recorder := httptest.NewRecorder()
|
||||
ctx, _ := gin.CreateTestContext(recorder)
|
||||
ctx.Params = gin.Params{{Key: "queue", Value: types.QueueDefault}}
|
||||
ctx.Request = httptest.NewRequest(http.MethodGet, "/tasks?state=unknown", nil)
|
||||
|
||||
handler.ListRuntimeTasks(ctx)
|
||||
if recorder.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want %d", recorder.Code, http.StatusBadRequest)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeTaskCancelUsesDomainCancellationWithTaskTenant(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
inspector := &runtimeTaskTestInspector{tasks: []types.RuntimeTaskInfo{{
|
||||
ID: "task-cancel", Queue: types.QueueDefault, Type: types.TypeDocumentProcess,
|
||||
State: types.RuntimeTaskActive, TenantID: 42, KnowledgeID: "knowledge-42",
|
||||
AllowedActions: []types.RuntimeTaskAction{types.RuntimeTaskActionCancel},
|
||||
}}}
|
||||
canceller := &runtimeKnowledgeCancelTest{}
|
||||
handler := &SystemHandler{taskInspector: inspector, knowledgeSvc: canceller}
|
||||
recorder := httptest.NewRecorder()
|
||||
ctx, _ := gin.CreateTestContext(recorder)
|
||||
ctx.Params = gin.Params{
|
||||
{Key: "queue", Value: types.QueueDefault},
|
||||
{Key: "task_id", Value: "task-cancel"},
|
||||
{Key: "action", Value: string(types.RuntimeTaskActionCancel)},
|
||||
}
|
||||
ctx.Request = httptest.NewRequest(http.MethodPost, "/cancel", nil)
|
||||
|
||||
handler.MutateRuntimeTask(ctx)
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
if canceller.tenantID != 42 || canceller.knowledgeID != "knowledge-42" {
|
||||
t.Fatalf("domain cancellation context mismatch: %+v", canceller)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeTaskCancelPurgesOrphanWhenKnowledgeGone(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
inspector := &runtimeTaskTestInspector{tasks: []types.RuntimeTaskInfo{{
|
||||
ID: "task-orphan", Queue: types.QueueMultimodal, Type: types.TypeImageMultimodal,
|
||||
State: types.RuntimeTaskRetry, TenantID: 42, KnowledgeID: "knowledge-gone",
|
||||
AllowedActions: []types.RuntimeTaskAction{types.RuntimeTaskActionCancel},
|
||||
}}}
|
||||
canceller := &runtimeKnowledgeCancelTest{err: repository.ErrKnowledgeNotFound}
|
||||
handler := &SystemHandler{taskInspector: inspector, knowledgeSvc: canceller}
|
||||
recorder := httptest.NewRecorder()
|
||||
ctx, _ := gin.CreateTestContext(recorder)
|
||||
ctx.Params = gin.Params{
|
||||
{Key: "queue", Value: types.QueueMultimodal},
|
||||
{Key: "task_id", Value: "task-orphan"},
|
||||
{Key: "action", Value: string(types.RuntimeTaskActionCancel)},
|
||||
}
|
||||
ctx.Request = httptest.NewRequest(http.MethodPost, "/cancel", nil)
|
||||
|
||||
handler.MutateRuntimeTask(ctx)
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
if inspector.forceDeleted != "task-orphan" {
|
||||
t.Fatalf("expected force delete, got deleted=%q force=%q cancel=%q",
|
||||
inspector.deletedTask, inspector.forceDeleted, inspector.cancelKnowledge)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user