diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cb756f..ec40fe3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Added +- 新增解析监控统一工作台:顶部运行指标与依赖卡片、下方单一文件处理表;来源文件与目标测试库归并最新版本,动作、阶段、状态分列,后端统一筛选、计数及分页。支持单条/跨页勾选重试和版本、失败、重试详情;测试库移除重试独立核验治理规则,不误走上传流程。取消六分页和手动开始入口,生产授权仍在发布管理;本次不改变后台自动调度策略,自动处理闭环需另行端到端验收。 + - 问答记录新增“本周”快捷查询,默认展示北京时间本周数据;所选范围自动读取全部记录,校验完整数量后展示,不再需要分页或加载更多。固定展示完整字段,空列不再隐藏,中文原文直接展示、外语未翻译明确标记;保留源端有界传输、取消和失败保护。 - 问答记录新增年份与自然周选择,周次展示起止日期,支持跨年及每年实际周数;按北京时间周一至周日填入查询范围,默认本周且仅查询截至当前的数据。保留自定义时间与原查询按钮,浏览器时区不改变查询边界。 diff --git a/api/openapi.yaml b/api/openapi.yaml index ef4e0eb..2f83c5f 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -3386,6 +3386,323 @@ "operationId": "cancelPublicationsBatch" } }, + "/api/v1/monitor/tasks": { + "get": { + "operationId": "listMonitorTasks", + "summary": "统一文件处理表(来源文件×目标测试库)", + "description": "按最新未替代版本归并后在数据库内筛选、分页和计数。id筛选附带最近50条版本状态、失败和重试记录。只读,不触发处理。", + "parameters": [ + { + "name": "q", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "source", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "scope", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "target", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "action", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "status", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "stage", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "since", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "until", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "id", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "limit", + "in": "query", + "schema": { + "type": "integer", + "minimum": 1, + "maximum": 100, + "default": 20 + } + }, + { + "name": "offset", + "in": "query", + "schema": { + "type": "integer", + "minimum": 0 + } + } + ], + "responses": { + "400": { + "description": "筛选参数错误" + }, + "401": { + "description": "需要登录" + }, + "403": { + "description": "权限不足" + }, + "500": { + "description": "读取或提交失败" + }, + "501": { + "description": "存储未配置" + }, + "200": { + "description": "同一快照的筛选结果、总数、可重试数及全局筛选项", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "data": { + "type": "object", + "required": [ + "items", + "total", + "limit", + "offset", + "retryable", + "sources", + "targets" + ], + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "title", + "path", + "source_id", + "source_name", + "target_scope", + "target_kb", + "action", + "stage", + "status", + "detail", + "updated_at", + "retry_kind", + "failure_id", + "version_id", + "retry_count", + "history" + ], + "properties": { + "id": { + "type": "string" + }, + "title": { + "type": "string" + }, + "path": { + "type": "string" + }, + "source_id": { + "type": "string" + }, + "source_name": { + "type": "string" + }, + "target_scope": { + "type": "string" + }, + "target_kb": { + "type": "string" + }, + "action": { + "type": "string" + }, + "stage": { + "type": "string" + }, + "status": { + "type": "string" + }, + "detail": { + "type": "string" + }, + "updated_at": { + "type": "string" + }, + "retry_kind": { + "type": "string" + }, + "failure_id": { + "type": "string" + }, + "version_id": { + "type": "string" + }, + "retry_count": { + "type": "integer", + "minimum": 0 + }, + "history": { + "type": "array", + "items": { + "type": "object", + "properties": { + "at": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "detail": { + "type": "string" + } + } + } + } + } + } + }, + "total": { + "type": "integer", + "minimum": 0 + }, + "limit": { + "type": "integer", + "minimum": 0 + }, + "offset": { + "type": "integer", + "minimum": 0 + }, + "retryable": { + "type": "integer", + "minimum": 0 + }, + "sources": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + } + } + } + }, + "targets": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + } + } + } + } + } + } + } + } + } + } + } + } + } + }, + "/api/v1/test-publications/{id}/removal-retry": { + "post": { + "operationId": "retryMonitorRemoval", + "summary": "重试失败的测试库移除任务", + "description": "仅工程师或管理员;事务内核验禁止导入规则、未被替代及无运行任务,恢复原移除任务并登记审计。重复调用冲突不重复入队,不上传知识、不操作来源及生产。", + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "400": { + "description": "筛选参数错误" + }, + "401": { + "description": "需要登录" + }, + "403": { + "description": "权限不足" + }, + "500": { + "description": "读取或提交失败" + }, + "501": { + "description": "存储未配置" + }, + "202": { + "description": "已受理,不代表移除已完成" + }, + "409": { + "description": "规则或状态已变化,不能重试" + } + } + } + }, "/api/v1/monitor/metrics": { "get": { "operationId": "getMonitorMetrics", diff --git a/internal/httpapi/monitor_tasks.go b/internal/httpapi/monitor_tasks.go new file mode 100644 index 0000000..99adad7 --- /dev/null +++ b/internal/httpapi/monitor_tasks.go @@ -0,0 +1,102 @@ +package httpapi + +import ( + "context" + "errors" + postgresstore "knowledge-governance/internal/platform/postgres" + "net/http" + "strconv" + "strings" + "time" +) + +type monitorRemovalRetrier interface { + RetryMonitorRemoval(context.Context, string, string, time.Time) error +} + +func retryMonitorRemoval(w http.ResponseWriter, r *http.Request, config Config) { + identity, ok := requireSessionIdentity(w, r) + if !ok { + return + } + if identity.Role != roleEngineer && identity.Role != roleAdmin { + writeJSON(w, 403, map[string]string{"error": "forbidden"}) + return + } + if r.Header.Get("X-Governance-Action") != "monitor-removal-retry" { + writeJSON(w, 403, map[string]string{"error": "write intent required"}) + return + } + id := r.PathValue("id") + if id == "" || len(id) > 128 { + writeJSON(w, 400, map[string]string{"error": "invalid publication"}) + return + } + store, ok := config.Store.(monitorRemovalRetrier) + if !ok { + writeJSON(w, 501, map[string]string{"error": "removal retry not configured"}) + return + } + err := store.RetryMonitorRemoval(r.Context(), id, actorFor(r), time.Now().UTC()) + if errors.Is(err, postgresstore.ErrPublicationStateConflict) { + writeJSON(w, 409, map[string]string{"error": "状态或治理规则已变化,不能重复提交"}) + return + } + if err != nil { + writeJSON(w, 500, map[string]string{"error": "测试库移除重试失败"}) + return + } + writeJSON(w, 202, map[string]string{"status": "accepted"}) +} + +type monitorTaskReader interface { + ListMonitorTasks(context.Context, postgresstore.MonitorTaskFilter) (postgresstore.MonitorTaskPage, error) +} + +func listMonitorTasks(w http.ResponseWriter, r *http.Request, config Config) { + if _, ok := requireSessionIdentity(w, r); !ok { + return + } + q := r.URL.Query() + f := postgresstore.MonitorTaskFilter{Search: q.Get("q"), Source: q.Get("source"), Scope: q.Get("scope"), Action: q.Get("action"), Status: q.Get("status"), Stage: q.Get("stage"), ID: q.Get("id"), Target: q.Get("target"), Since: q.Get("since"), Until: q.Get("until"), Limit: 20} + for k, p := range map[string]*int{"limit": &f.Limit, "offset": &f.Offset} { + if q.Get(k) != "" { + n, e := strconv.Atoi(q.Get(k)) + if e != nil { + writeJSON(w, 400, map[string]string{"error": "invalid pagination"}) + return + } + *p = n + } + } + if f.Limit < 1 || f.Limit > 100 || f.Offset < 0 || len(f.Search) > 512 || len(f.Source) > 128 || len(f.ID) > 128 { + writeJSON(w, 400, map[string]string{"error": "invalid filter"}) + return + } + for _, pair := range [][2]string{{f.Scope, "|internal|external"}, {f.Action, "|add|update|remove"}, {f.Status, "|active|waiting|running|retrying|attention|completed|cancelled"}, {f.Stage, "|waiting|fetch|upload|parsing|indexing|removal|finished"}} { + if !strings.Contains("|"+pair[1]+"|", "|"+pair[0]+"|") { + writeJSON(w, 400, map[string]string{"error": "invalid filter"}) + return + } + } + for _, v := range []string{f.Since, f.Until} { + if v != "" { + if _, e := time.Parse(time.RFC3339, v); e != nil { + writeJSON(w, 400, map[string]string{"error": "invalid date"}) + return + } + } + } + store, ok := config.Store.(monitorTaskReader) + if !ok { + writeJSON(w, 501, map[string]string{"error": "monitor tasks not configured"}) + return + } + page, e := store.ListMonitorTasks(r.Context(), f) + if e != nil { + writeJSON(w, 500, map[string]string{"error": "无法读取文件处理状态"}) + return + } + w.Header().Set("Cache-Control", "no-store") + writeJSON(w, 200, map[string]any{"data": page}) +} diff --git a/internal/httpapi/monitor_tasks_test.go b/internal/httpapi/monitor_tasks_test.go new file mode 100644 index 0000000..38623a0 --- /dev/null +++ b/internal/httpapi/monitor_tasks_test.go @@ -0,0 +1,53 @@ +package httpapi + +import ( + "context" + postgresstore "knowledge-governance/internal/platform/postgres" + "net/http/httptest" + "testing" +) + +type monitorTasksTestStore struct { + fakeJobStore + filter postgresstore.MonitorTaskFilter +} + +func (s *monitorTasksTestStore) ListMonitorTasks(_ context.Context, f postgresstore.MonitorTaskFilter) (postgresstore.MonitorTaskPage, error) { + s.filter = f + return postgresstore.MonitorTaskPage{Items: []postgresstore.MonitorTask{}, Total: 145, Limit: f.Limit, Offset: f.Offset}, nil +} +func TestMonitorTasksRoute(t *testing.T) { + s := &monitorTasksTestStore{} + req := httptest.NewRequest("GET", "/api/v1/monitor/tasks?status=active&limit=20&offset=40", nil) + req = req.WithContext(context.WithValue(req.Context(), sessionKey{}, sessionIdentity{Username: "tester", Role: roleViewer})) + w := httptest.NewRecorder() + h, err := NewHandler(Config{Store: s}) + if err != nil { + t.Fatal(err) + } + h.ServeHTTP(w, req) + if w.Code != 200 { + t.Fatalf("status %d: %s", w.Code, w.Body.String()) + } + if s.filter.Status != "active" || s.filter.Offset != 40 { + t.Fatalf("filter %#v", s.filter) + } + for _, query := range []string{"status=garbage", "stage=internal&scope=internal", "limit=101", "offset=-1", "since=bad"} { + r := httptest.NewRequest("GET", "/api/v1/monitor/tasks?"+query, nil).WithContext(req.Context()) + rr := httptest.NewRecorder() + h.ServeHTTP(rr, r) + if rr.Code != 400 { + t.Fatalf("invalid %s: %d", query, rr.Code) + } + } + unauth := httptest.NewRecorder() + h.ServeHTTP(unauth, httptest.NewRequest("GET", "/api/v1/monitor/tasks", nil)) + if unauth.Code != 401 { + t.Fatalf("unauthenticated %d", unauth.Code) + } + removal := httptest.NewRecorder() + h.ServeHTTP(removal, httptest.NewRequest("POST", "/api/v1/test-publications/example/removal-retry", nil).WithContext(req.Context())) + if removal.Code != 403 { + t.Fatalf("viewer removal %d", removal.Code) + } +} diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index 550cdd5..541f479 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -398,6 +398,9 @@ func NewHandler(config Config) (http.Handler, error) { mux.HandleFunc("GET /api/v1/monitor/knowledge-bases/{id}/files", func(w http.ResponseWriter, r *http.Request) { monitorFiles(w, r, config.Monitor) }) + mux.HandleFunc("GET /api/v1/monitor/tasks", func(w http.ResponseWriter, r *http.Request) { + listMonitorTasks(w, r, config) + }) mux.HandleFunc("GET /api/v1/monitor/metrics", func(w http.ResponseWriter, r *http.Request) { monitorMetrics(w, r, config.Monitor) }) @@ -557,6 +560,9 @@ func NewHandler(config Config) (http.Handler, error) { }) // 需下架条目的移除入口:把「被禁止导入但仍存活在测试库」的知识从测试库移除 // (置 disabling + 入队既有 test.remove 工作流)。与开始处理同级:engineer 即可。 + mux.HandleFunc("POST /api/v1/test-publications/{id}/removal-retry", func(w http.ResponseWriter, r *http.Request) { + retryMonitorRemoval(w, r, config) + }) mux.HandleFunc("POST /api/v1/test-publications/{id}/removal", func(w http.ResponseWriter, r *http.Request) { startTestPublicationRemoval(w, r, config) }) diff --git a/internal/platform/postgres/monitor_removal_retry.go b/internal/platform/postgres/monitor_removal_retry.go new file mode 100644 index 0000000..2675784 --- /dev/null +++ b/internal/platform/postgres/monitor_removal_retry.go @@ -0,0 +1,37 @@ +package postgres + +import ( + "context" + "time" +) + +func (s *Store) RetryMonitorRemoval(ctx context.Context, id, actor string, now time.Time) error { + var count int + err := s.db.QueryRow(ctx, `WITH candidate AS ( + SELECT p.id,j.id AS job_id FROM test_publications p + JOIN source_item_versions v ON v.id=p.source_item_version_id + JOIN source_items si ON si.id=v.source_item_id + JOIN jobs j ON j.payload->>'publication_id'=p.id AND j.type='test.remove' AND j.state='failed' + WHERE p.id=$1 AND p.superseded_at IS NULL AND p.knowledge_id<>'' AND p.retrieval_state IN ('failed','disabling') AND si.policy_state='blocked' + AND NOT EXISTS(SELECT 1 FROM jobs live WHERE live.payload->>'publication_id'=p.id AND live.state IN ('pending','running','retry')) + ORDER BY j.updated_at DESC,j.id LIMIT 1 FOR UPDATE OF p,si,j + ), retried AS ( + UPDATE jobs SET state='pending',attempt=0,available_at=$2,lease_owner='',lease_until=NULL,last_error='',last_error_code='',updated_at=$2 + FROM candidate c WHERE jobs.id=c.job_id AND jobs.state='failed' RETURNING c.id,jobs.id AS job_id + ), changed AS ( + UPDATE test_publications SET retrieval_state='disabling',last_error='',retry_count=retry_count+1,updated_at=$2 + FROM retried r WHERE test_publications.id=r.id RETURNING test_publications.id,r.job_id + ), closed AS ( + UPDATE job_failures SET state='requeued',resolved_at=$2,resolution_actor_ref=$4,resolution_reason='测试库移除重试' FROM changed c WHERE job_failures.job_id=c.job_id AND job_failures.state='open' + ), audited AS ( + INSERT INTO job_retry_operations(id,batch_key,publication_id,actor_ref,reason,outcome,created_at) + SELECT $3,$3,c.id,$4,'测试库移除重试','started',$2 FROM changed c + ) SELECT count(*) FROM changed`, id, now, orchestrationJobID("monitor-removal-retry", id, now.String()), actor).Scan(&count) + if err != nil { + return err + } + if count != 1 { + return ErrPublicationStateConflict + } + return nil +} diff --git a/internal/platform/postgres/monitor_tasks.go b/internal/platform/postgres/monitor_tasks.go new file mode 100644 index 0000000..23c52b6 --- /dev/null +++ b/internal/platform/postgres/monitor_tasks.go @@ -0,0 +1,140 @@ +package postgres + +import ( + "context" + "encoding/json" + "errors" +) + +type MonitorTaskFilter struct { + Search, Source, Scope, Action, Status, Stage, ID, Target, Since, Until string + Limit, Offset int +} +type MonitorTaskEvent struct { + At string `json:"at"` + Kind string `json:"kind"` + Detail string `json:"detail"` +} +type MonitorTask struct { + History []MonitorTaskEvent `json:"history"` + ID string `json:"id"` + Title string `json:"title"` + Path string `json:"path"` + SourceID string `json:"source_id"` + SourceName string `json:"source_name"` + TargetScope string `json:"target_scope"` + TargetKB string `json:"target_kb"` + Action string `json:"action"` + Stage string `json:"stage"` + Status string `json:"status"` + Detail string `json:"detail"` + UpdatedAt string `json:"updated_at"` + RetryKind string `json:"retry_kind"` + FailureID string `json:"failure_id"` + RetryCount int `json:"retry_count"` + VersionID string `json:"version_id"` +} +type MonitorTaskPage struct { + Items []MonitorTask `json:"items"` + Total int `json:"total"` + Limit int `json:"limit"` + Offset int `json:"offset"` + Sources []MonitorOption `json:"sources"` + Targets []MonitorOption `json:"targets"` + Retryable int `json:"retryable"` +} +type MonitorOption struct { + ID string `json:"id"` + Name string `json:"name"` +} + +// Reduce publication versions before filtering; totals and pages share one snapshot. +const monitorTasksCTE = `WITH history_publications AS ( + SELECT hp.* FROM test_publications hp JOIN source_item_versions hv ON hv.id=hp.source_item_version_id + WHERE $7<>'' AND (hv.source_item_id,hp.test_kb_id)=(SELECT v.source_item_id,p.test_kb_id FROM test_publications p JOIN source_item_versions v ON v.id=p.source_item_version_id WHERE p.id=$7) +), history_events AS ( + SELECT updated_at AS at,'版本状态' AS kind,source_item_version_id || ' · ' || processing_state || ' / ' || retrieval_state AS detail FROM history_publications + UNION ALL SELECT f.created_at,'失败记录',f.job_type || ' · ' || f.error_code || ' · 尝试 ' || f.attempt FROM job_failures f WHERE f.publication_id IN(SELECT id FROM history_publications) OR (f.job_type='artifact.fetch' AND f.business_version_id IN(SELECT source_item_version_id FROM history_publications)) + UNION ALL SELECT r.created_at,'重试记录',r.reason || ' · ' || r.outcome FROM job_retry_operations r WHERE r.publication_id IN(SELECT id FROM history_publications) +), latest AS ( + SELECT DISTINCT ON(v.source_item_id,p.test_kb_id) p.*,v.source_item_id,v.content_object_id + FROM test_publications p JOIN source_item_versions v ON v.id=p.source_item_version_id + WHERE p.superseded_at IS NULL + ORDER BY v.source_item_id,p.test_kb_id,p.created_at DESC,p.id DESC +), base AS ( + SELECT p.id,si.title,si.normalized_path AS path,c.id AS source_id,c.name AS source_name, + p.target_scope,p.test_kb_id AS target_kb,p.source_item_version_id AS version_id,p.retry_count, + GREATEST(p.updated_at,j.updated_at,f.created_at) AS updated_at, + p.processing_state,p.ingest_state,p.retrieval_state,p.last_error,p.knowledge_id, + co.content_state,co.local_spool_ref,si.policy_state, + CASE WHEN p.retrieval_state IN ('disabling','disabled') OR (p.knowledge_id<>'' AND si.policy_state='blocked') OR j.type='test.remove' THEN 'remove' + WHEN EXISTS(SELECT 1 FROM test_publications prev JOIN source_item_versions pv ON pv.id=prev.source_item_version_id WHERE pv.source_item_id=si.id AND prev.test_kb_id=p.test_kb_id AND prev.id<>p.id AND prev.created_at>'publication_id'=p.id OR (j.type='artifact.fetch' AND j.payload->>'source_item_version_id'=p.source_item_version_id)) + AND j.type IN ('artifact.fetch','test.upload','test.poll','test.remove') + ORDER BY CASE WHEN j.state IN ('pending','running','retry') THEN 0 ELSE 1 END,j.updated_at DESC,j.id DESC LIMIT 1) j ON TRUE + LEFT JOIN LATERAL(SELECT f.id,f.job_type,f.error_code,f.created_at FROM job_failures f WHERE f.state='open' AND (f.publication_id=p.id OR (f.job_type='artifact.fetch' AND f.business_version_id=p.source_item_version_id)) ORDER BY f.created_at DESC,f.id DESC LIMIT 1) f ON TRUE +), classified AS ( + SELECT *, + CASE WHEN action='remove' AND retrieval_state='disabled' THEN 'completed' + WHEN action<>'remove' AND processing_state='completed' THEN 'completed' + WHEN job_state IN ('pending','retry') AND job_attempt>0 THEN 'retrying' + WHEN job_state='running' THEN 'running' + WHEN job_state IN ('pending','retry') THEN 'waiting' + WHEN failure_id<>'' OR processing_state IN ('failed','stalled') OR ingest_state='failed' OR retrieval_state='failed' THEN 'attention' + WHEN processing_state='cancelled' THEN 'cancelled' + WHEN action='remove' THEN 'waiting' + WHEN processing_state IN ('processing','finalizing') THEN 'running' + ELSE 'waiting' END AS status, + CASE WHEN action='remove' AND retrieval_state='disabled' THEN 'finished' + WHEN action='remove' THEN 'removal' + WHEN processing_state IN ('completed','cancelled') THEN 'finished' + WHEN job_type='artifact.fetch' AND job_state IN ('pending','running','retry') THEN 'fetch' + WHEN processing_state='finalizing' THEN 'indexing' + WHEN processing_state='processing' OR job_type='test.poll' THEN 'parsing' + WHEN ingest_state='uploading' OR job_type='test.upload' THEN 'upload' + WHEN failure_type='artifact.fetch' OR COALESCE(content_state,'')<>'verified' THEN 'fetch' + ELSE 'waiting' END AS stage + FROM base +), projected AS ( + SELECT id,title,path,source_id,source_name,target_scope,target_kb,version_id,retry_count,updated_at,action,stage,status,failure_id, + CASE WHEN status<>'attention' THEN '' + WHEN action='remove' THEN CASE WHEN job_type='test.remove' AND job_state='failed' AND retrieval_state IN ('failed','disabling') AND policy_state='blocked' THEN 'remove' ELSE '' END + WHEN failure_type='artifact.fetch' THEN 'fetch' + WHEN content_state='verified' AND local_spool_ref<>'' AND processing_state IN ('failed','stalled','pending') THEN 'upload' ELSE '' END AS retry_kind, + CASE WHEN status='completed' THEN CASE WHEN action='remove' THEN '已从测试库移除;来源文件保留' ELSE '测试库处理完成;未代表已发布生产' END + WHEN status='cancelled' THEN '处理已取消,记录保留' + WHEN status='retrying' THEN '等待自动重试;下次调度 ' || COALESCE(to_char(available_at AT TIME ZONE 'Asia/Shanghai','YYYY-MM-DD HH24:MI:SS'),'未提供') + WHEN status='attention' THEN COALESCE(NULLIF(last_error,''),NULLIF(failure_error,''),'处理异常,请查看详情') + WHEN stage='removal' THEN CASE WHEN status='running' THEN '正在移除测试库知识' ELSE '等待测试库移除任务' END + WHEN stage='fetch' THEN '等待或正在获取源文件' + WHEN stage='waiting' THEN '文件已就绪,等待后台处理调度' + WHEN stage='upload' THEN '等待或正在上传测试知识库' + WHEN stage='indexing' THEN '正在索引与增强' + ELSE '正在解析处理' END AS detail + FROM classified +), filtered AS ( + SELECT * FROM projected WHERE ($1='' OR strpos(lower(title || ' ' || path),lower($1))>0) + AND ($2='' OR source_id=$2) AND ($3='' OR target_scope=$3) AND ($4='' OR action=$4) + AND ($5='' OR ($5='active' AND status NOT IN ('completed','cancelled')) OR status=$5) + AND ($6='' OR stage=$6) AND ($7='' OR id=$7) AND ($10='' OR target_kb=$10) + AND ($11='' OR updated_at>=NULLIF($11,'')::timestamptz) AND ($12='' OR updated_at<=NULLIF($12,'')::timestamptz) +) ` + +func (s *Store) ListMonitorTasks(ctx context.Context, f MonitorTaskFilter) (MonitorTaskPage, error) { + if f.Limit < 1 || f.Limit > 100 || f.Offset < 0 { + return MonitorTaskPage{}, errors.New("invalid monitor pagination") + } + var raw []byte + err := s.db.QueryRow(ctx, monitorTasksCTE+`SELECT jsonb_build_object('items',COALESCE((SELECT jsonb_agg(to_jsonb(x)||jsonb_build_object('history',COALESCE((SELECT jsonb_agg(e ORDER BY e.at DESC,e.kind,e.detail) FROM (SELECT * FROM history_events ORDER BY at DESC,kind,detail LIMIT 50)e),'[]'::jsonb)) ORDER BY x.updated_at DESC,x.id) FROM (SELECT * FROM filtered ORDER BY updated_at DESC,id LIMIT $8 OFFSET $9)x),'[]'::jsonb),'total',(SELECT count(*) FROM filtered),'limit',$8::int,'offset',$9::int,'retryable',(SELECT count(*) FROM filtered WHERE retry_kind<>''),'sources',COALESCE((SELECT jsonb_agg(x ORDER BY x.name) FROM (SELECT DISTINCT source_id AS id,source_name AS name FROM projected)x),'[]'::jsonb),'targets',COALESCE((SELECT jsonb_agg(x ORDER BY x.name) FROM (SELECT DISTINCT target_kb AS id,target_kb AS name FROM projected)x),'[]'::jsonb))`, f.Search, f.Source, f.Scope, f.Action, f.Status, f.Stage, f.ID, f.Limit, f.Offset, f.Target, f.Since, f.Until).Scan(&raw) + if err != nil { + return MonitorTaskPage{}, err + } + var result MonitorTaskPage + err = json.Unmarshal(raw, &result) + return result, err +} diff --git a/internal/platform/postgres/monitor_tasks_integration_test.go b/internal/platform/postgres/monitor_tasks_integration_test.go new file mode 100644 index 0000000..d3ba7e7 --- /dev/null +++ b/internal/platform/postgres/monitor_tasks_integration_test.go @@ -0,0 +1,105 @@ +package postgres + +import ( + "context" + "github.com/jackc/pgx/v5/pgxpool" + "os" + "testing" + "time" +) + +func TestMonitorTasksUnifiedPaginationIntegration(t *testing.T) { + url := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL")) + if url == "" { + t.Skip("TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, url) + if err != nil { + t.Fatal(err) + } + defer pool.Close() + if err = ApplyMigrations(ctx, pool); err != nil { + t.Fatal(err) + } + exec := func(q string) { + t.Helper() + if _, e := pool.Exec(ctx, q); e != nil { + t.Fatal(e) + } + } + exec(`INSERT INTO source_connections (id,source_type,name) VALUES ('conn-manual','manual','Manual')`) + now := time.Date(2026, 10, 27, 12, 0, 0, 0, time.UTC) + insertReviewFixture(t, ctx, pool, 4, "mt-ready", now) + insertReviewFixture(t, ctx, pool, 5, "mt-remove", now) + insertReviewFixture(t, ctx, pool, 6, "mt-removed", now) + exec(`UPDATE source_items SET policy_state='blocked' WHERE id='si-review-5'`) + exec(`UPDATE test_publications SET retrieval_state='disabled' WHERE id='mt-removed'`) + store := NewStore(pool) + page, err := store.ListMonitorTasks(ctx, MonitorTaskFilter{Limit: 1}) + if err != nil { + t.Fatal(err) + } + if page.Total != 3 || len(page.Items) != 1 { + t.Fatalf("global pagination: %#v", page) + } + active, err := store.ListMonitorTasks(ctx, MonitorTaskFilter{Status: "active", Limit: 100}) + if err != nil { + t.Fatal(err) + } + if active.Total != 1 || active.Items[0].ID != "mt-remove" || active.Items[0].Action != "remove" || active.Items[0].Status != "waiting" { + t.Fatalf("removal is not completed: %#v", active) + } + done, err := store.ListMonitorTasks(ctx, MonitorTaskFilter{Action: "remove", Status: "completed", Limit: 100}) + if err != nil { + t.Fatal(err) + } + if done.Total != 1 || done.Items[0].ID != "mt-removed" || done.Items[0].RetryKind != "" { + t.Fatalf("removed is not ready/retryable: %#v", done) + } + empty, err := store.ListMonitorTasks(ctx, MonitorTaskFilter{Limit: 1, Offset: 999}) + if err != nil || empty.Total != 3 || len(empty.Items) != 0 { + t.Fatalf("empty last page lost total: %#v %v", empty, err) + } + exec(`UPDATE test_publications SET retrieval_state='failed' WHERE id='mt-remove'`) + exec(`INSERT INTO jobs(id,type,idempotency_key,payload,state) VALUES('mt-removal-job','test.remove','mt-removal-key','{"publication_id":"mt-remove"}','failed')`) + exec(`INSERT INTO job_failures(id,job_id,job_type,publication_id,error_code,attempt,max_attempts_at_failure,last_error) VALUES('mt-failure','mt-removal-job','test.remove','mt-remove','removal_failed',8,8,'removal_failed')`) + if err = store.RetryMonitorRemoval(ctx, "mt-remove", "tester", now); err != nil { + t.Fatalf("retry removal: %v", err) + } + if err = store.RetryMonitorRemoval(ctx, "mt-remove", "tester", now); err != ErrPublicationStateConflict { + t.Fatalf("duplicate retry accepted: %v", err) + } + detail, e := store.ListMonitorTasks(ctx, MonitorTaskFilter{ID: "mt-remove", Limit: 1}) + if e != nil || len(detail.Items) != 1 || len(detail.Items[0].History) < 3 { + t.Fatalf("missing version/failure/retry history: %#v %v", detail, e) + } + var n int + if err = pool.QueryRow(ctx, `SELECT count(*) FROM jobs WHERE type='test.upload'`).Scan(&n); err != nil || n != 0 { + t.Fatalf("removal created upload: %d %v", n, err) + } + if err = pool.QueryRow(ctx, `SELECT count(*) FROM jobs WHERE id='mt-removal-job' AND state='pending'`).Scan(&n); err != nil || n != 1 { + t.Fatalf("not enqueued: %d %v", n, err) + } + exec(`UPDATE jobs SET state='failed' WHERE id='mt-removal-job'`) + exec(`UPDATE source_items SET policy_state='internal' WHERE id='si-review-5'`) + // A new version replaces the main row; routing that version to another KB adds one row. + exec(`INSERT INTO source_item_versions(id,source_item_id,revision,content_state,raw_record_summary) VALUES('mt-v-new','si-review-4','new-revision','verified','{}')`) + exec(`INSERT INTO test_publications(id,source_item_version_id,test_kb_id,target_scope,processing_state,created_at) SELECT 'mt-new','mt-v-new',test_kb_id,'internal','processing',created_at+interval '1 second' FROM test_publications WHERE id='mt-ready'`) + exec(`INSERT INTO test_publications(id,source_item_version_id,test_kb_id,target_scope,processing_state) VALUES('mt-external','mt-v-new','external-kb','external','planned')`) + versions, e := store.ListMonitorTasks(ctx, MonitorTaskFilter{Search: "mt-ready", Limit: 20}) + if e != nil || versions.Total != 2 { + t.Fatalf("file-target identity: %#v %v", versions, e) + } + for _, v := range versions.Items { + if v.ID == "mt-ready" { + t.Fatal("old version escaped dedup") + } + if v.ID == "mt-new" && (v.Action != "update" || v.Stage != "parsing") { + t.Fatalf("latest action/stage: %#v", v) + } + } + if err = store.RetryMonitorRemoval(ctx, "mt-remove", "tester", now); err != ErrPublicationStateConflict { + t.Fatalf("changed policy accepted: %v", err) + } +} diff --git a/web/src/WeKnoraMonitor.test.tsx b/web/src/WeKnoraMonitor.test.tsx index eba2060..bb5dd46 100644 --- a/web/src/WeKnoraMonitor.test.tsx +++ b/web/src/WeKnoraMonitor.test.tsx @@ -1,99 +1,105 @@ import { renderToStaticMarkup } from 'react-dom/server' -import { describe,expect,it,vi } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { WeKnoraMonitorView } from './WeKnoraMonitor' -import type { MonitorFilePage,MonitorOverview,RunMetrics } from './monitor' -import type { TestPublicationPage } from './testPublications' +import type { MonitorTaskPage } from './monitorTasks' -const overview:MonitorOverview={mode:'simulation',simulated:true,observed_at:'2026-09-08T12:00:00Z',host:{cpu_percent:42,memory_used_bytes:19971597926,memory_total_bytes:34359738368,disk_used_bytes:128849018880,disk_total_bytes:536870912000,gpu_state:'unknown',gpu_detail:'未接入真实GPU监控',simulated:true},services:[{key:'weknora',name:'WeKnora API',state:'healthy',detail:'模拟连接正常',latency_ms:86,observed_at:'2026-09-08T12:00:00Z',simulated:true},{key:'mineru',name:'MinerU',state:'healthy',detail:'模拟处理中2,队列5',latency_ms:24,observed_at:'2026-09-08T12:00:00Z',simulated:true},{key:'postgres',name:'PostgreSQL',state:'unknown',detail:'未接入真实监控',latency_ms:0,observed_at:'2026-09-08T12:00:00Z',simulated:true},{key:'redis',name:'Redis / MQ',state:'unknown',detail:'未接入真实监控',latency_ms:0,observed_at:'2026-09-08T12:00:00Z',simulated:true},{key:'oss',name:'OSS',state:'unknown',detail:'未接入真实监控',latency_ms:0,observed_at:'2026-09-08T12:00:00Z',simulated:true}],knowledge_bases:[{id:'kb-sim',name:'公司级技术支持知识库',visibility:'mixed',imported:5750,completed:4530,pending:450,processing:50,finalizing:714,failed:6,cancelled:0,planned:120,needs_removal:3,stale_planned:0,internal_imported:4200,internal_completed:3300,external_imported:1550,external_completed:1230}],pipeline:{fetch_pending:4,fetch_failed:1,fetch_missing:0,concurrency:2},health:[]} -const files:MonitorFilePage={items:[{name:'[模拟] 产品参数总表.xlsx',directory:'02|产品资料',visibility:'external',status:'finalizing',stage_detail:'基础检索可用,Wiki与图谱仍在生成',elapsed_seconds:2100,error_code:''}],total:1,limit:20,offset:0,sample_only:true} -const metrics:RunMetrics={observed_at:'2026-09-12T00:00:00Z',active_workers:2,worker_source:'pg_lease',queue:{pending:3,running:4,retry:1,failed:2,oldest_pending_age_seconds:120},short_window:{window_seconds:3600,jobs_started:50,jobs_completed:48,jobs_failed:6,success_rate:88.9,failure_rate:11.1,duration_avg_seconds:55,duration_p50_seconds:30,duration_p95_seconds:1800},long_window:{window_seconds:86400,jobs_started:500,jobs_completed:482,jobs_failed:18,success_rate:96.4,failure_rate:3.6,duration_avg_seconds:70,duration_p50_seconds:45,duration_p95_seconds:2400}} -const queue:TestPublicationPage={items:[{id:'tp-plan',title:'API.docx',ingest_state:'pending',processing_state:'planned',retrieval_state:'unpublished',target_scope: 'internal', change_set_id: '', change_type:'added',can_start:true,retry_count:0,last_error:'',can_retry:false,updated_at:'2026-09-09T10:01:00Z'},{id:'tp-fetch',title:'焊接手册.pdf',ingest_state:'pending',processing_state:'planned',retrieval_state:'unpublished',target_scope:'external',change_set_id:'',change_type:'added',can_start:false,retry_count:0,last_error:'',can_retry:false,updated_at:'2026-09-09T10:02:00Z'},{id:'tp-failed',title:'CR20A.pdf',ingest_state:'uploaded',processing_state:'failed',retrieval_state:'unpublished',target_scope: 'external', change_set_id: '', change_type:'updated',can_start:false,retry_count:1,last_error:'remote_parse_failed',can_retry:true,updated_at:'2026-09-09T10:00:00Z'}],total:3,limit:50,offset:0} -const plans=queue.items.filter((item)=>item.can_start) -const baseProps={overview,files,queue,plans,selectedKB:overview.knowledge_bases[0],status:'' as const,queueStatus:'' as const,queueScope:'' as const,loading:false,error:null as string|null,retryingIDs:[] as string[],starting:false,startMessage:'',onSelectKB:vi.fn(),onStatus:vi.fn(),onQueueStatus:vi.fn(),onQueueScope:vi.fn(),lane:'queue',onLaneChange:vi.fn(),metrics,onRetry:vi.fn(),onBatchRetry:vi.fn(),onStart:vi.fn(),onRefresh:vi.fn()} -describe('WeKnoraMonitorView',()=>{ - it('filters the processing queue by target knowledge base and shows each task target',()=>{ - const html=renderToStaticMarkup() - expect(html).toContain('aria-label="队列目标知识库"') - expect(html).toContain('对内知识库') - expect(html).toContain('对外知识库') - expect(html).toContain('