fix(monitor): keep failing publications visible outside the queue page

- The attention lane was derived from page 1 of the queue (limit 50, newest
  by updated_at), so failures ranked past 50 silently disappeared from
  "needs attention" as the completed backlog grew.
- Fetch attention items with a dedicated request: the API now accepts a
  comma-separated processing_state list (failed,stalled) and the view prefers
  that result set over the paged queue.
- Batch retry/cancel targets are computed from the same failure set instead of
  the visible page, so actions reach every failing item.
- 全部任务 shows 已显示 N / 共 M plus 加载更多 so rows past 50 are reachable.
- Fix pre-existing ui_settings integration test: strict JSON decoding rejects
  the legacy updated_by field (actor comes from the session).
- Tests: view regression (attention item outside the page, load-more
  affordance) + handler multi-state filter + isolated-DB set filter.
This commit is contained in:
姚章浩
2026-09-16 08:53:18 +08:00
parent 553eff5f7f
commit 1d6ba6dc2e
10 changed files with 149 additions and 21 deletions
+2 -9
View File
@@ -1414,17 +1414,10 @@
{
"name": "processing_state",
"in": "query",
"description": "解析状态过滤。可传单个状态,或用逗号拼接多个状态(例如 failed,stalled)一次取出全部失败项。",
"schema": {
"type": "string",
"enum": [
"pending",
"processing",
"finalizing",
"completed",
"failed",
"stalled",
"cancelled"
]
"pattern": "^(planned|pending|processing|finalizing|completed|failed|stalled|cancelled)(,(planned|pending|processing|finalizing|completed|failed|stalled|cancelled))*$"
}
},
{
+14 -2
View File
@@ -1725,8 +1725,8 @@ func getReleaseBatch(w http.ResponseWriter, r *http.Request, store Store) {
func listTestPublications(w http.ResponseWriter, r *http.Request, store Store) {
filter := governance.TestPublicationFilter{ProcessingState: r.URL.Query().Get("processing_state"), TargetScope: r.URL.Query().Get("target_scope"), Limit: 50}
if filter.ProcessingState != "" && !validProcessingState(filter.ProcessingState) {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "processing_state must be planned, pending, processing, finalizing, completed, failed, or stalled"})
if filter.ProcessingState != "" && !validProcessingStateFilter(filter.ProcessingState) {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "processing_state must be a comma-separated list of planned, pending, processing, finalizing, completed, failed, stalled, or cancelled"})
return
}
if filter.TargetScope != "" && filter.TargetScope != "internal" && filter.TargetScope != "external" {
@@ -2032,6 +2032,18 @@ func validProcessingState(state string) bool {
}
}
// validProcessingStateFilter accepts a single state or a comma-separated list so
// callers can ask for several lanes at once (e.g. the attention lane needs
// failed+stalled in one query instead of paging the whole queue).
func validProcessingStateFilter(filter string) bool {
for _, state := range strings.Split(filter, ",") {
if !validProcessingState(state) {
return false
}
}
return true
}
func newID() (string, error) {
var raw [16]byte
if _, err := rand.Read(raw[:]); err != nil {
+27
View File
@@ -7,6 +7,7 @@ import (
"errors"
"net/http"
"net/http/httptest"
"net/url"
"slices"
"strings"
"testing"
@@ -1335,6 +1336,32 @@ func TestHandlerListsTestPublicationsSafeProjection(t *testing.T) {
}
}
func TestHandlerAcceptsMultiStateAttentionFilter(t *testing.T) {
store := &fakeJobStore{}
handler, err := NewHandler(Config{Store: store})
if err != nil {
t.Fatal(err)
}
response := httptest.NewRecorder()
handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/api/v1/test-publications?processing_state=failed,stalled&limit=100", nil))
if response.Code != http.StatusOK {
t.Fatalf("status = %d; body=%s", response.Code, response.Body.String())
}
if store.publicationFilter.ProcessingState != "failed,stalled" {
t.Fatalf("filter = %#v", store.publicationFilter)
}
// One bad entry anywhere in the list is still rejected.
for _, value := range []string{"failed,bogus", "failed,", ",stalled"} {
response = httptest.NewRecorder()
handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/api/v1/test-publications?processing_state="+url.QueryEscape(value), nil))
if response.Code != http.StatusBadRequest {
t.Fatalf("processing_state=%q status = %d; want 400", value, response.Code)
}
}
}
func TestHandlerConfirmsChangeSetItems(t *testing.T) {
now := time.Date(2026, 9, 12, 5, 0, 0, 0, time.UTC)
store := &fakeJobStore{confirmChangeSetResult: governance.ConfirmCatalogChangeSetResult{ChangeSetID: "cs-1", ConfirmedItems: 2, UploadsCreated: 2}}
@@ -37,7 +37,7 @@ func TestUISettingsWriteReadBackIntegration(t *testing.T) {
}
create := httptest.NewRecorder()
createRequest := httptest.NewRequest(http.MethodPost, "/api/v1/ui-settings/candidates", bytes.NewBufferString(`{"document_refresh_seconds":30,"resource_refresh_seconds":60,"stalled_after_seconds":3600,"finalizing_after_seconds":14400,"updated_by":"engineer-e2e"}`))
createRequest := httptest.NewRequest(http.MethodPost, "/api/v1/ui-settings/candidates", bytes.NewBufferString(`{"document_refresh_seconds":30,"resource_refresh_seconds":60,"stalled_after_seconds":3600,"finalizing_after_seconds":14400}`))
createRequest.Header.Set("X-Governance-Action", "ui-settings")
createRequest.Header.Set("Idempotency-Key", "op-e2e-uis-create")
handler.ServeHTTP(create, createRequest)
@@ -133,3 +133,52 @@ VALUES('job-cancel-dlq','test.upload','tp-cancel-1','remote_parse_failed','remot
t.Fatal("cancelled publication lost its version link")
}
}
// The attention lane lists every failing publication in one request, so the
// processing_state filter has to behave as a set — a single literal would make
// failures silently vanish again as soon as the queue grows past one page.
func TestListPublicationsAcceptsStateSet(t *testing.T) {
databaseURL := isolatedIntegrationDatabaseURL(t, os.Getenv("TEST_DATABASE_URL"))
if databaseURL == "" {
t.Skip("TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, databaseURL)
if err != nil {
t.Fatal(err)
}
defer pool.Close()
if err := ApplyMigrations(ctx, pool); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `
INSERT INTO source_connections(id,source_type,name,enabled) VALUES('conn-set','dingtalk','集合来源',true);
INSERT INTO content_objects(id,sha256,size_bytes,local_spool_ref,content_state) VALUES('co-set',repeat('d',64),4891,'verified/set.docx','verified');
INSERT INTO source_items(id,source_connection_id,stable_external_id,canonical_uri_hash,title,item_type,revision,visibility,visibility_origin)
VALUES('item-set','conn-set','ext-set',repeat('e',64),'FAQ.adoc','document','r1','internal','manual');
INSERT INTO source_item_versions(id,source_item_id,revision,content_object_id,content_state) VALUES('siv-set','item-set','r1','co-set','verified');
INSERT INTO test_publications(id,source_item_version_id,test_kb_id,ingest_state,processing_state,retrieval_state,target_scope,last_error,updated_at) VALUES
('tp-set-failed','siv-set','kb-set-1','uploaded','failed','unpublished','internal','remote_parse_failed',NOW()),
('tp-set-stalled','siv-set','kb-set-2','uploaded','stalled','unpublished','internal','stalled',NOW()),
('tp-set-done','siv-set','kb-set-3','uploaded','completed','enabled','internal','',NOW());`); err != nil {
t.Fatal(err)
}
store := NewStore(pool)
for _, testCase := range []struct {
filter string
want int
}{
{"failed,stalled", 2},
{"failed", 1},
{"cancelled", 0},
{"", 3},
} {
page, err := store.ListTestPublications(ctx, governance.TestPublicationFilter{ProcessingState: testCase.filter, Limit: 50})
if err != nil {
t.Fatal(err)
}
if page.Total != testCase.want || len(page.Items) != testCase.want {
t.Fatalf("filter %q = %d items / total %d, want %d", testCase.filter, len(page.Items), page.Total, testCase.want)
}
}
}
@@ -116,7 +116,7 @@ ORDER BY p.id
const countTestPublicationsSQL = `
SELECT COUNT(*)
FROM test_publications
WHERE ($1 = '' OR processing_state = $1) AND ($2 = '' OR target_scope = $2)
WHERE ($1 = '' OR processing_state = ANY(string_to_array($1, ','))) AND ($2 = '' OR target_scope = $2)
`
const listTestPublicationsSQL = `
@@ -143,7 +143,7 @@ FROM test_publications p
JOIN source_item_versions siv ON siv.id = p.source_item_version_id
JOIN source_items si ON si.id = siv.source_item_id
LEFT JOIN content_objects co ON co.id = siv.content_object_id
WHERE ($1 = '' OR p.processing_state = $1) AND ($2 = '' OR p.target_scope = $2) AND p.superseded_at IS NULL
WHERE ($1 = '' OR p.processing_state = ANY(string_to_array($1, ','))) AND ($2 = '' OR p.target_scope = $2) AND p.superseded_at IS NULL
ORDER BY p.updated_at DESC, p.id
LIMIT $3 OFFSET $4
`
+14
View File
@@ -56,4 +56,18 @@ describe('WeKnoraMonitorView',()=>{
expect(html).toContain('加载失败')
expect(html).not.toContain('completed 0')
})
it('keeps attention items visible when they are outside the first queue page',()=>{
const firstPage:TestPublicationPage={items:[queue.items[0]],total:115,limit:50,offset:0}
const attentionItem=queue.items[2]
const html=renderToStaticMarkup(<WeKnoraMonitorView {...baseProps} queue={firstPage} attention={[attentionItem]} lane='attention' />)
expect(html).toContain('需要你出手(1)')
expect(html).toContain('CR20A.pdf')
expect(html).toContain('批量重试所选(1)')
})
it('offers to page through the full queue when more rows exist',()=>{
const firstPage:TestPublicationPage={items:queue.items,total:115,limit:50,offset:0}
const html=renderToStaticMarkup(<WeKnoraMonitorView {...baseProps} queue={firstPage} lane='queue' onLoadMore={vi.fn()} />)
expect(html).toContain('已显示 3 / 共 115')
expect(html).toContain('加载更多')
})
})
+36 -5
View File
@@ -78,6 +78,8 @@ interface ViewProps {
files: MonitorFilePage | null
queue?: TestPublicationPage
plans?: TestPublication[]
attention?: TestPublication[]
loadingMore?: boolean
selectedKB: KnowledgeProgress | null
status: MonitorStatus | ''
queueStatus?: ProcessingState | ''
@@ -101,6 +103,7 @@ interface ViewProps {
onBatchRetry?: (ids: string[]) => void
onBatchCancel?: (ids: string[]) => void
onStart?: (ids: string[]) => void
onLoadMore?: () => void
onRefresh: () => void
}
@@ -109,6 +112,7 @@ export function WeKnoraMonitorView({
files,
queue = emptyQueue,
plans = queue.items,
attention,
selectedKB,
status,
queueStatus = '',
@@ -120,6 +124,7 @@ export function WeKnoraMonitorView({
retryingIDs = [],
cancellingIDs = [],
starting = false,
loadingMore = false,
startMessage = '',
batchMessage = '',
failures = [],
@@ -132,6 +137,7 @@ export function WeKnoraMonitorView({
onBatchRetry = () => undefined,
onBatchCancel = () => undefined,
onStart = () => undefined,
onLoadMore = () => undefined,
onRefresh,
}: ViewProps) {
const [selectedStartIDs, setSelectedStartIDs] = useState<string[]>([])
@@ -146,11 +152,15 @@ export function WeKnoraMonitorView({
const priority = (item: TestPublication) => item.processing_state === 'failed' || item.processing_state === 'stalled' ? 0 : 1
return priority(left) - priority(right)
}), [queue.items])
const retryableIDs = queue.items.filter((item) => item.can_retry).map((item) => item.id)
// Batch actions must target every failing item the lane shows, not just the
// rows that happen to sit inside the current queue page.
const retryableIDs = (attention ?? queue.items).filter((item) => item.can_retry).map((item) => item.id)
const batchTargets = selectedRetryIDs.length ? retryableIDs.filter((id) => selectedRetryIDs.includes(id)) : retryableIDs
const startablePlans = plans.filter((item) => item.can_start)
const startTargets = selectedStartIDs.length ? startablePlans.filter((item) => selectedStartIDs.includes(item.id)).map((item) => item.id) : startablePlans.map((item) => item.id)
const attentionItems = queue.items.filter((item) => item.processing_state === 'failed' || item.processing_state === 'stalled')
// Failures needing a human must never be hidden by the queue page window:
// prefer the dedicated attention fetch, fall back to the visible page.
const attentionItems = (attention ?? queue.items).filter((item) => item.processing_state === 'failed' || item.processing_state === 'stalled')
const inflightItems = queue.items.filter((item) => item.processing_state === 'processing' || item.processing_state === 'finalizing' || (item.processing_state === 'pending' && !item.can_start))
const doneItems = queue.items.filter((item) => item.processing_state === 'completed')
const plannedItems = queue.items.filter((item) => item.processing_state === 'planned')
@@ -230,6 +240,7 @@ export function WeKnoraMonitorView({
{lane === 'queue' ? <section className="inbox-section" aria-label="全部任务">
<div className="section-head"><h2>全部任务({queue.total})</h2><span className="spacer" /><label>目标知识库<select aria-label="队列目标知识库" value={queueScope} onChange={(event) => onQueueScope(event.target.value as 'internal' | 'external' | '')}><option value="">内/外全部</option><option value="internal">对内知识库</option><option value="external">对外知识库</option></select><select aria-label="队列状态" value={queueStatus} onChange={(event) => onQueueStatus(event.target.value as ProcessingState | '')}><option value="">全部状态</option><option value="planned">待入库</option><option value="pending">等待解析</option><option value="processing">解析中</option><option value="finalizing">索引与增强</option><option value="completed">成功</option><option value="failed">失败</option><option value="stalled">停滞</option><option value="cancelled">已取消</option></select></label></div>
{orderedQueue.length ? orderedQueue.map(actionTask) : !loading ? emptyLane('当前筛选没有解析任务。') : null}
{queue.items.length < queue.total ? <div className="queue-more"><span>已显示 {queue.items.length} / 共 {queue.total}</span><button type="button" disabled={loadingMore} onClick={() => onLoadMore?.()}>{loadingMore ? '加载中…' : '加载更多'}</button></div> : null}
</section> : null}
{metrics ? <footer className="monitor-strip" aria-label="真实运行指标"><div>1小时·成功 <b className={metrics.short_window.failure_rate !== null && metrics.short_window.failure_rate > 5 ? 'bad' : 'good'}>{metrics.short_window.success_rate === null ? '数据不足' : `${metrics.short_window.success_rate}%`}</b></div><div>24小时·成功 <b>{metrics.long_window.success_rate === null ? '数据不足' : `${metrics.long_window.success_rate}%`}</b></div><div>p50 {metrics.short_window.duration_p50_seconds === null ? '—' : duration(Math.round(metrics.short_window.duration_p50_seconds))} · p95 {metrics.short_window.duration_p95_seconds === null ? '—' : duration(Math.round(metrics.short_window.duration_p95_seconds))}</div><div>Worker {metrics.active_workers} · 等待 {metrics.queue.pending} · 运行 {metrics.queue.running}{metrics.queue.failed ? <span className="bad"> · 失败 {metrics.queue.failed}</span> : null}</div>{overview?.health.map((service) => <div key={service.key}>{service.name} <b className={service.readiness === 'ready' ? 'good' : service.readiness === 'blocked' || service.readiness === 'down' ? 'bad' : ''}>{service.readiness === 'ready' ? '可接单' : service.readiness === 'degraded' ? '降级' : service.readiness === 'not_configured' ? '未配置' : service.readiness === 'down' ? '不可达' : '阻断'}</b></div>)}{overview ? <div>CPU {overview.host.cpu_percent.toFixed(0)}% · 内存 {gb(overview.host.memory_used_bytes)}/{gb(overview.host.memory_total_bytes)}GB</div> : null}</footer> : null}
</div>
@@ -259,6 +270,8 @@ export function WeKnoraMonitor() {
const [startMessage, setStartMessage] = useState('')
const [batchMessage, setBatchMessage] = useState('')
const [failures, setFailures] = useState<JobFailure[]>([])
const [attention, setAttention] = useState<TestPublication[] | null>(null)
const [loadingMore, setLoadingMore] = useState(false)
const [metrics, setMetrics] = useState<RunMetrics | null>(null)
const setContextTabs = useContext(WorkspaceContextTabsSetter)
@@ -287,8 +300,9 @@ export function WeKnoraMonitor() {
void Promise.all([
fetchTestPublications({ processing_state: queueStatus, target_scope: queueScope, limit: 50, offset: 0 }, controller.signal),
fetchTestPublications({ processing_state: 'planned', limit: 100, offset: 0 }, controller.signal),
]).then(([queuePage, planPage]) => {
if (!controller.signal.aborted) { setQueue(queuePage); setPlans(planPage.items); setError(null) }
fetchTestPublications({ processing_state: ['failed', 'stalled'], limit: 100, offset: 0 }, controller.signal),
]).then(([queuePage, planPage, attentionPage]) => {
if (!controller.signal.aborted) { setQueue(queuePage); setPlans(planPage.items); setAttention(attentionPage.items); setError(null) }
}).catch((cause) => { if (!controller.signal.aborted) setError(cause instanceof Error ? cause.message : '加载解析队列失败') }).finally(() => { if (!controller.signal.aborted) setLoading(false) })
return () => controller.abort()
}, [queueStatus, queueScope, documentRevision])
@@ -343,6 +357,23 @@ export function WeKnoraMonitor() {
}
}
const loadMoreQueue = async () => {
if (loadingMore) return
setLoadingMore(true)
try {
const next = await fetchTestPublications({ processing_state: queueStatus, target_scope: queueScope, limit: 50, offset: queue.items.length })
setQueue((current) => {
const seen = new Set(current.items.map((item) => item.id))
return { ...next, offset: 0, items: [...current.items, ...next.items.filter((item) => !seen.has(item.id))] }
})
setError(null)
} catch (cause) {
setError(cause instanceof Error ? cause.message : '加载更多失败')
} finally {
setLoadingMore(false)
}
}
const retryBatch = async (ids: string[]) => {
setRetryingIDs(ids)
setBatchMessage('')
@@ -388,5 +419,5 @@ export function WeKnoraMonitor() {
setStarting(false)
}
}
return <WeKnoraMonitorView overview={overview} files={files} queue={queue} plans={plans} selectedKB={selected} status={status} queueStatus={queueStatus} queueScope={queueScope} loading={loading} error={error} retryingIDs={retryingIDs} cancellingIDs={cancellingIDs} starting={starting} startMessage={startMessage} batchMessage={batchMessage} failures={failures} metrics={metrics} onSelectKB={(kb) => { setSelected(kb); setStatus(''); setFiles(null) }} onStatus={(value) => { setStatus(value); setFiles(null) }} onQueueStatus={setQueueStatus} onQueueScope={setQueueScope} lane={lane} onLaneChange={(value) => setLane(value as typeof lane)} onRetry={(id) => void retryOne(id)} onBatchRetry={(ids) => void retryBatch(ids)} onBatchCancel={(ids) => void cancelBatch(ids)} onStart={(ids) => void startPlans(ids)} onRefresh={() => { setDocumentRevision((value) => value + 1); setResourceRevision((value) => value + 1) }} />
return <WeKnoraMonitorView overview={overview} files={files} queue={queue} plans={plans} selectedKB={selected} status={status} queueStatus={queueStatus} queueScope={queueScope} loading={loading} error={error} retryingIDs={retryingIDs} cancellingIDs={cancellingIDs} starting={starting} startMessage={startMessage} batchMessage={batchMessage} failures={failures} metrics={metrics} onSelectKB={(kb) => { setSelected(kb); setStatus(''); setFiles(null) }} onStatus={(value) => { setStatus(value); setFiles(null) }} onQueueStatus={setQueueStatus} onQueueScope={setQueueScope} lane={lane} onLaneChange={(value) => setLane(value as typeof lane)} onRetry={(id) => void retryOne(id)} onBatchRetry={(ids) => void retryBatch(ids)} onBatchCancel={(ids) => void cancelBatch(ids)} onStart={(ids) => void startPlans(ids)} attention={attention ?? undefined} loadingMore={loadingMore} onLoadMore={() => void loadMoreQueue()} onRefresh={() => { setDocumentRevision((value) => value + 1); setResourceRevision((value) => value + 1) }} />
}
+3 -2
View File
@@ -34,7 +34,8 @@ export interface AcceptanceRecord {
}
export interface TestPublicationQuery {
processing_state: ProcessingState | ''
// A single state, or several at once (joined into a comma list server-side).
processing_state: ProcessingState | '' | ProcessingState[]
target_scope?: 'internal' | 'external' | ''
limit: number
offset: number
@@ -93,7 +94,7 @@ export function parseTestPublicationsResponse(value: unknown): TestPublicationPa
export async function fetchTestPublications(query: TestPublicationQuery, signal?: AbortSignal): Promise<TestPublicationPage> {
const parameters = new URLSearchParams()
if (query.processing_state) parameters.set('processing_state', query.processing_state)
if (query.processing_state) parameters.set('processing_state', Array.isArray(query.processing_state) ? query.processing_state.join(',') : query.processing_state)
if (query.target_scope) parameters.set('target_scope', query.target_scope)
parameters.set('limit', String(query.limit))
parameters.set('offset', String(query.offset))
+1
View File
@@ -57,6 +57,7 @@
button.danger{background:var(--c-fail);color:#fff;border-color:var(--c-fail)}
button.danger:hover:not(:disabled){background:#991b1b;border-color:#991b1b}
button.danger:disabled{background:var(--c-fail-border);color:#fff;border-color:var(--c-fail-border)}
.queue-more{display:flex;align-items:center;gap:12px;justify-content:center;padding:12px 0;font-size:13px;color:var(--c-dead)}
.operator-input{font:inherit;padding:5px 8px;border:1px solid var(--border-strong);border-radius:6px;width:120px}
.tiny{font-size:12px;color:var(--c-dead)}
.monitor-strip{display:flex;overflow-x:auto;background:var(--bg-panel);border:1px solid var(--border-strong);border-radius:8px;font-size:12px}