a1b46d2b7c
背景:来源侧被判「禁止导入」后,已建好的测试库知识不会自动消失;此前只有变更集
删除路径会入队 test.remove,页面既列不出条目也没有按钮,运维无法闭环(车道只有按
库计数)。
新增:
- GET /api/v1/test-publications?removal=required:条目级列表,判据与车道计数同源
(superseded_at IS NULL / 有 knowledge_id / completed / retrieval_state NOT IN
(disabling,disabled) / policy_state='blocked'),过滤是叠加而不是替换。
- POST /api/v1/test-publications/{id}/removal:engineer 级写路由,要求 Idempotency-Key,
按条目幂等(operator-remove:<id>);置 disabling 并交既有 test.remove 工作流去删除
远端知识并读回校验。错误映射 404/409(已移除)/409(不可移除)/501(未接线)。
- 界面:需下架车道逐条显示条目 + 「下架(从测试库移除)」,页内二次确认,成功提示,
完成后计数归零。
修掉过程中踩到的两个真问题:
1. 二次确认的确认态原先经容器 props 传给视图,一次容器改动把三个 props 丢了 ——
视图永远渲染原按钮、确认步骤静默消失,而所有静态单元测试仍然通过。改为把确认态
放在视图内部;并把「取消」放在原按钮位置(真实指针连点两下只会落到取消)。
2. jobs 上有部分唯一索引 UNIQUE (business_change_set_id, business_version_id,
business_target_kb_id, business_action) WHERE business_change_set_id IS NOT NULL。
本路径给这三列写空串(非 NULL),('','','','remove') 对所有条目是同一个键 ——
第一条之后每次 INSERT 都撞唯一索引、被 ON CONFLICT DO NOTHING 静默吞掉,留下
「状态已 disabling、没有任务」的永久卡死条目,而车道恰好隐藏 disabling,运维也
看不到。改为写 NULL(语义:无变更集身份)+ 冲突目标收敛到 idempotency_key +
校验 RowsAffected()(0 行回查既有任务;查不到就报错,不提交无任务的 disabling)。
卡死状态在重试时可自愈。
测试:internal/platform/postgres/test_publication_removal_integration_test.go(两条发布
各自入队、business_* 为 NULL、卡死自愈;把修复退回旧写法该测试即 FAIL,已实测)、
internal/httpapi/test_publication_removal_test.go(权限/幂等键/错误映射/未接线 501/列表
过滤)、前端 testPublications.test.ts + WeKnoraMonitor.test.tsx。
验收(纯界面,两条都走完):解析监控 → 需下架(2)→ 逐条「下架」+ 二次确认 →
retrieval_state=disabled(PDF 03:57:03Z、md 04:08:55Z),jobs completed(attempt 2,
首次 test_remove_pending=远端仍在处理,重试通过;工作流以 GetKnowledge 读回 404 判
成功)→ 车道需下架(0)。核验「确认前不执行」:首次点击后仍 unpublished、移除任务数仍 1。
267 lines
14 KiB
TypeScript
267 lines
14 KiB
TypeScript
import { newOperationID } from './operationKey'
|
||
export type IngestState = 'pending' | 'uploading' | 'uploaded' | 'failed'
|
||
export type ProcessingState = 'planned' | 'pending' | 'processing' | 'finalizing' | 'completed' | 'failed' | 'stalled' | 'cancelled'
|
||
|
||
export interface TestPublication {
|
||
id: string
|
||
title: string
|
||
ingest_state: IngestState
|
||
processing_state: ProcessingState
|
||
retrieval_state: string
|
||
target_scope: 'internal' | 'external'
|
||
change_set_id: string
|
||
change_type: 'added' | 'updated'
|
||
can_start: boolean
|
||
retry_count: number
|
||
last_error: string
|
||
can_retry: boolean
|
||
updated_at: string
|
||
}
|
||
|
||
export interface TestPublicationPage {
|
||
items: TestPublication[]
|
||
total: number
|
||
limit: number
|
||
offset: number
|
||
}
|
||
|
||
export interface AcceptanceRecord {
|
||
check_type: string
|
||
state: string
|
||
policy_version: string
|
||
evidence_summary: unknown
|
||
observed_at: string
|
||
}
|
||
|
||
export interface TestPublicationQuery {
|
||
/** removal: 'required' 只取「被禁止导入但仍存活在测试库」的条目(解析监控 → 需下架)。 */
|
||
removal?: 'required'
|
||
// A single state, or several at once (joined into a comma list server-side).
|
||
processing_state: ProcessingState | '' | ProcessingState[]
|
||
target_scope?: 'internal' | 'external' | ''
|
||
limit: number
|
||
offset: number
|
||
}
|
||
|
||
export interface StartTestPublicationsResult {
|
||
selected: number
|
||
started: number
|
||
already_started: number
|
||
unavailable: number
|
||
}
|
||
|
||
const isRecord = (value: unknown): value is Record<string, unknown> => typeof value === 'object' && value !== null
|
||
const isCount = (value: unknown): value is number => typeof value === 'number' && Number.isInteger(value) && value >= 0
|
||
const ingestStates = new Set<IngestState>(['pending', 'uploading', 'uploaded', 'failed'])
|
||
const processingStates = new Set<ProcessingState>(['planned', 'pending', 'processing', 'finalizing', 'completed', 'failed', 'stalled', 'cancelled'])
|
||
|
||
function parseTestPublication(value: unknown): TestPublication {
|
||
if (!isRecord(value) || typeof value.id !== 'string' || typeof value.title !== 'string' ||
|
||
typeof value.ingest_state !== 'string' || !ingestStates.has(value.ingest_state as IngestState) ||
|
||
typeof value.processing_state !== 'string' || !processingStates.has(value.processing_state as ProcessingState) ||
|
||
typeof value.retrieval_state !== 'string' || (value.target_scope !== 'internal' && value.target_scope !== 'external') ||
|
||
typeof value.change_set_id !== 'string' ||
|
||
(value.change_type !== 'added' && value.change_type !== 'updated') || typeof value.can_start !== 'boolean' ||
|
||
!isCount(value.retry_count) || typeof value.last_error !== 'string' || typeof value.can_retry !== 'boolean' || typeof value.updated_at !== 'string') {
|
||
throw new Error('invalid test publication response')
|
||
}
|
||
return {
|
||
id: value.id,
|
||
title: value.title,
|
||
ingest_state: value.ingest_state as IngestState,
|
||
processing_state: value.processing_state as ProcessingState,
|
||
retrieval_state: value.retrieval_state,
|
||
target_scope: value.target_scope,
|
||
change_set_id: value.change_set_id,
|
||
change_type: value.change_type,
|
||
can_start: value.can_start,
|
||
retry_count: value.retry_count,
|
||
last_error: value.last_error,
|
||
can_retry: value.can_retry,
|
||
updated_at: value.updated_at,
|
||
}
|
||
}
|
||
|
||
export function parseTestPublicationsResponse(value: unknown): TestPublicationPage {
|
||
if (!isRecord(value) || !Array.isArray(value.data) || !isCount(value.total) || !isCount(value.limit) || !isCount(value.offset)) {
|
||
throw new Error('invalid test publications response')
|
||
}
|
||
return {
|
||
items: value.data.map(parseTestPublication),
|
||
total: value.total,
|
||
limit: value.limit,
|
||
offset: value.offset,
|
||
}
|
||
}
|
||
|
||
export async function fetchTestPublications(query: TestPublicationQuery, signal?: AbortSignal): Promise<TestPublicationPage> {
|
||
const parameters = new URLSearchParams()
|
||
if (query.processing_state) parameters.set('processing_state', Array.isArray(query.processing_state) ? query.processing_state.join(',') : query.processing_state)
|
||
if (query.target_scope) parameters.set('target_scope', query.target_scope)
|
||
if (query.removal) parameters.set('removal', query.removal)
|
||
parameters.set('limit', String(query.limit))
|
||
parameters.set('offset', String(query.offset))
|
||
const response = await fetch(`/api/v1/test-publications?${parameters.toString()}`, { headers: { Accept: 'application/json' }, signal })
|
||
if (!response.ok) throw new Error(`加载测试发布失败(HTTP ${response.status})`)
|
||
return parseTestPublicationsResponse(await response.json())
|
||
}
|
||
|
||
/**
|
||
* 需下架车道用:列出「来源侧被判禁止导入、但知识仍存活在测试库里」的具体条目。
|
||
* 此前车道只有按库计数,运维看不到条目、也没有处置入口。
|
||
*/
|
||
export async function fetchRemovalRequiredPublications(signal?: AbortSignal): Promise<TestPublicationPage> {
|
||
return fetchTestPublications({ processing_state: '', removal: 'required', limit: 50, offset: 0 }, signal)
|
||
}
|
||
|
||
/**
|
||
* 触发移除:把该条目置为下架中,交服务端既有的 test.remove 工作流去删远端知识并读回校验。
|
||
* 服务端按条目幂等(重复点击不会重复入队),因此这里每次生成新的幂等键是安全的。
|
||
*/
|
||
export async function startPublicationRemoval(publicationId: string): Promise<{ job_id: string }> {
|
||
const response = await fetch(`/api/v1/test-publications/${encodeURIComponent(publicationId)}/removal`, {
|
||
method: 'POST',
|
||
headers: { Accept: 'application/json', 'Idempotency-Key': newOperationID() },
|
||
})
|
||
if (response.status !== 202) {
|
||
const body = (await response.json().catch(() => null)) as { error?: string } | null
|
||
throw new Error(body?.error ?? `触发下架失败(HTTP ${response.status})`)
|
||
}
|
||
const value = await response.json() as unknown
|
||
if (!isRecord(value) || typeof value.job_id !== 'string') throw new Error('下架响应无效')
|
||
return { job_id: value.job_id }
|
||
}
|
||
|
||
export async function fetchAcceptanceResults(publicationId: string, signal?: AbortSignal): Promise<AcceptanceRecord[]> {
|
||
const response = await fetch(`/api/v1/test-publications/${encodeURIComponent(publicationId)}/acceptance`, { headers: { Accept: 'application/json' }, signal })
|
||
if (!response.ok) throw new Error(`加载验收结果失败(HTTP ${response.status})`)
|
||
const value = await response.json() as unknown
|
||
if (!isRecord(value) || !Array.isArray(value.data)) throw new Error('invalid acceptance response')
|
||
return value.data as AcceptanceRecord[]
|
||
}
|
||
|
||
export async function retryPublication(publicationId: string): Promise<void> {
|
||
const response = await fetch(`/api/v1/test-publications/${encodeURIComponent(publicationId)}/retry`, { method: 'POST', headers: { Accept: 'application/json', 'Idempotency-Key': newOperationID() } })
|
||
if (!response.ok) {
|
||
const body = (await response.json().catch(() => null)) as { error?: string } | null
|
||
throw new Error(body?.error ?? `重试失败(HTTP ${response.status})`)
|
||
}
|
||
}
|
||
|
||
export async function startTestPublications(publicationIds: string[], reason: string): Promise<StartTestPublicationsResult> {
|
||
const response = await fetch('/api/v1/test-publications/start', {
|
||
method: 'POST',
|
||
headers: { Accept: 'application/json', 'Content-Type': 'application/json', 'Idempotency-Key': newOperationID() },
|
||
body: JSON.stringify({ publication_ids: publicationIds, reason }),
|
||
})
|
||
if (!response.ok) {
|
||
const body = (await response.json().catch(() => null)) as { error?: string } | null
|
||
throw new Error(body?.error ?? `开始处理失败(HTTP ${response.status})`)
|
||
}
|
||
const value = await response.json() as unknown
|
||
if (!isRecord(value) || !isRecord(value.data) || !isCount(value.data.selected) || !isCount(value.data.started) || !isCount(value.data.already_started) || !isCount(value.data.unavailable)) {
|
||
throw new Error('invalid processing start response')
|
||
}
|
||
return {
|
||
selected: value.data.selected,
|
||
started: value.data.started,
|
||
already_started: value.data.already_started,
|
||
unavailable: value.data.unavailable,
|
||
}
|
||
}
|
||
export interface RetryOutcome { publication_id: string; result: 'started' | 'already_running' | 'not_retryable'; detail?: string }
|
||
export interface RetryPublicationsBatchResult { batch_key: string; total: number; started: number; already_running: number; not_retryable: number; outcomes: RetryOutcome[] }
|
||
|
||
export async function retryPublicationsBatch(publicationIds: string[], reason: string): Promise<RetryPublicationsBatchResult> {
|
||
const response = await fetch('/api/v1/test-publications/retry-batch', {
|
||
method: 'POST',
|
||
headers: { Accept: 'application/json', 'Content-Type': 'application/json', 'Idempotency-Key': newOperationID() },
|
||
body: JSON.stringify({ publication_ids: publicationIds, reason }),
|
||
})
|
||
if (!response.ok) {
|
||
const body = (await response.json().catch(() => null)) as { error?: string } | null
|
||
throw new Error(body?.error ?? `批量重试失败(HTTP ${response.status})`)
|
||
}
|
||
const value = await response.json() as unknown
|
||
if (!isRecord(value) || !isRecord(value.data) || !Array.isArray(value.data.outcomes)) throw new Error('批量重试响应无效')
|
||
const data = value.data as Record<string, unknown>
|
||
const outcomes = Array.isArray(data.outcomes) ? data.outcomes : []
|
||
return {
|
||
batch_key: typeof data.batch_key === 'string' ? data.batch_key : '',
|
||
total: isCount(data.total) ? data.total : 0,
|
||
started: isCount(data.started) ? data.started : 0,
|
||
already_running: isCount(data.already_running) ? data.already_running : 0,
|
||
not_retryable: isCount(data.not_retryable) ? data.not_retryable : 0,
|
||
outcomes: outcomes.filter(isRecord).map((item) => ({
|
||
publication_id: typeof item.publication_id === 'string' ? item.publication_id : '',
|
||
result: item.result === 'started' || item.result === 'already_running' || item.result === 'not_retryable' ? item.result : 'not_retryable',
|
||
detail: typeof item.detail === 'string' ? item.detail : undefined,
|
||
})),
|
||
}
|
||
}
|
||
|
||
export interface JobFailure { id: string; job_id: string; job_type: string; publication_id?: string; source_item_id?: string; error_code: string; attempt: number; max_attempts_at_failure: number; change_set_id?: string; action?: string; target_kb_id?: string; state: 'open' | 'requeued' | 'abandoned'; created_at: string; doc_name?: string; last_error?: string }
|
||
|
||
export async function cancelPublicationsBatch(publicationIds: string[], reason: string): Promise<RetryPublicationsBatchResult> {
|
||
const response = await fetch('/api/v1/test-publications/cancel-batch', {
|
||
method: 'POST',
|
||
headers: { Accept: 'application/json', 'Content-Type': 'application/json', 'Idempotency-Key': newOperationID() },
|
||
body: JSON.stringify({ publication_ids: publicationIds, reason }),
|
||
})
|
||
if (!response.ok) {
|
||
const body = (await response.json().catch(() => null)) as { error?: string } | null
|
||
throw new Error(body?.error ?? `批量取消失败(HTTP ${response.status})`)
|
||
}
|
||
const value = await response.json() as unknown
|
||
if (!isRecord(value) || !isRecord(value.data) || !Array.isArray(value.data.outcomes)) throw new Error('批量取消响应无效')
|
||
const data = value.data as Record<string, unknown>
|
||
const outcomes = Array.isArray(data.outcomes) ? data.outcomes : []
|
||
return {
|
||
batch_key: typeof data.batch_key === 'string' ? data.batch_key : '',
|
||
total: isCount(data.total) ? data.total : 0,
|
||
started: isCount(data.started) ? data.started : 0,
|
||
already_running: isCount(data.already_running) ? data.already_running : 0,
|
||
not_retryable: isCount(data.not_retryable) ? data.not_retryable : 0,
|
||
outcomes: outcomes.filter(isRecord).map((item) => ({
|
||
publication_id: typeof item.publication_id === 'string' ? item.publication_id : '',
|
||
result: item.result === 'started' || item.result === 'already_running' || item.result === 'not_retryable' ? item.result : 'not_retryable',
|
||
detail: typeof item.detail === 'string' ? item.detail : undefined,
|
||
})),
|
||
}
|
||
}
|
||
|
||
export async function fetchJobFailures(signal?: AbortSignal): Promise<JobFailure[]> {
|
||
const response = await fetch('/api/v1/job-failures?state=open&limit=50', { headers: { Accept: 'application/json' }, signal })
|
||
if (!response.ok) throw new Error(`加载失败队列失败(HTTP ${response.status})`)
|
||
const value = await response.json() as unknown
|
||
if (!isRecord(value) || !isRecord(value.data) || !Array.isArray(value.data.items)) throw new Error('失败队列响应无效')
|
||
return value.data.items as JobFailure[]
|
||
}
|
||
|
||
export interface RequeuedArtifactFetch { failure_id: string; job_id: string; version_id: string; round: number; doc_name?: string; already_queued: boolean }
|
||
|
||
// 源头取件死信重排:取件任务的幂等键绑在版本上,失败那次已占住它,所以重新标注
|
||
// 同一文件不会再入队;这里让后端按新轮次重新排队(新键),并复位意图状态。
|
||
export async function requeueJobFailure(failureId: string, reason: string): Promise<RequeuedArtifactFetch> {
|
||
const response = await fetch(`/api/v1/job-failures/${encodeURIComponent(failureId)}/requeue`, {
|
||
method: 'POST',
|
||
headers: { Accept: 'application/json', 'Content-Type': 'application/json', 'Idempotency-Key': newOperationID() },
|
||
body: JSON.stringify({ reason }),
|
||
})
|
||
if (!response.ok) {
|
||
const body = (await response.json().catch(() => null)) as { error?: string } | null
|
||
throw new Error(body?.error ?? `重新取件失败(HTTP ${response.status})`)
|
||
}
|
||
const value = await response.json() as unknown
|
||
if (!isRecord(value) || !isRecord(value.data)) throw new Error('重新取件响应无效')
|
||
const data = value.data as Record<string, unknown>
|
||
return {
|
||
failure_id: typeof data.failure_id === 'string' ? data.failure_id : '',
|
||
job_id: typeof data.job_id === 'string' ? data.job_id : '',
|
||
version_id: typeof data.version_id === 'string' ? data.version_id : '',
|
||
round: isCount(data.round) ? data.round : 0,
|
||
doc_name: typeof data.doc_name === 'string' ? data.doc_name : undefined,
|
||
already_queued: data.already_queued === true,
|
||
}
|
||
}
|