feat(knowledge): instrument postprocess subspans + polish timeline UI
Backend: summary, question, and graph-extract async tasks now record real processing time as subspans hanging off the (closed) postprocess stage, so the trace viewer no longer caps the postprocess row at the ~10 ms enqueue duration. Carries Attempt through SummaryGeneration / QuestionGeneration / ExtractChunk payloads so cross-process workers can resolve the right parent attempt. Frontend: drawer now uses attach="body" so the secondary 820px detail drawer escapes the 654px container of the main drawer; timeline timestamps include date prefix (MM-DD, or YYYY-MM-DD across years); "updated X ago" caption only shows during live polling. Tests: 4 new cases covering postprocess subspan attaching under a closed parent, missing-parent fallthrough, and Attempt JSON round-trip on the three task payloads. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -64,6 +64,35 @@ mermaid.initialize({
|
||||
const props = defineProps(["visible", "details", "knowledgeType", "sourceInfo", "canEditKB", "parse_status"]);
|
||||
const emit = defineEmits(["closeDoc", "getDoc", "questionDeleted"]);
|
||||
|
||||
const hasTimelineSpans = ref(false);
|
||||
const timelineDrawerVisible = ref(false);
|
||||
const timelineSummary = ref<{ totalMs: number; status: string; stageIndex: number; stageTotal: number; stageLabel: string }>({
|
||||
totalMs: 0, status: '', stageIndex: 0, stageTotal: 0, stageLabel: '',
|
||||
});
|
||||
|
||||
watch(() => props.details?.id, () => {
|
||||
hasTimelineSpans.value = false;
|
||||
timelineDrawerVisible.value = false;
|
||||
timelineSummary.value = { totalMs: 0, status: '', stageIndex: 0, stageTotal: 0, stageLabel: '' };
|
||||
});
|
||||
|
||||
function formatTimelineDuration(ms: number): string {
|
||||
if (!ms || ms < 0) return '—';
|
||||
if (ms < 1000) return `${Math.round(ms)}ms`;
|
||||
if (ms < 60000) return `${(ms / 1000).toFixed(2)}s`;
|
||||
const mins = Math.floor(ms / 60000);
|
||||
const rem = ((ms % 60000) / 1000).toFixed(1);
|
||||
return `${mins}m${rem}s`;
|
||||
}
|
||||
|
||||
function openTimeline() {
|
||||
timelineDrawerVisible.value = true;
|
||||
}
|
||||
|
||||
function closeTimeline() {
|
||||
timelineDrawerVisible.value = false;
|
||||
}
|
||||
|
||||
marked.use({
|
||||
breaks: true, // 启用单行换行转 <br>
|
||||
gfm: true, // 启用 GitHub Flavored Markdown
|
||||
@@ -778,7 +807,7 @@ const handleDetailsScroll = () => {
|
||||
</script>
|
||||
<template>
|
||||
<div class="doc_content" ref="mdContentWrap">
|
||||
<t-drawer :visible="visible" :zIndex="2000" :closeBtn="true" :footer="false" @close="handleClose">
|
||||
<t-drawer :visible="visible" :zIndex="2000" size="654px" attach="body" :closeBtn="true" :footer="false" @close="handleClose">
|
||||
<template #header>
|
||||
<div class="drawer-header">
|
||||
<span class="header-title">{{ getDisplayTitle() }}</span>
|
||||
@@ -788,17 +817,94 @@ const handleDetailsScroll = () => {
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<!-- 解析阶段时间线(仅在解析中/失败时显示) -->
|
||||
<div
|
||||
v-if="details.id"
|
||||
class="parse_timeline_box"
|
||||
<!-- 解析阶段时间线 trigger(点击打开二级抽屉) -->
|
||||
<button
|
||||
v-if="details.id && hasTimelineSpans"
|
||||
type="button"
|
||||
class="kp-trigger"
|
||||
:class="['kp-trigger-' + (timelineSummary.status || 'unknown')]"
|
||||
@click="openTimeline"
|
||||
>
|
||||
<span class="kp-trigger-status">
|
||||
<span class="kp-trigger-dot" :class="['kp-trigger-dot-' + (timelineSummary.status || 'unknown')]" />
|
||||
<span class="kp-trigger-label">{{ $t('knowledgeStages.title') }}</span>
|
||||
</span>
|
||||
<span class="kp-trigger-divider" />
|
||||
<span class="kp-trigger-meta">
|
||||
<template v-if="timelineSummary.totalMs > 0">
|
||||
<span class="kp-trigger-meta-key">{{ $t('knowledgeStages.head.duration') }}</span>
|
||||
<span class="kp-trigger-meta-val kp-trigger-mono">{{ formatTimelineDuration(timelineSummary.totalMs) }}</span>
|
||||
</template>
|
||||
<template v-else-if="timelineSummary.stageTotal > 0">
|
||||
<span class="kp-trigger-meta-key">{{ $t('knowledgeStages.head.stage') }}</span>
|
||||
<span class="kp-trigger-meta-val kp-trigger-mono">
|
||||
{{ timelineSummary.stageIndex }}/{{ timelineSummary.stageTotal }}
|
||||
</span>
|
||||
<span class="kp-trigger-meta-key kp-trigger-meta-stage">· {{ timelineSummary.stageLabel }}</span>
|
||||
</template>
|
||||
</span>
|
||||
<span class="kp-trigger-tail">
|
||||
<span
|
||||
v-if="timelineSummary.status === 'running' || timelineSummary.status === 'processing' || timelineSummary.status === 'pending'"
|
||||
class="kp-trigger-live"
|
||||
>
|
||||
<span class="kp-trigger-live-dot" />
|
||||
<span class="kp-trigger-live-text">LIVE</span>
|
||||
</span>
|
||||
<span class="kp-trigger-cta">{{ $t('knowledgeStages.viewTrace') }}</span>
|
||||
<t-icon name="chevron-right" size="14px" class="kp-trigger-arrow" />
|
||||
</span>
|
||||
</button>
|
||||
|
||||
<!-- Hidden mount: keeps the timeline fetching data so trigger summary
|
||||
stays live even before the user opens the secondary drawer. The
|
||||
drawer itself shows a separate, fully-interactive instance. -->
|
||||
<div class="kp-trigger-shadow" aria-hidden="true">
|
||||
<KnowledgeProcessingTimeline
|
||||
v-if="details.id"
|
||||
:knowledge-id="details.id"
|
||||
:parse-status="details.parse_status"
|
||||
:compact="true"
|
||||
@update:has-spans="hasTimelineSpans = $event"
|
||||
@update:summary="timelineSummary = $event"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<!-- 二级抽屉:完整 Langfuse-style waterfall -->
|
||||
<t-drawer
|
||||
:visible="timelineDrawerVisible"
|
||||
:zIndex="2100"
|
||||
size="820px"
|
||||
attach="body"
|
||||
:closeBtn="false"
|
||||
:footer="false"
|
||||
:header="false"
|
||||
:showOverlay="true"
|
||||
:closeOnOverlayClick="true"
|
||||
placement="right"
|
||||
class="kp-secondary-drawer"
|
||||
@close="closeTimeline"
|
||||
>
|
||||
<div class="kp-drawer-shell">
|
||||
<div class="kp-drawer-titlebar">
|
||||
<div class="kp-drawer-titlebar-left">
|
||||
<span class="kp-drawer-titlebar-kind">trace</span>
|
||||
<span class="kp-drawer-titlebar-title">{{ $t('knowledgeStages.title') }}</span>
|
||||
</div>
|
||||
<button type="button" class="kp-drawer-titlebar-close" @click="closeTimeline" :aria-label="$t('knowledgeStages.close')">
|
||||
<t-icon name="close" size="16px" />
|
||||
</button>
|
||||
</div>
|
||||
<div class="kp-drawer-body">
|
||||
<KnowledgeProcessingTimeline
|
||||
v-if="details.id && timelineDrawerVisible"
|
||||
:knowledge-id="details.id"
|
||||
:parse-status="details.parse_status"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</t-drawer>
|
||||
|
||||
<!-- 文件类型专属区域 -->
|
||||
<div v-if="details.type === 'file'" class="doc_box">
|
||||
<a :href="url" style="display: none" ref="down" :download="details.title"></a>
|
||||
@@ -1015,18 +1121,12 @@ const handleDetailsScroll = () => {
|
||||
<style scoped lang="less">
|
||||
@import "./css/markdown.less";
|
||||
|
||||
:deep(.t-drawer .t-drawer__content-wrapper) {
|
||||
width: min(654px, 85vw) !important; // 减少到85%视口宽度,给左侧留更多空间
|
||||
max-width: 654px !important;
|
||||
}
|
||||
|
||||
// 在小屏幕上进一步调整
|
||||
@media (max-width: 768px) {
|
||||
:deep(.t-drawer .t-drawer__content-wrapper) {
|
||||
width: 90vw !important; // 小屏幕上使用90%宽度
|
||||
max-width: none !important;
|
||||
}
|
||||
}
|
||||
/* Drawer widths are now driven by the `:size` prop on each <t-drawer>
|
||||
(see mainDrawerSize / timelineDrawerSize in <script>). CSS rules with
|
||||
!important were removed because they fought each other across the
|
||||
scoped/non-scoped boundary and had no clean specificity ordering in
|
||||
dev mode (Vite injects scoped <style> tags later than non-scoped,
|
||||
inverting prod). Inline width via the prop is unambiguous. */
|
||||
|
||||
// 代码块样式
|
||||
:deep(.code-block-wrapper) {
|
||||
@@ -1108,6 +1208,301 @@ const handleDetailsScroll = () => {
|
||||
border-radius: 6px;
|
||||
}
|
||||
|
||||
/* ============== Timeline trigger pill ==============
|
||||
Designed to read like an information row in the doc drawer, not a
|
||||
loud CTA. Soft 6px corners, neutral border, left strip carries the
|
||||
status color so the pipeline state is glanceable without dominating
|
||||
the layout. Mirrors the KBInfoPopover/manual-knowledge-editor card
|
||||
feel that the rest of the app uses. */
|
||||
.kp-trigger {
|
||||
margin: 8px 0 16px;
|
||||
width: 100%;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
padding: 10px 14px;
|
||||
border: 1px solid var(--td-component-stroke);
|
||||
border-radius: var(--td-radius-medium);
|
||||
background: var(--td-bg-color-container);
|
||||
color: var(--td-text-color-primary);
|
||||
cursor: pointer;
|
||||
font-family: var(--app-font-family);
|
||||
text-align: left;
|
||||
position: relative;
|
||||
overflow: hidden;
|
||||
transition: border-color 150ms ease, background 150ms ease, transform 100ms ease;
|
||||
}
|
||||
|
||||
.kp-trigger::before {
|
||||
content: "";
|
||||
position: absolute;
|
||||
left: 0;
|
||||
top: 8px;
|
||||
bottom: 8px;
|
||||
width: 3px;
|
||||
border-radius: 0 2px 2px 0;
|
||||
background: var(--td-component-border);
|
||||
transition: background 150ms ease;
|
||||
}
|
||||
|
||||
.kp-trigger:hover {
|
||||
background: var(--td-bg-color-secondarycontainer);
|
||||
border-color: var(--td-component-border);
|
||||
}
|
||||
.kp-trigger:active { transform: translateY(1px); }
|
||||
|
||||
.kp-trigger-done::before,
|
||||
.kp-trigger-completed::before { background: var(--td-success-color); }
|
||||
.kp-trigger-failed::before { background: var(--td-error-color); }
|
||||
.kp-trigger-running::before,
|
||||
.kp-trigger-processing::before,
|
||||
.kp-trigger-pending::before { background: var(--td-brand-color); }
|
||||
.kp-trigger-unknown::before { background: var(--td-component-border); }
|
||||
|
||||
.kp-trigger-status {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.kp-trigger-dot {
|
||||
width: 8px;
|
||||
height: 8px;
|
||||
border-radius: 50%;
|
||||
background: var(--td-text-color-placeholder);
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.kp-trigger-dot-done,
|
||||
.kp-trigger-dot-completed { background: var(--td-success-color); }
|
||||
.kp-trigger-dot-failed { background: var(--td-error-color); }
|
||||
.kp-trigger-dot-running,
|
||||
.kp-trigger-dot-processing,
|
||||
.kp-trigger-dot-pending {
|
||||
background: var(--td-brand-color);
|
||||
animation: kpTriggerPulse 1.6s ease-in-out infinite;
|
||||
}
|
||||
.kp-trigger-dot-unknown { background: var(--td-text-color-placeholder); }
|
||||
|
||||
@keyframes kpTriggerPulse {
|
||||
0%, 100% { box-shadow: 0 0 0 0 var(--td-brand-color-light); }
|
||||
50% { box-shadow: 0 0 0 4px transparent; }
|
||||
}
|
||||
|
||||
.kp-trigger-label {
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
color: var(--td-text-color-primary);
|
||||
}
|
||||
|
||||
.kp-trigger-divider {
|
||||
width: 1px;
|
||||
height: 14px;
|
||||
background: var(--td-component-stroke);
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.kp-trigger-meta {
|
||||
display: inline-flex;
|
||||
align-items: baseline;
|
||||
gap: 6px;
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
overflow: hidden;
|
||||
white-space: nowrap;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
|
||||
.kp-trigger-meta-key {
|
||||
color: var(--td-text-color-placeholder);
|
||||
font-size: 11px;
|
||||
font-weight: 500;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.5px;
|
||||
}
|
||||
|
||||
.kp-trigger-meta-stage {
|
||||
text-transform: none;
|
||||
letter-spacing: 0;
|
||||
font-weight: 400;
|
||||
color: var(--td-text-color-secondary);
|
||||
margin-left: 2px;
|
||||
}
|
||||
|
||||
.kp-trigger-meta-val {
|
||||
color: var(--td-text-color-primary);
|
||||
font-size: 12px;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.kp-trigger-mono {
|
||||
font-family: var(--app-font-family-mono);
|
||||
font-size: 12px;
|
||||
letter-spacing: 0;
|
||||
}
|
||||
|
||||
.kp-trigger-tail {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
/* LIVE badge — pulses while the parser is mid-run, so the user knows
|
||||
opening the trace will show fresh data. */
|
||||
.kp-trigger-live {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 4px;
|
||||
padding: 2px 7px;
|
||||
background: var(--td-brand-color-light);
|
||||
color: var(--td-brand-color);
|
||||
border-radius: var(--td-radius-default);
|
||||
font-size: 10px;
|
||||
font-weight: 600;
|
||||
letter-spacing: 0.06em;
|
||||
line-height: 1;
|
||||
}
|
||||
|
||||
.kp-trigger-live-dot {
|
||||
width: 5px;
|
||||
height: 5px;
|
||||
border-radius: 50%;
|
||||
background: var(--td-brand-color);
|
||||
animation: kpTriggerLivePulse 1.4s ease-in-out infinite;
|
||||
}
|
||||
|
||||
.kp-trigger-live-text {
|
||||
font-family: var(--app-font-family-mono);
|
||||
}
|
||||
|
||||
@keyframes kpTriggerLivePulse {
|
||||
0%, 100% { opacity: 1; transform: scale(1); }
|
||||
50% { opacity: 0.4; transform: scale(0.8); }
|
||||
}
|
||||
|
||||
.kp-trigger-cta {
|
||||
font-size: 12px;
|
||||
color: var(--td-text-color-secondary);
|
||||
}
|
||||
|
||||
.kp-trigger:hover .kp-trigger-cta {
|
||||
color: var(--td-brand-color);
|
||||
}
|
||||
|
||||
.kp-trigger-arrow {
|
||||
color: var(--td-text-color-placeholder);
|
||||
transition: transform 150ms ease, color 150ms ease;
|
||||
}
|
||||
|
||||
.kp-trigger:hover .kp-trigger-arrow {
|
||||
color: var(--td-brand-color);
|
||||
transform: translateX(2px);
|
||||
}
|
||||
|
||||
/* Hidden mount keeps fetcher live without showing UI */
|
||||
.kp-trigger-shadow {
|
||||
display: none;
|
||||
}
|
||||
|
||||
/* ============== Secondary drawer shell ============== */
|
||||
.kp-drawer-shell {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
height: 100%;
|
||||
width: 100%;
|
||||
background: var(--td-bg-color-container);
|
||||
/* Belt-and-suspenders: even if some inner element forgets a min-width:0
|
||||
declaration in a flex chain, clip rather than overflow the drawer. */
|
||||
overflow: hidden;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.kp-drawer-titlebar {
|
||||
flex: 0 0 auto;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
padding: 14px 20px 12px;
|
||||
border-bottom: 1px solid var(--td-component-stroke);
|
||||
background: var(--td-bg-color-container);
|
||||
}
|
||||
|
||||
.kp-drawer-titlebar-left {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.kp-drawer-titlebar-kind {
|
||||
font-family: var(--app-font-family-mono);
|
||||
font-size: 10px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.5px;
|
||||
color: var(--td-text-color-placeholder);
|
||||
background: var(--td-bg-color-secondarycontainer);
|
||||
border-radius: var(--td-radius-default);
|
||||
padding: 2px 6px;
|
||||
line-height: 1.2;
|
||||
}
|
||||
|
||||
.kp-drawer-titlebar-title {
|
||||
font-size: 15px;
|
||||
font-weight: 600;
|
||||
color: var(--td-text-color-primary);
|
||||
letter-spacing: -0.005em;
|
||||
}
|
||||
|
||||
.kp-drawer-titlebar-close {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 28px;
|
||||
height: 28px;
|
||||
border: none;
|
||||
background: transparent;
|
||||
cursor: pointer;
|
||||
color: var(--td-text-color-placeholder);
|
||||
border-radius: var(--td-radius-default);
|
||||
transition: background 150ms ease, color 150ms ease;
|
||||
}
|
||||
|
||||
.kp-drawer-titlebar-close:hover {
|
||||
background: var(--td-bg-color-secondarycontainer);
|
||||
color: var(--td-text-color-primary);
|
||||
}
|
||||
|
||||
.kp-drawer-body {
|
||||
flex: 1 1 auto;
|
||||
min-height: 0;
|
||||
min-width: 0;
|
||||
overflow: hidden;
|
||||
/* Plain block layout — flex was unnecessary and risked the timeline
|
||||
becoming a flex item with auto-sized basis, leaking past the drawer
|
||||
bounds when its internal min-widths exceeded the parent. */
|
||||
display: block;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.kp-drawer-body > :deep(.kp-timeline) {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
/* Width is set via the :size prop on the <t-drawer>, not CSS — see the
|
||||
<script> for mainDrawerSize / timelineDrawerSize. Only padding +
|
||||
background are still tweaked here so the timeline fills the secondary
|
||||
drawer cleanly, edge-to-edge. */
|
||||
:deep(.kp-secondary-drawer .t-drawer__body) {
|
||||
padding: 0 !important;
|
||||
}
|
||||
|
||||
:deep(.kp-secondary-drawer .t-drawer__content) {
|
||||
background: var(--td-bg-color-container);
|
||||
}
|
||||
|
||||
// 文档摘要区域
|
||||
.summary_box {
|
||||
display: flex;
|
||||
@@ -1514,3 +1909,20 @@ const handleDetailsScroll = () => {
|
||||
margin-top: 12px;
|
||||
}
|
||||
</style>
|
||||
|
||||
<!-- Non-scoped padding/background overrides for the secondary drawer.
|
||||
Width is now controlled via the :size prop on <t-drawer> (see
|
||||
timelineDrawerSize in <script>) — that puts width on element.style
|
||||
rather than fighting !important CSS rules. We only keep these
|
||||
non-scoped rules because TDesign's default body padding and
|
||||
content background need to be flushed for the timeline to fill
|
||||
edge-to-edge. -->
|
||||
<style lang="less">
|
||||
.t-drawer.kp-secondary-drawer .t-drawer__body {
|
||||
padding: 0 !important;
|
||||
}
|
||||
|
||||
.t-drawer.kp-secondary-drawer .t-drawer__content {
|
||||
background: var(--td-bg-color-container);
|
||||
}
|
||||
</style>
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -394,21 +394,55 @@ export default {
|
||||
attempt: 'Attempt {n}',
|
||||
attemptLatest: 'Attempt {n} (latest)',
|
||||
retry: 'Retry parsing',
|
||||
refresh: 'Refresh now',
|
||||
copy: 'Copy',
|
||||
copyDetails: 'Copy details',
|
||||
copied: 'Copied to clipboard',
|
||||
close: 'Close',
|
||||
live: 'LIVE',
|
||||
liveTooltip: 'Parsing in progress, refreshes every 2s (next in {n}s)',
|
||||
viewTrace: 'View trace',
|
||||
justNow: 'just now',
|
||||
secondsAgo: '{n}s ago',
|
||||
minutesAgo: '{n}m ago',
|
||||
noActivity: 'No parsing activity yet',
|
||||
totalDuration: 'Total: {d}',
|
||||
total: 'Total {d}',
|
||||
head: {
|
||||
duration: 'Duration',
|
||||
stages: 'Stages',
|
||||
stage: 'Stage',
|
||||
status: 'Status',
|
||||
attempt: 'Attempt',
|
||||
updated: 'Updated',
|
||||
},
|
||||
tab: {
|
||||
overview: 'Overview',
|
||||
raw: 'Raw JSON',
|
||||
},
|
||||
detail: {
|
||||
started: 'Started',
|
||||
finished: 'Finished',
|
||||
duration: 'Duration',
|
||||
offset: 'Offset',
|
||||
timing: 'Timing',
|
||||
identity: 'Identity',
|
||||
stageBreakdown: 'Stage breakdown',
|
||||
stageOrder: 'Stage order',
|
||||
childCount: 'Child spans',
|
||||
kind: 'Kind',
|
||||
status: 'Status',
|
||||
name: 'Name',
|
||||
input: 'Input',
|
||||
output: 'Output',
|
||||
metadata: 'Metadata',
|
||||
error: 'Error',
|
||||
showJson: 'Show JSON',
|
||||
hideJson: 'Hide JSON',
|
||||
empty: 'No data',
|
||||
inProgress: 'In progress',
|
||||
elapsed: 'Elapsed',
|
||||
placeholderHint: 'This stage has no detailed span record; only the inferred state is shown.',
|
||||
showJson: 'Expand JSON',
|
||||
hideJson: 'Collapse JSON',
|
||||
},
|
||||
stage: {
|
||||
docreader: 'Document parsing',
|
||||
|
||||
@@ -400,21 +400,55 @@ export default {
|
||||
attempt: "{n}번째 시도",
|
||||
attemptLatest: "{n}번째 시도 (최신)",
|
||||
retry: "다시 파싱",
|
||||
refresh: "지금 새로고침",
|
||||
copy: "복사",
|
||||
copyDetails: "세부정보 복사",
|
||||
copied: "클립보드에 복사됨",
|
||||
close: "닫기",
|
||||
live: "LIVE",
|
||||
liveTooltip: "파싱 중, 2초마다 자동 새로고침 ({n}s 후)",
|
||||
viewTrace: "Trace 보기",
|
||||
justNow: "방금",
|
||||
secondsAgo: "{n}초 전",
|
||||
minutesAgo: "{n}분 전",
|
||||
noActivity: "파싱 활동 없음",
|
||||
totalDuration: "총 소요시간: {d}",
|
||||
total: "총 {d}",
|
||||
head: {
|
||||
duration: "소요시간",
|
||||
stages: "단계",
|
||||
stage: "단계",
|
||||
status: "상태",
|
||||
attempt: "시도",
|
||||
updated: "갱신",
|
||||
},
|
||||
tab: {
|
||||
overview: "개요",
|
||||
raw: "원본 JSON",
|
||||
},
|
||||
detail: {
|
||||
started: "시작",
|
||||
finished: "종료",
|
||||
duration: "소요시간",
|
||||
offset: "시작 오프셋",
|
||||
timing: "타이밍",
|
||||
identity: "식별 정보",
|
||||
stageBreakdown: "단계 분석",
|
||||
stageOrder: "단계 순서",
|
||||
childCount: "하위 span 수",
|
||||
kind: "유형",
|
||||
status: "상태",
|
||||
name: "이름",
|
||||
input: "입력",
|
||||
output: "출력",
|
||||
metadata: "메타데이터",
|
||||
error: "오류",
|
||||
showJson: "JSON 표시",
|
||||
hideJson: "JSON 숨기기",
|
||||
empty: "데이터 없음",
|
||||
inProgress: "진행 중",
|
||||
elapsed: "경과",
|
||||
placeholderHint: "이 단계에는 상세한 span 기록이 없으며, 추정된 상태만 표시됩니다.",
|
||||
showJson: "JSON 펼치기",
|
||||
hideJson: "JSON 접기",
|
||||
},
|
||||
stage: {
|
||||
docreader: "문서 파싱",
|
||||
|
||||
@@ -397,21 +397,55 @@ export default {
|
||||
attempt: "第 {n} 次尝试",
|
||||
attemptLatest: "第 {n} 次尝试(最新)",
|
||||
retry: "重新解析",
|
||||
refresh: "立即刷新",
|
||||
copy: "复制",
|
||||
copyDetails: "复制详情",
|
||||
copied: "已复制到剪贴板",
|
||||
close: "关闭",
|
||||
live: "LIVE",
|
||||
liveTooltip: "解析中,每 2 秒自动刷新({n}s 后)",
|
||||
viewTrace: "查看 Trace",
|
||||
justNow: "刚刚",
|
||||
secondsAgo: "{n} 秒前",
|
||||
minutesAgo: "{n} 分钟前",
|
||||
noActivity: "暂无解析记录",
|
||||
totalDuration: "总耗时:{d}",
|
||||
total: "总耗时 {d}",
|
||||
head: {
|
||||
duration: "总耗时",
|
||||
stages: "阶段",
|
||||
stage: "阶段",
|
||||
status: "状态",
|
||||
attempt: "尝试",
|
||||
updated: "更新于",
|
||||
},
|
||||
tab: {
|
||||
overview: "概览",
|
||||
raw: "原始 JSON",
|
||||
},
|
||||
detail: {
|
||||
started: "开始",
|
||||
finished: "结束",
|
||||
duration: "耗时",
|
||||
offset: "起点偏移",
|
||||
timing: "时序",
|
||||
identity: "身份信息",
|
||||
stageBreakdown: "阶段分解",
|
||||
stageOrder: "阶段顺序",
|
||||
childCount: "子 span 数",
|
||||
kind: "类型",
|
||||
status: "状态",
|
||||
name: "名称",
|
||||
input: "输入",
|
||||
output: "输出",
|
||||
metadata: "元数据",
|
||||
error: "错误",
|
||||
showJson: "显示 JSON",
|
||||
hideJson: "隐藏 JSON",
|
||||
empty: "暂无数据",
|
||||
inProgress: "进行中",
|
||||
elapsed: "已用",
|
||||
placeholderHint: "此阶段未记录详细 span,仅展示推断状态。",
|
||||
showJson: "展开 JSON",
|
||||
hideJson: "收起 JSON",
|
||||
},
|
||||
stage: {
|
||||
docreader: "文档解析",
|
||||
|
||||
@@ -87,15 +87,21 @@ func NewChunkExtractTask(
|
||||
tenantID uint64,
|
||||
chunkID string,
|
||||
modelID string,
|
||||
knowledgeID string,
|
||||
attempt int,
|
||||
chunkIndex int,
|
||||
) error {
|
||||
if strings.ToLower(os.Getenv("NEO4J_ENABLE")) != "true" {
|
||||
logger.Warn(ctx, "NEO4J is not enabled, skip chunk extract task")
|
||||
return nil
|
||||
}
|
||||
taskPayload := types.ExtractChunkPayload{
|
||||
TenantID: tenantID,
|
||||
ChunkID: chunkID,
|
||||
ModelID: modelID,
|
||||
TenantID: tenantID,
|
||||
ChunkID: chunkID,
|
||||
ModelID: modelID,
|
||||
KnowledgeID: knowledgeID,
|
||||
Attempt: attempt,
|
||||
ChunkIndex: chunkIndex,
|
||||
}
|
||||
langfuse.InjectTracing(ctx, &taskPayload)
|
||||
payload, err := json.Marshal(taskPayload)
|
||||
@@ -150,6 +156,10 @@ type ChunkExtractService struct {
|
||||
knowledgeBaseRepo interfaces.KnowledgeBaseRepository
|
||||
chunkRepo interfaces.ChunkRepository
|
||||
graphEngine interfaces.RetrieveGraphRepository
|
||||
// spanTracker records this graph-extract task's subspan under the
|
||||
// parent attempt's postprocess stage so the trace viewer shows real
|
||||
// per-chunk graph extraction time rather than the upstream's enqueue.
|
||||
spanTracker SpanTracker
|
||||
}
|
||||
|
||||
// NewChunkExtractService creates a new chunk extract service
|
||||
@@ -159,20 +169,25 @@ func NewChunkExtractService(
|
||||
knowledgeBaseRepo interfaces.KnowledgeBaseRepository,
|
||||
chunkRepo interfaces.ChunkRepository,
|
||||
graphEngine interfaces.RetrieveGraphRepository,
|
||||
spanTracker SpanTracker,
|
||||
) interfaces.TaskHandler {
|
||||
// generator := chatpipeline.NewQAPromptGenerator(chatpipeline.NewFormater(), config.ExtractManager.ExtractGraph)
|
||||
// ctx := context.Background()
|
||||
// logger.Debugf(ctx, "chunk extract system prompt: %s", generator.System(ctx))
|
||||
// logger.Debugf(ctx, "chunk extract user prompt: %s", generator.User(ctx, "demo"))
|
||||
return &ChunkExtractService{
|
||||
template: config.ExtractManager.ExtractGraph,
|
||||
modelService: modelService,
|
||||
knowledgeBaseRepo: knowledgeBaseRepo,
|
||||
chunkRepo: chunkRepo,
|
||||
graphEngine: graphEngine,
|
||||
spanTracker: spanTracker,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ChunkExtractService) tracker() SpanTracker {
|
||||
if s.spanTracker == nil {
|
||||
return noopSpanTracker{}
|
||||
}
|
||||
return s.spanTracker
|
||||
}
|
||||
|
||||
// Handle handles the chunk extraction task
|
||||
func (s *ChunkExtractService) Handle(ctx context.Context, t *asynq.Task) error {
|
||||
var p types.ExtractChunkPayload
|
||||
@@ -184,24 +199,59 @@ func (s *ChunkExtractService) Handle(ctx context.Context, t *asynq.Task) error {
|
||||
ctx = logger.WithField(ctx, "extract", p.ChunkID)
|
||||
ctx = context.WithValue(ctx, types.TenantIDContextKey, p.TenantID)
|
||||
|
||||
// Open a postprocess subspan keyed by chunk ordinal so the trace
|
||||
// shows real per-chunk graph extraction time. Skipped silently when
|
||||
// upstream didn't pass the parent attempt (legacy in-flight tasks)
|
||||
// or when the postprocess stage span isn't found.
|
||||
var gSpan *Span
|
||||
if p.KnowledgeID != "" && p.Attempt > 0 {
|
||||
parent := s.tracker().LookupStage(ctx, p.KnowledgeID, p.Attempt, types.StagePostProcess)
|
||||
if parent != nil {
|
||||
gSpan = s.tracker().BeginSubSpan(ctx, parent,
|
||||
fmt.Sprintf("postprocess.graph.chunk[%d]", p.ChunkIndex),
|
||||
types.SpanKindSubSpan,
|
||||
types.JSONMap{
|
||||
"chunk_id": p.ChunkID,
|
||||
"chunk_index": p.ChunkIndex,
|
||||
"model_id": p.ModelID,
|
||||
})
|
||||
}
|
||||
}
|
||||
var handleErr error
|
||||
graphOut := types.JSONMap{}
|
||||
defer func() {
|
||||
if gSpan == nil {
|
||||
return
|
||||
}
|
||||
if handleErr != nil {
|
||||
s.tracker().FailSpan(ctx, gSpan, "GRAPH_EXTRACT_FAILED", handleErr.Error(), handleErr)
|
||||
} else {
|
||||
s.tracker().EndSpan(ctx, gSpan, graphOut)
|
||||
}
|
||||
}()
|
||||
|
||||
chunk, err := s.chunkRepo.GetChunkByID(ctx, p.TenantID, p.ChunkID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "failed to get chunk: %v", err)
|
||||
handleErr = err
|
||||
return err
|
||||
}
|
||||
kb, err := s.knowledgeBaseRepo.GetKnowledgeBaseByID(ctx, chunk.KnowledgeBaseID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "failed to get knowledge base: %v", err)
|
||||
handleErr = err
|
||||
return err
|
||||
}
|
||||
if kb.ExtractConfig == nil {
|
||||
logger.Warnf(ctx, "failed to get extract config")
|
||||
graphOut["skipped"] = "no_extract_config"
|
||||
return err
|
||||
}
|
||||
|
||||
chatModel, err := s.modelService.GetChatModel(ctx, p.ModelID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "failed to get chat model: %v", err)
|
||||
handleErr = err
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -219,12 +269,14 @@ func (s *ChunkExtractService) Handle(ctx context.Context, t *asynq.Task) error {
|
||||
extractor := chatpipeline.NewExtractor(chatModel, template)
|
||||
graph, err := extractor.Extract(ctx, chunk.Content)
|
||||
if err != nil {
|
||||
handleErr = err
|
||||
return err
|
||||
}
|
||||
|
||||
chunk, err = s.chunkRepo.GetChunkByID(ctx, p.TenantID, p.ChunkID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "graph ignore chunk %s: %v", p.ChunkID, err)
|
||||
graphOut["skipped"] = "chunk_disappeared"
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -236,8 +288,11 @@ func (s *ChunkExtractService) Handle(ctx context.Context, t *asynq.Task) error {
|
||||
[]*types.GraphData{graph},
|
||||
); err != nil {
|
||||
logger.Errorf(ctx, "failed to add graph: %v", err)
|
||||
handleErr = err
|
||||
return err
|
||||
}
|
||||
graphOut["nodes_added"] = len(graph.Node)
|
||||
graphOut["relations_added"] = len(graph.Relation)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -219,6 +219,48 @@ func (s *knowledgeService) skipStage(ctx context.Context, kid, name, reason stri
|
||||
s.tracker().SkipSpan(ctx, span, reason)
|
||||
}
|
||||
|
||||
// beginPostprocessSubspan opens a subspan beneath the postprocess stage
|
||||
// span for (kid, attempt). Async post-pipeline tasks (summary, question,
|
||||
// graph, wiki) call this on entry so their actual processing time shows
|
||||
// up in the trace tree under postprocess instead of the stage looking
|
||||
// like an instant ~10ms enqueue.
|
||||
//
|
||||
// Returns nil when:
|
||||
// - attempt <= 0 (legacy in-flight task without span tracking)
|
||||
// - the postprocess stage span is missing (parse predates tracker, or
|
||||
// the upstream BeginStage call failed)
|
||||
//
|
||||
// Callers must tolerate nil — pair every begin with a deferred
|
||||
// endPostprocessSubspan / failPostprocessSubspan that no-ops on nil.
|
||||
func (s *knowledgeService) beginPostprocessSubspan(
|
||||
ctx context.Context, knowledgeID string, attempt int, name string, input types.JSONMap,
|
||||
) *Span {
|
||||
if attempt <= 0 || knowledgeID == "" || name == "" {
|
||||
return nil
|
||||
}
|
||||
parent := s.tracker().LookupStage(ctx, knowledgeID, attempt, types.StagePostProcess)
|
||||
if parent == nil {
|
||||
return nil
|
||||
}
|
||||
return s.tracker().BeginSubSpan(ctx, parent, name, types.SpanKindSubSpan, input)
|
||||
}
|
||||
|
||||
func (s *knowledgeService) endPostprocessSubspan(ctx context.Context, span *Span, output types.JSONMap) {
|
||||
if span == nil {
|
||||
return
|
||||
}
|
||||
s.tracker().EndSpan(ctx, span, output)
|
||||
}
|
||||
|
||||
func (s *knowledgeService) failPostprocessSubspan(
|
||||
ctx context.Context, span *Span, code, msg string, err error,
|
||||
) {
|
||||
if span == nil {
|
||||
return
|
||||
}
|
||||
s.tracker().FailSpan(ctx, span, code, msg, err)
|
||||
}
|
||||
|
||||
// getParserEngineOverridesFromContext returns parser engine overrides from tenant in context (e.g. MinerU endpoint, API key).
|
||||
// Used when building document ReadRequest so UI-configured values take precedence over env.
|
||||
func (s *knowledgeService) getParserEngineOverridesFromContext(ctx context.Context) map[string]string {
|
||||
|
||||
@@ -143,12 +143,12 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta
|
||||
enqueuedSummary := false
|
||||
enqueuedQuestion := false
|
||||
if len(textChunks) > 0 {
|
||||
s.enqueueSummaryGenerationTask(ctx, payload)
|
||||
s.enqueueSummaryGenerationTask(ctx, payload, attempt)
|
||||
enqueuedSummary = true
|
||||
// Question generation only makes sense for RAG indexing (improves chunk recall).
|
||||
// Skip when only Wiki/Graph is enabled without vector/keyword search.
|
||||
if kb.NeedsEmbeddingModel() {
|
||||
enqueuedQuestion = s.enqueueQuestionGenerationIfEnabled(ctx, payload, kb)
|
||||
enqueuedQuestion = s.enqueueQuestionGenerationIfEnabled(ctx, payload, kb, attempt)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -156,8 +156,9 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta
|
||||
enqueuedGraph := false
|
||||
if kb.IsGraphEnabled() {
|
||||
logger.Infof(ctx, "[KnowledgePostProcess] Spawning Graph RAG extract tasks for %d text-like chunks", len(textChunks))
|
||||
for _, chunk := range textChunks {
|
||||
err := NewChunkExtractTask(ctx, s.taskEnqueuer, payload.TenantID, chunk.ID, kb.SummaryModelID)
|
||||
for i, chunk := range textChunks {
|
||||
err := NewChunkExtractTask(ctx, s.taskEnqueuer, payload.TenantID, chunk.ID, kb.SummaryModelID,
|
||||
payload.KnowledgeID, attempt, i)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "[KnowledgePostProcess] Failed to create chunk extract task for %s: %v", chunk.ID, err)
|
||||
}
|
||||
@@ -190,7 +191,7 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *KnowledgePostProcessService) enqueueSummaryGenerationTask(ctx context.Context, payload types.KnowledgePostProcessPayload) {
|
||||
func (s *KnowledgePostProcessService) enqueueSummaryGenerationTask(ctx context.Context, payload types.KnowledgePostProcessPayload, attempt int) {
|
||||
if s.taskEnqueuer == nil {
|
||||
return
|
||||
}
|
||||
@@ -200,6 +201,7 @@ func (s *KnowledgePostProcessService) enqueueSummaryGenerationTask(ctx context.C
|
||||
KnowledgeBaseID: payload.KnowledgeBaseID,
|
||||
KnowledgeID: payload.KnowledgeID,
|
||||
Language: payload.Language,
|
||||
Attempt: attempt,
|
||||
}
|
||||
langfuse.InjectTracing(ctx, &taskPayload)
|
||||
payloadBytes, err := json.Marshal(taskPayload)
|
||||
@@ -216,7 +218,7 @@ func (s *KnowledgePostProcessService) enqueueSummaryGenerationTask(ctx context.C
|
||||
}
|
||||
}
|
||||
|
||||
func (s *KnowledgePostProcessService) enqueueQuestionGenerationIfEnabled(ctx context.Context, payload types.KnowledgePostProcessPayload, kb *types.KnowledgeBase) bool {
|
||||
func (s *KnowledgePostProcessService) enqueueQuestionGenerationIfEnabled(ctx context.Context, payload types.KnowledgePostProcessPayload, kb *types.KnowledgeBase, attempt int) bool {
|
||||
if s.taskEnqueuer == nil {
|
||||
return false
|
||||
}
|
||||
@@ -239,6 +241,7 @@ func (s *KnowledgePostProcessService) enqueueQuestionGenerationIfEnabled(ctx con
|
||||
KnowledgeID: payload.KnowledgeID,
|
||||
QuestionCount: questionCount,
|
||||
Language: payload.Language,
|
||||
Attempt: attempt,
|
||||
}
|
||||
langfuse.InjectTracing(ctx, &taskPayload)
|
||||
payloadBytes, err := json.Marshal(taskPayload)
|
||||
|
||||
@@ -887,15 +887,37 @@ func (s *knowledgeService) ProcessSummaryGeneration(ctx context.Context, t *asyn
|
||||
ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language)
|
||||
}
|
||||
|
||||
// Open a subspan under the parent attempt's postprocess stage so the
|
||||
// trace surface shows the real summary-generation duration (LLM call
|
||||
// + chunk write + index) instead of just the upstream's enqueue time.
|
||||
// Closes via the deferred handler below — every return path lands in
|
||||
// the defer, including the early returns ahead.
|
||||
span := s.beginPostprocessSubspan(ctx, payload.KnowledgeID, payload.Attempt, "postprocess.summary",
|
||||
types.JSONMap{"language": payload.Language})
|
||||
var summaryErr error
|
||||
summaryOut := types.JSONMap{}
|
||||
defer func() {
|
||||
if span == nil {
|
||||
return
|
||||
}
|
||||
if summaryErr != nil {
|
||||
s.failPostprocessSubspan(ctx, span, "SUMMARY_FAILED", summaryErr.Error(), summaryErr)
|
||||
} else {
|
||||
s.endPostprocessSubspan(ctx, span, summaryOut)
|
||||
}
|
||||
}()
|
||||
|
||||
// Get knowledge base
|
||||
kb, err := s.kbService.GetKnowledgeBaseByID(ctx, payload.KnowledgeBaseID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to get knowledge base: %v", err)
|
||||
summaryErr = err
|
||||
return nil
|
||||
}
|
||||
|
||||
if kb.SummaryModelID == "" {
|
||||
logger.Warn(ctx, "Knowledge base summary model ID is empty, skipping summary generation")
|
||||
summaryOut["skipped"] = "no_summary_model"
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -903,6 +925,7 @@ func (s *knowledgeService) ProcessSummaryGeneration(ctx context.Context, t *asyn
|
||||
knowledge, err := s.repo.GetKnowledgeByID(ctx, payload.TenantID, payload.KnowledgeID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to get knowledge: %v", err)
|
||||
summaryErr = err
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -927,6 +950,7 @@ func (s *knowledgeService) ProcessSummaryGeneration(ctx context.Context, t *asyn
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to get chunks: %v", err)
|
||||
markSummaryFailed()
|
||||
summaryErr = err
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -937,6 +961,7 @@ func (s *knowledgeService) ProcessSummaryGeneration(ctx context.Context, t *asyn
|
||||
textChunks = append(textChunks, chunk)
|
||||
}
|
||||
}
|
||||
summaryOut["text_chunks"] = len(textChunks)
|
||||
|
||||
if len(textChunks) == 0 {
|
||||
logger.Infof(ctx, "No text chunks found for knowledge: %s", payload.KnowledgeID)
|
||||
@@ -944,6 +969,7 @@ func (s *knowledgeService) ProcessSummaryGeneration(ctx context.Context, t *asyn
|
||||
knowledge.SummaryStatus = types.SummaryStatusCompleted
|
||||
knowledge.UpdatedAt = time.Now()
|
||||
s.repo.UpdateKnowledge(ctx, knowledge)
|
||||
summaryOut["skipped"] = "no_text_chunks"
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -957,6 +983,7 @@ func (s *knowledgeService) ProcessSummaryGeneration(ctx context.Context, t *asyn
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to get chat model: %v", err)
|
||||
markSummaryFailed()
|
||||
summaryErr = err
|
||||
return fmt.Errorf("failed to get chat model: %w", err)
|
||||
}
|
||||
|
||||
@@ -974,8 +1001,11 @@ func (s *knowledgeService) ProcessSummaryGeneration(ctx context.Context, t *asyn
|
||||
knowledge.UpdatedAt = time.Now()
|
||||
if updateErr := s.repo.UpdateKnowledge(ctx, knowledge); updateErr != nil {
|
||||
logger.Errorf(ctx, "Failed to mark summary as failed: %v", updateErr)
|
||||
summaryErr = updateErr
|
||||
return fmt.Errorf("failed to update knowledge: %w", updateErr)
|
||||
}
|
||||
summaryOut["fallback"] = "insufficient_content"
|
||||
summaryErr = err
|
||||
return nil
|
||||
}
|
||||
// For other errors (LLM API issues etc.), fall back to the first chunk.
|
||||
@@ -987,6 +1017,7 @@ func (s *knowledgeService) ProcessSummaryGeneration(ctx context.Context, t *asyn
|
||||
summary = string(runes[:500])
|
||||
}
|
||||
}
|
||||
summaryOut["fallback"] = "first_chunk"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -994,8 +1025,10 @@ func (s *knowledgeService) ProcessSummaryGeneration(ctx context.Context, t *asyn
|
||||
knowledge.Description = summary
|
||||
knowledge.SummaryStatus = types.SummaryStatusCompleted
|
||||
knowledge.UpdatedAt = time.Now()
|
||||
summaryOut["summary_chars"] = len([]rune(summary))
|
||||
if err := s.repo.UpdateKnowledge(ctx, knowledge); err != nil {
|
||||
logger.Errorf(ctx, "Failed to update knowledge description: %v", err)
|
||||
summaryErr = err
|
||||
return fmt.Errorf("failed to update knowledge: %w", err)
|
||||
}
|
||||
|
||||
@@ -1034,6 +1067,7 @@ func (s *knowledgeService) ProcessSummaryGeneration(ctx context.Context, t *asyn
|
||||
// Save summary chunk
|
||||
if err := s.chunkService.CreateChunks(ctx, []*types.Chunk{summaryChunk}); err != nil {
|
||||
logger.Errorf(ctx, "Failed to create summary chunk: %v", err)
|
||||
summaryErr = err
|
||||
return fmt.Errorf("failed to create summary chunk: %w", err)
|
||||
}
|
||||
|
||||
@@ -1041,6 +1075,7 @@ func (s *knowledgeService) ProcessSummaryGeneration(ctx context.Context, t *asyn
|
||||
tenantInfo, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to get tenant info: %v", err)
|
||||
summaryErr = err
|
||||
return fmt.Errorf("failed to get tenant info: %w", err)
|
||||
}
|
||||
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo)
|
||||
@@ -1049,12 +1084,14 @@ func (s *knowledgeService) ProcessSummaryGeneration(ctx context.Context, t *asyn
|
||||
ctx, s.retrieveEngine, s.ownership, tenantInfo.ID, kb.VectorStoreID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to init retrieve engine: %v", err)
|
||||
summaryErr = err
|
||||
return fmt.Errorf("failed to init retrieve engine: %w", err)
|
||||
}
|
||||
|
||||
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to get embedding model: %v", err)
|
||||
summaryErr = err
|
||||
return fmt.Errorf("failed to get embedding model: %w", err)
|
||||
}
|
||||
|
||||
@@ -1070,13 +1107,16 @@ func (s *knowledgeService) ProcessSummaryGeneration(ctx context.Context, t *asyn
|
||||
|
||||
if err := retrieveEngine.BatchIndex(ctx, embeddingModel, indexInfo); err != nil {
|
||||
logger.Errorf(ctx, "Failed to index summary chunk: %v", err)
|
||||
summaryErr = err
|
||||
return fmt.Errorf("failed to index summary chunk: %w", err)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Successfully created and indexed summary chunk for knowledge: %s", payload.KnowledgeID)
|
||||
summaryOut["summary_chunk_indexed"] = true
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Successfully generated summary for knowledge: %s", payload.KnowledgeID)
|
||||
summaryOut["status"] = "completed"
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1101,6 +1141,12 @@ func (s *knowledgeService) ProcessQuestionGeneration(ctx context.Context, t *asy
|
||||
indexEntriesPrepared := 0
|
||||
indexBatchAttempted := false
|
||||
indexBatchSucceeded := false
|
||||
// Postprocess subspan for the trace viewer. Opened lazily after we
|
||||
// unmarshal the payload (so we have payload.Attempt) and closed in
|
||||
// the defer below alongside the stats log so the span output mirrors
|
||||
// what we already log to stdout.
|
||||
var qSpan *Span
|
||||
var qErr error
|
||||
defer func() {
|
||||
logger.Infof(
|
||||
ctx,
|
||||
@@ -1125,6 +1171,39 @@ func (s *knowledgeService) ProcessQuestionGeneration(ctx context.Context, t *asy
|
||||
indexBatchAttempted,
|
||||
indexBatchSucceeded,
|
||||
)
|
||||
if qSpan != nil {
|
||||
out := types.JSONMap{
|
||||
"status": exitStatus,
|
||||
"total_chunks": totalChunks,
|
||||
"text_chunks": totalTextChunks,
|
||||
"empty_content_chunks": emptyContentChunks,
|
||||
"llm_attempts": llmCallAttempts,
|
||||
"llm_success": llmCallSuccess,
|
||||
"llm_empty": llmCallEmpty,
|
||||
"llm_failed": llmCallFailed,
|
||||
"questions_generated": generatedQuestionsTotal,
|
||||
"chunk_update_failed": chunkUpdateFailed,
|
||||
"metadata_set_failed": chunkMetadataSetFailed,
|
||||
"index_entries_prepared": indexEntriesPrepared,
|
||||
"index_batch_attempted": indexBatchAttempted,
|
||||
"index_batch_succeeded": indexBatchSucceeded,
|
||||
"retry": retryCount,
|
||||
"max_retry": maxRetry,
|
||||
}
|
||||
// Treat any non-success exitStatus as a failed run; the
|
||||
// existing stats-string already enumerates them. qErr stays
|
||||
// optional for callers that want to surface a Go error.
|
||||
if exitStatus != "success" || qErr != nil {
|
||||
msg := exitStatus
|
||||
var detailErr error = qErr
|
||||
if qErr != nil {
|
||||
msg = qErr.Error()
|
||||
}
|
||||
s.failPostprocessSubspan(ctx, qSpan, "QUESTION_FAILED", msg, detailErr)
|
||||
} else {
|
||||
s.endPostprocessSubspan(ctx, qSpan, out)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
if err := json.Unmarshal(t.Payload(), &payload); err != nil {
|
||||
@@ -1135,6 +1214,14 @@ func (s *knowledgeService) ProcessQuestionGeneration(ctx context.Context, t *asy
|
||||
|
||||
logger.Infof(ctx, "Processing question generation for knowledge: %s", payload.KnowledgeID)
|
||||
|
||||
// Open the postprocess.question subspan now that we have payload.Attempt.
|
||||
// Closes via the defer above.
|
||||
qSpan = s.beginPostprocessSubspan(ctx, payload.KnowledgeID, payload.Attempt, "postprocess.question",
|
||||
types.JSONMap{
|
||||
"question_count": payload.QuestionCount,
|
||||
"language": payload.Language,
|
||||
})
|
||||
|
||||
// Set tenant context
|
||||
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
|
||||
if payload.Language != "" {
|
||||
@@ -1144,7 +1231,8 @@ func (s *knowledgeService) ProcessQuestionGeneration(ctx context.Context, t *asy
|
||||
if strings.TrimSpace(s.config.Conversation.GenerateQuestionsPrompt) == "" {
|
||||
exitStatus = "prompt_not_configured"
|
||||
logger.Errorf(ctx, "GenerateQuestionsPrompt is empty: configure conversation.generate_questions_prompt_id")
|
||||
return fmt.Errorf("generate questions prompt not configured")
|
||||
qErr = fmt.Errorf("generate questions prompt not configured")
|
||||
return qErr
|
||||
}
|
||||
|
||||
// Get knowledge base
|
||||
@@ -1152,6 +1240,7 @@ func (s *knowledgeService) ProcessQuestionGeneration(ctx context.Context, t *asy
|
||||
if err != nil {
|
||||
exitStatus = "kb_not_found"
|
||||
logger.Errorf(ctx, "Failed to get knowledge base: %v", err)
|
||||
qErr = err
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1160,6 +1249,7 @@ func (s *knowledgeService) ProcessQuestionGeneration(ctx context.Context, t *asy
|
||||
if err != nil {
|
||||
exitStatus = "knowledge_not_found"
|
||||
logger.Errorf(ctx, "Failed to get knowledge: %v", err)
|
||||
qErr = err
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
@@ -271,3 +272,140 @@ func TestSpanTracker_FailSpan_CascadesDependentSubspans(t *testing.T) {
|
||||
assert.Equal(t, types.SpanStatusCancelled, statusBy["embedding.batch[0]"],
|
||||
"subspan under the cascaded stage must also be cancelled")
|
||||
}
|
||||
|
||||
// TestPostprocessSubspan_AttachesUnderPostProcessStage covers the contract
|
||||
// that the async post-pipeline tasks (summary, question, graph) rely on:
|
||||
// after the parsing pipeline closes the postprocess stage span, an
|
||||
// out-of-band worker can still LookupStage + BeginSubSpan to record its
|
||||
// real processing time as a child of postprocess. Without this guarantee
|
||||
// the trace viewer's postprocess row stays at the ~10ms enqueue duration
|
||||
// even when summary generation takes 20 s.
|
||||
func TestPostprocessSubspan_AttachesUnderPostProcessStage(t *testing.T) {
|
||||
tracker, db := setupSpanTrackerTest(t)
|
||||
ctx := context.Background()
|
||||
repo := repository.NewKnowledgeSpanRepository(db)
|
||||
|
||||
// Set up the parent attempt with a closed postprocess stage — the
|
||||
// async worker must still find it via LookupStage.
|
||||
_, attempt, err := tracker.OpenAttempt(ctx, "kid", "lf-trace")
|
||||
require.NoError(t, err)
|
||||
|
||||
post := tracker.BeginStage(ctx, "kid", attempt, types.StagePostProcess, types.JSONMap{
|
||||
"chunks_total": 20,
|
||||
})
|
||||
require.NotNil(t, post)
|
||||
tracker.EndSpan(ctx, post, types.JSONMap{"enqueued_summary": true})
|
||||
|
||||
// Simulate ProcessSummaryGeneration entering: lookup parent +
|
||||
// BeginSubSpan (the same call shape as beginPostprocessSubspan).
|
||||
parent := tracker.LookupStage(ctx, "kid", attempt, types.StagePostProcess)
|
||||
require.NotNil(t, parent, "lookup must succeed even after EndSpan closed the parent")
|
||||
assert.Equal(t, types.StagePostProcess, parent.Name)
|
||||
assert.Equal(t, types.SpanKindStage, parent.Kind)
|
||||
|
||||
sumSpan := tracker.BeginSubSpan(ctx, parent, "postprocess.summary", types.SpanKindSubSpan,
|
||||
types.JSONMap{"language": "zh-CN"})
|
||||
require.NotNil(t, sumSpan)
|
||||
assert.Equal(t, parent.SpanID, sumSpan.ParentSpanID,
|
||||
"subspan must hang off the postprocess stage's span_id")
|
||||
assert.Equal(t, types.SpanKindSubSpan, sumSpan.Kind)
|
||||
|
||||
tracker.EndSpan(ctx, sumSpan, types.JSONMap{
|
||||
"text_chunks": 20,
|
||||
"summary_chars": 142,
|
||||
})
|
||||
|
||||
// Verify the row landed under the right parent with the right name.
|
||||
rows, err := repo.ListByAttempt(ctx, "kid", attempt)
|
||||
require.NoError(t, err)
|
||||
var found *types.KnowledgeProcessingSpan
|
||||
for i := range rows {
|
||||
if rows[i].Name == "postprocess.summary" {
|
||||
cp := rows[i]
|
||||
found = &cp
|
||||
break
|
||||
}
|
||||
}
|
||||
require.NotNil(t, found, "summary subspan row must persist")
|
||||
assert.Equal(t, parent.SpanID, found.ParentSpanID,
|
||||
"persisted parent_span_id matches LookupStage result")
|
||||
assert.Equal(t, types.SpanStatusDone, found.Status)
|
||||
assert.NotNil(t, found.Output, "EndSpan must record the output map")
|
||||
}
|
||||
|
||||
// TestPostprocessSubspan_MissingParentFallsThrough covers the legacy
|
||||
// path: an in-flight async task may carry attempt=0 (queued before the
|
||||
// span-tracking field was added) or hit a knowledge whose postprocess
|
||||
// stage row is missing (parse predates tracker). LookupStage returning
|
||||
// nil must NOT crash the handler — the caller is expected to skip span
|
||||
// recording and continue normal processing.
|
||||
func TestPostprocessSubspan_MissingParentFallsThrough(t *testing.T) {
|
||||
tracker, _ := setupSpanTrackerTest(t)
|
||||
ctx := context.Background()
|
||||
|
||||
// No OpenAttempt → no rows for kid. LookupStage must return nil.
|
||||
parent := tracker.LookupStage(ctx, "kid-without-attempt", 7, types.StagePostProcess)
|
||||
assert.Nil(t, parent, "missing parent attempt yields nil, not an error")
|
||||
|
||||
// Open an attempt but never begin postprocess. Lookup must still nil.
|
||||
_, attempt, err := tracker.OpenAttempt(ctx, "kid-no-postprocess", "")
|
||||
require.NoError(t, err)
|
||||
parent = tracker.LookupStage(ctx, "kid-no-postprocess", attempt, types.StagePostProcess)
|
||||
assert.Nil(t, parent, "missing postprocess stage row yields nil")
|
||||
}
|
||||
|
||||
// TestChunkExtractPayload_AttemptRoundTrip verifies the new fields
|
||||
// added to ExtractChunkPayload survive JSON marshal/unmarshal so a
|
||||
// cross-process asynq worker can recover the parent attempt + chunk
|
||||
// ordinal on the receiving side. Skipping this would let a typo in
|
||||
// the JSON tag silently zero the attempt and disable span recording.
|
||||
func TestChunkExtractPayload_AttemptRoundTrip(t *testing.T) {
|
||||
in := types.ExtractChunkPayload{
|
||||
TenantID: 42,
|
||||
ChunkID: "chunk-x",
|
||||
ModelID: "m1",
|
||||
KnowledgeID: "kid-7",
|
||||
Attempt: 3,
|
||||
ChunkIndex: 9,
|
||||
}
|
||||
bytes, err := json.Marshal(in)
|
||||
require.NoError(t, err)
|
||||
|
||||
var out types.ExtractChunkPayload
|
||||
require.NoError(t, json.Unmarshal(bytes, &out))
|
||||
|
||||
assert.Equal(t, in.KnowledgeID, out.KnowledgeID)
|
||||
assert.Equal(t, in.Attempt, out.Attempt)
|
||||
assert.Equal(t, in.ChunkIndex, out.ChunkIndex)
|
||||
}
|
||||
|
||||
// TestSummaryQuestionPayload_AttemptRoundTrip mirrors the above for the
|
||||
// summary + question payloads to keep the contract documented.
|
||||
func TestSummaryQuestionPayload_AttemptRoundTrip(t *testing.T) {
|
||||
sumIn := types.SummaryGenerationPayload{
|
||||
TenantID: 42,
|
||||
KnowledgeBaseID: "kb-1",
|
||||
KnowledgeID: "kid-7",
|
||||
Language: "zh-CN",
|
||||
Attempt: 5,
|
||||
}
|
||||
sumBytes, err := json.Marshal(sumIn)
|
||||
require.NoError(t, err)
|
||||
var sumOut types.SummaryGenerationPayload
|
||||
require.NoError(t, json.Unmarshal(sumBytes, &sumOut))
|
||||
assert.Equal(t, 5, sumOut.Attempt)
|
||||
|
||||
qIn := types.QuestionGenerationPayload{
|
||||
TenantID: 42,
|
||||
KnowledgeBaseID: "kb-1",
|
||||
KnowledgeID: "kid-7",
|
||||
QuestionCount: 3,
|
||||
Language: "zh-CN",
|
||||
Attempt: 5,
|
||||
}
|
||||
qBytes, err := json.Marshal(qIn)
|
||||
require.NoError(t, err)
|
||||
var qOut types.QuestionGenerationPayload
|
||||
require.NoError(t, json.Unmarshal(qBytes, &qOut))
|
||||
assert.Equal(t, 5, qOut.Attempt)
|
||||
}
|
||||
|
||||
@@ -25,6 +25,16 @@ type ExtractChunkPayload struct {
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
ChunkID string `json:"chunk_id"`
|
||||
ModelID string `json:"model_id"`
|
||||
// KnowledgeID + Attempt link the per-chunk extract back to the parent
|
||||
// parse attempt's postprocess stage so the worker can record a
|
||||
// postprocess.graph.chunk[i] subspan. 0 / "" means "skip span
|
||||
// recording" for legacy in-flight tasks.
|
||||
KnowledgeID string `json:"knowledge_id,omitempty"`
|
||||
Attempt int `json:"attempt,omitempty"`
|
||||
// ChunkIndex is the 0-based ordinal of this chunk inside the parent
|
||||
// knowledge's text-chunk set, used as the subspan name suffix
|
||||
// ("postprocess.graph.chunk[3]") so the timeline preserves order.
|
||||
ChunkIndex int `json:"chunk_index,omitempty"`
|
||||
}
|
||||
|
||||
// DocumentProcessPayload represents the document process task payload
|
||||
@@ -77,6 +87,12 @@ type QuestionGenerationPayload struct {
|
||||
QuestionCount int `json:"question_count"`
|
||||
// Language is the request locale (e.g. zh-CN, en-US) when the task was enqueued, used for {{language}} / {{lang}} in templates.
|
||||
Language string `json:"language,omitempty"`
|
||||
// Attempt links this task to the parent parse attempt so the worker
|
||||
// can record a postprocess.question subspan under the right attempt's
|
||||
// postprocess stage. 0 means "skip span recording" (legacy in-flight
|
||||
// tasks queued before this field shipped, or callers without a
|
||||
// tracker).
|
||||
Attempt int `json:"attempt,omitempty"`
|
||||
}
|
||||
|
||||
// SummaryGenerationPayload represents the summary generation task payload
|
||||
@@ -86,6 +102,10 @@ type SummaryGenerationPayload struct {
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||||
KnowledgeID string `json:"knowledge_id"`
|
||||
Language string `json:"language,omitempty"`
|
||||
// Attempt links this task to the parent parse attempt so the worker
|
||||
// can record a postprocess.summary subspan under the right attempt's
|
||||
// postprocess stage. See QuestionGenerationPayload.Attempt notes.
|
||||
Attempt int `json:"attempt,omitempty"`
|
||||
}
|
||||
|
||||
// KBClonePayload represents the knowledge base clone task payload
|
||||
|
||||
Reference in New Issue
Block a user