Weknora/Enterprise-Knowledge-Base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(knowledge): harden batch reparse UX and async processing

Browse Source 
Move batch reparse work to an asynq task to avoid HTTP timeouts, add i18n
for all locales, guard in-flight documents, and prevent concurrent delete/reparse.
This commit is contained in:
wizardchen
2026-06-25 23:51:38 +08:00
committed by lyingbug
parent a14c1d87aa
commit 97b7bb9890
13 changed files with 214 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend/src/api/knowledge-base/index.ts
+2 -2
View File
@@ -482,10 +482,10 @@ export function knowledgeSemanticSearch(data: {
return post('/api/v1/knowledge-search', data);
}
export function batchReparseKnowledge(kbId: string, ids: string[], processConfig?: any) {
export function batchReparseKnowledge(kbId: string, ids: string[], processConfig?: KnowledgeProcessOverrides) {
return post(`/api/v1/knowledge/batch-reparse`, {
kb_id: kbId,
ids,
process_config: processConfig
process_config: processConfig,
});
}
frontend/src/i18n/locales/en-US.ts
+5
View File
@@ -575,6 +575,11 @@ export default {
confirmBatchDeleteDocument: 'Delete {count} selected documents? This action cannot be undone.',
batchDeleteSuccess: 'Deleted {count} documents',
batchDeleteFailed: 'Batch delete failed',
confirmBatchReparseDocument: 'Rebuild {count} selected documents? Existing content will be cleared and each document will be re-parsed.',
confirmBatchReparse: 'Confirm and reparse',
batchReparseSuccess: 'Submitted {count} rebuild tasks',
batchReparseFailed: 'Batch rebuild failed',
batchReparseSkippedInFlight: 'Skipped {count} document(s) still being parsed',
statusCompleted: 'Completed',
statusProcessing: 'Processing',
statusFinalizing: 'Optimizing',
frontend/src/i18n/locales/ko-KR.ts
+5
View File
@@ -578,6 +578,11 @@ export default {
confirmBatchDeleteDocument: "선택한 {count}개 문서를 삭제하시겠습니까? 삭제 후 복구할 수 없습니다.",
batchDeleteSuccess: "{count}개 문서가 삭제되었습니다",
batchDeleteFailed: "일괄 삭제 실패",
confirmBatchReparseDocument: "선택한 {count}개 문서를 재구축하시겠습니까? 기존 내용이 삭제되고 각 문서가 다시 파싱됩니다.",
confirmBatchReparse: "확인 후 재파싱",
batchReparseSuccess: "재구축 작업 {count}개가 제출되었습니다",
batchReparseFailed: "일괄 재구축 실패",
batchReparseSkippedInFlight: "파싱 중인 문서 {count}개를 건너뛰었습니다",
statusCompleted: "완료",
statusProcessing: "처리 중",
statusFinalizing: "최적화 중",
frontend/src/i18n/locales/ru-RU.ts
+5
View File
@@ -500,6 +500,11 @@ export default {
confirmBatchDeleteDocument: 'Удалить {count} выбранных документов? Это действие нельзя отменить.',
batchDeleteSuccess: 'Удалено документов: {count}',
batchDeleteFailed: 'Ошибка пакетного удаления',
confirmBatchReparseDocument: 'Пересобрать {count} выбранных документов? Существующее содержимое будет удалено, и каждый документ будет обработан заново.',
confirmBatchReparse: 'Подтвердить и обработать заново',
batchReparseSuccess: 'Отправлено задач пересборки: {count}',
batchReparseFailed: 'Ошибка пакетной пересборки',
batchReparseSkippedInFlight: 'Пропущено документов, которые ещё обрабатываются: {count}',
statusCompleted: 'Завершено',
statusProcessing: 'Обработка',
statusFinalizing: 'Оптимизация',
frontend/src/i18n/locales/zh-CN.ts
+5
View File
@@ -576,6 +576,11 @@ export default {
confirmBatchDeleteDocument: "确认删除选中的 {count} 个文档？删除后将无法恢复。",
batchDeleteSuccess: "成功删除 {count} 个文档",
batchDeleteFailed: "批量删除失败",
confirmBatchReparseDocument: "确认重建选中的 {count} 个文档？现有内容将被清除并重新解析。",
confirmBatchReparse: "确认并重新解析",
batchReparseSuccess: "已提交 {count} 个重建任务",
batchReparseFailed: "批量重建失败",
batchReparseSkippedInFlight: "已跳过 {count} 个正在解析中的文档",
statusCompleted: "已完成",
statusProcessing: "解析中",
statusFinalizing: "优化中",
frontend/src/views/knowledge/KnowledgeBase.vue
+66 -115
View File
@@ -61,7 +61,7 @@ import { formatFileSize } from '@/utils/files';
import { useMarqueeSelect } from '@/hooks/useMarqueeSelect';
import type { ParserEngineInfo } from '@/api/system';
const route = useRoute();
const { t, te } = useI18n();
const { t } = useI18n();
const kbId = computed(() => (route.params as any).kbId as string || '');
const kbInfo = ref<any>(null);
const uploadSourceRef = ref<InstanceType<typeof KbUploadSourceDropdown> | null>(null);
@@ -419,28 +419,34 @@ const batchDeleting = ref(false);
const batchReparsing = ref(false);
const confirmBatchReparse = async () => {
if (batchReparsing.value || selectedIds.value.size === 0) return;
const ids = Array.from(selectedIds.value);
if (batchReparsing.value || batchDeleting.value || selectedIds.value.size === 0) return;
const allIds = Array.from(selectedIds.value);
const ids = allIds.filter((id) => {
const item = cardList.value.find((c) => c.id === id);
return !item || !isParseInFlight(item.parse_status);
});
const skipped = allIds.length - ids.length;
if (ids.length === 0) {
MessagePlugin.info(t('knowledgeBase.rebuildInProgress'));
return;
}
if (skipped > 0) {
MessagePlugin.warning(t('knowledgeBase.batchReparseSkippedInFlight', { count: skipped }));
}
batchReparsing.value = true;
try {
const res: any = await batchReparseKnowledge(kbId.value, ids);
if (res?.success) {
if (res.partial && res.failed_ids && res.failed_ids.length > 0) {
MessagePlugin.warning(`部分重构任务提交成功，失败 ${res.failed_ids.length} 个`);
} else {
const successMsg = te('knowledgeBase.batchReparseSuccess')
? t('knowledgeBase.batchReparseSuccess', { count: ids.length })
: `已提交 ${ids.length} 个重构任务`;
MessagePlugin.success(successMsg);
}
MessagePlugin.success(t('knowledgeBase.batchReparseSuccess', { count: ids.length }));
clearSelection();
batchMode.value = false;
loadKnowledgeFiles(kbId.value);
scheduleWikiStatusProbes();
} else {
MessagePlugin.error(res?.message || '批量重建失败');
MessagePlugin.error(res?.message || t('knowledgeBase.batchReparseFailed'));
}
} catch (e: any) {
MessagePlugin.error(e?.message || '批量重建失败');
MessagePlugin.error(e?.message || t('knowledgeBase.batchReparseFailed'));
} finally {
batchReparsing.value = false;
}
@@ -1897,7 +1903,7 @@ const onCardClick = (item: KnowledgeCard) => {
};
const confirmBatchDelete = async () => {
if (batchDeleting.value || selectedIds.value.size === 0) return;
if (batchDeleting.value || batchReparsing.value || selectedIds.value.size === 0) return;
const ids = Array.from(selectedIds.value);
const deletedIdSet = new Set(ids);
batchDeleting.value = true;
@@ -2149,8 +2155,7 @@ async function createNewSession(value: string): Promise<void> {
<span class="tag-hash-icon">#</span>
<div class="tag-edit-input">
<t-input ref="newTagInputRef" v-model="newTagName" size="small" :maxlength="40"
:placeholder="$t('knowledgeBase.tagNamePlaceholder')"
@enter="submitCreateTag"
:placeholder="$t('knowledgeBase.tagNamePlaceholder')" @enter="submitCreateTag"
@keydown="(_v, ctx) => { if (ctx?.e?.key === 'Escape') { ctx.e.stopPropagation(); ctx.e.preventDefault(); cancelCreateTag() } }" />
</div>
</div>
@@ -2267,31 +2272,19 @@ async function createNewSession(value: string): Promise<void> {
</t-tooltip>
</div>
<div v-if="canEdit" class="doc-filter-actions">
<KbUploadSourceDropdown
ref="uploadSourceRef"
:accept-file-types="acceptFileTypes"
:supported-file-types="[...supportedFileTypes]"
include-manual
trigger-icon="file-add"
trigger-class="content-bar-icon-btn"
data-guide="kb-detail-add-doc"
:tooltip="t('knowledgeBase.addDocument')"
placement="bottom-right"
@files="handleUploadSourceFiles"
@url="handleUploadSourceUrl"
@manual="handleManualCreate"
/>
<KbUploadSourceDropdown ref="uploadSourceRef" :accept-file-types="acceptFileTypes"
:supported-file-types="[...supportedFileTypes]" include-manual trigger-icon="file-add"
trigger-class="content-bar-icon-btn" data-guide="kb-detail-add-doc"
:tooltip="t('knowledgeBase.addDocument')" placement="bottom-right" @files="handleUploadSourceFiles"
@url="handleUploadSourceUrl" @manual="handleManualCreate" />
</div>
</div>
<div class="doc-scroll-container" :class="{ 'is-empty': !cardList.length && !docListLoading, 'is-marquee-active': docMarqueeVisible }" ref="knowledgeScroll"
@scroll="handleScroll" @mousedown="onDocMarqueeMouseDown">
<div
v-if="docMarqueeVisible"
class="doc-marquee-box"
<div class="doc-scroll-container"
:class="{ 'is-empty': !cardList.length && !docListLoading, 'is-marquee-active': docMarqueeVisible }"
ref="knowledgeScroll" @scroll="handleScroll" @mousedown="onDocMarqueeMouseDown">
<div v-if="docMarqueeVisible" class="doc-marquee-box"
:class="{ 'is-add': docMarqueeMode === 'add', 'is-subtract': docMarqueeMode === 'subtract' }"
:style="docMarqueeBoxStyle"
aria-hidden="true"
/>
:style="docMarqueeBoxStyle" aria-hidden="true" />
<!-- 文档骨架屏 -->
<div v-if="docListLoading && cardList.length === 0" class="doc-card-list doc-card-list-animated">
<div v-for="n in 8" :key="'doc-skel-' + n" class="knowledge-card knowledge-card-skeleton">
@@ -2313,10 +2306,9 @@ async function createNewSession(value: string): Promise<void> {
<!-- 现有文档卡片 -->
<div class="knowledge-card"
:class="{ 'is-selected': selectedIds.has(item.id), 'batch-mode': batchMode }"
:data-select-id="item.id"
v-for="(item, index) in cardList" :key="item.id" @click="onCardClick(item)"
@mouseenter="onCardMouseEnter($event, item)" @mousemove="onCardMouseMove($event)"
@mouseleave="onCardMouseLeave">
:data-select-id="item.id" v-for="(item, index) in cardList" :key="item.id"
@click="onCardClick(item)" @mouseenter="onCardMouseEnter($event, item)"
@mousemove="onCardMouseMove($event)" @mouseleave="onCardMouseLeave">
<div class="card-content">
<div class="card-content-nav">
<div v-if="canEdit && batchMode" class="card-nav-check" @click.stop>
@@ -2327,8 +2319,7 @@ async function createNewSession(value: string): Promise<void> {
<span class="card-content-title" :title="item.file_name">{{ item.file_name }}</span>
<t-popup v-if="canEdit" v-model="item.isMore" overlayClassName="card-more"
:on-visible-change="(v: boolean) => onCardMoreVisibleChange(v, item)" trigger="click"
destroy-on-close
placement="bottom-right">
destroy-on-close placement="bottom-right">
<div variant="outline" class="more-wrap" @click.stop="openMore(index)"
:class="[moreIndex == index ? 'active-more' : '']">
<img class="more-icon" src="@/assets/img/more.png" alt="" />
@@ -2346,31 +2337,20 @@ async function createNewSession(value: string): Promise<void> {
<t-icon class="icon" name="chart-bar" />
<span>{{ t('knowledgeStages.viewTrace') }}</span>
</div>
<div
v-if="isParseInFlight(item.parse_status)"
class="card-menu-item"
@click.stop="onReparseMenuClick(index, item)"
>
<div v-if="isParseInFlight(item.parse_status)" class="card-menu-item"
@click.stop="onReparseMenuClick(index, item)">
<t-icon class="icon" name="refresh" />
<span>{{ t('knowledgeBase.rebuildDocument') }}</span>
</div>
<div
v-else
class="card-menu-item"
@click.stop="confirmRebuildKnowledge(index, item)"
>
<div v-else class="card-menu-item" @click.stop="confirmRebuildKnowledge(index, item)">
<t-icon class="icon" name="refresh" />
<span>{{ t('knowledgeBase.rebuildDocument') }}</span>
</div>
<t-popconfirm
v-if="isParseInFlight(item.parse_status)"
theme="warning"
<t-popconfirm v-if="isParseInFlight(item.parse_status)" theme="warning"
:content="t('knowledgeBase.cancelParseConfirmBody', { title: item.file_name || item.title || item.id })"
:confirm-btn="{ content: t('knowledgeBase.cancelParse'), theme: 'danger' }"
:cancel-btn="{ content: t('common.cancel') }"
placement="left"
@confirm="confirmCancelParseKnowledge(item)"
>
:cancel-btn="{ content: t('common.cancel') }" placement="left"
@confirm="confirmCancelParseKnowledge(item)">
<div class="card-menu-item danger" @click.stop>
<t-icon class="icon" name="close-circle" />
<span>{{ t('knowledgeBase.cancelParse') }}</span>
@@ -2386,14 +2366,11 @@ async function createNewSession(value: string): Promise<void> {
<t-icon class="icon" name="queue" />
<span>{{ t('menu.batchManage') }}</span>
</div>
<t-popconfirm
theme="warning"
<t-popconfirm theme="warning"
:content="t('knowledgeBase.confirmDeleteDocument', { fileName: item.file_name || '' })"
:confirm-btn="{ content: t('knowledgeBase.confirmDelete'), theme: 'danger' }"
:cancel-btn="{ content: t('common.cancel') }"
placement="left"
@confirm="confirmDeleteKnowledge(index, item)"
>
:cancel-btn="{ content: t('common.cancel') }" placement="left"
@confirm="confirmDeleteKnowledge(index, item)">
<div class="card-menu-item danger" @click.stop>
<t-icon class="icon" name="delete" />
<span>{{ t('knowledgeBase.deleteDocument') }}</span>
@@ -2441,7 +2418,7 @@ async function createNewSession(value: string): Promise<void> {
<div class="move-mode-text">
<span class="move-mode-label">{{ t('knowledgeBase.moveModeReuseVectors') }}</span>
<span class="move-mode-desc">{{ t('knowledgeBase.moveModeReuseVectorsDesc')
}}</span>
}}</span>
</div>
</div>
<div class="move-mode-item" :class="{ active: moveMode === 'reparse' }"
@@ -2464,57 +2441,33 @@ async function createNewSession(value: string): Promise<void> {
</template>
</t-popup>
</div>
<div
v-if="isParseInFlight(item.parse_status)"
class="card-analyze card-analyze-trace"
>
<div v-if="isParseInFlight(item.parse_status)" class="card-analyze card-analyze-trace">
<t-icon name="loading" class="card-analyze-loading"></t-icon>
<span
class="card-analyze-txt card-analyze-trace-link"
role="button"
tabindex="0"
:title="t('knowledgeStages.viewTrace')"
@click.stop="handleViewTrace(index, item)"
<span class="card-analyze-txt card-analyze-trace-link" role="button" tabindex="0"
:title="t('knowledgeStages.viewTrace')" @click.stop="handleViewTrace(index, item)"
@keydown.enter.stop="handleViewTrace(index, item)"
@keydown.space.prevent.stop="handleViewTrace(index, item)"
>{{ inFlightCardStatusText(item) }}</span>
<button
type="button"
class="card-analyze-trace-btn"
:title="t('knowledgeStages.viewTrace')"
:aria-label="t('knowledgeStages.viewTrace')"
@click.stop="handleViewTrace(index, item)"
>
@keydown.space.prevent.stop="handleViewTrace(index, item)">{{
inFlightCardStatusText(item) }}</span>
<button type="button" class="card-analyze-trace-btn" :title="t('knowledgeStages.viewTrace')"
:aria-label="t('knowledgeStages.viewTrace')" @click.stop="handleViewTrace(index, item)">
<t-icon name="chart-line" />
</button>
</div>
<div
v-else-if="item.parse_status === 'failed'"
class="card-analyze failure card-analyze-trace"
>
<div v-else-if="item.parse_status === 'failed'" class="card-analyze failure card-analyze-trace">
<t-icon name="close-circle" class="card-analyze-loading failure"></t-icon>
<span
class="card-analyze-txt failure card-analyze-trace-link"
role="button"
tabindex="0"
:title="t('knowledgeStages.viewTrace')"
@click.stop="handleViewTrace(index, item)"
<span class="card-analyze-txt failure card-analyze-trace-link" role="button" tabindex="0"
:title="t('knowledgeStages.viewTrace')" @click.stop="handleViewTrace(index, item)"
@keydown.enter.stop="handleViewTrace(index, item)"
@keydown.space.prevent.stop="handleViewTrace(index, item)"
>{{ t('knowledgeBase.parsingFailed') }}</span>
<button
type="button"
class="card-analyze-trace-btn"
:title="t('knowledgeStages.viewTrace')"
:aria-label="t('knowledgeStages.viewTrace')"
@click.stop="handleViewTrace(index, item)"
>
@keydown.space.prevent.stop="handleViewTrace(index, item)">{{
t('knowledgeBase.parsingFailed') }}</span>
<button type="button" class="card-analyze-trace-btn" :title="t('knowledgeStages.viewTrace')"
:aria-label="t('knowledgeStages.viewTrace')" @click.stop="handleViewTrace(index, item)">
<t-icon name="chart-bar" />
</button>
</div>
<div v-else-if="item.parse_status === 'draft'" class="card-draft">
<t-tag size="small" theme="warning" variant="light-outline">{{ t('knowledgeBase.draft')
}}</t-tag>
}}</t-tag>
<span class="card-draft-tip">{{ t('knowledgeBase.draftTip') }}</span>
</div>
<div
@@ -2578,9 +2531,7 @@ async function createNewSession(value: string): Promise<void> {
:style="{ left: cardPopoverPos.x + 'px', top: cardPopoverPos.y + 'px' }">
<template v-if="hoveredCardItem">
<div class="card-popover-title">{{ hoveredCardItem.file_name }}</div>
<div
v-if="isParseInFlight(hoveredCardItem.parse_status)"
class="card-popover-status parsing">
<div v-if="isParseInFlight(hoveredCardItem.parse_status)" class="card-popover-status parsing">
<KnowledgeProcessingTimeline :knowledge-id="hoveredCardItem.id"
:parse-status="hoveredCardItem.parse_status" :auto-poll="false" :compact="true" />
</div>
@@ -2611,12 +2562,12 @@ async function createNewSession(value: string): Promise<void> {
<div class="card-popover-meta">
<span class="card-popover-time">{{ t('knowledgeBase.updatedAt') }}：{{
formatDocTime(hoveredCardItem.updated_at)
}}</span>
}}</span>
<span v-if="(hoveredCardItem as any).channel && (hoveredCardItem as any).channel !== 'web'"
class="card-popover-channel">{{ getChannelLabel((hoveredCardItem as any).channel) }}</span>
<span v-if="getTagName(hoveredCardItem.tag_id)" class="card-popover-tag">{{
getTagName(hoveredCardItem.tag_id)
}}</span>
}}</span>
<span class="card-popover-type">{{ getKnowledgeType(hoveredCardItem) }}</span>
</div>
<div class="card-popover-hint">{{ t('knowledgeBase.clickToViewFull') }}</div>
@@ -2637,9 +2588,9 @@ async function createNewSession(value: string): Promise<void> {
</template>
</div>
<div class="doc-batch-bar-anchor" v-show="batchMode || selectedIds.size > 0">
<DocumentBatchBar :count="selectedIds.size" :loading="batchDeleting || batchReparsing"
:visible="batchMode || selectedIds.size > 0" @cancel="handleBatchCancel"
@delete="confirmBatchDelete" @reparse="confirmBatchReparse" />
<DocumentBatchBar :count="selectedIds.size" :delete-loading="batchDeleting"
:reparse-loading="batchReparsing" :visible="batchMode || selectedIds.size > 0"
@cancel="handleBatchCancel" @delete="confirmBatchDelete" @reparse="confirmBatchReparse" />
</div>
</div>
</div>
frontend/src/views/knowledge/components/DocumentBatchBar.vue
+15 -46
View File
@@ -3,7 +3,8 @@ import { useI18n } from 'vue-i18n';
defineProps<{
count: number;
loading?: boolean;
deleteLoading?: boolean;
reparseLoading?: boolean;
// When true the bar stays visible even with 0 selections, so users can exit
// batch mode from here without selecting anything first.
visible?: boolean;
@@ -15,68 +16,36 @@ const emit = defineEmits<{
(e: 'reparse'): void;
}>();
const { t, te } = useI18n();
const { t } = useI18n();
</script>
<template>
<transition name="batch-bar-fade">
<div
v-if="visible || count > 0"
class="doc-batch-bar"
role="region"
:aria-label="t('knowledgeBase.selectedCount', { count })"
>
<div v-if="visible || count > 0" class="doc-batch-bar" role="region"
:aria-label="t('knowledgeBase.selectedCount', { count })">
<div class="batch-bar-inner">
<div class="batch-bar-left">
<span class="batch-bar-count">{{ t('knowledgeBase.selectedCount', { count }) }}</span>
<t-button
variant="text"
theme="default"
size="small"
class="batch-bar-clear"
@click="emit('cancel')"
>
<t-button variant="text" theme="default" size="small" class="batch-bar-clear" @click="emit('cancel')">
{{ t('knowledgeBase.clearSelection') }}
</t-button>
</div>
<div class="batch-bar-actions">
<t-popconfirm
theme="warning"
:content="te('knowledgeBase.overviewDescReparse') ? t('knowledgeBase.overviewDescReparse') : '确认后现有内容将被清除并重新解析该文档'"
:confirm-btn="{ content: te('knowledgeBase.confirmReparse') ? t('knowledgeBase.confirmReparse') : '确认并重新解析', theme: 'warning' }"
:cancel-btn="{ content: t('common.cancel') }"
placement="top"
@confirm="emit('reparse')"
>
<t-button
theme="default"
variant="outline"
size="small"
:disabled="count === 0"
:loading="loading"
@click.stop
>
<t-popconfirm theme="warning" :content="t('knowledgeBase.confirmBatchReparseDocument', { count })"
:confirm-btn="{ content: t('knowledgeBase.confirmBatchReparse'), theme: 'warning' }"
:cancel-btn="{ content: t('common.cancel') }" placement="top" @confirm="emit('reparse')">
<t-button theme="default" variant="outline" size="small"
:disabled="count === 0 || deleteLoading || reparseLoading" :loading="reparseLoading" @click.stop>
<template #icon><t-icon name="refresh" size="14px" /></template>
{{ t('knowledgeBase.rebuildDocument') }}
</t-button>
</t-popconfirm>
<t-popconfirm
theme="warning"
:content="t('knowledgeBase.confirmBatchDeleteDocument', { count })"
<t-popconfirm theme="warning" :content="t('knowledgeBase.confirmBatchDeleteDocument', { count })"
:confirm-btn="{ content: t('knowledgeBase.confirmDelete'), theme: 'danger' }"
:cancel-btn="{ content: t('common.cancel') }"
placement="top"
@confirm="emit('delete')"
>
<t-button
theme="danger"
variant="outline"
size="small"
:disabled="count === 0"
:loading="loading"
@click.stop
>
:cancel-btn="{ content: t('common.cancel') }" placement="top" @confirm="emit('delete')">
<t-button theme="danger" variant="outline" size="small"
:disabled="count === 0 || deleteLoading || reparseLoading" :loading="deleteLoading" @click.stop>
<template #icon><t-icon name="delete" size="14px" /></template>
{{ t('knowledgeBase.batchDelete') }}
</t-button>
internal/application/service/knowledge_process.go
+35
View File
@@ -3377,3 +3377,38 @@ func (s *knowledgeService) enqueueImageMultimodalTasks(
}
}
}
// ProcessKnowledgeListReparse handles Asynq knowledge list reparse tasks.
func (s *knowledgeService) ProcessKnowledgeListReparse(ctx context.Context, t *asynq.Task) error {
var payload types.KnowledgeListReparsePayload
if err := json.Unmarshal(t.Payload(), &payload); err != nil {
logger.Errorf(ctx, "Failed to unmarshal knowledge list reparse payload: %v", err)
return err
}
logger.Infof(ctx, "Processing knowledge list reparse task for %d knowledge items", len(payload.KnowledgeIDs))
tenant, err := s.tenantRepo.GetTenantByID(ctx, payload.TenantID)
if err != nil {
logger.Errorf(ctx, "Failed to get tenant %d: %v", payload.TenantID, err)
return err
}
ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID)
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenant)
var failed int
for _, id := range payload.KnowledgeIDs {
if _, err := s.ReparseKnowledge(ctx, id, payload.ProcessConfig); err != nil {
logger.Errorf(ctx, "Failed to reparse knowledge %s: %v", id, err)
failed++
}
}
if failed > 0 {
logger.Warnf(ctx, "Knowledge list reparse completed with %d failures out of %d", failed, len(payload.KnowledgeIDs))
}
logger.Infof(ctx, "Knowledge list reparse task finished: %d submitted, %d failed",
len(payload.KnowledgeIDs)-failed, failed)
return nil
}
internal/handler/knowledge.go
+61 -36
View File
@@ -226,6 +226,30 @@ func (h *KnowledgeHandler) enqueueKnowledgeListDelete(
return info.ID, nil
}
// enqueueKnowledgeListReparse enqueues an async batch-reparse task for the
// given knowledge IDs and returns the asynq task ID.
func (h *KnowledgeHandler) enqueueKnowledgeListReparse(
ctx context.Context, tenantID uint64, ids []string, processConfig *types.KnowledgeProcessOverrides,
) (string, error) {
payload := types.KnowledgeListReparsePayload{
TenantID: tenantID,
KnowledgeIDs: ids,
ProcessConfig: processConfig,
}
langfuse.InjectTracing(ctx, &payload)
payloadBytes, err := json.Marshal(payload)
if err != nil {
return "", fmt.Errorf("marshal payload: %w", err)
}
task := asynq.NewTask(types.TypeKnowledgeListReparse, payloadBytes,
asynq.Queue("low"), asynq.MaxRetry(3))
info, err := h.asynqClient.Enqueue(task)
if err != nil {
return "", fmt.Errorf("enqueue task: %w", err)
}
return info.ID, nil
}
// CreateKnowledgeFromFile godoc
// @Summary 从文件创建知识
// @Description 上传文件并创建知识条目
@@ -2258,27 +2282,31 @@ func (h *KnowledgeHandler) BatchReparseKnowledge(c *gin.Context) {
return
}
var ids []string
seen := make(map[string]struct{})
for _, id := range req.IDs {
if id != "" {
if _, ok := seen[id]; !ok {
ids = append(ids, id)
seen[id] = struct{}{}
}
seen := make(map[string]struct{}, len(req.IDs))
ids := make([]string, 0, len(req.IDs))
for _, raw := range req.IDs {
id := strings.TrimSpace(raw)
if id == "" {
continue
}
if _, ok := seen[id]; ok {
continue
}
seen[id] = struct{}{}
ids = append(ids, id)
}
if len(ids) == 0 {
c.Error(errors.NewBadRequestError("no knowledge IDs provided for batch reparse"))
return
}
if len(ids) > 200 {
c.Error(errors.NewBadRequestError("the number of IDs exceeds the maximum batch size of 200"))
const maxBatch = 200
if len(ids) > maxBatch {
c.Error(errors.NewBadRequestError(fmt.Sprintf("too many ids (max %d per batch)", maxBatch)))
return
}
_, _, effectiveTenantID, permission, err := h.validateKnowledgeBaseAccessWithKBID(c, req.KBID)
_, kbID, effectiveTenantID, permission, err := h.validateKnowledgeBaseAccessWithKBID(c, req.KBID)
if err != nil {
c.Error(err)
return
@@ -2288,45 +2316,42 @@ func (h *KnowledgeHandler) BatchReparseKnowledge(c *gin.Context) {
return
}
ctx = context.WithValue(ctx, types.TenantIDContextKey, effectiveTenantID)
knows, err := h.kgService.GetKnowledgeBatch(ctx, effectiveTenantID, ids)
knowledgeList, err := h.kgService.GetKnowledgeBatch(ctx, effectiveTenantID, ids)
if err != nil {
logger.Errorf(ctx, "failed to get knowledge batch, kb_id: %s, size: %d, err: %v", req.KBID, len(ids), err)
logger.Errorf(ctx, "failed to get knowledge batch, kb_id: %s, size: %d, err: %v", kbID, len(ids), err)
c.Error(errors.NewInternalServerError("failed to get knowledge batch"))
return
}
if len(knows) != len(ids) {
if len(knowledgeList) != len(ids) {
c.Error(errors.NewBadRequestError("some knowledge entries were not found"))
return
}
for _, know := range knows {
if know.KnowledgeBaseID != req.KBID {
c.Error(errors.NewBadRequestError("some knowledge entries do not belong to the specified knowledge base"))
for _, k := range knowledgeList {
if k.KnowledgeBaseID != kbID {
c.Error(errors.NewBadRequestError(
fmt.Sprintf("Knowledge %s does not belong to knowledge base %s",
secutils.SanitizeForLog(k.ID), secutils.SanitizeForLog(kbID))))
return
}
}
var failedIDs []string
for _, id := range ids {
_, err := h.kgService.ReparseKnowledge(ctx, id, req.ProcessConfig)
if err != nil {
logger.Errorf(ctx, "Failed to reparse knowledge %s: %v", id, err)
failedIDs = append(failedIDs, id)
continue
}
}
if len(failedIDs) > 0 {
c.JSON(200, gin.H{
"success": len(failedIDs) < len(ids),
"partial": len(failedIDs) < len(ids),
"message": "Batch reparse completed with some failures",
"failed_ids": failedIDs,
})
taskID, err := h.enqueueKnowledgeListReparse(ctx, effectiveTenantID, ids, req.ProcessConfig)
if err != nil {
logger.Errorf(ctx, "Failed to enqueue batch knowledge reparse task: %v", err)
c.Error(errors.NewInternalServerError("Failed to enqueue batch reparse task"))
return
}
c.JSON(200, gin.H{
logger.Infof(ctx, "Batch knowledge reparse task enqueued: %s, kb_id: %s, count: %d",
taskID, secutils.SanitizeForLog(kbID), len(ids))
c.JSON(http.StatusOK, gin.H{
"success": true,
"message": "Batch reparse task submitted successfully",
"message": "Batch reparse task submitted",
"data": gin.H{
"task_id": taskID,
"reparse_count": len(ids),
},
})
}
internal/router/sync_task.go
+1
View File
@@ -139,6 +139,7 @@ func RegisterSyncHandlers(params SyncTaskParams) {
params.Executor.RegisterHandler(types.TypeKBClone, params.KnowledgeService.ProcessKBClone)
params.Executor.RegisterHandler(types.TypeKnowledgeMove, params.KnowledgeService.ProcessKnowledgeMove)
params.Executor.RegisterHandler(types.TypeKnowledgeListDelete, params.KnowledgeService.ProcessKnowledgeListDelete)
params.Executor.RegisterHandler(types.TypeKnowledgeListReparse, params.KnowledgeService.ProcessKnowledgeListReparse)
params.Executor.RegisterHandler(types.TypeIndexDelete, params.TagService.ProcessIndexDelete)
params.Executor.RegisterHandler(types.TypeKBDelete, params.KnowledgeBaseService.ProcessKBDelete)
params.Executor.RegisterHandler(types.TypeImageMultimodal, params.ImageMultimodal.Handle)
internal/router/task.go
+3
View File
@@ -205,6 +205,9 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux {
// Register knowledge list delete handler
mux.HandleFunc(types.TypeKnowledgeListDelete, params.KnowledgeService.ProcessKnowledgeListDelete)
// Register knowledge list reparse handler
mux.HandleFunc(types.TypeKnowledgeListReparse, params.KnowledgeService.ProcessKnowledgeListReparse)
// Register index delete handler
mux.HandleFunc(types.TypeIndexDelete, params.TagService.ProcessIndexDelete)
internal/types/interfaces/knowledge.go
+2
View File
@@ -174,6 +174,8 @@ type KnowledgeService interface {
ProcessKnowledgeMove(ctx context.Context, t *asynq.Task) error
// ProcessKnowledgeListDelete handles Asynq knowledge list delete tasks
ProcessKnowledgeListDelete(ctx context.Context, t *asynq.Task) error
// ProcessKnowledgeListReparse handles Asynq knowledge list reparse tasks
ProcessKnowledgeListReparse(ctx context.Context, t *asynq.Task) error
// GetKBCloneProgress retrieves the progress of a knowledge base clone task
GetKBCloneProgress(ctx context.Context, taskID string) (*types.KBCloneProgress, error)
// SaveKBCloneProgress saves the progress of a knowledge base clone task
internal/types/task.go
+9
View File
@@ -33,6 +33,7 @@ const (
TypeIndexDelete = "index:delete" // 索引删除任务
TypeKBDelete = "kb:delete" // 知识库删除任务
TypeKnowledgeListDelete = "knowledge:list_delete" // 批量删除知识任务
TypeKnowledgeListReparse = "knowledge:list_reparse" // 批量重解析知识任务
TypeKnowledgeMove = "knowledge:move" // 知识移动任务
TypeDataTableSummary = "datatable:summary" // 表格摘要任务
TypeImageMultimodal = "image:multimodal" // 图片多模态处理任务（OCR + VLM Caption）
@@ -201,6 +202,14 @@ type KnowledgeListDeletePayload struct {
KnowledgeIDs []string `json:"knowledge_ids"`
}
// KnowledgeListReparsePayload represents the batch knowledge reparse task payload
type KnowledgeListReparsePayload struct {
TracingContext
TenantID uint64 `json:"tenant_id"`
KnowledgeIDs []string `json:"knowledge_ids"`
ProcessConfig *KnowledgeProcessOverrides `json:"process_config,omitempty"`
}
// KnowledgeMovePayload represents the knowledge move task payload
type KnowledgeMovePayload struct {
TracingContext