diff --git a/frontend/src/api/chunker/index.ts b/frontend/src/api/chunker/index.ts new file mode 100644 index 00000000..cb49974b --- /dev/null +++ b/frontend/src/api/chunker/index.ts @@ -0,0 +1,15 @@ +// Chunker debug / preview API. Single endpoint that runs the adaptive +// chunker over a sample text without touching DB or embeddings. Used by +// the KB editor debug panel. + +import { post } from '../../utils/request' +import type { + PreviewChunkingRequest, + PreviewChunkingResponse +} from '../../types/chunker' + +export function previewChunking( + body: PreviewChunkingRequest +): Promise<{ success: boolean; data: PreviewChunkingResponse }> { + return post('/api/v1/chunker/preview', body) +} diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index 87272316..652d3067 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -1951,6 +1951,30 @@ export default { de: 'German', en: 'English', zh: 'Chinese' + }, + debug: { + toggle: 'Test with sample text', + toggleHint: 'Run the chunker against sample text without re-uploading', + sampleLabel: 'Sample text', + samplePlaceholder: 'Paste a Markdown / plain-text snippet to see how the current configuration would chunk it…', + runButton: 'Run preview', + errorPrefix: 'Preview failed', + selectedTier: 'Selected strategy', + rejected: 'Rejected tiers', + contextHeader: 'Context header', + fallbackWarning: 'Strategy chain fell through — content does not split intelligently with current settings', + profile: { + lines: 'lines', + chars: 'chars', + headings: 'Markdown headings', + pageBreaks: 'page breaks', + chapterMarkers: 'chapter markers', + languages: 'languages' + }, + stats: { + chunks: 'chunks', + truncated: 'truncated; total {total}' + } } }, multimodal: { diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts index 4a111acf..3c5369de 100755 --- a/frontend/src/i18n/locales/ko-KR.ts +++ b/frontend/src/i18n/locales/ko-KR.ts @@ -2560,6 +2560,30 @@ export default { de: "독일어", en: "영어", zh: "중국어" + }, + debug: { + toggle: "샘플 텍스트로 테스트", + toggleHint: "재업로드 없이 샘플 텍스트에 대해 청커 실행", + sampleLabel: "샘플 텍스트", + samplePlaceholder: "현재 구성으로 어떻게 청크되는지 보려면 Markdown / 일반 텍스트 스니펫을 붙여넣으세요…", + runButton: "미리보기 실행", + errorPrefix: "미리보기 실패", + selectedTier: "선택된 전략", + rejected: "거부된 계층", + contextHeader: "컨텍스트 헤더", + fallbackWarning: "전략 체인이 모두 실패함 — 현재 설정으로는 콘텐츠를 지능적으로 분할할 수 없음", + profile: { + lines: "줄", + chars: "문자", + headings: "Markdown 제목", + pageBreaks: "페이지 구분", + chapterMarkers: "챕터 마커", + languages: "언어" + }, + stats: { + chunks: "청크", + truncated: "잘림; 총 {total}" + } } }, multimodal: { diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index 29c7612a..e7351324 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -2163,6 +2163,30 @@ export default { de: 'Немецкий', en: 'Английский', zh: 'Китайский' + }, + debug: { + toggle: 'Проверить на примере текста', + toggleHint: 'Запустить разбиение для примера текста без повторной загрузки', + sampleLabel: 'Пример текста', + samplePlaceholder: 'Вставьте фрагмент Markdown / простого текста, чтобы увидеть, как его разобьет текущая конфигурация…', + runButton: 'Запустить предпросмотр', + errorPrefix: 'Ошибка предпросмотра', + selectedTier: 'Выбранная стратегия', + rejected: 'Отклоненные уровни', + contextHeader: 'Контекстный заголовок', + fallbackWarning: 'Цепочка стратегий полностью исчерпана — текущие настройки не позволяют разумно разбить контент', + profile: { + lines: 'строк', + chars: 'символов', + headings: 'заголовков Markdown', + pageBreaks: 'разрывов страниц', + chapterMarkers: 'маркеров глав', + languages: 'языки' + }, + stats: { + chunks: 'блоков', + truncated: 'обрезано; всего {total}' + } } }, multimodal: { diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index 93f1e911..69778435 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -2519,6 +2519,30 @@ export default { de: "德语", en: "英语", zh: "中文" + }, + debug: { + toggle: "使用示例文本测试", + toggleHint: "无需重新上传即可对示例文本运行分块器", + sampleLabel: "示例文本", + samplePlaceholder: "粘贴 Markdown / 纯文本片段以查看当前配置的分块结果…", + runButton: "运行预览", + errorPrefix: "预览失败", + selectedTier: "选定策略", + rejected: "被拒绝的层级", + contextHeader: "上下文标题", + fallbackWarning: "策略链已穷尽 — 当前设置无法智能分块此内容", + profile: { + lines: "行", + chars: "字符", + headings: "Markdown 标题", + pageBreaks: "分页符", + chapterMarkers: "章节标记", + languages: "语言" + }, + stats: { + chunks: "块", + truncated: "已截断;总数 {total}" + } } }, multimodal: { diff --git a/frontend/src/types/chunker.ts b/frontend/src/types/chunker.ts new file mode 100644 index 00000000..7065fa51 --- /dev/null +++ b/frontend/src/types/chunker.ts @@ -0,0 +1,72 @@ +// Types for the /api/v1/chunker/preview endpoint. Mirrors the JSON shape +// produced by internal/handler/chunker_debug.go. Used by the KB editor's +// chunking debug panel to render tier-info / chunk-cards / size stats. + +export type StrategyTier = 'heading' | 'heuristic' | 'recursive' | 'legacy' + +export interface TierRejection { + tier: StrategyTier + reason: string +} + +export interface DocProfile { + total_chars: number + total_lines: number + avg_line_len: number + std_line_len: number + md_heading_counts: Record + md_heading_total: number + numbered_section_count: number + all_caps_short_line_count: number + blank_paragraph_breaks: number + form_feed_count: number + visual_sep_count: number + german_chapter_count: number + english_chapter_count: number + chinese_chapter_count: number + repeated_footer_count: number + has_tables: boolean + has_code: boolean + code_ratio: number + detected_langs: string[] +} + +export interface PreviewChunk { + seq: number + start: number + end: number + size_chars: number + size_tokens_approx: number + context_header?: string + content: string +} + +export interface PreviewChunkingStats { + count: number + avg_chars: number + min_chars: number + max_chars: number + stddev_chars: number + truncated_to?: number +} + +export interface PreviewChunkingResponse { + selected_tier: StrategyTier + tier_chain: StrategyTier[] + rejected: TierRejection[] + profile: DocProfile + chunks: PreviewChunk[] + stats: PreviewChunkingStats +} + +export interface PreviewChunkingRequest { + text: string + chunking_config: { + chunk_size: number + chunk_overlap: number + separators: string[] + strategy?: string + token_limit?: number + languages?: string[] + } +} diff --git a/frontend/src/views/knowledge/settings/KBChunkingDebug.vue b/frontend/src/views/knowledge/settings/KBChunkingDebug.vue new file mode 100644 index 00000000..e563f33b --- /dev/null +++ b/frontend/src/views/knowledge/settings/KBChunkingDebug.vue @@ -0,0 +1,494 @@ + + + + + diff --git a/frontend/src/views/knowledge/settings/KBChunkingSettings.vue b/frontend/src/views/knowledge/settings/KBChunkingSettings.vue index 8b1e2558..c5492c8d 100644 --- a/frontend/src/views/knowledge/settings/KBChunkingSettings.vue +++ b/frontend/src/views/knowledge/settings/KBChunkingSettings.vue @@ -200,6 +200,9 @@ + + + @@ -207,6 +210,7 @@