From 83eeec0ddf3c67fd61fa0a7b1c0ac86bc76645cf Mon Sep 17 00:00:00 2001 From: wizardchen Date: Tue, 16 Jun 2026 21:07:28 +0800 Subject: [PATCH] feat(i18n): add optional embedding model descriptions for multiple locales Enhance the internationalization support by adding optional descriptions for the embedding model in English, Korean, Russian, and Chinese. Update the UI to reflect these changes, ensuring users are informed about the optional nature of the embedding model when configuring knowledge base settings. --- frontend/src/i18n/locales/en-US.ts | 2 + frontend/src/i18n/locales/ko-KR.ts | 2 + frontend/src/i18n/locales/ru-RU.ts | 2 + frontend/src/i18n/locales/zh-CN.ts | 2 + .../knowledge/settings/KBModelConfig.vue | 27 +++++++--- internal/agent/prompts_wiki.go | 12 +++-- internal/application/service/wiki_ingest.go | 44 ---------------- .../application/service/wiki_ingest_batch.go | 5 -- .../service/wiki_ingest_taxonomy.go | 5 +- .../service/wiki_ingest_taxonomy_test.go | 52 ------------------- 10 files changed, 41 insertions(+), 112 deletions(-) diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index b4d1ce2c..c3f34ba7 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -2760,6 +2760,8 @@ export default { embeddingDesc: 'Embedding model used for text vectorization', embeddingPlaceholder: 'Select an embedding model', embeddingLocked: 'Knowledge base already has files. Embedding model cannot be modified', + embeddingOptional: '(Optional)', + embeddingWikiOptionalDesc: 'Optional. When set, it powers similarity matching for Wiki directory classification so new pages reuse existing folders better; otherwise the full directory tree is used.', rerankLabel: 'ReRank Model', rerankDesc: 'Model for re-ranking search results (optional)', rerankPlaceholder: 'Select a ReRank model (optional)' diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts index 15cd55c3..1e1ea8d9 100755 --- a/frontend/src/i18n/locales/ko-KR.ts +++ b/frontend/src/i18n/locales/ko-KR.ts @@ -3631,6 +3631,8 @@ export default { embeddingDesc: "텍스트 벡터화를 위한 임베딩 모델", embeddingPlaceholder: "Embedding 모델을 선택해주세요", embeddingLocked: "지식베이스에 이미 파일이 있어 Embedding 모델을 변경할 수 없습니다", + embeddingOptional: "(선택)", + embeddingWikiOptionalDesc: "선택 사항입니다. 설정하면 Wiki 디렉터리 분류의 유사도 매칭에 사용되어 새 페이지가 기존 폴더를 더 잘 재사용합니다. 설정하지 않으면 전체 디렉터리 트리를 기준으로 분류합니다.", rerankLabel: "ReRank 재정렬 모델", rerankDesc: "검색 결과 재정렬을 위한 모델 (선택)", rerankPlaceholder: "ReRank 모델을 선택해주세요 (선택)", diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index 300ae3d6..5ad93d41 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -3117,6 +3117,8 @@ export default { embeddingDesc: 'Модель встраивания для векторизации текста', embeddingPlaceholder: 'Выберите модель встраивания', embeddingLocked: 'В базе знаний уже есть файлы. Модель встраивания нельзя изменить', + embeddingOptional: '(необязательно)', + embeddingWikiOptionalDesc: 'Необязательно. Если задано, используется для сопоставления по сходству при классификации каталога Wiki, чтобы новые страницы лучше повторно использовали существующие папки; иначе классификация выполняется по полному дереву каталога.', rerankLabel: 'Модель ReRank', rerankDesc: 'Модель для переранжирования результатов поиска (необязательно)', rerankPlaceholder: 'Выберите модель ReRank (необязательно)' diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index cfd1caf1..67c3f0ee 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -3624,6 +3624,8 @@ export default { embeddingDesc: "用于文本向量化的嵌入模型", embeddingPlaceholder: "请选择 Embedding 模型", embeddingLocked: "知识库中已有文件，无法修改 Embedding 模型", + embeddingOptional: "（可选）", + embeddingWikiOptionalDesc: "可选。配置后用于 Wiki 目录归类的相似度匹配，让新页面更好地复用已有目录；不配置则按完整目录树归类。", rerankLabel: "ReRank 重排序模型", rerankDesc: "用于搜索结果重排序的模型（可选）", rerankPlaceholder: "请选择 ReRank 模型（可选）", diff --git a/frontend/src/views/knowledge/settings/KBModelConfig.vue b/frontend/src/views/knowledge/settings/KBModelConfig.vue index 4207da04..1e6a440b 100644 --- a/frontend/src/views/knowledge/settings/KBModelConfig.vue +++ b/frontend/src/views/knowledge/settings/KBModelConfig.vue @@ -25,13 +25,21 @@ - -
+ +
- -

{{ $t('knowledgeEditor.models.embeddingDesc') }}

+ +

+ {{ (wikiEnabled && ragEnabled === false) + ? $t('knowledgeEditor.models.embeddingWikiOptionalDesc') + : $t('knowledgeEditor.models.embeddingDesc') }} +

{ color: var(--td-error-color); margin-left: 2px; } + + .optional { + color: var(--td-text-color-placeholder); + font-size: 12px; + font-weight: 400; + margin-left: 4px; + } } .desc { diff --git a/internal/agent/prompts_wiki.go b/internal/agent/prompts_wiki.go index 6fa0c05d..c375240f 100644 --- a/internal/agent/prompts_wiki.go +++ b/internal/agent/prompts_wiki.go @@ -7,7 +7,7 @@ package agent // batch, when the KB still has no folders to anchor on). The result is applied // in reduce only to pages that don't already have a category, so user edits and // previously-filed pages are never churned. -const WikiTaxonomyPlanPrompt = `You are organizing a wiki knowledge base into a navigation directory. Assign each item below to a directory path (category) so the whole set lands on ONE coherent tree that reuses existing folders. +const WikiTaxonomyPlanPrompt = `You are organizing a wiki knowledge base into a navigation directory. Assign each item below to a directory path (category) so the whole set lands on ONE coherent tree. {{.ExistingTaxonomy}} @@ -20,12 +20,16 @@ const WikiTaxonomyPlanPrompt = `You are organizing a wiki knowledge base into a For every item, output a category path: an array of folder labels from broad to narrow (at most 2 levels). The category classifies WHAT the item fundamentally IS (the stable library "shelf" it always sits on), never the role it plays in one document. -Rules: -- Reuse the EXACT folder labels listed in (character-for-character) whenever an item belongs under the same domain. Do NOT invent synonym folders (e.g. do NOT create "春节习俗" when "春节 / 传统习俗" already fits). +How to choose a path for each item: +1. If an existing folder in fits, REUSE its EXACT label (character-for-character). Do NOT invent a synonym folder (e.g. do NOT create "春节习俗" when "春节 / 传统习俗" already fits). +2. If NO existing folder fits, CREATE a new, broad, durable folder for it (e.g. an organization → "组织", a legal idea → "法律概念", a place → "地点"). The directory does not have to stay small — most items DO have a natural home, so coin a sensible top-level folder rather than leaving them unfiled. Group items of the SAME kind under the SAME new folder so the tree stays coherent. +3. Only give an empty path [] when an item genuinely belongs to NO durable subject at all. This must be RARE. The absence of a matching existing folder is NOT a reason for []; create a folder instead. + +Other rules: - Group items of the SAME kind under the SAME folder at the SAME depth. Do not file one equivalent item a level deeper than its siblings (e.g. avoid "地点 / 地址 / Address1" next to "地点 / Address2" — pick one consistent depth for equivalent items). - Prefer a single broad top-level folder; add a second level only for a genuinely durable sub-domain shared by several items. - Do NOT use the item type ("entity"/"concept") as a folder. Do NOT put slashes inside a single label. -- Every item slug in MUST appear exactly once in the output. If an item truly cannot be classified under any durable folder, give it an empty path []. +- Every item slug in MUST appear exactly once in the output. - Write ALL folder labels in {{.Language}}. ### JSON Formatting Rules diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go index afb340e2..591d291b 100644 --- a/internal/application/service/wiki_ingest.go +++ b/internal/application/service/wiki_ingest.go @@ -1536,50 +1536,6 @@ func splitSummaryLine(raw string) (summary string, content string) { return "", raw } -// splitCategoryLine extracts a leading "CATEGORY: a / b" line from content -// (the page body AFTER the SUMMARY line has been stripped) and returns the -// parsed folder labels plus the remaining content with that line removed. -// -// Category assignment lives in the page-modify output (reduce time) rather -// than per-document extraction so it reflects WHAT the whole entity IS, not -// the role it played in one source document. An empty/absent CATEGORY line -// yields nil so callers keep the page's existing category instead of wiping it. -func splitCategoryLine(content string) (category []string, rest string) { - trimmed := strings.TrimLeft(content, " \t\r\n") - if !strings.HasPrefix(trimmed, "CATEGORY:") && !strings.HasPrefix(trimmed, "CATEGORY：") { - return nil, content - } - var line, remainder string - if idx := strings.IndexByte(trimmed, '\n'); idx < 0 { - line = trimmed - } else { - line = trimmed[:idx] - remainder = trimmed[idx+1:] - } - line = strings.TrimPrefix(line, "CATEGORY:") - line = strings.TrimPrefix(line, "CATEGORY：") - return parseCategoryPath(line), strings.TrimSpace(remainder) -} - -// parseCategoryPath splits a "a / b" breadcrumb string into clean folder -// labels. Separator variants and wrapping quotes/brackets are normalized; the -// deeper structural cleaning (type-label stripping, depth cap, wiki_path) is -// left to normalizeWikiHierarchy on the page. -func parseCategoryPath(line string) []string { - line = strings.NewReplacer("／", "/", "｜", "/", "|", "/", "›", "/", "»", "/", ">", "/").Replace(line) - parts := strings.Split(line, "/") - out := make([]string, 0, len(parts)) - for _, p := range parts { - label := strings.TrimSpace(p) - label = strings.Trim(label, `"'“”‘’[]（）()`) - label = strings.TrimSpace(label) - if label != "" { - out = append(out, label) - } - } - return out -} - // buildLogEntry builds a WikiLogEntry struct for the current batch. It is // pure (no DB access) so callers can accumulate entries cheaply under their // lock and flush them in a single AppendBatch call at the end of the batch. diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go index 74454d82..4be3ebdb 100644 --- a/internal/application/service/wiki_ingest_batch.go +++ b/internal/application/service/wiki_ingest_batch.go @@ -1610,11 +1610,6 @@ func (s *wikiIngestService) reduceSlugUpdates( if err == nil && updatedContent != "" { updatedSummary, updatedBody := splitSummaryLine(updatedContent) - // Category now comes from the batch taxonomy plan, not this prompt. - // Defensively strip any stray leading CATEGORY line the model may - // still emit so it never leaks into the page body; discard the value. - _, bodyAfterCategory := splitCategoryLine(updatedBody) - updatedBody = bodyAfterCategory if updatedBody != "" { page.Content = updatedBody } else { diff --git a/internal/application/service/wiki_ingest_taxonomy.go b/internal/application/service/wiki_ingest_taxonomy.go index edb34745..4b3c19b9 100644 --- a/internal/application/service/wiki_ingest_taxonomy.go +++ b/internal/application/service/wiki_ingest_taxonomy.go @@ -132,7 +132,10 @@ func (s *wikiIngestService) selectRelevantFolders( } } - if !kb.NeedsEmbeddingModel() || strings.TrimSpace(kb.EmbeddingModelID) == "" || len(deeper) == 0 { + // Gate purely on whether an embedding model is configured — NOT on + // NeedsEmbeddingModel(), which is false for wiki-only KBs that may still + // opt into an embedding model purely for directory/taxonomy similarity. + if strings.TrimSpace(kb.EmbeddingModelID) == "" || len(deeper) == 0 { return capFolders(pool, wikiTaxonomyPromptMaxPaths) } diff --git a/internal/application/service/wiki_ingest_taxonomy_test.go b/internal/application/service/wiki_ingest_taxonomy_test.go index 85440cdb..21465dc3 100644 --- a/internal/application/service/wiki_ingest_taxonomy_test.go +++ b/internal/application/service/wiki_ingest_taxonomy_test.go @@ -7,58 +7,6 @@ import ( "github.com/Tencent/WeKnora/internal/types" ) -func TestSplitCategoryLine(t *testing.T) { - tests := []struct { - name string - content string - wantCat []string - wantRest string - }{ - { - name: "two level category", - content: "CATEGORY: 组织 / 企业\n# 标题\n正文", - wantCat: []string{"组织", "企业"}, - wantRest: "# 标题\n正文", - }, - { - name: "single level category", - content: "CATEGORY: 人物\n# 陈洋", - wantCat: []string{"人物"}, - wantRest: "# 陈洋", - }, - { - name: "empty category keeps existing (nil) and strips line", - content: "CATEGORY:\n# 标题", - wantCat: nil, - wantRest: "# 标题", - }, - { - name: "no category line left untouched", - content: "# 标题\n正文", - wantCat: nil, - wantRest: "# 标题\n正文", - }, - { - name: "fullwidth colon and separators", - content: "CATEGORY：组织｜企业\n正文", - wantCat: []string{"组织", "企业"}, - wantRest: "正文", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - cat, rest := splitCategoryLine(tt.content) - if strings.Join(cat, "/") != strings.Join(tt.wantCat, "/") { - t.Fatalf("splitCategoryLine() cat = %v, want %v", cat, tt.wantCat) - } - if rest != tt.wantRest { - t.Fatalf("splitCategoryLine() rest = %q, want %q", rest, tt.wantRest) - } - }) - } -} - func TestFormatExistingTaxonomyForPrompt(t *testing.T) { got := formatExistingTaxonomyForPrompt([][]string{ {"春节", "传统习俗"},