feat(i18n): add optional embedding model descriptions for multiple locales

Enhance the internationalization support by adding optional descriptions for the embedding model in English, Korean, Russian, and Chinese. Update the UI to reflect these changes, ensuring users are informed about the optional nature of the embedding model when configuring knowledge base settings.
This commit is contained in:
wizardchen
2026-06-16 21:07:28 +08:00
committed by lyingbug
parent 07e0f6f21a
commit 83eeec0ddf
10 changed files with 41 additions and 112 deletions
+8 -4
View File
@@ -7,7 +7,7 @@ package agent
// batch, when the KB still has no folders to anchor on). The result is applied
// in reduce only to pages that don't already have a category, so user edits and
// previously-filed pages are never churned.
const WikiTaxonomyPlanPrompt = `You are organizing a wiki knowledge base into a navigation directory. Assign each item below to a directory path (category) so the whole set lands on ONE coherent tree that reuses existing folders.
const WikiTaxonomyPlanPrompt = `You are organizing a wiki knowledge base into a navigation directory. Assign each item below to a directory path (category) so the whole set lands on ONE coherent tree.
<existing_folders>
{{.ExistingTaxonomy}}
@@ -20,12 +20,16 @@ const WikiTaxonomyPlanPrompt = `You are organizing a wiki knowledge base into a
<instructions>
For every item, output a category path: an array of folder labels from broad to narrow (at most 2 levels). The category classifies WHAT the item fundamentally IS (the stable library "shelf" it always sits on), never the role it plays in one document.
Rules:
- Reuse the EXACT folder labels listed in <existing_folders> (character-for-character) whenever an item belongs under the same domain. Do NOT invent synonym folders (e.g. do NOT create "春节习俗" when "春节 / 传统习俗" already fits).
How to choose a path for each item:
1. If an existing folder in <existing_folders> fits, REUSE its EXACT label (character-for-character). Do NOT invent a synonym folder (e.g. do NOT create "春节习俗" when "春节 / 传统习俗" already fits).
2. If NO existing folder fits, CREATE a new, broad, durable folder for it (e.g. an organization → "组织", a legal idea → "法律概念", a place → "地点"). The directory does not have to stay small — most items DO have a natural home, so coin a sensible top-level folder rather than leaving them unfiled. Group items of the SAME kind under the SAME new folder so the tree stays coherent.
3. Only give an empty path [] when an item genuinely belongs to NO durable subject at all. This must be RARE. The absence of a matching existing folder is NOT a reason for []; create a folder instead.
Other rules:
- Group items of the SAME kind under the SAME folder at the SAME depth. Do not file one equivalent item a level deeper than its siblings (e.g. avoid "地点 / 地址 / Address1" next to "地点 / Address2" — pick one consistent depth for equivalent items).
- Prefer a single broad top-level folder; add a second level only for a genuinely durable sub-domain shared by several items.
- Do NOT use the item type ("entity"/"concept") as a folder. Do NOT put slashes inside a single label.
- Every item slug in <items> MUST appear exactly once in the output. If an item truly cannot be classified under any durable folder, give it an empty path [].
- Every item slug in <items> MUST appear exactly once in the output.
- Write ALL folder labels in {{.Language}}.
### JSON Formatting Rules
@@ -1536,50 +1536,6 @@ func splitSummaryLine(raw string) (summary string, content string) {
return "", raw
}
// splitCategoryLine extracts a leading "CATEGORY: a / b" line from content
// (the page body AFTER the SUMMARY line has been stripped) and returns the
// parsed folder labels plus the remaining content with that line removed.
//
// Category assignment lives in the page-modify output (reduce time) rather
// than per-document extraction so it reflects WHAT the whole entity IS, not
// the role it played in one source document. An empty/absent CATEGORY line
// yields nil so callers keep the page's existing category instead of wiping it.
func splitCategoryLine(content string) (category []string, rest string) {
trimmed := strings.TrimLeft(content, " \t\r\n")
if !strings.HasPrefix(trimmed, "CATEGORY:") && !strings.HasPrefix(trimmed, "CATEGORY:") {
return nil, content
}
var line, remainder string
if idx := strings.IndexByte(trimmed, '\n'); idx < 0 {
line = trimmed
} else {
line = trimmed[:idx]
remainder = trimmed[idx+1:]
}
line = strings.TrimPrefix(line, "CATEGORY:")
line = strings.TrimPrefix(line, "CATEGORY:")
return parseCategoryPath(line), strings.TrimSpace(remainder)
}
// parseCategoryPath splits a "a / b" breadcrumb string into clean folder
// labels. Separator variants and wrapping quotes/brackets are normalized; the
// deeper structural cleaning (type-label stripping, depth cap, wiki_path) is
// left to normalizeWikiHierarchy on the page.
func parseCategoryPath(line string) []string {
line = strings.NewReplacer("/", "/", "|", "/", "|", "/", "›", "/", "»", "/", ">", "/").Replace(line)
parts := strings.Split(line, "/")
out := make([]string, 0, len(parts))
for _, p := range parts {
label := strings.TrimSpace(p)
label = strings.Trim(label, `"'“”‘’[]()()`)
label = strings.TrimSpace(label)
if label != "" {
out = append(out, label)
}
}
return out
}
// buildLogEntry builds a WikiLogEntry struct for the current batch. It is
// pure (no DB access) so callers can accumulate entries cheaply under their
// lock and flush them in a single AppendBatch call at the end of the batch.
@@ -1610,11 +1610,6 @@ func (s *wikiIngestService) reduceSlugUpdates(
if err == nil && updatedContent != "" {
updatedSummary, updatedBody := splitSummaryLine(updatedContent)
// Category now comes from the batch taxonomy plan, not this prompt.
// Defensively strip any stray leading CATEGORY line the model may
// still emit so it never leaks into the page body; discard the value.
_, bodyAfterCategory := splitCategoryLine(updatedBody)
updatedBody = bodyAfterCategory
if updatedBody != "" {
page.Content = updatedBody
} else {
@@ -132,7 +132,10 @@ func (s *wikiIngestService) selectRelevantFolders(
}
}
if !kb.NeedsEmbeddingModel() || strings.TrimSpace(kb.EmbeddingModelID) == "" || len(deeper) == 0 {
// Gate purely on whether an embedding model is configured — NOT on
// NeedsEmbeddingModel(), which is false for wiki-only KBs that may still
// opt into an embedding model purely for directory/taxonomy similarity.
if strings.TrimSpace(kb.EmbeddingModelID) == "" || len(deeper) == 0 {
return capFolders(pool, wikiTaxonomyPromptMaxPaths)
}
@@ -7,58 +7,6 @@ import (
"github.com/Tencent/WeKnora/internal/types"
)
func TestSplitCategoryLine(t *testing.T) {
tests := []struct {
name string
content string
wantCat []string
wantRest string
}{
{
name: "two level category",
content: "CATEGORY: 组织 / 企业\n# 标题\n正文",
wantCat: []string{"组织", "企业"},
wantRest: "# 标题\n正文",
},
{
name: "single level category",
content: "CATEGORY: 人物\n# 陈洋",
wantCat: []string{"人物"},
wantRest: "# 陈洋",
},
{
name: "empty category keeps existing (nil) and strips line",
content: "CATEGORY:\n# 标题",
wantCat: nil,
wantRest: "# 标题",
},
{
name: "no category line left untouched",
content: "# 标题\n正文",
wantCat: nil,
wantRest: "# 标题\n正文",
},
{
name: "fullwidth colon and separators",
content: "CATEGORY:组织|企业\n正文",
wantCat: []string{"组织", "企业"},
wantRest: "正文",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cat, rest := splitCategoryLine(tt.content)
if strings.Join(cat, "/") != strings.Join(tt.wantCat, "/") {
t.Fatalf("splitCategoryLine() cat = %v, want %v", cat, tt.wantCat)
}
if rest != tt.wantRest {
t.Fatalf("splitCategoryLine() rest = %q, want %q", rest, tt.wantRest)
}
})
}
}
func TestFormatExistingTaxonomyForPrompt(t *testing.T) {
got := formatExistingTaxonomyForPrompt([][]string{
{"春节", "传统习俗"},