feat(i18n): add optional embedding model descriptions for multiple locales

Enhance the internationalization support by adding optional descriptions for the embedding model in English, Korean, Russian, and Chinese. Update the UI to reflect these changes, ensuring users are informed about the optional nature of the embedding model when configuring knowledge base settings.
This commit is contained in:
wizardchen
2026-06-16 21:07:28 +08:00
committed by lyingbug
parent 07e0f6f21a
commit 83eeec0ddf
10 changed files with 41 additions and 112 deletions
@@ -1536,50 +1536,6 @@ func splitSummaryLine(raw string) (summary string, content string) {
return "", raw
}
// splitCategoryLine extracts a leading "CATEGORY: a / b" line from content
// (the page body AFTER the SUMMARY line has been stripped) and returns the
// parsed folder labels plus the remaining content with that line removed.
//
// Category assignment lives in the page-modify output (reduce time) rather
// than per-document extraction so it reflects WHAT the whole entity IS, not
// the role it played in one source document. An empty/absent CATEGORY line
// yields nil so callers keep the page's existing category instead of wiping it.
func splitCategoryLine(content string) (category []string, rest string) {
trimmed := strings.TrimLeft(content, " \t\r\n")
if !strings.HasPrefix(trimmed, "CATEGORY:") && !strings.HasPrefix(trimmed, "CATEGORY:") {
return nil, content
}
var line, remainder string
if idx := strings.IndexByte(trimmed, '\n'); idx < 0 {
line = trimmed
} else {
line = trimmed[:idx]
remainder = trimmed[idx+1:]
}
line = strings.TrimPrefix(line, "CATEGORY:")
line = strings.TrimPrefix(line, "CATEGORY:")
return parseCategoryPath(line), strings.TrimSpace(remainder)
}
// parseCategoryPath splits a "a / b" breadcrumb string into clean folder
// labels. Separator variants and wrapping quotes/brackets are normalized; the
// deeper structural cleaning (type-label stripping, depth cap, wiki_path) is
// left to normalizeWikiHierarchy on the page.
func parseCategoryPath(line string) []string {
line = strings.NewReplacer("/", "/", "|", "/", "|", "/", "›", "/", "»", "/", ">", "/").Replace(line)
parts := strings.Split(line, "/")
out := make([]string, 0, len(parts))
for _, p := range parts {
label := strings.TrimSpace(p)
label = strings.Trim(label, `"'“”‘’[]()()`)
label = strings.TrimSpace(label)
if label != "" {
out = append(out, label)
}
}
return out
}
// buildLogEntry builds a WikiLogEntry struct for the current batch. It is
// pure (no DB access) so callers can accumulate entries cheaply under their
// lock and flush them in a single AppendBatch call at the end of the batch.
@@ -1610,11 +1610,6 @@ func (s *wikiIngestService) reduceSlugUpdates(
if err == nil && updatedContent != "" {
updatedSummary, updatedBody := splitSummaryLine(updatedContent)
// Category now comes from the batch taxonomy plan, not this prompt.
// Defensively strip any stray leading CATEGORY line the model may
// still emit so it never leaks into the page body; discard the value.
_, bodyAfterCategory := splitCategoryLine(updatedBody)
updatedBody = bodyAfterCategory
if updatedBody != "" {
page.Content = updatedBody
} else {
@@ -132,7 +132,10 @@ func (s *wikiIngestService) selectRelevantFolders(
}
}
if !kb.NeedsEmbeddingModel() || strings.TrimSpace(kb.EmbeddingModelID) == "" || len(deeper) == 0 {
// Gate purely on whether an embedding model is configured — NOT on
// NeedsEmbeddingModel(), which is false for wiki-only KBs that may still
// opt into an embedding model purely for directory/taxonomy similarity.
if strings.TrimSpace(kb.EmbeddingModelID) == "" || len(deeper) == 0 {
return capFolders(pool, wikiTaxonomyPromptMaxPaths)
}
@@ -7,58 +7,6 @@ import (
"github.com/Tencent/WeKnora/internal/types"
)
func TestSplitCategoryLine(t *testing.T) {
tests := []struct {
name string
content string
wantCat []string
wantRest string
}{
{
name: "two level category",
content: "CATEGORY: 组织 / 企业\n# 标题\n正文",
wantCat: []string{"组织", "企业"},
wantRest: "# 标题\n正文",
},
{
name: "single level category",
content: "CATEGORY: 人物\n# 陈洋",
wantCat: []string{"人物"},
wantRest: "# 陈洋",
},
{
name: "empty category keeps existing (nil) and strips line",
content: "CATEGORY:\n# 标题",
wantCat: nil,
wantRest: "# 标题",
},
{
name: "no category line left untouched",
content: "# 标题\n正文",
wantCat: nil,
wantRest: "# 标题\n正文",
},
{
name: "fullwidth colon and separators",
content: "CATEGORY:组织|企业\n正文",
wantCat: []string{"组织", "企业"},
wantRest: "正文",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cat, rest := splitCategoryLine(tt.content)
if strings.Join(cat, "/") != strings.Join(tt.wantCat, "/") {
t.Fatalf("splitCategoryLine() cat = %v, want %v", cat, tt.wantCat)
}
if rest != tt.wantRest {
t.Fatalf("splitCategoryLine() rest = %q, want %q", rest, tt.wantRest)
}
})
}
}
func TestFormatExistingTaxonomyForPrompt(t *testing.T) {
got := formatExistingTaxonomyForPrompt([][]string{
{"春节", "传统习俗"},