diff --git a/internal/agent/prompts_wiki.go b/internal/agent/prompts_wiki.go
index 330af022..6fa0c05d 100644
--- a/internal/agent/prompts_wiki.go
+++ b/internal/agent/prompts_wiki.go
@@ -1,24 +1,45 @@
package agent
-// wikiExistingTaxonomyXML is injected into the page-modify prompt so each page's
-// CATEGORY line can reuse folder labels already present in the knowledge base
-// instead of inventing synonyms. Category assignment happens at page-write
-// (reduce) time, where the LLM sees the whole entity — not per source document.
-const wikiExistingTaxonomyXML = `
-
-{{.ExistingTaxonomy}}
-
-`
+// WikiTaxonomyPlanPrompt assigns a directory path (category) to every entity /
+// concept page produced by ONE ingest batch in a single call, so the whole set
+// lands on one coherent tree that reuses existing folders — instead of each page
+// inventing its own folders in parallel (which diverges worst on the founding
+// batch, when the KB still has no folders to anchor on). The result is applied
+// in reduce only to pages that don't already have a category, so user edits and
+// previously-filed pages are never churned.
+const WikiTaxonomyPlanPrompt = `You are organizing a wiki knowledge base into a navigation directory. Assign each item below to a directory path (category) so the whole set lands on ONE coherent tree that reuses existing folders.
-// wikiExistingTaxonomyContinuityRules extends Directory Taxonomy Rules with
-// cross-document folder reuse requirements.
-const wikiExistingTaxonomyContinuityRules = `
-- If above lists folders already used in this knowledge base, you MUST reuse those **exact folder labels** (character-for-character) when the new item belongs under the same domain. Do NOT invent synonym folders (e.g. do NOT create "春节习俗" when "春节" already has a "传统习俗" subfolder and that path fits semantically).
-- Prefer nesting under an existing parent folder over creating a new top-level folder with a similar name.
-- When multiple existing paths could fit, pick the most specific existing path whose labels match the document's section hierarchy.
-- Only introduce a new folder label when no existing folder is a reasonable match.
-- You may extend an existing path with one new trailing folder only when the document clearly introduces a genuinely new sub-section.
-`
+
+{{.ExistingTaxonomy}}
+
+
+
+{{.Items}}
+
+
+
+For every item, output a category path: an array of folder labels from broad to narrow (at most 2 levels). The category classifies WHAT the item fundamentally IS (the stable library "shelf" it always sits on), never the role it plays in one document.
+
+Rules:
+- Reuse the EXACT folder labels listed in (character-for-character) whenever an item belongs under the same domain. Do NOT invent synonym folders (e.g. do NOT create "春节习俗" when "春节 / 传统习俗" already fits).
+- Group items of the SAME kind under the SAME folder at the SAME depth. Do not file one equivalent item a level deeper than its siblings (e.g. avoid "地点 / 地址 / Address1" next to "地点 / Address2" — pick one consistent depth for equivalent items).
+- Prefer a single broad top-level folder; add a second level only for a genuinely durable sub-domain shared by several items.
+- Do NOT use the item type ("entity"/"concept") as a folder. Do NOT put slashes inside a single label.
+- Every item slug in MUST appear exactly once in the output. If an item truly cannot be classified under any durable folder, give it an empty path [].
+- Write ALL folder labels in {{.Language}}.
+
+### JSON Formatting Rules
+- Output ONLY valid JSON, no preamble.
+- Do NOT use literal newlines inside JSON string values.
+
+
+Output format:
+{
+ "assignments": [
+ {"slug": "entity/zhang-san", "path": ["人物"]},
+ {"slug": "concept/spring-festival", "path": ["节日", "传统节日"]}
+ ]
+}`
// Wiki ingest prompt templates for LLM-powered wiki page generation.
// These prompts are used by the wiki ingest pipeline to extract structured
@@ -333,22 +354,8 @@ The block above is assembled from VERBATIM source chunks that
{{.AvailableSlugs}}
-
-{{.CurrentCategory}}
-
-` + wikiExistingTaxonomyXML + `
1. The FIRST line of your output MUST be: SUMMARY: {one sentence, 15-40 words, describing what this page is about after the update — for wiki index listing}
-1b. The SECOND line of your output MUST be: CATEGORY: {a navigation taxonomy path for this page, folders from broad to narrow separated by " / ", e.g. "CATEGORY: 人物" or "CATEGORY: 组织 / 企业"}. Follow the Directory Taxonomy Rules below. Almost every page can be filed under at least one broad top-level folder, so leaving it empty should be rare.
-
-### Directory Taxonomy Rules (for the CATEGORY line)
-The CATEGORY classifies WHAT this {{.PageType}} fundamentally IS — a label that stays true across the whole knowledge base — never the role it happens to play in one document.
-- Litmus test: keep a label only if it would still fit when this same {{.PageType}} shows up in a completely unrelated document. If a label only describes how it relates to THIS document's topic (who organized, sponsored, acknowledged, authored, or took part in something), that is a ROLE, not a category — leave it out and let the page body and [[wiki-links]] carry the relationship.
-- Always assign at least the single broadest top-level folder that fits (think "which shelf would this always sit on in a library"). A coarse one-level category is expected for most pages; reserve a second level for a genuinely durable sub-domain.
-- Do NOT use the item type itself as a folder (e.g. "实体", "概念", "摘要").
-- Do NOT put slashes inside a single label. Use at most 2 levels.
-- Treat as the default and keep it unless new evidence clearly points to a better stable classification.
-- Only output an empty "CATEGORY:" in the rare case where the page genuinely cannot be classified under any broad, durable folder.` + wikiExistingTaxonomyContinuityRules + `
{{if .HasRetractions}}
2. REMOVE facts/claims that were ONLY sourced from the and are NOT present in any or .
{{end}}
@@ -366,51 +373,12 @@ The CATEGORY classifies WHAT this {{.PageType}} fundamentally IS — a label tha
6. Maintain the existing page structure and formatting style. Use "# {{.PageTitle}}" as the top-level heading if the page does not already have one. Do NOT introduce new heading levels beyond what the source or existing page justifies.
7. **Image rule**: Include relevant images using Markdown syntax:  from new information if applicable. The URL inside  is an opaque token; reproduce it EXACTLY and VERBATIM, do not alter, shorten, or normalize it.
{{if .HasRetractions}}
-8. If after removing deleted content the page becomes nearly empty and there is no new information to add, output just: "SUMMARY: (empty page)\nCATEGORY:\n# {{.PageTitle}}\n\n*This page's primary source document was removed.*"
+8. If after removing deleted content the page becomes nearly empty and there is no new information to add, output just: "SUMMARY: (empty page)\n# {{.PageTitle}}\n\n*This page's primary source document was removed.*"
{{end}}
9. Write in {{.Language}}.
-Output the SUMMARY line first, then the CATEGORY line, then the updated Markdown content. Do not include any other preamble.`
-
-// WikiTaxonomyReconcilePrompt is a per-knowledge-base maintenance prompt. It is
-// given the full set of distinct category_path folders currently in use and asks
-// the LLM to collapse synonymous folders onto a single canonical path. The
-// result is applied in Go by re-pointing matching pages' category_path — slugs
-// never change. This runs occasionally (after a batch introduces new folders, or
-// on manual trigger), NOT per page, so its cost is O(folder count).
-const WikiTaxonomyReconcilePrompt = `You are a taxonomy curator for a wiki knowledge base. Below is the complete list of directory paths (category_path) currently used to file wiki pages. Some of them are synonyms or near-duplicates that should share ONE canonical path.
-
-
-{{.CurrentTaxonomy}}
-
-
-
-Identify groups of paths that mean the SAME thing and pick the single best canonical path for each group, then output a remap from every non-canonical path to its canonical path.
-
-Rules:
-- Only merge paths that are genuinely synonymous or where one is clearly a less-canonical spelling/wording of another (e.g. "春节习俗" → "春节 / 传统习俗"; "AI厂商" and "AI行业 / 厂商" → one of them).
-- Prefer the more standard, concise, and reusable label as the canonical target. Prefer an existing path over inventing a new one.
-- Do NOT merge paths that refer to DIFFERENT things, even if they look similar.
-- Do NOT merge across clearly different domains.
-- Do NOT output an entry whose "from" equals its "to".
-- A canonical "to" path may itself be one of the listed paths; never map a path onto a synonym chain (every "to" must be a final canonical path, not another "from").
-- Keep canonical paths to at most 2 levels.
-- Each path is an array of folder labels from broad to narrow. Write ALL labels in {{.Language}}.
-- If nothing should be merged, return {"remap": []}.
-
-### JSON Formatting Rules
-- Output ONLY valid JSON, no preamble.
-- Do NOT use literal newlines inside JSON string values.
-
-
-Output format:
-{
- "remap": [
- {"from": ["春节习俗"], "to": ["春节", "传统习俗"]},
- {"from": ["AI厂商"], "to": ["AI行业", "厂商"]}
- ]
-}`
+Output the SUMMARY line first, then the updated Markdown content. Do not include any other preamble.`
// WikiIndexIntroPrompt generates the introduction for a NEW index page (first time only).
const WikiIndexIntroPrompt = `You are a wiki editor. Write a brief introduction for a wiki knowledge base index page.
diff --git a/internal/application/repository/wiki_page.go b/internal/application/repository/wiki_page.go
index 44e22cfe..f8e50777 100644
--- a/internal/application/repository/wiki_page.go
+++ b/internal/application/repository/wiki_page.go
@@ -153,35 +153,12 @@ func (r *wikiPageRepository) GetBySlug(ctx context.Context, kbID string, slug st
return &page, nil
}
-// splitWikiPageTypes parses a page_type filter that may carry several
-// comma-separated types (e.g. "entity,concept") into a deduplicated slice.
-// An empty/whitespace-only input yields nil, meaning "no type filter".
-func splitWikiPageTypes(raw string) []string {
- if strings.TrimSpace(raw) == "" {
- return nil
- }
- seen := make(map[string]struct{})
- out := make([]string, 0, 4)
- for _, part := range strings.Split(raw, ",") {
- part = strings.TrimSpace(part)
- if part == "" {
- continue
- }
- if _, ok := seen[part]; ok {
- continue
- }
- seen[part] = struct{}{}
- out = append(out, part)
- }
- return out
-}
-
// List retrieves wiki pages with filtering and pagination
func (r *wikiPageRepository) List(ctx context.Context, req *types.WikiPageListRequest) ([]*types.WikiPage, int64, error) {
query := r.db.WithContext(ctx).Model(&types.WikiPage{}).
Where("knowledge_base_id = ?", req.KnowledgeBaseID)
- if pageTypes := splitWikiPageTypes(req.PageType); len(pageTypes) == 1 {
+ if pageTypes := types.SplitWikiPageTypes(req.PageType); len(pageTypes) == 1 {
query = query.Where("page_type = ?", pageTypes[0])
} else if len(pageTypes) > 1 {
query = query.Where("page_type IN ?", pageTypes)
@@ -437,9 +414,13 @@ func (r *wikiPageRepository) ListBySlugs(
return out, nil
}
-// ListDistinctCategoryPaths scans entity/concept pages and returns unique
-// non-empty category_path values. Rows are read in wiki_path order so the
-// prompt tree is stable; deduplication happens in Go with a maxPaths cap.
+// ListDistinctCategoryPaths returns the distinct non-empty category paths used
+// by entity/concept pages. The distinctness is pushed entirely to SQL over the
+// flat category_l1/l2/l3 columns (indexed by idx_wiki_pages_category_levels),
+// so the cost scales with the number of distinct folders — not the page count —
+// and there is no full-table scan to bound. maxPaths caps the returned folder
+// count. Used by the batch taxonomy planner as the candidate pool of existing
+// folders to reuse (similarity preprocessing then narrows it per batch).
func (r *wikiPageRepository) ListDistinctCategoryPaths(
ctx context.Context,
kbID string,
@@ -450,41 +431,39 @@ func (r *wikiPageRepository) ListDistinctCategoryPaths(
}
type row struct {
- CategoryPath types.StringArray `gorm:"column:category_path"`
+ L1 string `gorm:"column:category_l1"`
+ L2 string `gorm:"column:category_l2"`
+ L3 string `gorm:"column:category_l3"`
}
var rows []row
if err := r.db.WithContext(ctx).
Model(&types.WikiPage{}).
- Select("category_path").
- Where("knowledge_base_id = ? AND page_type IN ? AND status <> ?",
+ Distinct("category_l1", "category_l2", "category_l3").
+ Where("knowledge_base_id = ? AND page_type IN ? AND status <> ? AND category_l1 <> ?",
kbID,
[]string{types.WikiPageTypeEntity, types.WikiPageTypeConcept},
types.WikiPageStatusArchived,
+ "",
).
- Order(r.wikiCategoryRankOrder()).
- Order("wiki_path ASC").
+ Order("category_l1 ASC").
+ Order("category_l2 ASC").
+ Order("category_l3 ASC").
+ Limit(maxPaths).
Scan(&rows).Error; err != nil {
return nil, err
}
- seen := make(map[string]struct{}, len(rows))
out := make([][]string, 0, len(rows))
- for _, row := range rows {
- if len(row.CategoryPath) == 0 {
- continue
+ for _, rw := range rows {
+ path := make([]string, 0, 3)
+ for _, label := range []string{rw.L1, rw.L2, rw.L3} {
+ if label == "" {
+ break // levels are contiguous; an empty level ends the path
+ }
+ path = append(path, label)
}
- clean := types.CleanWikiCategoryPath(row.CategoryPath)
- if len(clean) == 0 {
- continue
- }
- key := strings.Join(clean, "\x00")
- if _, ok := seen[key]; ok {
- continue
- }
- seen[key] = struct{}{}
- out = append(out, clean)
- if len(out) >= maxPaths {
- break
+ if len(path) > 0 {
+ out = append(out, path)
}
}
return out, nil
diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go
index 447bb025..afb340e2 100644
--- a/internal/application/service/wiki_ingest.go
+++ b/internal/application/service/wiki_ingest.go
@@ -659,11 +659,12 @@ type WikiBatchContext struct {
// three valid values.
ExtractionGranularity types.WikiExtractionGranularity
- // ExistingTaxonomy returns a formatted directory tree of category_path
- // folders already used in this KB. Loaded once per batch and injected
- // into entity/concept extraction prompts so new pages reuse established
- // folder labels instead of inventing synonyms.
- ExistingTaxonomy func(ctx context.Context) string
+ // PlannedCategory holds the per-slug directory path assigned by the batch
+ // taxonomy planning pass (planBatchTaxonomy), keyed by page slug. Reduce
+ // applies it only to pages that don't already have a category, so the whole
+ // batch lands on one coherent tree without churning user-curated pages.
+ // Populated before the reduce phase and read-only during it.
+ PlannedCategory map[string][]string
}
// SlugUpdate represents a single update operation for a specific slug
@@ -1203,25 +1204,32 @@ func collectLinkRefs(pages []*types.WikiPage) []linkRef {
return refs
}
-// wikiTaxonomyScanRowLimit caps how many wiki_pages rows we scan when
-// building the existing-taxonomy prompt block. Distinct paths are deduped
-// in Go; this bound keeps the query cheap on large KBs.
-const wikiTaxonomyScanRowLimit = 2000
-
-// wikiTaxonomyPromptMaxPaths caps how many unique category_path prefixes
-// are rendered into the LLM prompt to control token usage.
+// wikiTaxonomyPromptMaxPaths caps how many existing folders are rendered into a
+// planning prompt as the set to reuse. Reached only for pathologically large
+// taxonomies; the similarity preprocessing keeps the fed set well under it.
const wikiTaxonomyPromptMaxPaths = 150
-func resolveExistingTaxonomyForPrompt(ctx context.Context, batchCtx *WikiBatchContext) string {
- if batchCtx == nil || batchCtx.ExistingTaxonomy == nil {
- return "(none — this knowledge base has no established folders yet)"
- }
- text := strings.TrimSpace(batchCtx.ExistingTaxonomy(ctx))
- if text == "" {
- return "(none — this knowledge base has no established folders yet)"
- }
- return text
-}
+// wikiTaxonomyFolderPoolMax bounds the existing folders pulled from the DB as the
+// candidate pool for similarity selection. Distinct folders are few even for
+// large KBs, so this only guards against a degenerate taxonomy.
+const wikiTaxonomyFolderPoolMax = 400
+
+// wikiTaxonomyFeedAllMaxFolders is the folder count at or below which the whole
+// folder set is fed to the planner as-is: a healthy navigation directory is
+// small, so feeding everything gives perfect reuse recall with no embedding cost
+// (similarity preprocessing only earns its keep once folders are numerous).
+const wikiTaxonomyFeedAllMaxFolders = 60
+
+// wikiTaxonomyRelevantTopK is how many nearest existing deeper folders each item
+// contributes to the reuse set when similarity preprocessing kicks in.
+const wikiTaxonomyRelevantTopK = 3
+
+// wikiTaxonomyPlanChunkSize caps how many items go into a single planning call.
+// Larger batches are split into chunks; folders assigned by earlier chunks are
+// fed forward as "existing folders" so later chunks converge onto the same tree.
+const wikiTaxonomyPlanChunkSize = 60
+
+const wikiTaxonomyEmptyTreeHint = "(none yet — this knowledge base has no folders, design a fresh directory)"
type wikiTaxonomyNode struct {
children map[string]*wikiTaxonomyNode
@@ -1290,39 +1298,6 @@ func formatExistingTaxonomyForPrompt(paths [][]string) string {
}
return strings.TrimSpace(buf.String())
}
-
-// dedupeWikiCategoryPaths returns unique non-empty category_path slices,
-// preserving first-seen order up to maxPaths.
-func dedupeWikiCategoryPaths(paths [][]string, maxPaths int) [][]string {
- if maxPaths <= 0 {
- maxPaths = wikiTaxonomyPromptMaxPaths
- }
- seen := make(map[string]struct{}, len(paths))
- out := make([][]string, 0, len(paths))
- for _, raw := range paths {
- clean := make([]string, 0, len(raw))
- for _, part := range raw {
- part = strings.TrimSpace(part)
- if part != "" {
- clean = append(clean, part)
- }
- }
- if len(clean) == 0 {
- continue
- }
- key := strings.Join(clean, "\x00")
- if _, ok := seen[key]; ok {
- continue
- }
- seen[key] = struct{}{}
- out = append(out, clean)
- if len(out) >= maxPaths {
- break
- }
- }
- return out
-}
-
// getExistingPageSlugsForKnowledge returns all page slugs that currently
// reference a given knowledge ID in their source_refs. Used to snapshot
// state before re-ingest so the reduce phase can reconcile additions vs
diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go
index 7023c857..74454d82 100644
--- a/internal/application/service/wiki_ingest_batch.go
+++ b/internal/application/service/wiki_ingest_batch.go
@@ -357,28 +357,6 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
ExtractionGranularity: granularity,
}
- var taxonomyOnce sync.Once
- var taxonomyText string
- batchCtx.ExistingTaxonomy = func(ctx context.Context) string {
- taxonomyOnce.Do(func() {
- if s.wikiService == nil {
- return
- }
- paths, err := s.wikiService.ListDistinctCategoryPaths(ctx, payload.KnowledgeBaseID, wikiTaxonomyPromptMaxPaths)
- if err != nil {
- logger.Warnf(ctx, "wiki ingest: ListDistinctCategoryPaths failed: %v", err)
- return
- }
- taxonomyText = formatExistingTaxonomyForPrompt(paths)
- })
- return taxonomyText
- }
-
- // Snapshot the KB's category folders before this batch runs so we can tell
- // afterwards whether new (possibly synonymous) folders were introduced and
- // a taxonomy reconcile pass is worth running.
- startCategoryKeys := s.snapshotCategoryKeys(ctx, payload.KnowledgeBaseID)
-
// 1. MAP PHASE (Parallel extraction and generation of updates)
var mapMu sync.Mutex
var failedOps []WikiPendingOp
@@ -507,6 +485,13 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
}
_ = eg.Wait()
+ // Plan the directory once for the whole batch BEFORE reduce. Reduce writes
+ // pages in parallel, so it can't converge on shared folders on its own; this
+ // single pass assigns every new entity/concept slug a coherent category_path
+ // that reuses existing folders. Reduce then only applies the plan to pages
+ // that don't already have a category (user-curated pages are never churned).
+ batchCtx.PlannedCategory = s.planBatchTaxonomy(ctx, chatModel, kb, slugUpdates, lang)
+
// 2. REDUCE PHASE (Parallel upserting grouped by Slug)
egReduce, reduceCtx := errgroup.WithContext(ctx)
egReduce.SetLimit(reduceParallel) // Reduce phase limit (LLM + DB concurrent connections, configurable)
@@ -650,14 +635,6 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
}
}
- // Collapse synonymous category folders introduced by this batch (cold
- // start, intra-batch races, long-term drift). Runs only when a genuinely
- // new folder appeared vs the pre-batch snapshot; re-points pages' category
- // without touching slugs, then refreshes the index.
- if len(ingestPagesAffected) > 0 {
- s.maybeReconcileTaxonomy(ctx, chatModel, payload.KnowledgeBaseID, lang, startCategoryKeys)
- }
-
// Clean dead [[slug]] references whenever ANY page was touched this
// batch (not just retracts). Reduce-phase failures can leave stale
// references in pages we just rewrote (e.g. summary pages cite
@@ -1615,11 +1592,6 @@ func (s *wikiIngestService) reduceSlugUpdates(
}
pageAliases := strings.Join(page.Aliases, ", ")
- currentCategory := strings.Join(page.CategoryPath, " / ")
- if strings.TrimSpace(currentCategory) == "" {
- currentCategory = "(none yet)"
- }
-
var updatedContent string
updatedContent, err = s.generateWithTemplate(ctx, chatModel, agent.WikiPageModifyPrompt, map[string]string{
"HasAdditions": hasAdditionsStr,
@@ -1633,22 +1605,16 @@ func (s *wikiIngestService) reduceSlugUpdates(
"DeletedContent": deletedContent.String(),
"RemainingSourcesContent": remainingSourcesContent.String(),
"AvailableSlugs": relatedSlugs.String(),
- "CurrentCategory": currentCategory,
- "ExistingTaxonomy": resolveExistingTaxonomyForPrompt(ctx, batchCtx),
"Language": language,
})
if err == nil && updatedContent != "" {
updatedSummary, updatedBody := splitSummaryLine(updatedContent)
- // CATEGORY is the page-level intrinsic classification produced by
- // the editor model. Only overwrite when the model actually emitted
- // a non-empty path so a lazy/empty line never wipes a good category.
- category, bodyAfterCategory := splitCategoryLine(updatedBody)
+ // Category now comes from the batch taxonomy plan, not this prompt.
+ // Defensively strip any stray leading CATEGORY line the model may
+ // still emit so it never leaks into the page body; discard the value.
+ _, bodyAfterCategory := splitCategoryLine(updatedBody)
updatedBody = bodyAfterCategory
- if len(category) > 0 {
- page.CategoryPath = types.StringArray(category)
- page.WikiPath = ""
- }
if updatedBody != "" {
page.Content = updatedBody
} else {
@@ -1676,6 +1642,16 @@ func (s *wikiIngestService) reduceSlugUpdates(
}
}
+ // Apply the batch taxonomy plan, but only to pages that don't already have a
+ // category — so brand-new pages get filed coherently while previously-filed
+ // or user-curated categories are preserved (manual edits are authoritative).
+ if len(page.CategoryPath) == 0 && batchCtx != nil {
+ if planned := types.CleanWikiCategoryPath(batchCtx.PlannedCategory[slug]); len(planned) > 0 {
+ page.CategoryPath = types.StringArray(planned)
+ page.WikiPath = ""
+ }
+ }
+
if changed {
// Refresh chunk refs in-place on the page so they persist alongside
// the rest of the row. Retract-only updates (no additions) preserve
diff --git a/internal/application/service/wiki_ingest_taxonomy.go b/internal/application/service/wiki_ingest_taxonomy.go
index eb78369f..edb34745 100644
--- a/internal/application/service/wiki_ingest_taxonomy.go
+++ b/internal/application/service/wiki_ingest_taxonomy.go
@@ -4,6 +4,8 @@ import (
"context"
"encoding/json"
"fmt"
+ "math"
+ "sort"
"strings"
"github.com/Tencent/WeKnora/internal/agent"
@@ -12,239 +14,269 @@ import (
"github.com/Tencent/WeKnora/internal/types"
)
-// wikiTaxonomyReconcileMaxPages caps how many pages a single reconcile pass
-// will rewrite, so an over-eager remap cannot fan out into an unbounded write
-// storm on a very large KB. Pages beyond the cap are picked up by the next run.
-const wikiTaxonomyReconcileMaxPages = 5000
-
-// wikiTaxonomyReconcilePageSize is the cursor page size used when scanning the
-// KB to apply a category remap.
-const wikiTaxonomyReconcilePageSize = 500
-
-type wikiTaxonomyRemapEntry struct {
- From []string `json:"from"`
- To []string `json:"to"`
+// wikiTaxonomyItem is one entity/concept page to be filed into the directory.
+type wikiTaxonomyItem struct {
+ slug string
+ title string
+ pageType string
+ about string
}
-type wikiTaxonomyRemapResult struct {
- Remap []wikiTaxonomyRemapEntry `json:"remap"`
-}
-
-// ReconcileTaxonomy is the manual entry point for collapsing synonymous
-// category folders across a wiki knowledge base. It resolves the KB's synthesis
-// chat model itself so it can be invoked outside an ingest batch (e.g. from a
-// future maintenance handler).
-func (s *wikiIngestService) ReconcileTaxonomy(ctx context.Context, kbID string) error {
- kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, kbID)
- if err != nil {
- return fmt.Errorf("wiki taxonomy reconcile: get KB: %w", err)
- }
- if !kb.IsWikiEnabled() {
- return fmt.Errorf("wiki taxonomy reconcile: KB %s is not wiki type", kbID)
- }
-
- synthesisModelID := ""
- if kb.WikiConfig != nil {
- synthesisModelID = kb.WikiConfig.SynthesisModelID
- }
- if synthesisModelID == "" {
- synthesisModelID = kb.SummaryModelID
- }
- if synthesisModelID == "" {
- return fmt.Errorf("wiki taxonomy reconcile: no synthesis model configured for KB %s", kbID)
- }
- chatModel, err := s.modelService.GetChatModel(ctx, synthesisModelID)
- if err != nil {
- return fmt.Errorf("wiki taxonomy reconcile: get chat model: %w", err)
- }
-
- return s.reconcileTaxonomy(ctx, chatModel, kbID, types.LanguageNameFromContext(ctx))
-}
-
-// reconcileTaxonomy collapses synonymous category_path folders across the KB
-// onto a single canonical path. It is per-KB (cost O(folder count), not page
-// count): one LLM call clusters the distinct folders, then matching pages are
-// re-pointed in Go. Slugs are NEVER touched — only the navigation category.
-func (s *wikiIngestService) reconcileTaxonomy(ctx context.Context, chatModel chat.Chat, kbID, lang string) error {
- if s.wikiService == nil || chatModel == nil {
- return nil
- }
- paths, err := s.wikiService.ListDistinctCategoryPaths(ctx, kbID, wikiTaxonomyPromptMaxPaths)
- if err != nil {
- return fmt.Errorf("list distinct category paths: %w", err)
- }
- if len(paths) < 2 {
- return nil // nothing to merge
- }
-
- remap := s.buildTaxonomyRemap(ctx, chatModel, paths, lang)
- if len(remap) == 0 {
- return nil
- }
-
- updated := s.applyTaxonomyRemap(ctx, kbID, remap)
- if updated > 0 {
- logger.Infof(ctx, "wiki taxonomy reconcile: re-pointed %d pages via %d rules (kb=%s)", updated, len(remap), kbID)
- if err := s.wikiService.RebuildIndexPage(ctx, kbID); err != nil {
- logger.Warnf(ctx, "wiki taxonomy reconcile: rebuild index failed: %v", err)
- }
- }
- return nil
-}
-
-// maybeReconcileTaxonomy runs a reconcile pass only when this batch introduced
-// at least one category folder that was not present in startKeys (the snapshot
-// taken before the batch ran). This keeps the extra LLM call off the hot path
-// for batches that only reuse established folders.
-func (s *wikiIngestService) maybeReconcileTaxonomy(
- ctx context.Context, chatModel chat.Chat, kbID, lang string, startKeys map[string]bool,
-) {
- if s.wikiService == nil {
- return
- }
- paths, err := s.wikiService.ListDistinctCategoryPaths(ctx, kbID, wikiTaxonomyPromptMaxPaths)
- if err != nil || len(paths) < 2 {
- return
- }
- introducedNewFolder := false
- for _, p := range paths {
- if !startKeys[categoryPathKey(cleanCategoryPathParts(p))] {
- introducedNewFolder = true
- break
- }
- }
- if !introducedNewFolder {
- return
- }
- if err := s.reconcileTaxonomy(ctx, chatModel, kbID, lang); err != nil {
- logger.Warnf(ctx, "wiki taxonomy reconcile failed: %v", err)
- }
-}
-
-// snapshotCategoryKeys returns the set of canonical category keys currently in
-// use, used as the "before" baseline for maybeReconcileTaxonomy.
-func (s *wikiIngestService) snapshotCategoryKeys(ctx context.Context, kbID string) map[string]bool {
- out := map[string]bool{}
- if s.wikiService == nil {
- return out
- }
- paths, err := s.wikiService.ListDistinctCategoryPaths(ctx, kbID, wikiTaxonomyPromptMaxPaths)
- if err != nil {
- return out
- }
- for _, p := range paths {
- out[categoryPathKey(cleanCategoryPathParts(p))] = true
- }
- return out
-}
-
-// buildTaxonomyRemap asks the LLM to cluster synonymous folders and returns a
-// map keyed by the canonical key of each non-canonical path to its replacement.
-func (s *wikiIngestService) buildTaxonomyRemap(
- ctx context.Context, chatModel chat.Chat, paths [][]string, lang string,
+// planBatchTaxonomy assigns a directory path to every entity/concept slug in the
+// batch in ONE planning pass (chunked for large batches), so the whole set lands
+// on a single coherent tree that reuses existing folders. This replaces per-page,
+// parallel CATEGORY invention — which couldn't converge, worst of all on the
+// founding batch when the KB has no folders to anchor on. The returned map is
+// keyed by slug; an entry may be an empty slice when the item is unclassifiable.
+// Reduce applies these only to pages that don't already have a category.
+func (s *wikiIngestService) planBatchTaxonomy(
+ ctx context.Context,
+ chatModel chat.Chat,
+ kb *types.KnowledgeBase,
+ slugUpdates map[string][]SlugUpdate,
+ lang string,
) map[string][]string {
- taxonomyText := formatExistingTaxonomyForPrompt(paths)
- if strings.TrimSpace(taxonomyText) == "" {
+ if kb == nil {
return nil
}
- raw, err := s.generateWithTemplate(ctx, chatModel, agent.WikiTaxonomyReconcilePrompt, map[string]string{
- "CurrentTaxonomy": taxonomyText,
- "Language": lang,
- })
- if err != nil {
- logger.Warnf(ctx, "wiki taxonomy reconcile: LLM call failed: %v", err)
- return nil
- }
- raw = cleanLLMJSON(raw)
- var parsed wikiTaxonomyRemapResult
- if jerr := json.Unmarshal([]byte(raw), &parsed); jerr != nil {
- logger.Warnf(ctx, "wiki taxonomy reconcile: parse failed: %v\nRaw: %s", jerr, raw)
+ items := collectTaxonomyItems(slugUpdates)
+ if len(items) == 0 {
return nil
}
- out := make(map[string][]string, len(parsed.Remap))
- for _, e := range parsed.Remap {
- from := cleanCategoryPathParts(e.From)
- to := cleanCategoryPathParts(e.To)
- if len(from) == 0 || len(to) == 0 {
+ // Existing folders anchor reuse. Errors (e.g. a dialect without the query)
+ // just mean the plan designs a fresh tree — no fatal dependency.
+ var pool [][]string
+ if s.wikiService != nil {
+ paths, err := s.wikiService.ListDistinctCategoryPaths(ctx, kb.ID, wikiTaxonomyFolderPoolMax)
+ if err != nil {
+ logger.Warnf(ctx, "wiki ingest: list category paths for plan failed: %v", err)
+ } else {
+ pool = paths
+ }
+ }
+
+ // Preprocess the pool down to the folders relevant to THIS batch, so the
+ // planner reuses established folders without every prompt carrying the whole
+ // directory. Small/healthy taxonomies are fed whole (best recall, no cost).
+ existing := s.selectRelevantFolders(ctx, kb, items, pool)
+
+ result := make(map[string][]string, len(items))
+ for start := 0; start < len(items); start += wikiTaxonomyPlanChunkSize {
+ end := start + wikiTaxonomyPlanChunkSize
+ if end > len(items) {
+ end = len(items)
+ }
+ chunk := items[start:end]
+
+ tree := formatExistingTaxonomyForPrompt(existing)
+ if strings.TrimSpace(tree) == "" {
+ tree = wikiTaxonomyEmptyTreeHint
+ }
+
+ var itemsBlock strings.Builder
+ for _, it := range chunk {
+ fmt.Fprintf(&itemsBlock, "- slug: %s | title: %s | type: %s | about: %s\n",
+ it.slug, it.title, it.pageType, previewText(it.about, 120))
+ }
+
+ raw, err := s.generateWithTemplate(ctx, chatModel, agent.WikiTaxonomyPlanPrompt, map[string]string{
+ "ExistingTaxonomy": tree,
+ "Items": itemsBlock.String(),
+ "Language": lang,
+ })
+ if err != nil {
+ logger.Warnf(ctx, "wiki ingest: taxonomy plan call failed (%d items): %v", len(chunk), err)
continue
}
- fromKey := categoryPathKey(from)
- if fromKey == categoryPathKey(to) {
- continue // no-op rule
+
+ for slug, path := range parseTaxonomyAssignments(raw) {
+ clean := types.CleanWikiCategoryPath(path)
+ result[slug] = clean
+ if len(clean) > 0 {
+ existing = append(existing, clean) // feed forward so later chunks converge
+ }
+ }
+ }
+ return result
+}
+
+// selectRelevantFolders narrows the existing folder pool to the subset worth
+// showing the planner for THIS batch. A healthy navigation directory is small,
+// so it is fed whole (perfect reuse recall, no embedding cost). Only once folders
+// are numerous does similarity preprocessing kick in: all level-1 folders are
+// always kept as coarse anchors, and each item pulls in its nearest deeper
+// folders by embedding similarity. KBs without an embedding model (wiki-only)
+// fall back to a capped feed-all.
+func (s *wikiIngestService) selectRelevantFolders(
+ ctx context.Context, kb *types.KnowledgeBase, items []wikiTaxonomyItem, pool [][]string,
+) [][]string {
+ if len(pool) <= wikiTaxonomyFeedAllMaxFolders {
+ return pool
+ }
+
+ // Split into always-kept level-1 anchors and the deeper candidate folders
+ // that similarity selects among.
+ l1Seen := make(map[string]struct{})
+ var l1Paths, deeper [][]string
+ for _, p := range pool {
+ if len(p) == 0 {
+ continue
+ }
+ if _, ok := l1Seen[p[0]]; !ok {
+ l1Seen[p[0]] = struct{}{}
+ l1Paths = append(l1Paths, []string{p[0]})
+ }
+ if len(p) >= 2 {
+ deeper = append(deeper, p)
+ }
+ }
+
+ if !kb.NeedsEmbeddingModel() || strings.TrimSpace(kb.EmbeddingModelID) == "" || len(deeper) == 0 {
+ return capFolders(pool, wikiTaxonomyPromptMaxPaths)
+ }
+
+ embedder, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID)
+ if err != nil {
+ logger.Warnf(ctx, "wiki ingest: taxonomy plan embed model unavailable, feeding all folders: %v", err)
+ return capFolders(pool, wikiTaxonomyPromptMaxPaths)
+ }
+
+ folderTexts := make([]string, len(deeper))
+ for i, p := range deeper {
+ folderTexts[i] = strings.Join(p, " / ")
+ }
+ itemTexts := make([]string, len(items))
+ for i, it := range items {
+ itemTexts[i] = strings.TrimSpace(it.title + " " + previewText(it.about, 120))
+ }
+
+ folderVecs, err := embedder.BatchEmbed(ctx, folderTexts)
+ if err != nil {
+ logger.Warnf(ctx, "wiki ingest: taxonomy plan folder embed failed, feeding all folders: %v", err)
+ return capFolders(pool, wikiTaxonomyPromptMaxPaths)
+ }
+ itemVecs, err := embedder.BatchEmbed(ctx, itemTexts)
+ if err != nil {
+ logger.Warnf(ctx, "wiki ingest: taxonomy plan item embed failed, feeding all folders: %v", err)
+ return capFolders(pool, wikiTaxonomyPromptMaxPaths)
+ }
+
+ selected := append(l1Paths, selectFoldersByVectors(deeper, folderVecs, itemVecs, wikiTaxonomyRelevantTopK)...)
+ return capFolders(selected, wikiTaxonomyPromptMaxPaths)
+}
+
+// selectFoldersByVectors returns the deeper folders that rank in any item's
+// top-K by cosine similarity, preserving the input order for determinism.
+func selectFoldersByVectors(deeper [][]string, folderVecs, itemVecs [][]float32, topK int) [][]string {
+ if len(deeper) != len(folderVecs) || len(itemVecs) == 0 || topK <= 0 {
+ return nil
+ }
+ chosen := make(map[int]struct{})
+ for _, iv := range itemVecs {
+ type scored struct {
+ idx int
+ sim float64
+ }
+ ranking := make([]scored, 0, len(folderVecs))
+ for fi, fv := range folderVecs {
+ ranking = append(ranking, scored{fi, cosineSimilarity(iv, fv)})
+ }
+ sort.SliceStable(ranking, func(a, b int) bool { return ranking[a].sim > ranking[b].sim })
+ for k := 0; k < topK && k < len(ranking); k++ {
+ chosen[ranking[k].idx] = struct{}{}
+ }
+ }
+ out := make([][]string, 0, len(chosen))
+ for i := range deeper {
+ if _, ok := chosen[i]; ok {
+ out = append(out, deeper[i])
}
- out[fromKey] = to
}
return out
}
-// applyTaxonomyRemap scans entity/concept pages and re-points any whose
-// category matches a remap rule. Returns the number of pages updated.
-func (s *wikiIngestService) applyTaxonomyRemap(ctx context.Context, kbID string, remap map[string][]string) int {
- var cursor string
- var updated int
- for updated < wikiTaxonomyReconcileMaxPages {
- pages, next, err := s.wikiService.ListPagesCursor(ctx, kbID, cursor, wikiTaxonomyReconcilePageSize)
- if err != nil {
- logger.Warnf(ctx, "wiki taxonomy reconcile: list pages failed: %v", err)
- break
- }
- if len(pages) == 0 {
- break
- }
- for _, p := range pages {
- if p.PageType != types.WikiPageTypeEntity && p.PageType != types.WikiPageTypeConcept {
- continue
- }
- newPath, ok := remapCategoryPath([]string(p.CategoryPath), remap)
- if !ok {
- continue
- }
- p.CategoryPath = types.StringArray(newPath)
- p.WikiPath = "" // force recompute in normalizeWikiHierarchy
- if err := s.wikiService.UpdatePageMeta(ctx, p); err != nil {
- logger.Warnf(ctx, "wiki taxonomy reconcile: update %s failed: %v", p.Slug, err)
- continue
- }
- updated++
- if updated >= wikiTaxonomyReconcileMaxPages {
- break
- }
- }
- if next == "" {
- break
- }
- cursor = next
+// cosineSimilarity returns the cosine of two equal-length vectors, or 0 for empty
+// / mismatched / zero-norm inputs.
+func cosineSimilarity(a, b []float32) float64 {
+ if len(a) == 0 || len(a) != len(b) {
+ return 0
}
- return updated
+ var dot, na, nb float64
+ for i := range a {
+ dot += float64(a[i]) * float64(b[i])
+ na += float64(a[i]) * float64(a[i])
+ nb += float64(b[i]) * float64(b[i])
+ }
+ if na == 0 || nb == 0 {
+ return 0
+ }
+ return dot / (math.Sqrt(na) * math.Sqrt(nb))
}
-// remapCategoryPath returns the canonical category for current when it matches
-// a remap "from" key, plus whether a change is needed. Pure (no DB) so it can
-// be unit-tested in isolation.
-func remapCategoryPath(current []string, remap map[string][]string) ([]string, bool) {
- cleaned := cleanCategoryPathParts(current)
- if len(cleaned) == 0 {
- return nil, false
+// capFolders truncates a folder list to at most max entries (max <= 0 = no cap).
+func capFolders(paths [][]string, max int) [][]string {
+ if max > 0 && len(paths) > max {
+ return paths[:max]
}
- to, ok := remap[categoryPathKey(cleaned)]
- if !ok {
- return nil, false
- }
- if categoryPathKey(cleaned) == categoryPathKey(cleanCategoryPathParts(to)) {
- return nil, false
- }
- return append([]string(nil), to...), true
+ return paths
}
-// cleanCategoryPathParts normalizes a category path the same way the page
-// hierarchy normalizer does (separator/quote cleanup, type-label stripping,
-// depth cap) so remap keys match what is stored on pages.
-func cleanCategoryPathParts(parts []string) []string {
- return types.CleanWikiCategoryPath(parts)
+// collectTaxonomyItems extracts the entity/concept pages from a batch's slug
+// updates, in deterministic slug order so chunk boundaries are stable. Summary
+// and retract-only slugs are skipped (they carry no directory category).
+func collectTaxonomyItems(slugUpdates map[string][]SlugUpdate) []wikiTaxonomyItem {
+ slugs := make([]string, 0, len(slugUpdates))
+ for slug := range slugUpdates {
+ slugs = append(slugs, slug)
+ }
+ sort.Strings(slugs)
+
+ items := make([]wikiTaxonomyItem, 0, len(slugs))
+ for _, slug := range slugs {
+ for _, u := range slugUpdates[slug] {
+ if u.Type != types.WikiPageTypeEntity && u.Type != types.WikiPageTypeConcept {
+ continue
+ }
+ title := strings.TrimSpace(u.Item.Name)
+ if title == "" {
+ title = slug
+ }
+ items = append(items, wikiTaxonomyItem{
+ slug: slug,
+ title: title,
+ pageType: u.Type,
+ about: strings.TrimSpace(u.Item.Description),
+ })
+ break // one entry per slug is enough for classification
+ }
+ }
+ return items
}
-// categoryPathKey builds a stable comparison key for a cleaned category path.
-func categoryPathKey(parts []string) string {
- return strings.Join(parts, "\u0000")
+// parseTaxonomyAssignments parses the planning LLM's JSON into a slug → path map.
+// Malformed output yields nil; individual entries with a blank slug are dropped.
+func parseTaxonomyAssignments(raw string) map[string][]string {
+ raw = cleanLLMJSON(raw)
+ if raw == "" {
+ return nil
+ }
+ var parsed struct {
+ Assignments []struct {
+ Slug string `json:"slug"`
+ Path []string `json:"path"`
+ } `json:"assignments"`
+ }
+ if err := json.Unmarshal([]byte(raw), &parsed); err != nil {
+ return nil
+ }
+ out := make(map[string][]string, len(parsed.Assignments))
+ for _, a := range parsed.Assignments {
+ slug := strings.TrimSpace(a.Slug)
+ if slug == "" {
+ continue
+ }
+ out[slug] = a.Path
+ }
+ return out
}
diff --git a/internal/application/service/wiki_ingest_taxonomy_test.go b/internal/application/service/wiki_ingest_taxonomy_test.go
index d8fab8f9..85440cdb 100644
--- a/internal/application/service/wiki_ingest_taxonomy_test.go
+++ b/internal/application/service/wiki_ingest_taxonomy_test.go
@@ -1,9 +1,10 @@
package service
import (
- "context"
"strings"
"testing"
+
+ "github.com/Tencent/WeKnora/internal/types"
)
func TestSplitCategoryLine(t *testing.T) {
@@ -82,104 +83,99 @@ func TestFormatExistingTaxonomyForPromptEmpty(t *testing.T) {
}
}
-func TestDedupeWikiCategoryPaths(t *testing.T) {
- got := dedupeWikiCategoryPaths([][]string{
- {"春节", "传统习俗"},
- {" 春节 ", "传统习俗"},
- {"春节", "文化习俗"},
- {"", "空"},
- {},
- }, 10)
+func TestParseTaxonomyAssignments(t *testing.T) {
+ raw := "```json\n{\"assignments\":[" +
+ "{\"slug\":\"entity/zhang-san\",\"path\":[\"人物\"]}," +
+ "{\"slug\":\"concept/spring\",\"path\":[\"节日\",\"传统节日\"]}," +
+ "{\"slug\":\" \",\"path\":[\"X\"]}," +
+ "{\"slug\":\"entity/unclassified\",\"path\":[]}" +
+ "]}\n```"
+
+ got := parseTaxonomyAssignments(raw)
if len(got) != 3 {
- t.Fatalf("dedupeWikiCategoryPaths() len = %d, want 3: %v", len(got), got)
+ t.Fatalf("parseTaxonomyAssignments() returned %d entries, want 3 (blank slug dropped): %v", len(got), got)
}
- if got[0][0] != "春节" || got[0][1] != "传统习俗" {
- t.Fatalf("dedupeWikiCategoryPaths()[0] = %v", got[0])
+ if strings.Join(got["entity/zhang-san"], "/") != "人物" {
+ t.Fatalf("zhang-san path = %v, want [人物]", got["entity/zhang-san"])
}
- if got[1][1] != "文化习俗" {
- t.Fatalf("dedupeWikiCategoryPaths()[1] = %v", got[1])
+ if strings.Join(got["concept/spring"], "/") != "节日/传统节日" {
+ t.Fatalf("spring path = %v, want [节日 传统节日]", got["concept/spring"])
+ }
+ if p, ok := got["entity/unclassified"]; !ok || len(p) != 0 {
+ t.Fatalf("unclassified path = %v (ok=%v), want empty slice present", p, ok)
}
}
-func TestResolveExistingTaxonomyForPrompt(t *testing.T) {
- t.Run("nil batch context", func(t *testing.T) {
- got := resolveExistingTaxonomyForPrompt(t.Context(), nil)
- if got == "" {
- t.Fatal("expected fallback text for nil batch context")
- }
- })
- t.Run("batch context with taxonomy", func(t *testing.T) {
- batchCtx := &WikiBatchContext{
- ExistingTaxonomy: func(ctx context.Context) string {
- return "春节\n 传统习俗"
- },
- }
- got := resolveExistingTaxonomyForPrompt(t.Context(), batchCtx)
- if got != "春节\n 传统习俗" {
- t.Fatalf("resolveExistingTaxonomyForPrompt() = %q", got)
- }
- })
-}
-
-func TestRemapCategoryPath(t *testing.T) {
- remap := map[string][]string{
- categoryPathKey([]string{"春节习俗"}): {"春节", "传统习俗"},
- categoryPathKey([]string{"组织单位", "致谢人员"}): {"组织"},
- }
-
- tests := []struct {
- name string
- current []string
- want []string
- wantHit bool
- }{
- {
- name: "single-label synonym remapped",
- current: []string{"春节习俗"},
- want: []string{"春节", "传统习俗"},
- wantHit: true,
- },
- {
- name: "document-role path collapsed to coarse folder",
- current: []string{"组织单位", "致谢人员"},
- want: []string{"组织"},
- wantHit: true,
- },
- {
- name: "noise normalized before matching",
- current: []string{"实体/春节习俗"},
- want: []string{"春节", "传统习俗"},
- wantHit: true,
- },
- {
- name: "unrelated path untouched",
- current: []string{"人物"},
- wantHit: false,
- },
- {
- name: "empty path untouched",
- current: nil,
- wantHit: false,
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, hit := remapCategoryPath(tt.current, remap)
- if hit != tt.wantHit {
- t.Fatalf("remapCategoryPath() hit = %v, want %v", hit, tt.wantHit)
- }
- if !hit {
- return
- }
- if len(got) != len(tt.want) {
- t.Fatalf("remapCategoryPath() = %v, want %v", got, tt.want)
- }
- for i := range tt.want {
- if got[i] != tt.want[i] {
- t.Fatalf("remapCategoryPath()[%d] = %q, want %q (full=%v)", i, got[i], tt.want[i], got)
- }
- }
- })
+func TestParseTaxonomyAssignmentsMalformed(t *testing.T) {
+ if got := parseTaxonomyAssignments("not json at all"); got != nil {
+ t.Fatalf("parseTaxonomyAssignments(garbage) = %v, want nil", got)
+ }
+}
+
+func TestCosineSimilarity(t *testing.T) {
+ if got := cosineSimilarity([]float32{1, 0}, []float32{1, 0}); got < 0.999 {
+ t.Fatalf("identical vectors sim = %v, want ~1", got)
+ }
+ if got := cosineSimilarity([]float32{1, 0}, []float32{0, 1}); got != 0 {
+ t.Fatalf("orthogonal vectors sim = %v, want 0", got)
+ }
+ if got := cosineSimilarity([]float32{1, 0}, nil); got != 0 {
+ t.Fatalf("mismatched length sim = %v, want 0", got)
+ }
+ if got := cosineSimilarity([]float32{0, 0}, []float32{1, 1}); got != 0 {
+ t.Fatalf("zero-norm sim = %v, want 0", got)
+ }
+}
+
+func TestSelectFoldersByVectors(t *testing.T) {
+ deeper := [][]string{
+ {"AI", "厂商"}, // 0
+ {"AI", "模型"}, // 1
+ {"地理", "城市"}, // 2
+ }
+ folderVecs := [][]float32{
+ {1, 0, 0},
+ {0.9, 0.1, 0},
+ {0, 0, 1},
+ }
+ // Item closest to folder 0 (and 1 as runner-up); far from 2.
+ itemVecs := [][]float32{{1, 0, 0}}
+
+ got := selectFoldersByVectors(deeper, folderVecs, itemVecs, 2)
+ if len(got) != 2 {
+ t.Fatalf("selectFoldersByVectors() returned %d folders, want 2: %v", len(got), got)
+ }
+ // Input order preserved: folders 0 and 1, not the orthogonal 2.
+ if strings.Join(got[0], "/") != "AI/厂商" || strings.Join(got[1], "/") != "AI/模型" {
+ t.Fatalf("selectFoldersByVectors() = %v, want [[AI 厂商] [AI 模型]]", got)
+ }
+}
+
+func TestSelectFoldersByVectorsGuards(t *testing.T) {
+ if got := selectFoldersByVectors([][]string{{"a"}}, [][]float32{{1}, {2}}, [][]float32{{1}}, 1); got != nil {
+ t.Fatalf("mismatched lengths should return nil, got %v", got)
+ }
+ if got := selectFoldersByVectors([][]string{{"a"}}, [][]float32{{1}}, nil, 1); got != nil {
+ t.Fatalf("no items should return nil, got %v", got)
+ }
+}
+
+func TestCollectTaxonomyItems(t *testing.T) {
+ slugUpdates := map[string][]SlugUpdate{
+ "entity/b": {{Type: types.WikiPageTypeEntity, Item: extractedItem{Name: "B", Description: "about B"}}},
+ "entity/a": {{Type: types.WikiPageTypeConcept, Item: extractedItem{Name: "A"}}},
+ "sum/x": {{Type: "summary"}},
+ "ret/y": {{Type: "retract"}},
+ }
+ items := collectTaxonomyItems(slugUpdates)
+ if len(items) != 2 {
+ t.Fatalf("collectTaxonomyItems() = %d items, want 2 (summary/retract skipped): %+v", len(items), items)
+ }
+ // Deterministic slug order so chunk boundaries are stable.
+ if items[0].slug != "entity/a" || items[1].slug != "entity/b" {
+ t.Fatalf("collectTaxonomyItems() order = [%s, %s], want [entity/a, entity/b]", items[0].slug, items[1].slug)
+ }
+ if items[1].about != "about B" {
+ t.Fatalf("items[1].about = %q, want %q", items[1].about, "about B")
}
}
diff --git a/internal/handler/wiki_page.go b/internal/handler/wiki_page.go
index a392e8a0..38a184c7 100644
--- a/internal/handler/wiki_page.go
+++ b/internal/handler/wiki_page.go
@@ -148,7 +148,7 @@ func (h *WikiPageHandler) ListCategories(c *gin.Context) {
return
}
- pageTypes := parseWikiPageTypes(c.Query("page_type"))
+ pageTypes := types.SplitWikiPageTypes(c.Query("page_type"))
if len(pageTypes) == 0 {
c.JSON(http.StatusBadRequest, gin.H{"error": "page_type is required"})
return
@@ -186,28 +186,6 @@ func (h *WikiPageHandler) ListCategories(c *gin.Context) {
})
}
-// parseWikiPageTypes splits a comma-separated page_type query value (e.g.
-// "entity,concept") into a deduplicated slice, dropping blanks.
-func parseWikiPageTypes(raw string) []string {
- if strings.TrimSpace(raw) == "" {
- return nil
- }
- seen := make(map[string]struct{})
- out := make([]string, 0, 4)
- for _, part := range strings.Split(raw, ",") {
- part = strings.TrimSpace(part)
- if part == "" {
- continue
- }
- if _, ok := seen[part]; ok {
- continue
- }
- seen[part] = struct{}{}
- out = append(out, part)
- }
- return out
-}
-
func parseWikiCategoryPath(raw string) []string {
if strings.TrimSpace(raw) == "" {
return nil
diff --git a/internal/types/wiki_page.go b/internal/types/wiki_page.go
index c7f16ee2..9439064f 100644
--- a/internal/types/wiki_page.go
+++ b/internal/types/wiki_page.go
@@ -63,6 +63,31 @@ func CleanWikiCategoryPath(parts []string) []string {
return cleaned
}
+// SplitWikiPageTypes parses a page_type value that may carry several
+// comma-separated types (e.g. "entity,concept") into a deduplicated slice,
+// dropping blanks. An empty/whitespace-only input yields nil ("no filter").
+// Shared by the handler (query parsing) and repository (List filter) so the
+// two layers split identically.
+func SplitWikiPageTypes(raw string) []string {
+ if strings.TrimSpace(raw) == "" {
+ return nil
+ }
+ seen := make(map[string]struct{})
+ out := make([]string, 0, 4)
+ for _, part := range strings.Split(raw, ",") {
+ part = strings.TrimSpace(part)
+ if part == "" {
+ continue
+ }
+ if _, ok := seen[part]; ok {
+ continue
+ }
+ seen[part] = struct{}{}
+ out = append(out, part)
+ }
+ return out
+}
+
// WikiCategoryLevels flattens a (already cleaned) category path into the three
// fixed level columns used for SQL grouping. Missing levels yield "".
func WikiCategoryLevels(path []string) (l1, l2, l3 string) {