From 588f00b18ac83b5aba04a3ed350a0b082a23b550 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Tue, 16 Jun 2026 20:23:54 +0800 Subject: [PATCH] refactor(wiki): enhance taxonomy planning and streamline category handling Revamp the wiki taxonomy planning process to assign coherent directory paths to entity/concept pages in a single pass, ensuring consistent reuse of existing folders. This change eliminates the need for per-page category assignments during the reduce phase, preserving user-curated categories. Consolidate category path parsing and deduplication into a shared utility, improving consistency across the service and repository layers. Update related functions and tests to reflect these changes, ensuring efficient handling of category paths in the wiki ingestion process. --- internal/agent/prompts_wiki.go | 114 ++--- internal/application/repository/wiki_page.go | 75 ++- internal/application/service/wiki_ingest.go | 85 ++-- .../application/service/wiki_ingest_batch.go | 66 +-- .../service/wiki_ingest_taxonomy.go | 456 ++++++++++-------- .../service/wiki_ingest_taxonomy_test.go | 184 ++++--- internal/handler/wiki_page.go | 24 +- internal/types/wiki_page.go | 25 + 8 files changed, 479 insertions(+), 550 deletions(-) diff --git a/internal/agent/prompts_wiki.go b/internal/agent/prompts_wiki.go index 330af022..6fa0c05d 100644 --- a/internal/agent/prompts_wiki.go +++ b/internal/agent/prompts_wiki.go @@ -1,24 +1,45 @@ package agent -// wikiExistingTaxonomyXML is injected into the page-modify prompt so each page's -// CATEGORY line can reuse folder labels already present in the knowledge base -// instead of inventing synonyms. Category assignment happens at page-write -// (reduce) time, where the LLM sees the whole entity — not per source document. -const wikiExistingTaxonomyXML = ` - -{{.ExistingTaxonomy}} - -` +// WikiTaxonomyPlanPrompt assigns a directory path (category) to every entity / +// concept page produced by ONE ingest batch in a single call, so the whole set +// lands on one coherent tree that reuses existing folders — instead of each page +// inventing its own folders in parallel (which diverges worst on the founding +// batch, when the KB still has no folders to anchor on). The result is applied +// in reduce only to pages that don't already have a category, so user edits and +// previously-filed pages are never churned. +const WikiTaxonomyPlanPrompt = `You are organizing a wiki knowledge base into a navigation directory. Assign each item below to a directory path (category) so the whole set lands on ONE coherent tree that reuses existing folders. -// wikiExistingTaxonomyContinuityRules extends Directory Taxonomy Rules with -// cross-document folder reuse requirements. -const wikiExistingTaxonomyContinuityRules = ` -- If above lists folders already used in this knowledge base, you MUST reuse those **exact folder labels** (character-for-character) when the new item belongs under the same domain. Do NOT invent synonym folders (e.g. do NOT create "春节习俗" when "春节" already has a "传统习俗" subfolder and that path fits semantically). -- Prefer nesting under an existing parent folder over creating a new top-level folder with a similar name. -- When multiple existing paths could fit, pick the most specific existing path whose labels match the document's section hierarchy. -- Only introduce a new folder label when no existing folder is a reasonable match. -- You may extend an existing path with one new trailing folder only when the document clearly introduces a genuinely new sub-section. -` + +{{.ExistingTaxonomy}} + + + +{{.Items}} + + + +For every item, output a category path: an array of folder labels from broad to narrow (at most 2 levels). The category classifies WHAT the item fundamentally IS (the stable library "shelf" it always sits on), never the role it plays in one document. + +Rules: +- Reuse the EXACT folder labels listed in (character-for-character) whenever an item belongs under the same domain. Do NOT invent synonym folders (e.g. do NOT create "春节习俗" when "春节 / 传统习俗" already fits). +- Group items of the SAME kind under the SAME folder at the SAME depth. Do not file one equivalent item a level deeper than its siblings (e.g. avoid "地点 / 地址 / Address1" next to "地点 / Address2" — pick one consistent depth for equivalent items). +- Prefer a single broad top-level folder; add a second level only for a genuinely durable sub-domain shared by several items. +- Do NOT use the item type ("entity"/"concept") as a folder. Do NOT put slashes inside a single label. +- Every item slug in MUST appear exactly once in the output. If an item truly cannot be classified under any durable folder, give it an empty path []. +- Write ALL folder labels in {{.Language}}. + +### JSON Formatting Rules +- Output ONLY valid JSON, no preamble. +- Do NOT use literal newlines inside JSON string values. + + +Output format: +{ + "assignments": [ + {"slug": "entity/zhang-san", "path": ["人物"]}, + {"slug": "concept/spring-festival", "path": ["节日", "传统节日"]} + ] +}` // Wiki ingest prompt templates for LLM-powered wiki page generation. // These prompts are used by the wiki ingest pipeline to extract structured @@ -333,22 +354,8 @@ The block above is assembled from VERBATIM source chunks that {{.AvailableSlugs}} - -{{.CurrentCategory}} - -` + wikiExistingTaxonomyXML + ` 1. The FIRST line of your output MUST be: SUMMARY: {one sentence, 15-40 words, describing what this page is about after the update — for wiki index listing} -1b. The SECOND line of your output MUST be: CATEGORY: {a navigation taxonomy path for this page, folders from broad to narrow separated by " / ", e.g. "CATEGORY: 人物" or "CATEGORY: 组织 / 企业"}. Follow the Directory Taxonomy Rules below. Almost every page can be filed under at least one broad top-level folder, so leaving it empty should be rare. - -### Directory Taxonomy Rules (for the CATEGORY line) -The CATEGORY classifies WHAT this {{.PageType}} fundamentally IS — a label that stays true across the whole knowledge base — never the role it happens to play in one document. -- Litmus test: keep a label only if it would still fit when this same {{.PageType}} shows up in a completely unrelated document. If a label only describes how it relates to THIS document's topic (who organized, sponsored, acknowledged, authored, or took part in something), that is a ROLE, not a category — leave it out and let the page body and [[wiki-links]] carry the relationship. -- Always assign at least the single broadest top-level folder that fits (think "which shelf would this always sit on in a library"). A coarse one-level category is expected for most pages; reserve a second level for a genuinely durable sub-domain. -- Do NOT use the item type itself as a folder (e.g. "实体", "概念", "摘要"). -- Do NOT put slashes inside a single label. Use at most 2 levels. -- Treat as the default and keep it unless new evidence clearly points to a better stable classification. -- Only output an empty "CATEGORY:" in the rare case where the page genuinely cannot be classified under any broad, durable folder.` + wikiExistingTaxonomyContinuityRules + ` {{if .HasRetractions}} 2. REMOVE facts/claims that were ONLY sourced from the and are NOT present in any or . {{end}} @@ -366,51 +373,12 @@ The CATEGORY classifies WHAT this {{.PageType}} fundamentally IS — a label tha 6. Maintain the existing page structure and formatting style. Use "# {{.PageTitle}}" as the top-level heading if the page does not already have one. Do NOT introduce new heading levels beyond what the source or existing page justifies. 7. **Image rule**: Include relevant images using Markdown syntax: ![caption](url) from new information if applicable. The URL inside ![caption](url) is an opaque token; reproduce it EXACTLY and VERBATIM, do not alter, shorten, or normalize it. {{if .HasRetractions}} -8. If after removing deleted content the page becomes nearly empty and there is no new information to add, output just: "SUMMARY: (empty page)\nCATEGORY:\n# {{.PageTitle}}\n\n*This page's primary source document was removed.*" +8. If after removing deleted content the page becomes nearly empty and there is no new information to add, output just: "SUMMARY: (empty page)\n# {{.PageTitle}}\n\n*This page's primary source document was removed.*" {{end}} 9. Write in {{.Language}}. -Output the SUMMARY line first, then the CATEGORY line, then the updated Markdown content. Do not include any other preamble.` - -// WikiTaxonomyReconcilePrompt is a per-knowledge-base maintenance prompt. It is -// given the full set of distinct category_path folders currently in use and asks -// the LLM to collapse synonymous folders onto a single canonical path. The -// result is applied in Go by re-pointing matching pages' category_path — slugs -// never change. This runs occasionally (after a batch introduces new folders, or -// on manual trigger), NOT per page, so its cost is O(folder count). -const WikiTaxonomyReconcilePrompt = `You are a taxonomy curator for a wiki knowledge base. Below is the complete list of directory paths (category_path) currently used to file wiki pages. Some of them are synonyms or near-duplicates that should share ONE canonical path. - - -{{.CurrentTaxonomy}} - - - -Identify groups of paths that mean the SAME thing and pick the single best canonical path for each group, then output a remap from every non-canonical path to its canonical path. - -Rules: -- Only merge paths that are genuinely synonymous or where one is clearly a less-canonical spelling/wording of another (e.g. "春节习俗" → "春节 / 传统习俗"; "AI厂商" and "AI行业 / 厂商" → one of them). -- Prefer the more standard, concise, and reusable label as the canonical target. Prefer an existing path over inventing a new one. -- Do NOT merge paths that refer to DIFFERENT things, even if they look similar. -- Do NOT merge across clearly different domains. -- Do NOT output an entry whose "from" equals its "to". -- A canonical "to" path may itself be one of the listed paths; never map a path onto a synonym chain (every "to" must be a final canonical path, not another "from"). -- Keep canonical paths to at most 2 levels. -- Each path is an array of folder labels from broad to narrow. Write ALL labels in {{.Language}}. -- If nothing should be merged, return {"remap": []}. - -### JSON Formatting Rules -- Output ONLY valid JSON, no preamble. -- Do NOT use literal newlines inside JSON string values. - - -Output format: -{ - "remap": [ - {"from": ["春节习俗"], "to": ["春节", "传统习俗"]}, - {"from": ["AI厂商"], "to": ["AI行业", "厂商"]} - ] -}` +Output the SUMMARY line first, then the updated Markdown content. Do not include any other preamble.` // WikiIndexIntroPrompt generates the introduction for a NEW index page (first time only). const WikiIndexIntroPrompt = `You are a wiki editor. Write a brief introduction for a wiki knowledge base index page. diff --git a/internal/application/repository/wiki_page.go b/internal/application/repository/wiki_page.go index 44e22cfe..f8e50777 100644 --- a/internal/application/repository/wiki_page.go +++ b/internal/application/repository/wiki_page.go @@ -153,35 +153,12 @@ func (r *wikiPageRepository) GetBySlug(ctx context.Context, kbID string, slug st return &page, nil } -// splitWikiPageTypes parses a page_type filter that may carry several -// comma-separated types (e.g. "entity,concept") into a deduplicated slice. -// An empty/whitespace-only input yields nil, meaning "no type filter". -func splitWikiPageTypes(raw string) []string { - if strings.TrimSpace(raw) == "" { - return nil - } - seen := make(map[string]struct{}) - out := make([]string, 0, 4) - for _, part := range strings.Split(raw, ",") { - part = strings.TrimSpace(part) - if part == "" { - continue - } - if _, ok := seen[part]; ok { - continue - } - seen[part] = struct{}{} - out = append(out, part) - } - return out -} - // List retrieves wiki pages with filtering and pagination func (r *wikiPageRepository) List(ctx context.Context, req *types.WikiPageListRequest) ([]*types.WikiPage, int64, error) { query := r.db.WithContext(ctx).Model(&types.WikiPage{}). Where("knowledge_base_id = ?", req.KnowledgeBaseID) - if pageTypes := splitWikiPageTypes(req.PageType); len(pageTypes) == 1 { + if pageTypes := types.SplitWikiPageTypes(req.PageType); len(pageTypes) == 1 { query = query.Where("page_type = ?", pageTypes[0]) } else if len(pageTypes) > 1 { query = query.Where("page_type IN ?", pageTypes) @@ -437,9 +414,13 @@ func (r *wikiPageRepository) ListBySlugs( return out, nil } -// ListDistinctCategoryPaths scans entity/concept pages and returns unique -// non-empty category_path values. Rows are read in wiki_path order so the -// prompt tree is stable; deduplication happens in Go with a maxPaths cap. +// ListDistinctCategoryPaths returns the distinct non-empty category paths used +// by entity/concept pages. The distinctness is pushed entirely to SQL over the +// flat category_l1/l2/l3 columns (indexed by idx_wiki_pages_category_levels), +// so the cost scales with the number of distinct folders — not the page count — +// and there is no full-table scan to bound. maxPaths caps the returned folder +// count. Used by the batch taxonomy planner as the candidate pool of existing +// folders to reuse (similarity preprocessing then narrows it per batch). func (r *wikiPageRepository) ListDistinctCategoryPaths( ctx context.Context, kbID string, @@ -450,41 +431,39 @@ func (r *wikiPageRepository) ListDistinctCategoryPaths( } type row struct { - CategoryPath types.StringArray `gorm:"column:category_path"` + L1 string `gorm:"column:category_l1"` + L2 string `gorm:"column:category_l2"` + L3 string `gorm:"column:category_l3"` } var rows []row if err := r.db.WithContext(ctx). Model(&types.WikiPage{}). - Select("category_path"). - Where("knowledge_base_id = ? AND page_type IN ? AND status <> ?", + Distinct("category_l1", "category_l2", "category_l3"). + Where("knowledge_base_id = ? AND page_type IN ? AND status <> ? AND category_l1 <> ?", kbID, []string{types.WikiPageTypeEntity, types.WikiPageTypeConcept}, types.WikiPageStatusArchived, + "", ). - Order(r.wikiCategoryRankOrder()). - Order("wiki_path ASC"). + Order("category_l1 ASC"). + Order("category_l2 ASC"). + Order("category_l3 ASC"). + Limit(maxPaths). Scan(&rows).Error; err != nil { return nil, err } - seen := make(map[string]struct{}, len(rows)) out := make([][]string, 0, len(rows)) - for _, row := range rows { - if len(row.CategoryPath) == 0 { - continue + for _, rw := range rows { + path := make([]string, 0, 3) + for _, label := range []string{rw.L1, rw.L2, rw.L3} { + if label == "" { + break // levels are contiguous; an empty level ends the path + } + path = append(path, label) } - clean := types.CleanWikiCategoryPath(row.CategoryPath) - if len(clean) == 0 { - continue - } - key := strings.Join(clean, "\x00") - if _, ok := seen[key]; ok { - continue - } - seen[key] = struct{}{} - out = append(out, clean) - if len(out) >= maxPaths { - break + if len(path) > 0 { + out = append(out, path) } } return out, nil diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go index 447bb025..afb340e2 100644 --- a/internal/application/service/wiki_ingest.go +++ b/internal/application/service/wiki_ingest.go @@ -659,11 +659,12 @@ type WikiBatchContext struct { // three valid values. ExtractionGranularity types.WikiExtractionGranularity - // ExistingTaxonomy returns a formatted directory tree of category_path - // folders already used in this KB. Loaded once per batch and injected - // into entity/concept extraction prompts so new pages reuse established - // folder labels instead of inventing synonyms. - ExistingTaxonomy func(ctx context.Context) string + // PlannedCategory holds the per-slug directory path assigned by the batch + // taxonomy planning pass (planBatchTaxonomy), keyed by page slug. Reduce + // applies it only to pages that don't already have a category, so the whole + // batch lands on one coherent tree without churning user-curated pages. + // Populated before the reduce phase and read-only during it. + PlannedCategory map[string][]string } // SlugUpdate represents a single update operation for a specific slug @@ -1203,25 +1204,32 @@ func collectLinkRefs(pages []*types.WikiPage) []linkRef { return refs } -// wikiTaxonomyScanRowLimit caps how many wiki_pages rows we scan when -// building the existing-taxonomy prompt block. Distinct paths are deduped -// in Go; this bound keeps the query cheap on large KBs. -const wikiTaxonomyScanRowLimit = 2000 - -// wikiTaxonomyPromptMaxPaths caps how many unique category_path prefixes -// are rendered into the LLM prompt to control token usage. +// wikiTaxonomyPromptMaxPaths caps how many existing folders are rendered into a +// planning prompt as the set to reuse. Reached only for pathologically large +// taxonomies; the similarity preprocessing keeps the fed set well under it. const wikiTaxonomyPromptMaxPaths = 150 -func resolveExistingTaxonomyForPrompt(ctx context.Context, batchCtx *WikiBatchContext) string { - if batchCtx == nil || batchCtx.ExistingTaxonomy == nil { - return "(none — this knowledge base has no established folders yet)" - } - text := strings.TrimSpace(batchCtx.ExistingTaxonomy(ctx)) - if text == "" { - return "(none — this knowledge base has no established folders yet)" - } - return text -} +// wikiTaxonomyFolderPoolMax bounds the existing folders pulled from the DB as the +// candidate pool for similarity selection. Distinct folders are few even for +// large KBs, so this only guards against a degenerate taxonomy. +const wikiTaxonomyFolderPoolMax = 400 + +// wikiTaxonomyFeedAllMaxFolders is the folder count at or below which the whole +// folder set is fed to the planner as-is: a healthy navigation directory is +// small, so feeding everything gives perfect reuse recall with no embedding cost +// (similarity preprocessing only earns its keep once folders are numerous). +const wikiTaxonomyFeedAllMaxFolders = 60 + +// wikiTaxonomyRelevantTopK is how many nearest existing deeper folders each item +// contributes to the reuse set when similarity preprocessing kicks in. +const wikiTaxonomyRelevantTopK = 3 + +// wikiTaxonomyPlanChunkSize caps how many items go into a single planning call. +// Larger batches are split into chunks; folders assigned by earlier chunks are +// fed forward as "existing folders" so later chunks converge onto the same tree. +const wikiTaxonomyPlanChunkSize = 60 + +const wikiTaxonomyEmptyTreeHint = "(none yet — this knowledge base has no folders, design a fresh directory)" type wikiTaxonomyNode struct { children map[string]*wikiTaxonomyNode @@ -1290,39 +1298,6 @@ func formatExistingTaxonomyForPrompt(paths [][]string) string { } return strings.TrimSpace(buf.String()) } - -// dedupeWikiCategoryPaths returns unique non-empty category_path slices, -// preserving first-seen order up to maxPaths. -func dedupeWikiCategoryPaths(paths [][]string, maxPaths int) [][]string { - if maxPaths <= 0 { - maxPaths = wikiTaxonomyPromptMaxPaths - } - seen := make(map[string]struct{}, len(paths)) - out := make([][]string, 0, len(paths)) - for _, raw := range paths { - clean := make([]string, 0, len(raw)) - for _, part := range raw { - part = strings.TrimSpace(part) - if part != "" { - clean = append(clean, part) - } - } - if len(clean) == 0 { - continue - } - key := strings.Join(clean, "\x00") - if _, ok := seen[key]; ok { - continue - } - seen[key] = struct{}{} - out = append(out, clean) - if len(out) >= maxPaths { - break - } - } - return out -} - // getExistingPageSlugsForKnowledge returns all page slugs that currently // reference a given knowledge ID in their source_refs. Used to snapshot // state before re-ingest so the reduce phase can reconcile additions vs diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go index 7023c857..74454d82 100644 --- a/internal/application/service/wiki_ingest_batch.go +++ b/internal/application/service/wiki_ingest_batch.go @@ -357,28 +357,6 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task ExtractionGranularity: granularity, } - var taxonomyOnce sync.Once - var taxonomyText string - batchCtx.ExistingTaxonomy = func(ctx context.Context) string { - taxonomyOnce.Do(func() { - if s.wikiService == nil { - return - } - paths, err := s.wikiService.ListDistinctCategoryPaths(ctx, payload.KnowledgeBaseID, wikiTaxonomyPromptMaxPaths) - if err != nil { - logger.Warnf(ctx, "wiki ingest: ListDistinctCategoryPaths failed: %v", err) - return - } - taxonomyText = formatExistingTaxonomyForPrompt(paths) - }) - return taxonomyText - } - - // Snapshot the KB's category folders before this batch runs so we can tell - // afterwards whether new (possibly synonymous) folders were introduced and - // a taxonomy reconcile pass is worth running. - startCategoryKeys := s.snapshotCategoryKeys(ctx, payload.KnowledgeBaseID) - // 1. MAP PHASE (Parallel extraction and generation of updates) var mapMu sync.Mutex var failedOps []WikiPendingOp @@ -507,6 +485,13 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task } _ = eg.Wait() + // Plan the directory once for the whole batch BEFORE reduce. Reduce writes + // pages in parallel, so it can't converge on shared folders on its own; this + // single pass assigns every new entity/concept slug a coherent category_path + // that reuses existing folders. Reduce then only applies the plan to pages + // that don't already have a category (user-curated pages are never churned). + batchCtx.PlannedCategory = s.planBatchTaxonomy(ctx, chatModel, kb, slugUpdates, lang) + // 2. REDUCE PHASE (Parallel upserting grouped by Slug) egReduce, reduceCtx := errgroup.WithContext(ctx) egReduce.SetLimit(reduceParallel) // Reduce phase limit (LLM + DB concurrent connections, configurable) @@ -650,14 +635,6 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task } } - // Collapse synonymous category folders introduced by this batch (cold - // start, intra-batch races, long-term drift). Runs only when a genuinely - // new folder appeared vs the pre-batch snapshot; re-points pages' category - // without touching slugs, then refreshes the index. - if len(ingestPagesAffected) > 0 { - s.maybeReconcileTaxonomy(ctx, chatModel, payload.KnowledgeBaseID, lang, startCategoryKeys) - } - // Clean dead [[slug]] references whenever ANY page was touched this // batch (not just retracts). Reduce-phase failures can leave stale // references in pages we just rewrote (e.g. summary pages cite @@ -1615,11 +1592,6 @@ func (s *wikiIngestService) reduceSlugUpdates( } pageAliases := strings.Join(page.Aliases, ", ") - currentCategory := strings.Join(page.CategoryPath, " / ") - if strings.TrimSpace(currentCategory) == "" { - currentCategory = "(none yet)" - } - var updatedContent string updatedContent, err = s.generateWithTemplate(ctx, chatModel, agent.WikiPageModifyPrompt, map[string]string{ "HasAdditions": hasAdditionsStr, @@ -1633,22 +1605,16 @@ func (s *wikiIngestService) reduceSlugUpdates( "DeletedContent": deletedContent.String(), "RemainingSourcesContent": remainingSourcesContent.String(), "AvailableSlugs": relatedSlugs.String(), - "CurrentCategory": currentCategory, - "ExistingTaxonomy": resolveExistingTaxonomyForPrompt(ctx, batchCtx), "Language": language, }) if err == nil && updatedContent != "" { updatedSummary, updatedBody := splitSummaryLine(updatedContent) - // CATEGORY is the page-level intrinsic classification produced by - // the editor model. Only overwrite when the model actually emitted - // a non-empty path so a lazy/empty line never wipes a good category. - category, bodyAfterCategory := splitCategoryLine(updatedBody) + // Category now comes from the batch taxonomy plan, not this prompt. + // Defensively strip any stray leading CATEGORY line the model may + // still emit so it never leaks into the page body; discard the value. + _, bodyAfterCategory := splitCategoryLine(updatedBody) updatedBody = bodyAfterCategory - if len(category) > 0 { - page.CategoryPath = types.StringArray(category) - page.WikiPath = "" - } if updatedBody != "" { page.Content = updatedBody } else { @@ -1676,6 +1642,16 @@ func (s *wikiIngestService) reduceSlugUpdates( } } + // Apply the batch taxonomy plan, but only to pages that don't already have a + // category — so brand-new pages get filed coherently while previously-filed + // or user-curated categories are preserved (manual edits are authoritative). + if len(page.CategoryPath) == 0 && batchCtx != nil { + if planned := types.CleanWikiCategoryPath(batchCtx.PlannedCategory[slug]); len(planned) > 0 { + page.CategoryPath = types.StringArray(planned) + page.WikiPath = "" + } + } + if changed { // Refresh chunk refs in-place on the page so they persist alongside // the rest of the row. Retract-only updates (no additions) preserve diff --git a/internal/application/service/wiki_ingest_taxonomy.go b/internal/application/service/wiki_ingest_taxonomy.go index eb78369f..edb34745 100644 --- a/internal/application/service/wiki_ingest_taxonomy.go +++ b/internal/application/service/wiki_ingest_taxonomy.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "fmt" + "math" + "sort" "strings" "github.com/Tencent/WeKnora/internal/agent" @@ -12,239 +14,269 @@ import ( "github.com/Tencent/WeKnora/internal/types" ) -// wikiTaxonomyReconcileMaxPages caps how many pages a single reconcile pass -// will rewrite, so an over-eager remap cannot fan out into an unbounded write -// storm on a very large KB. Pages beyond the cap are picked up by the next run. -const wikiTaxonomyReconcileMaxPages = 5000 - -// wikiTaxonomyReconcilePageSize is the cursor page size used when scanning the -// KB to apply a category remap. -const wikiTaxonomyReconcilePageSize = 500 - -type wikiTaxonomyRemapEntry struct { - From []string `json:"from"` - To []string `json:"to"` +// wikiTaxonomyItem is one entity/concept page to be filed into the directory. +type wikiTaxonomyItem struct { + slug string + title string + pageType string + about string } -type wikiTaxonomyRemapResult struct { - Remap []wikiTaxonomyRemapEntry `json:"remap"` -} - -// ReconcileTaxonomy is the manual entry point for collapsing synonymous -// category folders across a wiki knowledge base. It resolves the KB's synthesis -// chat model itself so it can be invoked outside an ingest batch (e.g. from a -// future maintenance handler). -func (s *wikiIngestService) ReconcileTaxonomy(ctx context.Context, kbID string) error { - kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, kbID) - if err != nil { - return fmt.Errorf("wiki taxonomy reconcile: get KB: %w", err) - } - if !kb.IsWikiEnabled() { - return fmt.Errorf("wiki taxonomy reconcile: KB %s is not wiki type", kbID) - } - - synthesisModelID := "" - if kb.WikiConfig != nil { - synthesisModelID = kb.WikiConfig.SynthesisModelID - } - if synthesisModelID == "" { - synthesisModelID = kb.SummaryModelID - } - if synthesisModelID == "" { - return fmt.Errorf("wiki taxonomy reconcile: no synthesis model configured for KB %s", kbID) - } - chatModel, err := s.modelService.GetChatModel(ctx, synthesisModelID) - if err != nil { - return fmt.Errorf("wiki taxonomy reconcile: get chat model: %w", err) - } - - return s.reconcileTaxonomy(ctx, chatModel, kbID, types.LanguageNameFromContext(ctx)) -} - -// reconcileTaxonomy collapses synonymous category_path folders across the KB -// onto a single canonical path. It is per-KB (cost O(folder count), not page -// count): one LLM call clusters the distinct folders, then matching pages are -// re-pointed in Go. Slugs are NEVER touched — only the navigation category. -func (s *wikiIngestService) reconcileTaxonomy(ctx context.Context, chatModel chat.Chat, kbID, lang string) error { - if s.wikiService == nil || chatModel == nil { - return nil - } - paths, err := s.wikiService.ListDistinctCategoryPaths(ctx, kbID, wikiTaxonomyPromptMaxPaths) - if err != nil { - return fmt.Errorf("list distinct category paths: %w", err) - } - if len(paths) < 2 { - return nil // nothing to merge - } - - remap := s.buildTaxonomyRemap(ctx, chatModel, paths, lang) - if len(remap) == 0 { - return nil - } - - updated := s.applyTaxonomyRemap(ctx, kbID, remap) - if updated > 0 { - logger.Infof(ctx, "wiki taxonomy reconcile: re-pointed %d pages via %d rules (kb=%s)", updated, len(remap), kbID) - if err := s.wikiService.RebuildIndexPage(ctx, kbID); err != nil { - logger.Warnf(ctx, "wiki taxonomy reconcile: rebuild index failed: %v", err) - } - } - return nil -} - -// maybeReconcileTaxonomy runs a reconcile pass only when this batch introduced -// at least one category folder that was not present in startKeys (the snapshot -// taken before the batch ran). This keeps the extra LLM call off the hot path -// for batches that only reuse established folders. -func (s *wikiIngestService) maybeReconcileTaxonomy( - ctx context.Context, chatModel chat.Chat, kbID, lang string, startKeys map[string]bool, -) { - if s.wikiService == nil { - return - } - paths, err := s.wikiService.ListDistinctCategoryPaths(ctx, kbID, wikiTaxonomyPromptMaxPaths) - if err != nil || len(paths) < 2 { - return - } - introducedNewFolder := false - for _, p := range paths { - if !startKeys[categoryPathKey(cleanCategoryPathParts(p))] { - introducedNewFolder = true - break - } - } - if !introducedNewFolder { - return - } - if err := s.reconcileTaxonomy(ctx, chatModel, kbID, lang); err != nil { - logger.Warnf(ctx, "wiki taxonomy reconcile failed: %v", err) - } -} - -// snapshotCategoryKeys returns the set of canonical category keys currently in -// use, used as the "before" baseline for maybeReconcileTaxonomy. -func (s *wikiIngestService) snapshotCategoryKeys(ctx context.Context, kbID string) map[string]bool { - out := map[string]bool{} - if s.wikiService == nil { - return out - } - paths, err := s.wikiService.ListDistinctCategoryPaths(ctx, kbID, wikiTaxonomyPromptMaxPaths) - if err != nil { - return out - } - for _, p := range paths { - out[categoryPathKey(cleanCategoryPathParts(p))] = true - } - return out -} - -// buildTaxonomyRemap asks the LLM to cluster synonymous folders and returns a -// map keyed by the canonical key of each non-canonical path to its replacement. -func (s *wikiIngestService) buildTaxonomyRemap( - ctx context.Context, chatModel chat.Chat, paths [][]string, lang string, +// planBatchTaxonomy assigns a directory path to every entity/concept slug in the +// batch in ONE planning pass (chunked for large batches), so the whole set lands +// on a single coherent tree that reuses existing folders. This replaces per-page, +// parallel CATEGORY invention — which couldn't converge, worst of all on the +// founding batch when the KB has no folders to anchor on. The returned map is +// keyed by slug; an entry may be an empty slice when the item is unclassifiable. +// Reduce applies these only to pages that don't already have a category. +func (s *wikiIngestService) planBatchTaxonomy( + ctx context.Context, + chatModel chat.Chat, + kb *types.KnowledgeBase, + slugUpdates map[string][]SlugUpdate, + lang string, ) map[string][]string { - taxonomyText := formatExistingTaxonomyForPrompt(paths) - if strings.TrimSpace(taxonomyText) == "" { + if kb == nil { return nil } - raw, err := s.generateWithTemplate(ctx, chatModel, agent.WikiTaxonomyReconcilePrompt, map[string]string{ - "CurrentTaxonomy": taxonomyText, - "Language": lang, - }) - if err != nil { - logger.Warnf(ctx, "wiki taxonomy reconcile: LLM call failed: %v", err) - return nil - } - raw = cleanLLMJSON(raw) - var parsed wikiTaxonomyRemapResult - if jerr := json.Unmarshal([]byte(raw), &parsed); jerr != nil { - logger.Warnf(ctx, "wiki taxonomy reconcile: parse failed: %v\nRaw: %s", jerr, raw) + items := collectTaxonomyItems(slugUpdates) + if len(items) == 0 { return nil } - out := make(map[string][]string, len(parsed.Remap)) - for _, e := range parsed.Remap { - from := cleanCategoryPathParts(e.From) - to := cleanCategoryPathParts(e.To) - if len(from) == 0 || len(to) == 0 { + // Existing folders anchor reuse. Errors (e.g. a dialect without the query) + // just mean the plan designs a fresh tree — no fatal dependency. + var pool [][]string + if s.wikiService != nil { + paths, err := s.wikiService.ListDistinctCategoryPaths(ctx, kb.ID, wikiTaxonomyFolderPoolMax) + if err != nil { + logger.Warnf(ctx, "wiki ingest: list category paths for plan failed: %v", err) + } else { + pool = paths + } + } + + // Preprocess the pool down to the folders relevant to THIS batch, so the + // planner reuses established folders without every prompt carrying the whole + // directory. Small/healthy taxonomies are fed whole (best recall, no cost). + existing := s.selectRelevantFolders(ctx, kb, items, pool) + + result := make(map[string][]string, len(items)) + for start := 0; start < len(items); start += wikiTaxonomyPlanChunkSize { + end := start + wikiTaxonomyPlanChunkSize + if end > len(items) { + end = len(items) + } + chunk := items[start:end] + + tree := formatExistingTaxonomyForPrompt(existing) + if strings.TrimSpace(tree) == "" { + tree = wikiTaxonomyEmptyTreeHint + } + + var itemsBlock strings.Builder + for _, it := range chunk { + fmt.Fprintf(&itemsBlock, "- slug: %s | title: %s | type: %s | about: %s\n", + it.slug, it.title, it.pageType, previewText(it.about, 120)) + } + + raw, err := s.generateWithTemplate(ctx, chatModel, agent.WikiTaxonomyPlanPrompt, map[string]string{ + "ExistingTaxonomy": tree, + "Items": itemsBlock.String(), + "Language": lang, + }) + if err != nil { + logger.Warnf(ctx, "wiki ingest: taxonomy plan call failed (%d items): %v", len(chunk), err) continue } - fromKey := categoryPathKey(from) - if fromKey == categoryPathKey(to) { - continue // no-op rule + + for slug, path := range parseTaxonomyAssignments(raw) { + clean := types.CleanWikiCategoryPath(path) + result[slug] = clean + if len(clean) > 0 { + existing = append(existing, clean) // feed forward so later chunks converge + } + } + } + return result +} + +// selectRelevantFolders narrows the existing folder pool to the subset worth +// showing the planner for THIS batch. A healthy navigation directory is small, +// so it is fed whole (perfect reuse recall, no embedding cost). Only once folders +// are numerous does similarity preprocessing kick in: all level-1 folders are +// always kept as coarse anchors, and each item pulls in its nearest deeper +// folders by embedding similarity. KBs without an embedding model (wiki-only) +// fall back to a capped feed-all. +func (s *wikiIngestService) selectRelevantFolders( + ctx context.Context, kb *types.KnowledgeBase, items []wikiTaxonomyItem, pool [][]string, +) [][]string { + if len(pool) <= wikiTaxonomyFeedAllMaxFolders { + return pool + } + + // Split into always-kept level-1 anchors and the deeper candidate folders + // that similarity selects among. + l1Seen := make(map[string]struct{}) + var l1Paths, deeper [][]string + for _, p := range pool { + if len(p) == 0 { + continue + } + if _, ok := l1Seen[p[0]]; !ok { + l1Seen[p[0]] = struct{}{} + l1Paths = append(l1Paths, []string{p[0]}) + } + if len(p) >= 2 { + deeper = append(deeper, p) + } + } + + if !kb.NeedsEmbeddingModel() || strings.TrimSpace(kb.EmbeddingModelID) == "" || len(deeper) == 0 { + return capFolders(pool, wikiTaxonomyPromptMaxPaths) + } + + embedder, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID) + if err != nil { + logger.Warnf(ctx, "wiki ingest: taxonomy plan embed model unavailable, feeding all folders: %v", err) + return capFolders(pool, wikiTaxonomyPromptMaxPaths) + } + + folderTexts := make([]string, len(deeper)) + for i, p := range deeper { + folderTexts[i] = strings.Join(p, " / ") + } + itemTexts := make([]string, len(items)) + for i, it := range items { + itemTexts[i] = strings.TrimSpace(it.title + " " + previewText(it.about, 120)) + } + + folderVecs, err := embedder.BatchEmbed(ctx, folderTexts) + if err != nil { + logger.Warnf(ctx, "wiki ingest: taxonomy plan folder embed failed, feeding all folders: %v", err) + return capFolders(pool, wikiTaxonomyPromptMaxPaths) + } + itemVecs, err := embedder.BatchEmbed(ctx, itemTexts) + if err != nil { + logger.Warnf(ctx, "wiki ingest: taxonomy plan item embed failed, feeding all folders: %v", err) + return capFolders(pool, wikiTaxonomyPromptMaxPaths) + } + + selected := append(l1Paths, selectFoldersByVectors(deeper, folderVecs, itemVecs, wikiTaxonomyRelevantTopK)...) + return capFolders(selected, wikiTaxonomyPromptMaxPaths) +} + +// selectFoldersByVectors returns the deeper folders that rank in any item's +// top-K by cosine similarity, preserving the input order for determinism. +func selectFoldersByVectors(deeper [][]string, folderVecs, itemVecs [][]float32, topK int) [][]string { + if len(deeper) != len(folderVecs) || len(itemVecs) == 0 || topK <= 0 { + return nil + } + chosen := make(map[int]struct{}) + for _, iv := range itemVecs { + type scored struct { + idx int + sim float64 + } + ranking := make([]scored, 0, len(folderVecs)) + for fi, fv := range folderVecs { + ranking = append(ranking, scored{fi, cosineSimilarity(iv, fv)}) + } + sort.SliceStable(ranking, func(a, b int) bool { return ranking[a].sim > ranking[b].sim }) + for k := 0; k < topK && k < len(ranking); k++ { + chosen[ranking[k].idx] = struct{}{} + } + } + out := make([][]string, 0, len(chosen)) + for i := range deeper { + if _, ok := chosen[i]; ok { + out = append(out, deeper[i]) } - out[fromKey] = to } return out } -// applyTaxonomyRemap scans entity/concept pages and re-points any whose -// category matches a remap rule. Returns the number of pages updated. -func (s *wikiIngestService) applyTaxonomyRemap(ctx context.Context, kbID string, remap map[string][]string) int { - var cursor string - var updated int - for updated < wikiTaxonomyReconcileMaxPages { - pages, next, err := s.wikiService.ListPagesCursor(ctx, kbID, cursor, wikiTaxonomyReconcilePageSize) - if err != nil { - logger.Warnf(ctx, "wiki taxonomy reconcile: list pages failed: %v", err) - break - } - if len(pages) == 0 { - break - } - for _, p := range pages { - if p.PageType != types.WikiPageTypeEntity && p.PageType != types.WikiPageTypeConcept { - continue - } - newPath, ok := remapCategoryPath([]string(p.CategoryPath), remap) - if !ok { - continue - } - p.CategoryPath = types.StringArray(newPath) - p.WikiPath = "" // force recompute in normalizeWikiHierarchy - if err := s.wikiService.UpdatePageMeta(ctx, p); err != nil { - logger.Warnf(ctx, "wiki taxonomy reconcile: update %s failed: %v", p.Slug, err) - continue - } - updated++ - if updated >= wikiTaxonomyReconcileMaxPages { - break - } - } - if next == "" { - break - } - cursor = next +// cosineSimilarity returns the cosine of two equal-length vectors, or 0 for empty +// / mismatched / zero-norm inputs. +func cosineSimilarity(a, b []float32) float64 { + if len(a) == 0 || len(a) != len(b) { + return 0 } - return updated + var dot, na, nb float64 + for i := range a { + dot += float64(a[i]) * float64(b[i]) + na += float64(a[i]) * float64(a[i]) + nb += float64(b[i]) * float64(b[i]) + } + if na == 0 || nb == 0 { + return 0 + } + return dot / (math.Sqrt(na) * math.Sqrt(nb)) } -// remapCategoryPath returns the canonical category for current when it matches -// a remap "from" key, plus whether a change is needed. Pure (no DB) so it can -// be unit-tested in isolation. -func remapCategoryPath(current []string, remap map[string][]string) ([]string, bool) { - cleaned := cleanCategoryPathParts(current) - if len(cleaned) == 0 { - return nil, false +// capFolders truncates a folder list to at most max entries (max <= 0 = no cap). +func capFolders(paths [][]string, max int) [][]string { + if max > 0 && len(paths) > max { + return paths[:max] } - to, ok := remap[categoryPathKey(cleaned)] - if !ok { - return nil, false - } - if categoryPathKey(cleaned) == categoryPathKey(cleanCategoryPathParts(to)) { - return nil, false - } - return append([]string(nil), to...), true + return paths } -// cleanCategoryPathParts normalizes a category path the same way the page -// hierarchy normalizer does (separator/quote cleanup, type-label stripping, -// depth cap) so remap keys match what is stored on pages. -func cleanCategoryPathParts(parts []string) []string { - return types.CleanWikiCategoryPath(parts) +// collectTaxonomyItems extracts the entity/concept pages from a batch's slug +// updates, in deterministic slug order so chunk boundaries are stable. Summary +// and retract-only slugs are skipped (they carry no directory category). +func collectTaxonomyItems(slugUpdates map[string][]SlugUpdate) []wikiTaxonomyItem { + slugs := make([]string, 0, len(slugUpdates)) + for slug := range slugUpdates { + slugs = append(slugs, slug) + } + sort.Strings(slugs) + + items := make([]wikiTaxonomyItem, 0, len(slugs)) + for _, slug := range slugs { + for _, u := range slugUpdates[slug] { + if u.Type != types.WikiPageTypeEntity && u.Type != types.WikiPageTypeConcept { + continue + } + title := strings.TrimSpace(u.Item.Name) + if title == "" { + title = slug + } + items = append(items, wikiTaxonomyItem{ + slug: slug, + title: title, + pageType: u.Type, + about: strings.TrimSpace(u.Item.Description), + }) + break // one entry per slug is enough for classification + } + } + return items } -// categoryPathKey builds a stable comparison key for a cleaned category path. -func categoryPathKey(parts []string) string { - return strings.Join(parts, "\u0000") +// parseTaxonomyAssignments parses the planning LLM's JSON into a slug → path map. +// Malformed output yields nil; individual entries with a blank slug are dropped. +func parseTaxonomyAssignments(raw string) map[string][]string { + raw = cleanLLMJSON(raw) + if raw == "" { + return nil + } + var parsed struct { + Assignments []struct { + Slug string `json:"slug"` + Path []string `json:"path"` + } `json:"assignments"` + } + if err := json.Unmarshal([]byte(raw), &parsed); err != nil { + return nil + } + out := make(map[string][]string, len(parsed.Assignments)) + for _, a := range parsed.Assignments { + slug := strings.TrimSpace(a.Slug) + if slug == "" { + continue + } + out[slug] = a.Path + } + return out } diff --git a/internal/application/service/wiki_ingest_taxonomy_test.go b/internal/application/service/wiki_ingest_taxonomy_test.go index d8fab8f9..85440cdb 100644 --- a/internal/application/service/wiki_ingest_taxonomy_test.go +++ b/internal/application/service/wiki_ingest_taxonomy_test.go @@ -1,9 +1,10 @@ package service import ( - "context" "strings" "testing" + + "github.com/Tencent/WeKnora/internal/types" ) func TestSplitCategoryLine(t *testing.T) { @@ -82,104 +83,99 @@ func TestFormatExistingTaxonomyForPromptEmpty(t *testing.T) { } } -func TestDedupeWikiCategoryPaths(t *testing.T) { - got := dedupeWikiCategoryPaths([][]string{ - {"春节", "传统习俗"}, - {" 春节 ", "传统习俗"}, - {"春节", "文化习俗"}, - {"", "空"}, - {}, - }, 10) +func TestParseTaxonomyAssignments(t *testing.T) { + raw := "```json\n{\"assignments\":[" + + "{\"slug\":\"entity/zhang-san\",\"path\":[\"人物\"]}," + + "{\"slug\":\"concept/spring\",\"path\":[\"节日\",\"传统节日\"]}," + + "{\"slug\":\" \",\"path\":[\"X\"]}," + + "{\"slug\":\"entity/unclassified\",\"path\":[]}" + + "]}\n```" + + got := parseTaxonomyAssignments(raw) if len(got) != 3 { - t.Fatalf("dedupeWikiCategoryPaths() len = %d, want 3: %v", len(got), got) + t.Fatalf("parseTaxonomyAssignments() returned %d entries, want 3 (blank slug dropped): %v", len(got), got) } - if got[0][0] != "春节" || got[0][1] != "传统习俗" { - t.Fatalf("dedupeWikiCategoryPaths()[0] = %v", got[0]) + if strings.Join(got["entity/zhang-san"], "/") != "人物" { + t.Fatalf("zhang-san path = %v, want [人物]", got["entity/zhang-san"]) } - if got[1][1] != "文化习俗" { - t.Fatalf("dedupeWikiCategoryPaths()[1] = %v", got[1]) + if strings.Join(got["concept/spring"], "/") != "节日/传统节日" { + t.Fatalf("spring path = %v, want [节日 传统节日]", got["concept/spring"]) + } + if p, ok := got["entity/unclassified"]; !ok || len(p) != 0 { + t.Fatalf("unclassified path = %v (ok=%v), want empty slice present", p, ok) } } -func TestResolveExistingTaxonomyForPrompt(t *testing.T) { - t.Run("nil batch context", func(t *testing.T) { - got := resolveExistingTaxonomyForPrompt(t.Context(), nil) - if got == "" { - t.Fatal("expected fallback text for nil batch context") - } - }) - t.Run("batch context with taxonomy", func(t *testing.T) { - batchCtx := &WikiBatchContext{ - ExistingTaxonomy: func(ctx context.Context) string { - return "春节\n 传统习俗" - }, - } - got := resolveExistingTaxonomyForPrompt(t.Context(), batchCtx) - if got != "春节\n 传统习俗" { - t.Fatalf("resolveExistingTaxonomyForPrompt() = %q", got) - } - }) -} - -func TestRemapCategoryPath(t *testing.T) { - remap := map[string][]string{ - categoryPathKey([]string{"春节习俗"}): {"春节", "传统习俗"}, - categoryPathKey([]string{"组织单位", "致谢人员"}): {"组织"}, - } - - tests := []struct { - name string - current []string - want []string - wantHit bool - }{ - { - name: "single-label synonym remapped", - current: []string{"春节习俗"}, - want: []string{"春节", "传统习俗"}, - wantHit: true, - }, - { - name: "document-role path collapsed to coarse folder", - current: []string{"组织单位", "致谢人员"}, - want: []string{"组织"}, - wantHit: true, - }, - { - name: "noise normalized before matching", - current: []string{"实体/春节习俗"}, - want: []string{"春节", "传统习俗"}, - wantHit: true, - }, - { - name: "unrelated path untouched", - current: []string{"人物"}, - wantHit: false, - }, - { - name: "empty path untouched", - current: nil, - wantHit: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, hit := remapCategoryPath(tt.current, remap) - if hit != tt.wantHit { - t.Fatalf("remapCategoryPath() hit = %v, want %v", hit, tt.wantHit) - } - if !hit { - return - } - if len(got) != len(tt.want) { - t.Fatalf("remapCategoryPath() = %v, want %v", got, tt.want) - } - for i := range tt.want { - if got[i] != tt.want[i] { - t.Fatalf("remapCategoryPath()[%d] = %q, want %q (full=%v)", i, got[i], tt.want[i], got) - } - } - }) +func TestParseTaxonomyAssignmentsMalformed(t *testing.T) { + if got := parseTaxonomyAssignments("not json at all"); got != nil { + t.Fatalf("parseTaxonomyAssignments(garbage) = %v, want nil", got) + } +} + +func TestCosineSimilarity(t *testing.T) { + if got := cosineSimilarity([]float32{1, 0}, []float32{1, 0}); got < 0.999 { + t.Fatalf("identical vectors sim = %v, want ~1", got) + } + if got := cosineSimilarity([]float32{1, 0}, []float32{0, 1}); got != 0 { + t.Fatalf("orthogonal vectors sim = %v, want 0", got) + } + if got := cosineSimilarity([]float32{1, 0}, nil); got != 0 { + t.Fatalf("mismatched length sim = %v, want 0", got) + } + if got := cosineSimilarity([]float32{0, 0}, []float32{1, 1}); got != 0 { + t.Fatalf("zero-norm sim = %v, want 0", got) + } +} + +func TestSelectFoldersByVectors(t *testing.T) { + deeper := [][]string{ + {"AI", "厂商"}, // 0 + {"AI", "模型"}, // 1 + {"地理", "城市"}, // 2 + } + folderVecs := [][]float32{ + {1, 0, 0}, + {0.9, 0.1, 0}, + {0, 0, 1}, + } + // Item closest to folder 0 (and 1 as runner-up); far from 2. + itemVecs := [][]float32{{1, 0, 0}} + + got := selectFoldersByVectors(deeper, folderVecs, itemVecs, 2) + if len(got) != 2 { + t.Fatalf("selectFoldersByVectors() returned %d folders, want 2: %v", len(got), got) + } + // Input order preserved: folders 0 and 1, not the orthogonal 2. + if strings.Join(got[0], "/") != "AI/厂商" || strings.Join(got[1], "/") != "AI/模型" { + t.Fatalf("selectFoldersByVectors() = %v, want [[AI 厂商] [AI 模型]]", got) + } +} + +func TestSelectFoldersByVectorsGuards(t *testing.T) { + if got := selectFoldersByVectors([][]string{{"a"}}, [][]float32{{1}, {2}}, [][]float32{{1}}, 1); got != nil { + t.Fatalf("mismatched lengths should return nil, got %v", got) + } + if got := selectFoldersByVectors([][]string{{"a"}}, [][]float32{{1}}, nil, 1); got != nil { + t.Fatalf("no items should return nil, got %v", got) + } +} + +func TestCollectTaxonomyItems(t *testing.T) { + slugUpdates := map[string][]SlugUpdate{ + "entity/b": {{Type: types.WikiPageTypeEntity, Item: extractedItem{Name: "B", Description: "about B"}}}, + "entity/a": {{Type: types.WikiPageTypeConcept, Item: extractedItem{Name: "A"}}}, + "sum/x": {{Type: "summary"}}, + "ret/y": {{Type: "retract"}}, + } + items := collectTaxonomyItems(slugUpdates) + if len(items) != 2 { + t.Fatalf("collectTaxonomyItems() = %d items, want 2 (summary/retract skipped): %+v", len(items), items) + } + // Deterministic slug order so chunk boundaries are stable. + if items[0].slug != "entity/a" || items[1].slug != "entity/b" { + t.Fatalf("collectTaxonomyItems() order = [%s, %s], want [entity/a, entity/b]", items[0].slug, items[1].slug) + } + if items[1].about != "about B" { + t.Fatalf("items[1].about = %q, want %q", items[1].about, "about B") } } diff --git a/internal/handler/wiki_page.go b/internal/handler/wiki_page.go index a392e8a0..38a184c7 100644 --- a/internal/handler/wiki_page.go +++ b/internal/handler/wiki_page.go @@ -148,7 +148,7 @@ func (h *WikiPageHandler) ListCategories(c *gin.Context) { return } - pageTypes := parseWikiPageTypes(c.Query("page_type")) + pageTypes := types.SplitWikiPageTypes(c.Query("page_type")) if len(pageTypes) == 0 { c.JSON(http.StatusBadRequest, gin.H{"error": "page_type is required"}) return @@ -186,28 +186,6 @@ func (h *WikiPageHandler) ListCategories(c *gin.Context) { }) } -// parseWikiPageTypes splits a comma-separated page_type query value (e.g. -// "entity,concept") into a deduplicated slice, dropping blanks. -func parseWikiPageTypes(raw string) []string { - if strings.TrimSpace(raw) == "" { - return nil - } - seen := make(map[string]struct{}) - out := make([]string, 0, 4) - for _, part := range strings.Split(raw, ",") { - part = strings.TrimSpace(part) - if part == "" { - continue - } - if _, ok := seen[part]; ok { - continue - } - seen[part] = struct{}{} - out = append(out, part) - } - return out -} - func parseWikiCategoryPath(raw string) []string { if strings.TrimSpace(raw) == "" { return nil diff --git a/internal/types/wiki_page.go b/internal/types/wiki_page.go index c7f16ee2..9439064f 100644 --- a/internal/types/wiki_page.go +++ b/internal/types/wiki_page.go @@ -63,6 +63,31 @@ func CleanWikiCategoryPath(parts []string) []string { return cleaned } +// SplitWikiPageTypes parses a page_type value that may carry several +// comma-separated types (e.g. "entity,concept") into a deduplicated slice, +// dropping blanks. An empty/whitespace-only input yields nil ("no filter"). +// Shared by the handler (query parsing) and repository (List filter) so the +// two layers split identically. +func SplitWikiPageTypes(raw string) []string { + if strings.TrimSpace(raw) == "" { + return nil + } + seen := make(map[string]struct{}) + out := make([]string, 0, 4) + for _, part := range strings.Split(raw, ",") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + if _, ok := seen[part]; ok { + continue + } + seen[part] = struct{}{} + out = append(out, part) + } + return out +} + // WikiCategoryLevels flattens a (already cleaned) category path into the three // fixed level columns used for SQL grouping. Missing levels yield "". func WikiCategoryLevels(path []string) (l1, l2, l3 string) {