feat(wiki): add page hierarchy/category metadata and retrieval APIs

Persist category and hierarchy metadata on wiki pages (new migration), expose
category-path retrieval and related repository/service/handler/router APIs, and
update wiki ingestion (batch, cite, taxonomy) plus agent wiki prompts to build
and consume the taxonomy. Regenerate swagger docs for the new endpoints.
This commit is contained in:
wizardchen
2026-06-16 16:35:06 +08:00
committed by lyingbug
parent ec66a3dac7
commit fe33db9493
20 changed files with 1893 additions and 25 deletions
+170
View File
@@ -658,6 +658,12 @@ type WikiBatchContext struct {
// Already Normalize()'d — consumers can assume it is one of the
// three valid values.
ExtractionGranularity types.WikiExtractionGranularity
// ExistingTaxonomy returns a formatted directory tree of category_path
// folders already used in this KB. Loaded once per batch and injected
// into entity/concept extraction prompts so new pages reuse established
// folder labels instead of inventing synonyms.
ExistingTaxonomy func(ctx context.Context) string
}
// SlugUpdate represents a single update operation for a specific slug
@@ -1197,6 +1203,126 @@ func collectLinkRefs(pages []*types.WikiPage) []linkRef {
return refs
}
// wikiTaxonomyScanRowLimit caps how many wiki_pages rows we scan when
// building the existing-taxonomy prompt block. Distinct paths are deduped
// in Go; this bound keeps the query cheap on large KBs.
const wikiTaxonomyScanRowLimit = 2000
// wikiTaxonomyPromptMaxPaths caps how many unique category_path prefixes
// are rendered into the LLM prompt to control token usage.
const wikiTaxonomyPromptMaxPaths = 150
func resolveExistingTaxonomyForPrompt(ctx context.Context, batchCtx *WikiBatchContext) string {
if batchCtx == nil || batchCtx.ExistingTaxonomy == nil {
return "(none — this knowledge base has no established folders yet)"
}
text := strings.TrimSpace(batchCtx.ExistingTaxonomy(ctx))
if text == "" {
return "(none — this knowledge base has no established folders yet)"
}
return text
}
type wikiTaxonomyNode struct {
children map[string]*wikiTaxonomyNode
}
func insertWikiTaxonomyPath(root *wikiTaxonomyNode, path []string) {
if root == nil || len(path) == 0 {
return
}
cur := root
for _, part := range path {
part = strings.TrimSpace(part)
if part == "" {
continue
}
if cur.children == nil {
cur.children = make(map[string]*wikiTaxonomyNode)
}
child := cur.children[part]
if child == nil {
child = &wikiTaxonomyNode{}
cur.children[part] = child
}
cur = child
}
}
func appendWikiTaxonomyNode(buf *strings.Builder, label string, node *wikiTaxonomyNode, depth int) {
if label != "" {
fmt.Fprintf(buf, "%s%s\n", strings.Repeat(" ", depth), label)
}
if node == nil || len(node.children) == 0 {
return
}
keys := make([]string, 0, len(node.children))
for k := range node.children {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
appendWikiTaxonomyNode(buf, k, node.children[k], depth+1)
}
}
// formatExistingTaxonomyForPrompt renders distinct category_path values as an
// indented folder tree for LLM extraction prompts.
func formatExistingTaxonomyForPrompt(paths [][]string) string {
if len(paths) == 0 {
return ""
}
root := &wikiTaxonomyNode{}
for _, path := range paths {
insertWikiTaxonomyPath(root, path)
}
if len(root.children) == 0 {
return ""
}
var buf strings.Builder
keys := make([]string, 0, len(root.children))
for k := range root.children {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
appendWikiTaxonomyNode(&buf, k, root.children[k], 0)
}
return strings.TrimSpace(buf.String())
}
// dedupeWikiCategoryPaths returns unique non-empty category_path slices,
// preserving first-seen order up to maxPaths.
func dedupeWikiCategoryPaths(paths [][]string, maxPaths int) [][]string {
if maxPaths <= 0 {
maxPaths = wikiTaxonomyPromptMaxPaths
}
seen := make(map[string]struct{}, len(paths))
out := make([][]string, 0, len(paths))
for _, raw := range paths {
clean := make([]string, 0, len(raw))
for _, part := range raw {
part = strings.TrimSpace(part)
if part != "" {
clean = append(clean, part)
}
}
if len(clean) == 0 {
continue
}
key := strings.Join(clean, "\x00")
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
out = append(out, clean)
if len(out) >= maxPaths {
break
}
}
return out
}
// getExistingPageSlugsForKnowledge returns all page slugs that currently
// reference a given knowledge ID in their source_refs. Used to snapshot
// state before re-ingest so the reduce phase can reconcile additions vs
@@ -1435,6 +1561,50 @@ func splitSummaryLine(raw string) (summary string, content string) {
return "", raw
}
// splitCategoryLine extracts a leading "CATEGORY: a / b" line from content
// (the page body AFTER the SUMMARY line has been stripped) and returns the
// parsed folder labels plus the remaining content with that line removed.
//
// Category assignment lives in the page-modify output (reduce time) rather
// than per-document extraction so it reflects WHAT the whole entity IS, not
// the role it played in one source document. An empty/absent CATEGORY line
// yields nil so callers keep the page's existing category instead of wiping it.
func splitCategoryLine(content string) (category []string, rest string) {
trimmed := strings.TrimLeft(content, " \t\r\n")
if !strings.HasPrefix(trimmed, "CATEGORY:") && !strings.HasPrefix(trimmed, "CATEGORY:") {
return nil, content
}
var line, remainder string
if idx := strings.IndexByte(trimmed, '\n'); idx < 0 {
line = trimmed
} else {
line = trimmed[:idx]
remainder = trimmed[idx+1:]
}
line = strings.TrimPrefix(line, "CATEGORY:")
line = strings.TrimPrefix(line, "CATEGORY:")
return parseCategoryPath(line), strings.TrimSpace(remainder)
}
// parseCategoryPath splits a "a / b" breadcrumb string into clean folder
// labels. Separator variants and wrapping quotes/brackets are normalized; the
// deeper structural cleaning (type-label stripping, depth cap, wiki_path) is
// left to normalizeWikiHierarchy on the page.
func parseCategoryPath(line string) []string {
line = strings.NewReplacer("/", "/", "|", "/", "|", "/", "›", "/", "»", "/", ">", "/").Replace(line)
parts := strings.Split(line, "/")
out := make([]string, 0, len(parts))
for _, p := range parts {
label := strings.TrimSpace(p)
label = strings.Trim(label, `"'“”‘’[]()()`)
label = strings.TrimSpace(label)
if label != "" {
out = append(out, label)
}
}
return out
}
// buildLogEntry builds a WikiLogEntry struct for the current batch. It is
// pure (no DB access) so callers can accumulate entries cheaply under their
// lock and flush them in a single AppendBatch call at the end of the batch.
@@ -357,6 +357,28 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
ExtractionGranularity: granularity,
}
var taxonomyOnce sync.Once
var taxonomyText string
batchCtx.ExistingTaxonomy = func(ctx context.Context) string {
taxonomyOnce.Do(func() {
if s.wikiService == nil {
return
}
paths, err := s.wikiService.ListDistinctCategoryPaths(ctx, payload.KnowledgeBaseID, wikiTaxonomyPromptMaxPaths)
if err != nil {
logger.Warnf(ctx, "wiki ingest: ListDistinctCategoryPaths failed: %v", err)
return
}
taxonomyText = formatExistingTaxonomyForPrompt(paths)
})
return taxonomyText
}
// Snapshot the KB's category folders before this batch runs so we can tell
// afterwards whether new (possibly synonymous) folders were introduced and
// a taxonomy reconcile pass is worth running.
startCategoryKeys := s.snapshotCategoryKeys(ctx, payload.KnowledgeBaseID)
// 1. MAP PHASE (Parallel extraction and generation of updates)
var mapMu sync.Mutex
var failedOps []WikiPendingOp
@@ -628,6 +650,14 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
}
}
// Collapse synonymous category folders introduced by this batch (cold
// start, intra-batch races, long-term drift). Runs only when a genuinely
// new folder appeared vs the pre-batch snapshot; re-points pages' category
// without touching slugs, then refreshes the index.
if len(ingestPagesAffected) > 0 {
s.maybeReconcileTaxonomy(ctx, chatModel, payload.KnowledgeBaseID, lang, startCategoryKeys)
}
// Clean dead [[slug]] references whenever ANY page was touched this
// batch (not just retracts). Reduce-phase failures can leave stale
// references in pages we just rewrote (e.g. summary pages cite
@@ -960,7 +990,7 @@ func (s *wikiIngestService) mapOneDocument(
return
}
candidatesXML := renderCandidateSlugsXML(extractedEntities, extractedConcepts)
citations, newSlugs, batchCount = s.classifyChunkCitations(ctx, chatModel, candidatesXML, chunks, lang)
citations, newSlugs, batchCount = s.classifyChunkCitations(ctx, chatModel, candidatesXML, chunks, lang, batchCtx)
s.tracker().EndSpan(ctx, classifySpan, types.JSONMap{
"cited_slugs": len(citations),
"new_slugs": len(newSlugs),
@@ -1585,6 +1615,11 @@ func (s *wikiIngestService) reduceSlugUpdates(
}
pageAliases := strings.Join(page.Aliases, ", ")
currentCategory := strings.Join(page.CategoryPath, " / ")
if strings.TrimSpace(currentCategory) == "" {
currentCategory = "(none yet)"
}
var updatedContent string
updatedContent, err = s.generateWithTemplate(ctx, chatModel, agent.WikiPageModifyPrompt, map[string]string{
"HasAdditions": hasAdditionsStr,
@@ -1598,11 +1633,22 @@ func (s *wikiIngestService) reduceSlugUpdates(
"DeletedContent": deletedContent.String(),
"RemainingSourcesContent": remainingSourcesContent.String(),
"AvailableSlugs": relatedSlugs.String(),
"CurrentCategory": currentCategory,
"ExistingTaxonomy": resolveExistingTaxonomyForPrompt(ctx, batchCtx),
"Language": language,
})
if err == nil && updatedContent != "" {
updatedSummary, updatedBody := splitSummaryLine(updatedContent)
// CATEGORY is the page-level intrinsic classification produced by
// the editor model. Only overwrite when the model actually emitted
// a non-empty path so a lazy/empty line never wipes a good category.
category, bodyAfterCategory := splitCategoryLine(updatedBody)
updatedBody = bodyAfterCategory
if len(category) > 0 {
page.CategoryPath = types.StringArray(category)
page.WikiPath = ""
}
if updatedBody != "" {
page.Content = updatedBody
} else {
@@ -279,6 +279,7 @@ func (s *wikiIngestService) classifyChunkCitations(
candidatesXML string,
chunks []*types.Chunk,
lang string,
batchCtx *WikiBatchContext,
) (map[string][]string, []newSlugFromCitation, int) {
batches := splitChunksIntoCitationBatches(chunks)
if len(batches) == 0 || strings.TrimSpace(candidatesXML) == "" {
@@ -0,0 +1,259 @@
package service
import (
"context"
"encoding/json"
"fmt"
"strings"
"github.com/Tencent/WeKnora/internal/agent"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/types"
)
// wikiTaxonomyReconcileMaxPages caps how many pages a single reconcile pass
// will rewrite, so an over-eager remap cannot fan out into an unbounded write
// storm on a very large KB. Pages beyond the cap are picked up by the next run.
const wikiTaxonomyReconcileMaxPages = 5000
// wikiTaxonomyReconcilePageSize is the cursor page size used when scanning the
// KB to apply a category remap.
const wikiTaxonomyReconcilePageSize = 500
type wikiTaxonomyRemapEntry struct {
From []string `json:"from"`
To []string `json:"to"`
}
type wikiTaxonomyRemapResult struct {
Remap []wikiTaxonomyRemapEntry `json:"remap"`
}
// ReconcileTaxonomy is the manual entry point for collapsing synonymous
// category folders across a wiki knowledge base. It resolves the KB's synthesis
// chat model itself so it can be invoked outside an ingest batch (e.g. from a
// future maintenance handler).
func (s *wikiIngestService) ReconcileTaxonomy(ctx context.Context, kbID string) error {
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, kbID)
if err != nil {
return fmt.Errorf("wiki taxonomy reconcile: get KB: %w", err)
}
if !kb.IsWikiEnabled() {
return fmt.Errorf("wiki taxonomy reconcile: KB %s is not wiki type", kbID)
}
synthesisModelID := ""
if kb.WikiConfig != nil {
synthesisModelID = kb.WikiConfig.SynthesisModelID
}
if synthesisModelID == "" {
synthesisModelID = kb.SummaryModelID
}
if synthesisModelID == "" {
return fmt.Errorf("wiki taxonomy reconcile: no synthesis model configured for KB %s", kbID)
}
chatModel, err := s.modelService.GetChatModel(ctx, synthesisModelID)
if err != nil {
return fmt.Errorf("wiki taxonomy reconcile: get chat model: %w", err)
}
return s.reconcileTaxonomy(ctx, chatModel, kbID, types.LanguageNameFromContext(ctx))
}
// reconcileTaxonomy collapses synonymous category_path folders across the KB
// onto a single canonical path. It is per-KB (cost O(folder count), not page
// count): one LLM call clusters the distinct folders, then matching pages are
// re-pointed in Go. Slugs are NEVER touched — only the navigation category.
func (s *wikiIngestService) reconcileTaxonomy(ctx context.Context, chatModel chat.Chat, kbID, lang string) error {
if s.wikiService == nil || chatModel == nil {
return nil
}
paths, err := s.wikiService.ListDistinctCategoryPaths(ctx, kbID, wikiTaxonomyPromptMaxPaths)
if err != nil {
return fmt.Errorf("list distinct category paths: %w", err)
}
if len(paths) < 2 {
return nil // nothing to merge
}
remap := s.buildTaxonomyRemap(ctx, chatModel, paths, lang)
if len(remap) == 0 {
return nil
}
updated := s.applyTaxonomyRemap(ctx, kbID, remap)
if updated > 0 {
logger.Infof(ctx, "wiki taxonomy reconcile: re-pointed %d pages via %d rules (kb=%s)", updated, len(remap), kbID)
if err := s.wikiService.RebuildIndexPage(ctx, kbID); err != nil {
logger.Warnf(ctx, "wiki taxonomy reconcile: rebuild index failed: %v", err)
}
}
return nil
}
// maybeReconcileTaxonomy runs a reconcile pass only when this batch introduced
// at least one category folder that was not present in startKeys (the snapshot
// taken before the batch ran). This keeps the extra LLM call off the hot path
// for batches that only reuse established folders.
func (s *wikiIngestService) maybeReconcileTaxonomy(
ctx context.Context, chatModel chat.Chat, kbID, lang string, startKeys map[string]bool,
) {
if s.wikiService == nil {
return
}
paths, err := s.wikiService.ListDistinctCategoryPaths(ctx, kbID, wikiTaxonomyPromptMaxPaths)
if err != nil || len(paths) < 2 {
return
}
introducedNewFolder := false
for _, p := range paths {
if !startKeys[categoryPathKey(cleanCategoryPathParts(p))] {
introducedNewFolder = true
break
}
}
if !introducedNewFolder {
return
}
if err := s.reconcileTaxonomy(ctx, chatModel, kbID, lang); err != nil {
logger.Warnf(ctx, "wiki taxonomy reconcile failed: %v", err)
}
}
// snapshotCategoryKeys returns the set of canonical category keys currently in
// use, used as the "before" baseline for maybeReconcileTaxonomy.
func (s *wikiIngestService) snapshotCategoryKeys(ctx context.Context, kbID string) map[string]bool {
out := map[string]bool{}
if s.wikiService == nil {
return out
}
paths, err := s.wikiService.ListDistinctCategoryPaths(ctx, kbID, wikiTaxonomyPromptMaxPaths)
if err != nil {
return out
}
for _, p := range paths {
out[categoryPathKey(cleanCategoryPathParts(p))] = true
}
return out
}
// buildTaxonomyRemap asks the LLM to cluster synonymous folders and returns a
// map keyed by the canonical key of each non-canonical path to its replacement.
func (s *wikiIngestService) buildTaxonomyRemap(
ctx context.Context, chatModel chat.Chat, paths [][]string, lang string,
) map[string][]string {
taxonomyText := formatExistingTaxonomyForPrompt(paths)
if strings.TrimSpace(taxonomyText) == "" {
return nil
}
raw, err := s.generateWithTemplate(ctx, chatModel, agent.WikiTaxonomyReconcilePrompt, map[string]string{
"CurrentTaxonomy": taxonomyText,
"Language": lang,
})
if err != nil {
logger.Warnf(ctx, "wiki taxonomy reconcile: LLM call failed: %v", err)
return nil
}
raw = cleanLLMJSON(raw)
var parsed wikiTaxonomyRemapResult
if jerr := json.Unmarshal([]byte(raw), &parsed); jerr != nil {
logger.Warnf(ctx, "wiki taxonomy reconcile: parse failed: %v\nRaw: %s", jerr, raw)
return nil
}
out := make(map[string][]string, len(parsed.Remap))
for _, e := range parsed.Remap {
from := cleanCategoryPathParts(e.From)
to := cleanCategoryPathParts(e.To)
if len(from) == 0 || len(to) == 0 {
continue
}
fromKey := categoryPathKey(from)
if fromKey == categoryPathKey(to) {
continue // no-op rule
}
out[fromKey] = to
}
return out
}
// applyTaxonomyRemap scans entity/concept pages and re-points any whose
// category matches a remap rule. Returns the number of pages updated.
func (s *wikiIngestService) applyTaxonomyRemap(ctx context.Context, kbID string, remap map[string][]string) int {
var cursor string
var updated int
for updated < wikiTaxonomyReconcileMaxPages {
pages, next, err := s.wikiService.ListPagesCursor(ctx, kbID, cursor, wikiTaxonomyReconcilePageSize)
if err != nil {
logger.Warnf(ctx, "wiki taxonomy reconcile: list pages failed: %v", err)
break
}
if len(pages) == 0 {
break
}
for _, p := range pages {
if p.PageType != types.WikiPageTypeEntity && p.PageType != types.WikiPageTypeConcept {
continue
}
newPath, ok := remapCategoryPath([]string(p.CategoryPath), remap)
if !ok {
continue
}
p.CategoryPath = types.StringArray(newPath)
p.WikiPath = "" // force recompute in normalizeWikiHierarchy
if err := s.wikiService.UpdatePageMeta(ctx, p); err != nil {
logger.Warnf(ctx, "wiki taxonomy reconcile: update %s failed: %v", p.Slug, err)
continue
}
updated++
if updated >= wikiTaxonomyReconcileMaxPages {
break
}
}
if next == "" {
break
}
cursor = next
}
return updated
}
// remapCategoryPath returns the canonical category for current when it matches
// a remap "from" key, plus whether a change is needed. Pure (no DB) so it can
// be unit-tested in isolation.
func remapCategoryPath(current []string, remap map[string][]string) ([]string, bool) {
cleaned := cleanCategoryPathParts(current)
if len(cleaned) == 0 {
return nil, false
}
to, ok := remap[categoryPathKey(cleaned)]
if !ok {
return nil, false
}
if categoryPathKey(cleaned) == categoryPathKey(cleanCategoryPathParts(to)) {
return nil, false
}
return append([]string(nil), to...), true
}
// cleanCategoryPathParts normalizes a category path the same way the page
// hierarchy normalizer does (separator/quote cleanup, type-label stripping,
// depth cap) so remap keys match what is stored on pages.
func cleanCategoryPathParts(parts []string) []string {
out := make([]string, 0, len(parts))
for _, part := range parts {
for _, c := range cleanWikiCategoryPart(part) {
out = append(out, c)
if len(out) >= 3 {
return out
}
}
}
return out
}
// categoryPathKey builds a stable comparison key for a cleaned category path.
func categoryPathKey(parts []string) string {
return strings.Join(parts, "\u0000")
}
@@ -0,0 +1,185 @@
package service
import (
"context"
"strings"
"testing"
)
func TestSplitCategoryLine(t *testing.T) {
tests := []struct {
name string
content string
wantCat []string
wantRest string
}{
{
name: "two level category",
content: "CATEGORY: 组织 / 企业\n# 标题\n正文",
wantCat: []string{"组织", "企业"},
wantRest: "# 标题\n正文",
},
{
name: "single level category",
content: "CATEGORY: 人物\n# 陈洋",
wantCat: []string{"人物"},
wantRest: "# 陈洋",
},
{
name: "empty category keeps existing (nil) and strips line",
content: "CATEGORY:\n# 标题",
wantCat: nil,
wantRest: "# 标题",
},
{
name: "no category line left untouched",
content: "# 标题\n正文",
wantCat: nil,
wantRest: "# 标题\n正文",
},
{
name: "fullwidth colon and separators",
content: "CATEGORY:组织|企业\n正文",
wantCat: []string{"组织", "企业"},
wantRest: "正文",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cat, rest := splitCategoryLine(tt.content)
if strings.Join(cat, "/") != strings.Join(tt.wantCat, "/") {
t.Fatalf("splitCategoryLine() cat = %v, want %v", cat, tt.wantCat)
}
if rest != tt.wantRest {
t.Fatalf("splitCategoryLine() rest = %q, want %q", rest, tt.wantRest)
}
})
}
}
func TestFormatExistingTaxonomyForPrompt(t *testing.T) {
got := formatExistingTaxonomyForPrompt([][]string{
{"春节", "传统习俗"},
{"春节", "文化习俗", "节日习俗"},
{"春节习俗"},
{"产品定位"},
})
want := "产品定位\n" +
"春节\n" +
" 传统习俗\n" +
" 文化习俗\n" +
" 节日习俗\n" +
"春节习俗"
if got != want {
t.Fatalf("formatExistingTaxonomyForPrompt():\n%q\nwant:\n%q", got, want)
}
}
func TestFormatExistingTaxonomyForPromptEmpty(t *testing.T) {
if got := formatExistingTaxonomyForPrompt(nil); got != "" {
t.Fatalf("formatExistingTaxonomyForPrompt(nil) = %q, want empty", got)
}
}
func TestDedupeWikiCategoryPaths(t *testing.T) {
got := dedupeWikiCategoryPaths([][]string{
{"春节", "传统习俗"},
{" 春节 ", "传统习俗"},
{"春节", "文化习俗"},
{"", "空"},
{},
}, 10)
if len(got) != 3 {
t.Fatalf("dedupeWikiCategoryPaths() len = %d, want 3: %v", len(got), got)
}
if got[0][0] != "春节" || got[0][1] != "传统习俗" {
t.Fatalf("dedupeWikiCategoryPaths()[0] = %v", got[0])
}
if got[1][1] != "文化习俗" {
t.Fatalf("dedupeWikiCategoryPaths()[1] = %v", got[1])
}
}
func TestResolveExistingTaxonomyForPrompt(t *testing.T) {
t.Run("nil batch context", func(t *testing.T) {
got := resolveExistingTaxonomyForPrompt(t.Context(), nil)
if got == "" {
t.Fatal("expected fallback text for nil batch context")
}
})
t.Run("batch context with taxonomy", func(t *testing.T) {
batchCtx := &WikiBatchContext{
ExistingTaxonomy: func(ctx context.Context) string {
return "春节\n 传统习俗"
},
}
got := resolveExistingTaxonomyForPrompt(t.Context(), batchCtx)
if got != "春节\n 传统习俗" {
t.Fatalf("resolveExistingTaxonomyForPrompt() = %q", got)
}
})
}
func TestRemapCategoryPath(t *testing.T) {
remap := map[string][]string{
categoryPathKey([]string{"春节习俗"}): {"春节", "传统习俗"},
categoryPathKey([]string{"组织单位", "致谢人员"}): {"组织"},
}
tests := []struct {
name string
current []string
want []string
wantHit bool
}{
{
name: "single-label synonym remapped",
current: []string{"春节习俗"},
want: []string{"春节", "传统习俗"},
wantHit: true,
},
{
name: "document-role path collapsed to coarse folder",
current: []string{"组织单位", "致谢人员"},
want: []string{"组织"},
wantHit: true,
},
{
name: "noise normalized before matching",
current: []string{"实体/春节习俗"},
want: []string{"春节", "传统习俗"},
wantHit: true,
},
{
name: "unrelated path untouched",
current: []string{"人物"},
wantHit: false,
},
{
name: "empty path untouched",
current: nil,
wantHit: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, hit := remapCategoryPath(tt.current, remap)
if hit != tt.wantHit {
t.Fatalf("remapCategoryPath() hit = %v, want %v", hit, tt.wantHit)
}
if !hit {
return
}
if len(got) != len(tt.want) {
t.Fatalf("remapCategoryPath() = %v, want %v", got, tt.want)
}
for i := range tt.want {
if got[i] != tt.want[i] {
t.Fatalf("remapCategoryPath()[%d] = %q, want %q (full=%v)", i, got[i], tt.want[i], got)
}
}
})
}
}
+144
View File
@@ -67,6 +67,7 @@ func (s *wikiPageService) CreatePage(ctx context.Context, page *types.WikiPage)
// Parse outbound links from content
page.OutLinks = s.parseOutLinks(page.Content)
normalizeWikiHierarchy(page)
now := time.Now()
page.CreatedAt = now
@@ -115,6 +116,11 @@ func (s *wikiPageService) UpdatePage(ctx context.Context, page *types.WikiPage)
existing.SourceRefs = page.SourceRefs
existing.ChunkRefs = page.ChunkRefs
existing.PageMetadata = page.PageMetadata
existing.ParentSlug = page.ParentSlug
existing.CategoryPath = page.CategoryPath
existing.WikiPath = page.WikiPath
existing.Depth = page.Depth
existing.SortOrder = page.SortOrder
existing.Status = page.Status
existing.UpdatedAt = time.Now()
@@ -122,11 +128,18 @@ func (s *wikiPageService) UpdatePage(ctx context.Context, page *types.WikiPage)
// when content shifts. Re-parse unconditionally to stay consistent with
// the stored body.
existing.OutLinks = s.parseOutLinks(existing.Content)
normalizeWikiHierarchy(existing)
if contentChanged {
if err := s.repo.Update(ctx, existing); err != nil {
return nil, fmt.Errorf("update wiki page: %w", err)
}
// GORM's struct Updates path skips zero values, so persist hierarchy
// metadata through the explicit map path as well. This keeps clearing
// parent/category fields deterministic without changing version again.
if err := s.repo.UpdateMeta(ctx, existing); err != nil {
return nil, fmt.Errorf("update wiki page hierarchy meta: %w", err)
}
} else {
// No user-visible change — persist bookkeeping fields but preserve
// the version so downstream consumers can rely on it.
@@ -145,6 +158,7 @@ func (s *wikiPageService) UpdatePage(ctx context.Context, page *types.WikiPage)
// UpdatePageMeta updates only metadata (status, source_refs) without version bump or link re-parse.
func (s *wikiPageService) UpdatePageMeta(ctx context.Context, page *types.WikiPage) error {
normalizeWikiHierarchy(page)
page.UpdatedAt = time.Now()
return s.repo.UpdateMeta(ctx, page)
}
@@ -192,6 +206,9 @@ func (s *wikiPageService) ListPages(ctx context.Context, req *types.WikiPageList
if err != nil {
return nil, err
}
for _, page := range pages {
normalizeWikiHierarchy(page)
}
pageSize := req.PageSize
if pageSize < 1 {
@@ -323,6 +340,9 @@ func (s *wikiPageService) GetIndexView(
if entries == nil {
entries = []types.WikiIndexEntry{}
}
for i := range entries {
normalizeWikiIndexEntryHierarchy(&entries[i], pt)
}
next := ""
// Only emit a cursor when a full page was returned AND more rows
// remain past `offset + limit`. A short page or one that exactly
@@ -811,6 +831,20 @@ func (s *wikiPageService) FindSimilarPages(ctx context.Context, kbID string, que
return s.repo.FindSimilarPages(ctx, kbID, query, pageTypes, limit)
}
// ListDistinctCategoryPaths returns unique category_path folders already used
// by entity/concept pages. Used by wiki ingest to ground extraction prompts.
func (s *wikiPageService) ListDistinctCategoryPaths(ctx context.Context, kbID string, maxPaths int) ([][]string, error) {
return s.repo.ListDistinctCategoryPaths(ctx, kbID, maxPaths)
}
// ListDistinctCategoryPathsByType returns unique category_path folders across
// the given page types. The browser uses this to render directory nodes before
// article pages are paged in; passing several types lets one tab (e.g. the
// merged knowledge tab) build a single directory skeleton server-side.
func (s *wikiPageService) ListDistinctCategoryPathsByType(ctx context.Context, kbID string, pageTypes []string, parentPath []string, maxPaths int) ([]types.WikiCategoryPath, error) {
return s.repo.ListDistinctCategoryPathsByType(ctx, kbID, pageTypes, parentPath, maxPaths)
}
// CountByType is a service-layer pass-through over the repo. Used by
// the index intro path to frame the LLM prompt's "showing N of M" hint.
func (s *wikiPageService) CountByType(ctx context.Context, kbID string) (map[string]int64, error) {
@@ -917,6 +951,7 @@ func (s *wikiPageService) createDefaultPage(ctx context.Context, kbID string, sl
Summary: title,
Version: 1,
}
normalizeWikiHierarchy(page)
if err := s.repo.Create(ctx, page); err != nil {
return nil, fmt.Errorf("create default %s page: %w", slug, err)
@@ -924,6 +959,115 @@ func (s *wikiPageService) createDefaultPage(ctx context.Context, kbID string, sl
return page, nil
}
func normalizeWikiHierarchy(page *types.WikiPage) {
if page == nil {
return
}
page.ParentSlug = strings.TrimSpace(page.ParentSlug)
cleanPath := make(types.StringArray, 0, len(page.CategoryPath))
for _, part := range page.CategoryPath {
for _, cleanPart := range cleanWikiCategoryPart(part) {
if !containsString(cleanPath, cleanPart) {
cleanPath = append(cleanPath, cleanPart)
}
if len(cleanPath) >= 3 {
break
}
}
if len(cleanPath) >= 3 {
break
}
}
page.CategoryPath = cleanPath
page.Depth = len(cleanPath)
parts := make([]string, 0, len(cleanPath)+2)
if page.PageType != "" {
parts = append(parts, page.PageType)
}
for _, part := range cleanPath {
parts = append(parts, part)
}
display := strings.TrimSpace(page.Title)
if display == "" {
display = strings.TrimSpace(page.Slug)
}
if display != "" {
parts = append(parts, display)
}
page.WikiPath = strings.Join(parts, "/")
}
func normalizeWikiIndexEntryHierarchy(entry *types.WikiIndexEntry, pageType string) {
if entry == nil {
return
}
cleanPath := make(types.StringArray, 0, len(entry.CategoryPath))
for _, part := range entry.CategoryPath {
for _, cleanPart := range cleanWikiCategoryPart(part) {
if !containsString(cleanPath, cleanPart) {
cleanPath = append(cleanPath, cleanPart)
}
if len(cleanPath) >= 3 {
break
}
}
if len(cleanPath) >= 3 {
break
}
}
entry.CategoryPath = cleanPath
entry.Depth = len(cleanPath)
parts := make([]string, 0, len(cleanPath)+2)
if strings.TrimSpace(pageType) != "" {
parts = append(parts, strings.TrimSpace(pageType))
}
parts = append(parts, cleanPath...)
display := strings.TrimSpace(entry.Title)
if display == "" {
display = strings.TrimSpace(entry.Slug)
}
if display != "" {
parts = append(parts, display)
}
entry.WikiPath = strings.Join(parts, "/")
}
func cleanWikiCategoryPart(part string) []string {
part = strings.TrimSpace(part)
if part == "" {
return nil
}
part = strings.NewReplacer("/", "/", "|", "/", "|", "/").Replace(part)
rawParts := strings.Split(part, "/")
cleaned := make([]string, 0, len(rawParts))
for _, raw := range rawParts {
label := strings.TrimSpace(raw)
label = strings.Trim(label, `"'“”‘’[]()()`)
label = strings.TrimSpace(label)
if label == "" || isWikiTypeCategoryLabel(label) {
continue
}
cleaned = append(cleaned, label)
}
return cleaned
}
func isWikiTypeCategoryLabel(label string) bool {
normalized := strings.ToLower(strings.TrimSpace(label))
normalized = strings.TrimSuffix(normalized, "s")
switch normalized {
case "entity", "实体", "實體", "concept", "概念", "summary", "摘要", "wiki", "页面", "頁面":
return true
default:
return false
}
}
// containsString checks if a string slice contains a given string
func containsString(slice []string, s string) bool {
for _, v := range slice {
@@ -99,6 +99,56 @@ func TestNormalizeSlug(t *testing.T) {
}
}
func TestNormalizeWikiHierarchyCleansModelCategoryNoise(t *testing.T) {
page := &types.WikiPage{
Slug: "concept/ai-knowledge",
Title: "AI时代的知识困境",
PageType: types.WikiPageTypeConcept,
CategoryPath: types.StringArray{"概念/爱护花草", " 实体/标牌 ", "摘要", "生态/平台", "多余层级"},
}
normalizeWikiHierarchy(page)
want := types.StringArray{"爱护花草", "标牌", "生态"}
if len(page.CategoryPath) != len(want) {
t.Fatalf("CategoryPath = %v, want %v", page.CategoryPath, want)
}
for i := range want {
if page.CategoryPath[i] != want[i] {
t.Fatalf("CategoryPath[%d] = %q, want %q; full path=%v", i, page.CategoryPath[i], want[i], page.CategoryPath)
}
}
if page.Depth != 3 {
t.Fatalf("Depth = %d, want 3", page.Depth)
}
if page.WikiPath != "concept/爱护花草/标牌/生态/AI时代的知识困境" {
t.Fatalf("WikiPath = %q", page.WikiPath)
}
}
func TestNormalizeWikiIndexEntryHierarchyCleansModelCategoryNoise(t *testing.T) {
entry := &types.WikiIndexEntry{
Slug: "entity/sign",
Title: "爱护花草标牌",
CategoryPath: types.StringArray{"实体/标牌", "概念", "生态/行为倡导"},
}
normalizeWikiIndexEntryHierarchy(entry, types.WikiPageTypeEntity)
want := types.StringArray{"标牌", "生态", "行为倡导"}
if len(entry.CategoryPath) != len(want) {
t.Fatalf("CategoryPath = %v, want %v", entry.CategoryPath, want)
}
for i := range want {
if entry.CategoryPath[i] != want[i] {
t.Fatalf("CategoryPath[%d] = %q, want %q; full path=%v", i, entry.CategoryPath[i], want[i], entry.CategoryPath)
}
}
if entry.WikiPath != "entity/标牌/生态/行为倡导/爱护花草标牌" {
t.Fatalf("WikiPath = %q", entry.WikiPath)
}
}
func TestContainsString(t *testing.T) {
slice := []string{"a", "b", "c"}
if !containsString(slice, "b") {