feat(wiki): add page hierarchy/category metadata and retrieval APIs
Persist category and hierarchy metadata on wiki pages (new migration), expose category-path retrieval and related repository/service/handler/router APIs, and update wiki ingestion (batch, cite, taxonomy) plus agent wiki prompts to build and consume the taxonomy. Regenerate swagger docs for the new endpoints.
This commit is contained in:
@@ -658,6 +658,12 @@ type WikiBatchContext struct {
|
||||
// Already Normalize()'d — consumers can assume it is one of the
|
||||
// three valid values.
|
||||
ExtractionGranularity types.WikiExtractionGranularity
|
||||
|
||||
// ExistingTaxonomy returns a formatted directory tree of category_path
|
||||
// folders already used in this KB. Loaded once per batch and injected
|
||||
// into entity/concept extraction prompts so new pages reuse established
|
||||
// folder labels instead of inventing synonyms.
|
||||
ExistingTaxonomy func(ctx context.Context) string
|
||||
}
|
||||
|
||||
// SlugUpdate represents a single update operation for a specific slug
|
||||
@@ -1197,6 +1203,126 @@ func collectLinkRefs(pages []*types.WikiPage) []linkRef {
|
||||
return refs
|
||||
}
|
||||
|
||||
// wikiTaxonomyScanRowLimit caps how many wiki_pages rows we scan when
|
||||
// building the existing-taxonomy prompt block. Distinct paths are deduped
|
||||
// in Go; this bound keeps the query cheap on large KBs.
|
||||
const wikiTaxonomyScanRowLimit = 2000
|
||||
|
||||
// wikiTaxonomyPromptMaxPaths caps how many unique category_path prefixes
|
||||
// are rendered into the LLM prompt to control token usage.
|
||||
const wikiTaxonomyPromptMaxPaths = 150
|
||||
|
||||
func resolveExistingTaxonomyForPrompt(ctx context.Context, batchCtx *WikiBatchContext) string {
|
||||
if batchCtx == nil || batchCtx.ExistingTaxonomy == nil {
|
||||
return "(none — this knowledge base has no established folders yet)"
|
||||
}
|
||||
text := strings.TrimSpace(batchCtx.ExistingTaxonomy(ctx))
|
||||
if text == "" {
|
||||
return "(none — this knowledge base has no established folders yet)"
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
type wikiTaxonomyNode struct {
|
||||
children map[string]*wikiTaxonomyNode
|
||||
}
|
||||
|
||||
func insertWikiTaxonomyPath(root *wikiTaxonomyNode, path []string) {
|
||||
if root == nil || len(path) == 0 {
|
||||
return
|
||||
}
|
||||
cur := root
|
||||
for _, part := range path {
|
||||
part = strings.TrimSpace(part)
|
||||
if part == "" {
|
||||
continue
|
||||
}
|
||||
if cur.children == nil {
|
||||
cur.children = make(map[string]*wikiTaxonomyNode)
|
||||
}
|
||||
child := cur.children[part]
|
||||
if child == nil {
|
||||
child = &wikiTaxonomyNode{}
|
||||
cur.children[part] = child
|
||||
}
|
||||
cur = child
|
||||
}
|
||||
}
|
||||
|
||||
func appendWikiTaxonomyNode(buf *strings.Builder, label string, node *wikiTaxonomyNode, depth int) {
|
||||
if label != "" {
|
||||
fmt.Fprintf(buf, "%s%s\n", strings.Repeat(" ", depth), label)
|
||||
}
|
||||
if node == nil || len(node.children) == 0 {
|
||||
return
|
||||
}
|
||||
keys := make([]string, 0, len(node.children))
|
||||
for k := range node.children {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
appendWikiTaxonomyNode(buf, k, node.children[k], depth+1)
|
||||
}
|
||||
}
|
||||
|
||||
// formatExistingTaxonomyForPrompt renders distinct category_path values as an
|
||||
// indented folder tree for LLM extraction prompts.
|
||||
func formatExistingTaxonomyForPrompt(paths [][]string) string {
|
||||
if len(paths) == 0 {
|
||||
return ""
|
||||
}
|
||||
root := &wikiTaxonomyNode{}
|
||||
for _, path := range paths {
|
||||
insertWikiTaxonomyPath(root, path)
|
||||
}
|
||||
if len(root.children) == 0 {
|
||||
return ""
|
||||
}
|
||||
var buf strings.Builder
|
||||
keys := make([]string, 0, len(root.children))
|
||||
for k := range root.children {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
appendWikiTaxonomyNode(&buf, k, root.children[k], 0)
|
||||
}
|
||||
return strings.TrimSpace(buf.String())
|
||||
}
|
||||
|
||||
// dedupeWikiCategoryPaths returns unique non-empty category_path slices,
|
||||
// preserving first-seen order up to maxPaths.
|
||||
func dedupeWikiCategoryPaths(paths [][]string, maxPaths int) [][]string {
|
||||
if maxPaths <= 0 {
|
||||
maxPaths = wikiTaxonomyPromptMaxPaths
|
||||
}
|
||||
seen := make(map[string]struct{}, len(paths))
|
||||
out := make([][]string, 0, len(paths))
|
||||
for _, raw := range paths {
|
||||
clean := make([]string, 0, len(raw))
|
||||
for _, part := range raw {
|
||||
part = strings.TrimSpace(part)
|
||||
if part != "" {
|
||||
clean = append(clean, part)
|
||||
}
|
||||
}
|
||||
if len(clean) == 0 {
|
||||
continue
|
||||
}
|
||||
key := strings.Join(clean, "\x00")
|
||||
if _, ok := seen[key]; ok {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
out = append(out, clean)
|
||||
if len(out) >= maxPaths {
|
||||
break
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// getExistingPageSlugsForKnowledge returns all page slugs that currently
|
||||
// reference a given knowledge ID in their source_refs. Used to snapshot
|
||||
// state before re-ingest so the reduce phase can reconcile additions vs
|
||||
@@ -1435,6 +1561,50 @@ func splitSummaryLine(raw string) (summary string, content string) {
|
||||
return "", raw
|
||||
}
|
||||
|
||||
// splitCategoryLine extracts a leading "CATEGORY: a / b" line from content
|
||||
// (the page body AFTER the SUMMARY line has been stripped) and returns the
|
||||
// parsed folder labels plus the remaining content with that line removed.
|
||||
//
|
||||
// Category assignment lives in the page-modify output (reduce time) rather
|
||||
// than per-document extraction so it reflects WHAT the whole entity IS, not
|
||||
// the role it played in one source document. An empty/absent CATEGORY line
|
||||
// yields nil so callers keep the page's existing category instead of wiping it.
|
||||
func splitCategoryLine(content string) (category []string, rest string) {
|
||||
trimmed := strings.TrimLeft(content, " \t\r\n")
|
||||
if !strings.HasPrefix(trimmed, "CATEGORY:") && !strings.HasPrefix(trimmed, "CATEGORY:") {
|
||||
return nil, content
|
||||
}
|
||||
var line, remainder string
|
||||
if idx := strings.IndexByte(trimmed, '\n'); idx < 0 {
|
||||
line = trimmed
|
||||
} else {
|
||||
line = trimmed[:idx]
|
||||
remainder = trimmed[idx+1:]
|
||||
}
|
||||
line = strings.TrimPrefix(line, "CATEGORY:")
|
||||
line = strings.TrimPrefix(line, "CATEGORY:")
|
||||
return parseCategoryPath(line), strings.TrimSpace(remainder)
|
||||
}
|
||||
|
||||
// parseCategoryPath splits a "a / b" breadcrumb string into clean folder
|
||||
// labels. Separator variants and wrapping quotes/brackets are normalized; the
|
||||
// deeper structural cleaning (type-label stripping, depth cap, wiki_path) is
|
||||
// left to normalizeWikiHierarchy on the page.
|
||||
func parseCategoryPath(line string) []string {
|
||||
line = strings.NewReplacer("/", "/", "|", "/", "|", "/", "›", "/", "»", "/", ">", "/").Replace(line)
|
||||
parts := strings.Split(line, "/")
|
||||
out := make([]string, 0, len(parts))
|
||||
for _, p := range parts {
|
||||
label := strings.TrimSpace(p)
|
||||
label = strings.Trim(label, `"'“”‘’[]()()`)
|
||||
label = strings.TrimSpace(label)
|
||||
if label != "" {
|
||||
out = append(out, label)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// buildLogEntry builds a WikiLogEntry struct for the current batch. It is
|
||||
// pure (no DB access) so callers can accumulate entries cheaply under their
|
||||
// lock and flush them in a single AppendBatch call at the end of the batch.
|
||||
|
||||
@@ -357,6 +357,28 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
ExtractionGranularity: granularity,
|
||||
}
|
||||
|
||||
var taxonomyOnce sync.Once
|
||||
var taxonomyText string
|
||||
batchCtx.ExistingTaxonomy = func(ctx context.Context) string {
|
||||
taxonomyOnce.Do(func() {
|
||||
if s.wikiService == nil {
|
||||
return
|
||||
}
|
||||
paths, err := s.wikiService.ListDistinctCategoryPaths(ctx, payload.KnowledgeBaseID, wikiTaxonomyPromptMaxPaths)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: ListDistinctCategoryPaths failed: %v", err)
|
||||
return
|
||||
}
|
||||
taxonomyText = formatExistingTaxonomyForPrompt(paths)
|
||||
})
|
||||
return taxonomyText
|
||||
}
|
||||
|
||||
// Snapshot the KB's category folders before this batch runs so we can tell
|
||||
// afterwards whether new (possibly synonymous) folders were introduced and
|
||||
// a taxonomy reconcile pass is worth running.
|
||||
startCategoryKeys := s.snapshotCategoryKeys(ctx, payload.KnowledgeBaseID)
|
||||
|
||||
// 1. MAP PHASE (Parallel extraction and generation of updates)
|
||||
var mapMu sync.Mutex
|
||||
var failedOps []WikiPendingOp
|
||||
@@ -628,6 +650,14 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
}
|
||||
}
|
||||
|
||||
// Collapse synonymous category folders introduced by this batch (cold
|
||||
// start, intra-batch races, long-term drift). Runs only when a genuinely
|
||||
// new folder appeared vs the pre-batch snapshot; re-points pages' category
|
||||
// without touching slugs, then refreshes the index.
|
||||
if len(ingestPagesAffected) > 0 {
|
||||
s.maybeReconcileTaxonomy(ctx, chatModel, payload.KnowledgeBaseID, lang, startCategoryKeys)
|
||||
}
|
||||
|
||||
// Clean dead [[slug]] references whenever ANY page was touched this
|
||||
// batch (not just retracts). Reduce-phase failures can leave stale
|
||||
// references in pages we just rewrote (e.g. summary pages cite
|
||||
@@ -960,7 +990,7 @@ func (s *wikiIngestService) mapOneDocument(
|
||||
return
|
||||
}
|
||||
candidatesXML := renderCandidateSlugsXML(extractedEntities, extractedConcepts)
|
||||
citations, newSlugs, batchCount = s.classifyChunkCitations(ctx, chatModel, candidatesXML, chunks, lang)
|
||||
citations, newSlugs, batchCount = s.classifyChunkCitations(ctx, chatModel, candidatesXML, chunks, lang, batchCtx)
|
||||
s.tracker().EndSpan(ctx, classifySpan, types.JSONMap{
|
||||
"cited_slugs": len(citations),
|
||||
"new_slugs": len(newSlugs),
|
||||
@@ -1585,6 +1615,11 @@ func (s *wikiIngestService) reduceSlugUpdates(
|
||||
}
|
||||
pageAliases := strings.Join(page.Aliases, ", ")
|
||||
|
||||
currentCategory := strings.Join(page.CategoryPath, " / ")
|
||||
if strings.TrimSpace(currentCategory) == "" {
|
||||
currentCategory = "(none yet)"
|
||||
}
|
||||
|
||||
var updatedContent string
|
||||
updatedContent, err = s.generateWithTemplate(ctx, chatModel, agent.WikiPageModifyPrompt, map[string]string{
|
||||
"HasAdditions": hasAdditionsStr,
|
||||
@@ -1598,11 +1633,22 @@ func (s *wikiIngestService) reduceSlugUpdates(
|
||||
"DeletedContent": deletedContent.String(),
|
||||
"RemainingSourcesContent": remainingSourcesContent.String(),
|
||||
"AvailableSlugs": relatedSlugs.String(),
|
||||
"CurrentCategory": currentCategory,
|
||||
"ExistingTaxonomy": resolveExistingTaxonomyForPrompt(ctx, batchCtx),
|
||||
"Language": language,
|
||||
})
|
||||
|
||||
if err == nil && updatedContent != "" {
|
||||
updatedSummary, updatedBody := splitSummaryLine(updatedContent)
|
||||
// CATEGORY is the page-level intrinsic classification produced by
|
||||
// the editor model. Only overwrite when the model actually emitted
|
||||
// a non-empty path so a lazy/empty line never wipes a good category.
|
||||
category, bodyAfterCategory := splitCategoryLine(updatedBody)
|
||||
updatedBody = bodyAfterCategory
|
||||
if len(category) > 0 {
|
||||
page.CategoryPath = types.StringArray(category)
|
||||
page.WikiPath = ""
|
||||
}
|
||||
if updatedBody != "" {
|
||||
page.Content = updatedBody
|
||||
} else {
|
||||
|
||||
@@ -279,6 +279,7 @@ func (s *wikiIngestService) classifyChunkCitations(
|
||||
candidatesXML string,
|
||||
chunks []*types.Chunk,
|
||||
lang string,
|
||||
batchCtx *WikiBatchContext,
|
||||
) (map[string][]string, []newSlugFromCitation, int) {
|
||||
batches := splitChunksIntoCitationBatches(chunks)
|
||||
if len(batches) == 0 || strings.TrimSpace(candidatesXML) == "" {
|
||||
|
||||
@@ -0,0 +1,259 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/agent"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/models/chat"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// wikiTaxonomyReconcileMaxPages caps how many pages a single reconcile pass
|
||||
// will rewrite, so an over-eager remap cannot fan out into an unbounded write
|
||||
// storm on a very large KB. Pages beyond the cap are picked up by the next run.
|
||||
const wikiTaxonomyReconcileMaxPages = 5000
|
||||
|
||||
// wikiTaxonomyReconcilePageSize is the cursor page size used when scanning the
|
||||
// KB to apply a category remap.
|
||||
const wikiTaxonomyReconcilePageSize = 500
|
||||
|
||||
type wikiTaxonomyRemapEntry struct {
|
||||
From []string `json:"from"`
|
||||
To []string `json:"to"`
|
||||
}
|
||||
|
||||
type wikiTaxonomyRemapResult struct {
|
||||
Remap []wikiTaxonomyRemapEntry `json:"remap"`
|
||||
}
|
||||
|
||||
// ReconcileTaxonomy is the manual entry point for collapsing synonymous
|
||||
// category folders across a wiki knowledge base. It resolves the KB's synthesis
|
||||
// chat model itself so it can be invoked outside an ingest batch (e.g. from a
|
||||
// future maintenance handler).
|
||||
func (s *wikiIngestService) ReconcileTaxonomy(ctx context.Context, kbID string) error {
|
||||
kb, err := s.kbService.GetKnowledgeBaseByIDOnly(ctx, kbID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("wiki taxonomy reconcile: get KB: %w", err)
|
||||
}
|
||||
if !kb.IsWikiEnabled() {
|
||||
return fmt.Errorf("wiki taxonomy reconcile: KB %s is not wiki type", kbID)
|
||||
}
|
||||
|
||||
synthesisModelID := ""
|
||||
if kb.WikiConfig != nil {
|
||||
synthesisModelID = kb.WikiConfig.SynthesisModelID
|
||||
}
|
||||
if synthesisModelID == "" {
|
||||
synthesisModelID = kb.SummaryModelID
|
||||
}
|
||||
if synthesisModelID == "" {
|
||||
return fmt.Errorf("wiki taxonomy reconcile: no synthesis model configured for KB %s", kbID)
|
||||
}
|
||||
chatModel, err := s.modelService.GetChatModel(ctx, synthesisModelID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("wiki taxonomy reconcile: get chat model: %w", err)
|
||||
}
|
||||
|
||||
return s.reconcileTaxonomy(ctx, chatModel, kbID, types.LanguageNameFromContext(ctx))
|
||||
}
|
||||
|
||||
// reconcileTaxonomy collapses synonymous category_path folders across the KB
|
||||
// onto a single canonical path. It is per-KB (cost O(folder count), not page
|
||||
// count): one LLM call clusters the distinct folders, then matching pages are
|
||||
// re-pointed in Go. Slugs are NEVER touched — only the navigation category.
|
||||
func (s *wikiIngestService) reconcileTaxonomy(ctx context.Context, chatModel chat.Chat, kbID, lang string) error {
|
||||
if s.wikiService == nil || chatModel == nil {
|
||||
return nil
|
||||
}
|
||||
paths, err := s.wikiService.ListDistinctCategoryPaths(ctx, kbID, wikiTaxonomyPromptMaxPaths)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list distinct category paths: %w", err)
|
||||
}
|
||||
if len(paths) < 2 {
|
||||
return nil // nothing to merge
|
||||
}
|
||||
|
||||
remap := s.buildTaxonomyRemap(ctx, chatModel, paths, lang)
|
||||
if len(remap) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
updated := s.applyTaxonomyRemap(ctx, kbID, remap)
|
||||
if updated > 0 {
|
||||
logger.Infof(ctx, "wiki taxonomy reconcile: re-pointed %d pages via %d rules (kb=%s)", updated, len(remap), kbID)
|
||||
if err := s.wikiService.RebuildIndexPage(ctx, kbID); err != nil {
|
||||
logger.Warnf(ctx, "wiki taxonomy reconcile: rebuild index failed: %v", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// maybeReconcileTaxonomy runs a reconcile pass only when this batch introduced
|
||||
// at least one category folder that was not present in startKeys (the snapshot
|
||||
// taken before the batch ran). This keeps the extra LLM call off the hot path
|
||||
// for batches that only reuse established folders.
|
||||
func (s *wikiIngestService) maybeReconcileTaxonomy(
|
||||
ctx context.Context, chatModel chat.Chat, kbID, lang string, startKeys map[string]bool,
|
||||
) {
|
||||
if s.wikiService == nil {
|
||||
return
|
||||
}
|
||||
paths, err := s.wikiService.ListDistinctCategoryPaths(ctx, kbID, wikiTaxonomyPromptMaxPaths)
|
||||
if err != nil || len(paths) < 2 {
|
||||
return
|
||||
}
|
||||
introducedNewFolder := false
|
||||
for _, p := range paths {
|
||||
if !startKeys[categoryPathKey(cleanCategoryPathParts(p))] {
|
||||
introducedNewFolder = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !introducedNewFolder {
|
||||
return
|
||||
}
|
||||
if err := s.reconcileTaxonomy(ctx, chatModel, kbID, lang); err != nil {
|
||||
logger.Warnf(ctx, "wiki taxonomy reconcile failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// snapshotCategoryKeys returns the set of canonical category keys currently in
|
||||
// use, used as the "before" baseline for maybeReconcileTaxonomy.
|
||||
func (s *wikiIngestService) snapshotCategoryKeys(ctx context.Context, kbID string) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
if s.wikiService == nil {
|
||||
return out
|
||||
}
|
||||
paths, err := s.wikiService.ListDistinctCategoryPaths(ctx, kbID, wikiTaxonomyPromptMaxPaths)
|
||||
if err != nil {
|
||||
return out
|
||||
}
|
||||
for _, p := range paths {
|
||||
out[categoryPathKey(cleanCategoryPathParts(p))] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// buildTaxonomyRemap asks the LLM to cluster synonymous folders and returns a
|
||||
// map keyed by the canonical key of each non-canonical path to its replacement.
|
||||
func (s *wikiIngestService) buildTaxonomyRemap(
|
||||
ctx context.Context, chatModel chat.Chat, paths [][]string, lang string,
|
||||
) map[string][]string {
|
||||
taxonomyText := formatExistingTaxonomyForPrompt(paths)
|
||||
if strings.TrimSpace(taxonomyText) == "" {
|
||||
return nil
|
||||
}
|
||||
raw, err := s.generateWithTemplate(ctx, chatModel, agent.WikiTaxonomyReconcilePrompt, map[string]string{
|
||||
"CurrentTaxonomy": taxonomyText,
|
||||
"Language": lang,
|
||||
})
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "wiki taxonomy reconcile: LLM call failed: %v", err)
|
||||
return nil
|
||||
}
|
||||
raw = cleanLLMJSON(raw)
|
||||
var parsed wikiTaxonomyRemapResult
|
||||
if jerr := json.Unmarshal([]byte(raw), &parsed); jerr != nil {
|
||||
logger.Warnf(ctx, "wiki taxonomy reconcile: parse failed: %v\nRaw: %s", jerr, raw)
|
||||
return nil
|
||||
}
|
||||
|
||||
out := make(map[string][]string, len(parsed.Remap))
|
||||
for _, e := range parsed.Remap {
|
||||
from := cleanCategoryPathParts(e.From)
|
||||
to := cleanCategoryPathParts(e.To)
|
||||
if len(from) == 0 || len(to) == 0 {
|
||||
continue
|
||||
}
|
||||
fromKey := categoryPathKey(from)
|
||||
if fromKey == categoryPathKey(to) {
|
||||
continue // no-op rule
|
||||
}
|
||||
out[fromKey] = to
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// applyTaxonomyRemap scans entity/concept pages and re-points any whose
|
||||
// category matches a remap rule. Returns the number of pages updated.
|
||||
func (s *wikiIngestService) applyTaxonomyRemap(ctx context.Context, kbID string, remap map[string][]string) int {
|
||||
var cursor string
|
||||
var updated int
|
||||
for updated < wikiTaxonomyReconcileMaxPages {
|
||||
pages, next, err := s.wikiService.ListPagesCursor(ctx, kbID, cursor, wikiTaxonomyReconcilePageSize)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "wiki taxonomy reconcile: list pages failed: %v", err)
|
||||
break
|
||||
}
|
||||
if len(pages) == 0 {
|
||||
break
|
||||
}
|
||||
for _, p := range pages {
|
||||
if p.PageType != types.WikiPageTypeEntity && p.PageType != types.WikiPageTypeConcept {
|
||||
continue
|
||||
}
|
||||
newPath, ok := remapCategoryPath([]string(p.CategoryPath), remap)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
p.CategoryPath = types.StringArray(newPath)
|
||||
p.WikiPath = "" // force recompute in normalizeWikiHierarchy
|
||||
if err := s.wikiService.UpdatePageMeta(ctx, p); err != nil {
|
||||
logger.Warnf(ctx, "wiki taxonomy reconcile: update %s failed: %v", p.Slug, err)
|
||||
continue
|
||||
}
|
||||
updated++
|
||||
if updated >= wikiTaxonomyReconcileMaxPages {
|
||||
break
|
||||
}
|
||||
}
|
||||
if next == "" {
|
||||
break
|
||||
}
|
||||
cursor = next
|
||||
}
|
||||
return updated
|
||||
}
|
||||
|
||||
// remapCategoryPath returns the canonical category for current when it matches
|
||||
// a remap "from" key, plus whether a change is needed. Pure (no DB) so it can
|
||||
// be unit-tested in isolation.
|
||||
func remapCategoryPath(current []string, remap map[string][]string) ([]string, bool) {
|
||||
cleaned := cleanCategoryPathParts(current)
|
||||
if len(cleaned) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
to, ok := remap[categoryPathKey(cleaned)]
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
if categoryPathKey(cleaned) == categoryPathKey(cleanCategoryPathParts(to)) {
|
||||
return nil, false
|
||||
}
|
||||
return append([]string(nil), to...), true
|
||||
}
|
||||
|
||||
// cleanCategoryPathParts normalizes a category path the same way the page
|
||||
// hierarchy normalizer does (separator/quote cleanup, type-label stripping,
|
||||
// depth cap) so remap keys match what is stored on pages.
|
||||
func cleanCategoryPathParts(parts []string) []string {
|
||||
out := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
for _, c := range cleanWikiCategoryPart(part) {
|
||||
out = append(out, c)
|
||||
if len(out) >= 3 {
|
||||
return out
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// categoryPathKey builds a stable comparison key for a cleaned category path.
|
||||
func categoryPathKey(parts []string) string {
|
||||
return strings.Join(parts, "\u0000")
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSplitCategoryLine(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
content string
|
||||
wantCat []string
|
||||
wantRest string
|
||||
}{
|
||||
{
|
||||
name: "two level category",
|
||||
content: "CATEGORY: 组织 / 企业\n# 标题\n正文",
|
||||
wantCat: []string{"组织", "企业"},
|
||||
wantRest: "# 标题\n正文",
|
||||
},
|
||||
{
|
||||
name: "single level category",
|
||||
content: "CATEGORY: 人物\n# 陈洋",
|
||||
wantCat: []string{"人物"},
|
||||
wantRest: "# 陈洋",
|
||||
},
|
||||
{
|
||||
name: "empty category keeps existing (nil) and strips line",
|
||||
content: "CATEGORY:\n# 标题",
|
||||
wantCat: nil,
|
||||
wantRest: "# 标题",
|
||||
},
|
||||
{
|
||||
name: "no category line left untouched",
|
||||
content: "# 标题\n正文",
|
||||
wantCat: nil,
|
||||
wantRest: "# 标题\n正文",
|
||||
},
|
||||
{
|
||||
name: "fullwidth colon and separators",
|
||||
content: "CATEGORY:组织|企业\n正文",
|
||||
wantCat: []string{"组织", "企业"},
|
||||
wantRest: "正文",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cat, rest := splitCategoryLine(tt.content)
|
||||
if strings.Join(cat, "/") != strings.Join(tt.wantCat, "/") {
|
||||
t.Fatalf("splitCategoryLine() cat = %v, want %v", cat, tt.wantCat)
|
||||
}
|
||||
if rest != tt.wantRest {
|
||||
t.Fatalf("splitCategoryLine() rest = %q, want %q", rest, tt.wantRest)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatExistingTaxonomyForPrompt(t *testing.T) {
|
||||
got := formatExistingTaxonomyForPrompt([][]string{
|
||||
{"春节", "传统习俗"},
|
||||
{"春节", "文化习俗", "节日习俗"},
|
||||
{"春节习俗"},
|
||||
{"产品定位"},
|
||||
})
|
||||
want := "产品定位\n" +
|
||||
"春节\n" +
|
||||
" 传统习俗\n" +
|
||||
" 文化习俗\n" +
|
||||
" 节日习俗\n" +
|
||||
"春节习俗"
|
||||
if got != want {
|
||||
t.Fatalf("formatExistingTaxonomyForPrompt():\n%q\nwant:\n%q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatExistingTaxonomyForPromptEmpty(t *testing.T) {
|
||||
if got := formatExistingTaxonomyForPrompt(nil); got != "" {
|
||||
t.Fatalf("formatExistingTaxonomyForPrompt(nil) = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDedupeWikiCategoryPaths(t *testing.T) {
|
||||
got := dedupeWikiCategoryPaths([][]string{
|
||||
{"春节", "传统习俗"},
|
||||
{" 春节 ", "传统习俗"},
|
||||
{"春节", "文化习俗"},
|
||||
{"", "空"},
|
||||
{},
|
||||
}, 10)
|
||||
if len(got) != 3 {
|
||||
t.Fatalf("dedupeWikiCategoryPaths() len = %d, want 3: %v", len(got), got)
|
||||
}
|
||||
if got[0][0] != "春节" || got[0][1] != "传统习俗" {
|
||||
t.Fatalf("dedupeWikiCategoryPaths()[0] = %v", got[0])
|
||||
}
|
||||
if got[1][1] != "文化习俗" {
|
||||
t.Fatalf("dedupeWikiCategoryPaths()[1] = %v", got[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveExistingTaxonomyForPrompt(t *testing.T) {
|
||||
t.Run("nil batch context", func(t *testing.T) {
|
||||
got := resolveExistingTaxonomyForPrompt(t.Context(), nil)
|
||||
if got == "" {
|
||||
t.Fatal("expected fallback text for nil batch context")
|
||||
}
|
||||
})
|
||||
t.Run("batch context with taxonomy", func(t *testing.T) {
|
||||
batchCtx := &WikiBatchContext{
|
||||
ExistingTaxonomy: func(ctx context.Context) string {
|
||||
return "春节\n 传统习俗"
|
||||
},
|
||||
}
|
||||
got := resolveExistingTaxonomyForPrompt(t.Context(), batchCtx)
|
||||
if got != "春节\n 传统习俗" {
|
||||
t.Fatalf("resolveExistingTaxonomyForPrompt() = %q", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRemapCategoryPath(t *testing.T) {
|
||||
remap := map[string][]string{
|
||||
categoryPathKey([]string{"春节习俗"}): {"春节", "传统习俗"},
|
||||
categoryPathKey([]string{"组织单位", "致谢人员"}): {"组织"},
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
current []string
|
||||
want []string
|
||||
wantHit bool
|
||||
}{
|
||||
{
|
||||
name: "single-label synonym remapped",
|
||||
current: []string{"春节习俗"},
|
||||
want: []string{"春节", "传统习俗"},
|
||||
wantHit: true,
|
||||
},
|
||||
{
|
||||
name: "document-role path collapsed to coarse folder",
|
||||
current: []string{"组织单位", "致谢人员"},
|
||||
want: []string{"组织"},
|
||||
wantHit: true,
|
||||
},
|
||||
{
|
||||
name: "noise normalized before matching",
|
||||
current: []string{"实体/春节习俗"},
|
||||
want: []string{"春节", "传统习俗"},
|
||||
wantHit: true,
|
||||
},
|
||||
{
|
||||
name: "unrelated path untouched",
|
||||
current: []string{"人物"},
|
||||
wantHit: false,
|
||||
},
|
||||
{
|
||||
name: "empty path untouched",
|
||||
current: nil,
|
||||
wantHit: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, hit := remapCategoryPath(tt.current, remap)
|
||||
if hit != tt.wantHit {
|
||||
t.Fatalf("remapCategoryPath() hit = %v, want %v", hit, tt.wantHit)
|
||||
}
|
||||
if !hit {
|
||||
return
|
||||
}
|
||||
if len(got) != len(tt.want) {
|
||||
t.Fatalf("remapCategoryPath() = %v, want %v", got, tt.want)
|
||||
}
|
||||
for i := range tt.want {
|
||||
if got[i] != tt.want[i] {
|
||||
t.Fatalf("remapCategoryPath()[%d] = %q, want %q (full=%v)", i, got[i], tt.want[i], got)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -67,6 +67,7 @@ func (s *wikiPageService) CreatePage(ctx context.Context, page *types.WikiPage)
|
||||
|
||||
// Parse outbound links from content
|
||||
page.OutLinks = s.parseOutLinks(page.Content)
|
||||
normalizeWikiHierarchy(page)
|
||||
|
||||
now := time.Now()
|
||||
page.CreatedAt = now
|
||||
@@ -115,6 +116,11 @@ func (s *wikiPageService) UpdatePage(ctx context.Context, page *types.WikiPage)
|
||||
existing.SourceRefs = page.SourceRefs
|
||||
existing.ChunkRefs = page.ChunkRefs
|
||||
existing.PageMetadata = page.PageMetadata
|
||||
existing.ParentSlug = page.ParentSlug
|
||||
existing.CategoryPath = page.CategoryPath
|
||||
existing.WikiPath = page.WikiPath
|
||||
existing.Depth = page.Depth
|
||||
existing.SortOrder = page.SortOrder
|
||||
existing.Status = page.Status
|
||||
existing.UpdatedAt = time.Now()
|
||||
|
||||
@@ -122,11 +128,18 @@ func (s *wikiPageService) UpdatePage(ctx context.Context, page *types.WikiPage)
|
||||
// when content shifts. Re-parse unconditionally to stay consistent with
|
||||
// the stored body.
|
||||
existing.OutLinks = s.parseOutLinks(existing.Content)
|
||||
normalizeWikiHierarchy(existing)
|
||||
|
||||
if contentChanged {
|
||||
if err := s.repo.Update(ctx, existing); err != nil {
|
||||
return nil, fmt.Errorf("update wiki page: %w", err)
|
||||
}
|
||||
// GORM's struct Updates path skips zero values, so persist hierarchy
|
||||
// metadata through the explicit map path as well. This keeps clearing
|
||||
// parent/category fields deterministic without changing version again.
|
||||
if err := s.repo.UpdateMeta(ctx, existing); err != nil {
|
||||
return nil, fmt.Errorf("update wiki page hierarchy meta: %w", err)
|
||||
}
|
||||
} else {
|
||||
// No user-visible change — persist bookkeeping fields but preserve
|
||||
// the version so downstream consumers can rely on it.
|
||||
@@ -145,6 +158,7 @@ func (s *wikiPageService) UpdatePage(ctx context.Context, page *types.WikiPage)
|
||||
|
||||
// UpdatePageMeta updates only metadata (status, source_refs) without version bump or link re-parse.
|
||||
func (s *wikiPageService) UpdatePageMeta(ctx context.Context, page *types.WikiPage) error {
|
||||
normalizeWikiHierarchy(page)
|
||||
page.UpdatedAt = time.Now()
|
||||
return s.repo.UpdateMeta(ctx, page)
|
||||
}
|
||||
@@ -192,6 +206,9 @@ func (s *wikiPageService) ListPages(ctx context.Context, req *types.WikiPageList
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, page := range pages {
|
||||
normalizeWikiHierarchy(page)
|
||||
}
|
||||
|
||||
pageSize := req.PageSize
|
||||
if pageSize < 1 {
|
||||
@@ -323,6 +340,9 @@ func (s *wikiPageService) GetIndexView(
|
||||
if entries == nil {
|
||||
entries = []types.WikiIndexEntry{}
|
||||
}
|
||||
for i := range entries {
|
||||
normalizeWikiIndexEntryHierarchy(&entries[i], pt)
|
||||
}
|
||||
next := ""
|
||||
// Only emit a cursor when a full page was returned AND more rows
|
||||
// remain past `offset + limit`. A short page or one that exactly
|
||||
@@ -811,6 +831,20 @@ func (s *wikiPageService) FindSimilarPages(ctx context.Context, kbID string, que
|
||||
return s.repo.FindSimilarPages(ctx, kbID, query, pageTypes, limit)
|
||||
}
|
||||
|
||||
// ListDistinctCategoryPaths returns unique category_path folders already used
|
||||
// by entity/concept pages. Used by wiki ingest to ground extraction prompts.
|
||||
func (s *wikiPageService) ListDistinctCategoryPaths(ctx context.Context, kbID string, maxPaths int) ([][]string, error) {
|
||||
return s.repo.ListDistinctCategoryPaths(ctx, kbID, maxPaths)
|
||||
}
|
||||
|
||||
// ListDistinctCategoryPathsByType returns unique category_path folders across
|
||||
// the given page types. The browser uses this to render directory nodes before
|
||||
// article pages are paged in; passing several types lets one tab (e.g. the
|
||||
// merged knowledge tab) build a single directory skeleton server-side.
|
||||
func (s *wikiPageService) ListDistinctCategoryPathsByType(ctx context.Context, kbID string, pageTypes []string, parentPath []string, maxPaths int) ([]types.WikiCategoryPath, error) {
|
||||
return s.repo.ListDistinctCategoryPathsByType(ctx, kbID, pageTypes, parentPath, maxPaths)
|
||||
}
|
||||
|
||||
// CountByType is a service-layer pass-through over the repo. Used by
|
||||
// the index intro path to frame the LLM prompt's "showing N of M" hint.
|
||||
func (s *wikiPageService) CountByType(ctx context.Context, kbID string) (map[string]int64, error) {
|
||||
@@ -917,6 +951,7 @@ func (s *wikiPageService) createDefaultPage(ctx context.Context, kbID string, sl
|
||||
Summary: title,
|
||||
Version: 1,
|
||||
}
|
||||
normalizeWikiHierarchy(page)
|
||||
|
||||
if err := s.repo.Create(ctx, page); err != nil {
|
||||
return nil, fmt.Errorf("create default %s page: %w", slug, err)
|
||||
@@ -924,6 +959,115 @@ func (s *wikiPageService) createDefaultPage(ctx context.Context, kbID string, sl
|
||||
return page, nil
|
||||
}
|
||||
|
||||
func normalizeWikiHierarchy(page *types.WikiPage) {
|
||||
if page == nil {
|
||||
return
|
||||
}
|
||||
page.ParentSlug = strings.TrimSpace(page.ParentSlug)
|
||||
|
||||
cleanPath := make(types.StringArray, 0, len(page.CategoryPath))
|
||||
for _, part := range page.CategoryPath {
|
||||
for _, cleanPart := range cleanWikiCategoryPart(part) {
|
||||
if !containsString(cleanPath, cleanPart) {
|
||||
cleanPath = append(cleanPath, cleanPart)
|
||||
}
|
||||
if len(cleanPath) >= 3 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(cleanPath) >= 3 {
|
||||
break
|
||||
}
|
||||
}
|
||||
page.CategoryPath = cleanPath
|
||||
page.Depth = len(cleanPath)
|
||||
|
||||
parts := make([]string, 0, len(cleanPath)+2)
|
||||
if page.PageType != "" {
|
||||
parts = append(parts, page.PageType)
|
||||
}
|
||||
for _, part := range cleanPath {
|
||||
parts = append(parts, part)
|
||||
}
|
||||
display := strings.TrimSpace(page.Title)
|
||||
if display == "" {
|
||||
display = strings.TrimSpace(page.Slug)
|
||||
}
|
||||
if display != "" {
|
||||
parts = append(parts, display)
|
||||
}
|
||||
page.WikiPath = strings.Join(parts, "/")
|
||||
}
|
||||
|
||||
func normalizeWikiIndexEntryHierarchy(entry *types.WikiIndexEntry, pageType string) {
|
||||
if entry == nil {
|
||||
return
|
||||
}
|
||||
|
||||
cleanPath := make(types.StringArray, 0, len(entry.CategoryPath))
|
||||
for _, part := range entry.CategoryPath {
|
||||
for _, cleanPart := range cleanWikiCategoryPart(part) {
|
||||
if !containsString(cleanPath, cleanPart) {
|
||||
cleanPath = append(cleanPath, cleanPart)
|
||||
}
|
||||
if len(cleanPath) >= 3 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(cleanPath) >= 3 {
|
||||
break
|
||||
}
|
||||
}
|
||||
entry.CategoryPath = cleanPath
|
||||
entry.Depth = len(cleanPath)
|
||||
|
||||
parts := make([]string, 0, len(cleanPath)+2)
|
||||
if strings.TrimSpace(pageType) != "" {
|
||||
parts = append(parts, strings.TrimSpace(pageType))
|
||||
}
|
||||
parts = append(parts, cleanPath...)
|
||||
display := strings.TrimSpace(entry.Title)
|
||||
if display == "" {
|
||||
display = strings.TrimSpace(entry.Slug)
|
||||
}
|
||||
if display != "" {
|
||||
parts = append(parts, display)
|
||||
}
|
||||
entry.WikiPath = strings.Join(parts, "/")
|
||||
}
|
||||
|
||||
func cleanWikiCategoryPart(part string) []string {
|
||||
part = strings.TrimSpace(part)
|
||||
if part == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
part = strings.NewReplacer("/", "/", "|", "/", "|", "/").Replace(part)
|
||||
rawParts := strings.Split(part, "/")
|
||||
cleaned := make([]string, 0, len(rawParts))
|
||||
for _, raw := range rawParts {
|
||||
label := strings.TrimSpace(raw)
|
||||
label = strings.Trim(label, `"'“”‘’[]()()`)
|
||||
label = strings.TrimSpace(label)
|
||||
if label == "" || isWikiTypeCategoryLabel(label) {
|
||||
continue
|
||||
}
|
||||
cleaned = append(cleaned, label)
|
||||
}
|
||||
return cleaned
|
||||
}
|
||||
|
||||
func isWikiTypeCategoryLabel(label string) bool {
|
||||
normalized := strings.ToLower(strings.TrimSpace(label))
|
||||
normalized = strings.TrimSuffix(normalized, "s")
|
||||
switch normalized {
|
||||
case "entity", "实体", "實體", "concept", "概念", "summary", "摘要", "wiki", "页面", "頁面":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// containsString checks if a string slice contains a given string
|
||||
func containsString(slice []string, s string) bool {
|
||||
for _, v := range slice {
|
||||
|
||||
@@ -99,6 +99,56 @@ func TestNormalizeSlug(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeWikiHierarchyCleansModelCategoryNoise(t *testing.T) {
|
||||
page := &types.WikiPage{
|
||||
Slug: "concept/ai-knowledge",
|
||||
Title: "AI时代的知识困境",
|
||||
PageType: types.WikiPageTypeConcept,
|
||||
CategoryPath: types.StringArray{"概念/爱护花草", " 实体/标牌 ", "摘要", "生态/平台", "多余层级"},
|
||||
}
|
||||
|
||||
normalizeWikiHierarchy(page)
|
||||
|
||||
want := types.StringArray{"爱护花草", "标牌", "生态"}
|
||||
if len(page.CategoryPath) != len(want) {
|
||||
t.Fatalf("CategoryPath = %v, want %v", page.CategoryPath, want)
|
||||
}
|
||||
for i := range want {
|
||||
if page.CategoryPath[i] != want[i] {
|
||||
t.Fatalf("CategoryPath[%d] = %q, want %q; full path=%v", i, page.CategoryPath[i], want[i], page.CategoryPath)
|
||||
}
|
||||
}
|
||||
if page.Depth != 3 {
|
||||
t.Fatalf("Depth = %d, want 3", page.Depth)
|
||||
}
|
||||
if page.WikiPath != "concept/爱护花草/标牌/生态/AI时代的知识困境" {
|
||||
t.Fatalf("WikiPath = %q", page.WikiPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeWikiIndexEntryHierarchyCleansModelCategoryNoise(t *testing.T) {
|
||||
entry := &types.WikiIndexEntry{
|
||||
Slug: "entity/sign",
|
||||
Title: "爱护花草标牌",
|
||||
CategoryPath: types.StringArray{"实体/标牌", "概念", "生态/行为倡导"},
|
||||
}
|
||||
|
||||
normalizeWikiIndexEntryHierarchy(entry, types.WikiPageTypeEntity)
|
||||
|
||||
want := types.StringArray{"标牌", "生态", "行为倡导"}
|
||||
if len(entry.CategoryPath) != len(want) {
|
||||
t.Fatalf("CategoryPath = %v, want %v", entry.CategoryPath, want)
|
||||
}
|
||||
for i := range want {
|
||||
if entry.CategoryPath[i] != want[i] {
|
||||
t.Fatalf("CategoryPath[%d] = %q, want %q; full path=%v", i, entry.CategoryPath[i], want[i], entry.CategoryPath)
|
||||
}
|
||||
}
|
||||
if entry.WikiPath != "entity/标牌/生态/行为倡导/爱护花草标牌" {
|
||||
t.Fatalf("WikiPath = %q", entry.WikiPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContainsString(t *testing.T) {
|
||||
slice := []string{"a", "b", "c"}
|
||||
if !containsString(slice, "b") {
|
||||
|
||||
Reference in New Issue
Block a user