Files
Enterprise-Knowledge-Base/internal/application/service/wiki_slug_alias.go
T
wizardchen 5d0326f987 fix(wiki): harden slug integrity across ingest and agent write paths
Weak LLMs routinely mangle high-entropy wiki slugs (especially UUID-based
summary/<id> slugs), producing dead cross-links and cross-item dedup
hallucinations. This tightens integrity end to end:

- Ingest dedup now groups each new item with only its own trigram
  candidates and validates merges against that per-item scope, rejecting
  cross-item mispairings; candidate top-K narrowed 20 -> 5.
- Editor path aliases out-link slugs behind low-entropy ref-N tokens and
  translates them back after generation, removing the chance to mangle.
- Agent write tool normalizes/validates slugs, blocks manual creation of
  system-owned summary pages, and auto-repairs dead [[slug]] links
  (rewrite-only, never strips) via bigram/display-text resolution.
- llmresource registry aliases summary/<uuid> slugs the same way as
  storage handles so they round-trip safely through chat.

Adds unit tests for slug validation, aliasing round-trip, dead-link
repair, and dedup rejection rules.
2026-07-24 15:57:08 +08:00

89 lines
2.9 KiB
Go

package service
import (
"fmt"
"strings"
)
// slugAliaser maps real wiki slugs to short, low-entropy reference tokens
// (ref-1, ref-2, …) so the ingest editor LLM never has to reproduce a
// high-entropy slug verbatim — most importantly the UUID-based summary slugs
// (summary/<knowledgeID>) that models routinely mangle by inserting or
// dropping hex digits. The model copies the tiny token; we translate tokens
// back to real slugs on output.
//
// This is the generation-stage counterpart to wiki_write_page's slug
// validation: rather than repairing a mangled slug after the fact, it removes
// the opportunity to mangle at the source. It mirrors the chunk-alias
// (c000/c001) indirection already used by the chunk-citation pass in
// wiki_ingest_cite.go.
//
// A slugAliaser is single-use and NOT safe for concurrent use; construct one
// per LLM call.
type slugAliaser struct {
toToken map[string]string // real slug -> token
toSlug map[string]string // token -> real slug
}
func newSlugAliaser() *slugAliaser {
return &slugAliaser{
toToken: make(map[string]string),
toSlug: make(map[string]string),
}
}
// token returns the stable token for a real slug, assigning a fresh one on
// first use. An empty slug returns "". Tokens have no '/' so they can never
// collide with a namespaced real slug (entity/…, summary/…).
func (a *slugAliaser) token(realSlug string) string {
realSlug = strings.TrimSpace(realSlug)
if realSlug == "" {
return ""
}
if t, ok := a.toToken[realSlug]; ok {
return t
}
t := fmt.Sprintf("ref-%d", len(a.toToken)+1)
a.toToken[realSlug] = t
a.toSlug[t] = realSlug
return t
}
// empty reports whether no tokens have been assigned yet (nothing to alias).
func (a *slugAliaser) empty() bool { return len(a.toToken) == 0 }
// aliasContent rewrites [[realSlug|disp]] / [[realSlug]] occurrences to their
// token form, but only for slugs present in `known`. Unknown links are left
// untouched. Assigns tokens on demand so a slug that appears only in the body
// (not in the listing) still gets a consistent token.
func (a *slugAliaser) aliasContent(content string, known map[string]struct{}) string {
if content == "" || len(known) == 0 {
return content
}
out, _ := rewriteDeadWikiLinks(content, func(norm, _ string) (string, bool) {
if _, ok := known[norm]; !ok {
return "", false
}
return a.token(norm), true
})
return out
}
// dealiasContent rewrites [[token|disp]] / [[token]] occurrences back to their
// real slug. Tokens with no mapping are left untouched — they fall through to
// the ordinary parse/dead-link-cleanup path, which is the correct behavior for
// anything the model invented that isn't a known reference.
func (a *slugAliaser) dealiasContent(content string) string {
if content == "" || a.empty() {
return content
}
out, _ := rewriteDeadWikiLinks(content, func(norm, _ string) (string, bool) {
real, ok := a.toSlug[norm]
if !ok {
return "", false
}
return real, true
})
return out
}