Files
Enterprise-Knowledge-Base/internal/application/service/wiki_ingest_dedup.go
T
wizardchen 5d0326f987 fix(wiki): harden slug integrity across ingest and agent write paths
Weak LLMs routinely mangle high-entropy wiki slugs (especially UUID-based
summary/<id> slugs), producing dead cross-links and cross-item dedup
hallucinations. This tightens integrity end to end:

- Ingest dedup now groups each new item with only its own trigram
  candidates and validates merges against that per-item scope, rejecting
  cross-item mispairings; candidate top-K narrowed 20 -> 5.
- Editor path aliases out-link slugs behind low-entropy ref-N tokens and
  translates them back after generation, removing the chance to mangle.
- Agent write tool normalizes/validates slugs, blocks manual creation of
  system-owned summary pages, and auto-repairs dead [[slug]] links
  (rewrite-only, never strips) via bigram/display-text resolution.
- llmresource registry aliases summary/<uuid> slugs the same way as
  storage handles so they round-trip safely through chat.

Adds unit tests for slug validation, aliasing round-trip, dead-link
repair, and dedup rejection rules.
2026-07-24 15:57:08 +08:00

300 lines
10 KiB
Go

package service
import (
"sort"
"strings"
"unicode"
"github.com/Tencent/WeKnora/internal/searchutil"
"github.com/Tencent/WeKnora/internal/types"
)
// Pre-filtering candidate existing pages before the dedup LLM call.
//
// Without pre-filtering, the dedup prompt packs the entire entity+concept
// page corpus into <existing_pages>. On knowledge bases with 100+ pages
// this inflates input tokens and — more importantly — gives weaker LLMs
// enough rope to hallucinate merges between totally unrelated slugs just
// because the output looked plausible. Observed cases include
// "城镇登记失业人员" → "中华优秀传统文化" (zero shared characters).
//
// The filter below keeps only pages that share at least some cheap
// surface-level signal with one of the new items. Fast to compute, no
// external calls, and it only ever *removes* candidates from the prompt —
// the downstream validMerge check still guards the final write.
const (
// dedupCandidateTopK bounds how many trigram-similar existing pages
// each new item's similarity probe returns (the LIMIT passed to
// FindSimilarPages, applied per query term = name + each alias). Kept
// deliberately small: the DB already orders by similarity desc behind a
// pg_trgm threshold, so a genuine same-entity target is essentially
// always the top hit. A tight K keeps each item's <candidates> list in
// the dedup prompt short, which improves the model's precision and cuts
// tokens, at negligible recall cost.
dedupCandidateTopK = 5
// dedupCandidateScoreFloor is the Jaccard floor. Pairs at or above
// this similarity are always included regardless of the top-K cap.
// Tuned so that "城镇登记失业人员" vs "中华优秀传统文化" (Jaccard 0)
// is excluded while "Acme Corp" vs "Acme Corporation" (Jaccard ≈ 0.5)
// clearly passes.
dedupCandidateScoreFloor = 0.08
// dedupSmallCorpusBypass skips pre-filtering entirely when the
// existing-page corpus is already small enough to fit in the prompt
// without degrading the LLM. The filter only earns its keep on large
// KBs; on small ones it risks cutting legitimate matches with no
// real token savings.
dedupSmallCorpusBypass = 25
)
// dedupSurface is the pre-computed similarity feature set for one side of
// a (new item, existing page) comparison.
type dedupSurface struct {
// slugTokens are the kebab-case tokens from the slug base (after "/").
// Slugs are an orthogonal signal to the surface names — Chinese pages
// carry their pinyin here, which keeps the filter useful on purely
// Latin-script new items too.
slugTokens map[string]struct{}
// nameGramSets holds one char-bigram set per surface form (name and
// each alias). We keep them separate so the pair score is max-over-
// surfaces — a rare alias match shouldn't be diluted by the primary
// name disagreeing.
nameGramSets []map[string]struct{}
}
// countEntityConceptPages returns how many of the given pages are
// entity- or concept-typed. Used only for logging the prefilter's
// reduction ratio.
func countEntityConceptPages(pages []*types.WikiPage) int {
n := 0
for _, p := range pages {
if p == nil {
continue
}
if p.PageType == types.WikiPageTypeEntity || p.PageType == types.WikiPageTypeConcept {
n++
}
}
return n
}
// selectDedupCandidatePages returns the subset of allPages plausibly
// related to at least one of newItems. Non-entity/concept pages are
// dropped unconditionally. The returned slice preserves the input order
// so the downstream prompt stays stable across runs.
//
// On small corpora (<= dedupSmallCorpusBypass entries) this is a no-op
// aside from the page-type filter.
func selectDedupCandidatePages(
newItems []extractedItem,
allPages []*types.WikiPage,
) []*types.WikiPage {
pages := make([]*types.WikiPage, 0, len(allPages))
for _, p := range allPages {
if p == nil {
continue
}
if p.PageType != types.WikiPageTypeEntity && p.PageType != types.WikiPageTypeConcept {
continue
}
pages = append(pages, p)
}
if len(pages) == 0 {
return pages
}
if len(newItems) == 0 || len(pages) <= dedupSmallCorpusBypass {
return pages
}
pageFeats := make([]dedupSurface, len(pages))
for i, p := range pages {
surfaces := make([]string, 0, 1+len(p.Aliases))
surfaces = append(surfaces, p.Title)
surfaces = append(surfaces, []string(p.Aliases)...)
pageFeats[i] = dedupSurface{
slugTokens: slugBaseTokens(p.Slug),
nameGramSets: gramsPerSurface(surfaces),
}
}
selected := make(map[int]bool, len(pages))
for _, it := range newItems {
surfaces := make([]string, 0, 1+len(it.Aliases))
surfaces = append(surfaces, it.Name)
surfaces = append(surfaces, it.Aliases...)
itemFeat := dedupSurface{
slugTokens: slugBaseTokens(it.Slug),
nameGramSets: gramsPerSurface(surfaces),
}
if len(itemFeat.slugTokens) == 0 && len(itemFeat.nameGramSets) == 0 {
continue
}
scores := make([]struct {
idx int
score float64
}, len(pageFeats))
for i := range pageFeats {
scores[i].idx = i
scores[i].score = dedupPairScore(itemFeat, pageFeats[i])
}
// Stable sort so ties break deterministically by original index.
sort.SliceStable(scores, func(i, j int) bool {
return scores[i].score > scores[j].score
})
topKRemaining := dedupCandidateTopK
for _, s := range scores {
if s.score >= dedupCandidateScoreFloor {
selected[s.idx] = true
continue
}
// Below the floor but we still owe the LLM some candidates so
// it can decline cleanly — fill the top-K budget with the
// highest-scoring remaining pages, as long as the score isn't
// flatly zero (a zero score means we have nothing in common
// with this page and including it just invites hallucination).
if topKRemaining > 0 && s.score > 0 {
selected[s.idx] = true
topKRemaining--
continue
}
break
}
}
out := make([]*types.WikiPage, 0, len(selected))
for i, p := range pages {
if selected[i] {
out = append(out, p)
}
}
return out
}
// dedupMergeRejectReason validates a single LLM-proposed merge (srcSlug →
// dstSlug) against deterministic, model-independent rules. It returns an
// empty string when the merge is allowed, or a short human-readable reason
// when it must be rejected. srcCandidates is the set of existing-page slugs
// that surfaced for srcSlug's OWN similarity probe (see itemCandidates in
// deduplicateExtractedBatch).
//
// The per-item scoping check is the key guard: the dedup prompt shows the
// model a flattened union of candidates across every new item, so nothing
// stops a weak model from pairing an item with a page that was only similar
// to a *different* item (observed: entity/tencent-open → entity/hiring-agent,
// which share no trigram signal). Requiring dstSlug to be one of srcSlug's
// own candidates rejects that entire class without depending on the model
// getting the semantic judgment right.
func dedupMergeRejectReason(srcSlug, dstSlug string, srcCandidates map[string]bool) string {
if !srcCandidates[dstSlug] {
// Covers both an outright hallucinated target (in no candidate
// set at all) and a real page that was only similar to another
// item. Either way the pair lacks a similarity signal for THIS
// item, so it is not a safe merge.
return "target is not a similarity candidate for this item"
}
srcSlash := strings.Index(srcSlug, "/")
dstSlash := strings.Index(dstSlug, "/")
if srcSlash <= 0 || dstSlash <= 0 {
// A type-prefixed slug must look like "entity/foo" or
// "concept/bar". An LLM that emits an un-prefixed slug here is
// hallucinating; reject rather than fall through the prefix-
// equality check (which would treat both empty prefixes as a
// match).
return "missing type prefix"
}
if srcSlug[:srcSlash+1] != dstSlug[:dstSlash+1] {
return "type mismatch: " + srcSlug[:srcSlash+1] + " vs " + dstSlug[:dstSlash+1]
}
return ""
}
// dedupPairScore is the max similarity between any surface form of a and
// b (plus the slug-token similarity). Slug and name signals live in
// different symbol spaces (ASCII pinyin vs raw surface form) so we take
// the max rather than e.g. their average.
func dedupPairScore(a, b dedupSurface) float64 {
best := searchutil.Jaccard(a.slugTokens, b.slugTokens)
for _, ag := range a.nameGramSets {
for _, bg := range b.nameGramSets {
if v := searchutil.Jaccard(ag, bg); v > best {
best = v
}
}
}
return best
}
// slugBaseTokens returns the kebab-case tokens of a slug's base component.
// "entity/beijing-nongshang-yinxing" → {"beijing", "nongshang", "yinxing"}.
func slugBaseTokens(slug string) map[string]struct{} {
if slug == "" {
return nil
}
base := slug
if i := strings.Index(slug, "/"); i >= 0 {
base = slug[i+1:]
}
base = strings.ToLower(base)
fields := strings.FieldsFunc(base, func(r rune) bool {
return r == '-' || r == '_' || r == '.' || unicode.IsSpace(r)
})
if len(fields) == 0 {
return nil
}
out := make(map[string]struct{}, len(fields))
for _, tok := range fields {
if tok == "" {
continue
}
out[tok] = struct{}{}
}
return out
}
// gramsPerSurface computes a gram set per non-empty surface form.
func gramsPerSurface(surfaces []string) []map[string]struct{} {
out := make([]map[string]struct{}, 0, len(surfaces))
for _, s := range surfaces {
g := surfaceGrams(s)
if len(g) > 0 {
out = append(out, g)
}
}
return out
}
// surfaceGrams returns a character-bigram set for a surface form after
// lowercasing and stripping non-letter/digit runes. Bigrams work well
// across both CJK (where each bigram approximates a word) and Latin
// (where they catch stem overlap like "corporation" ↔ "corp"). Single-
// rune strings fall back to a 1-gram so they still contribute a signal.
func surfaceGrams(s string) map[string]struct{} {
if s == "" {
return nil
}
var b strings.Builder
b.Grow(len(s))
for _, r := range strings.ToLower(s) {
if unicode.IsLetter(r) || unicode.IsDigit(r) {
b.WriteRune(r)
}
}
runes := []rune(b.String())
if len(runes) == 0 {
return nil
}
out := make(map[string]struct{}, len(runes))
if len(runes) == 1 {
out[string(runes)] = struct{}{}
return out
}
for i := 0; i < len(runes)-1; i++ {
out[string(runes[i:i+2])] = struct{}{}
}
return out
}