Files
Enterprise-Knowledge-Base/internal/application/service/wiki_ingest_dedup_test.go
T
wizardchen 5d0326f987 fix(wiki): harden slug integrity across ingest and agent write paths
Weak LLMs routinely mangle high-entropy wiki slugs (especially UUID-based
summary/<id> slugs), producing dead cross-links and cross-item dedup
hallucinations. This tightens integrity end to end:

- Ingest dedup now groups each new item with only its own trigram
  candidates and validates merges against that per-item scope, rejecting
  cross-item mispairings; candidate top-K narrowed 20 -> 5.
- Editor path aliases out-link slugs behind low-entropy ref-N tokens and
  translates them back after generation, removing the chance to mangle.
- Agent write tool normalizes/validates slugs, blocks manual creation of
  system-owned summary pages, and auto-repairs dead [[slug]] links
  (rewrite-only, never strips) via bigram/display-text resolution.
- llmresource registry aliases summary/<uuid> slugs the same way as
  storage handles so they round-trip safely through chat.

Adds unit tests for slug validation, aliasing round-trip, dead-link
repair, and dedup rejection rules.
2026-07-24 15:57:08 +08:00

247 lines
8.7 KiB
Go

package service
import (
"strings"
"testing"
"github.com/Tencent/WeKnora/internal/types"
)
func TestDedupMergeRejectReason(t *testing.T) {
cases := []struct {
name string
src, dst string
candidates map[string]bool
wantAllowed bool
}{
{
name: "allowed: dst is a candidate for this item, same type prefix",
src: "entity/acme-corp",
dst: "entity/acme-corporation",
candidates: map[string]bool{"entity/acme-corporation": true},
wantAllowed: true,
},
{
name: "rejected: dst similar to a DIFFERENT item (union hallucination)",
src: "entity/tencent-open",
dst: "entity/hiring-agent",
candidates: map[string]bool{"entity/tencent-ur": true}, // hiring-agent not here
wantAllowed: false,
},
{
name: "rejected: dst is not a candidate at all (pure hallucination)",
src: "entity/hy3-preview",
dst: "entity/llm-cli-tool",
candidates: nil,
wantAllowed: false,
},
{
name: "rejected: type mismatch even when dst is a candidate",
src: "entity/foo",
dst: "concept/foo",
candidates: map[string]bool{"concept/foo": true},
wantAllowed: false,
},
{
name: "rejected: missing type prefix",
src: "foo",
dst: "entity/foo",
candidates: map[string]bool{"entity/foo": true},
wantAllowed: false,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
reason := dedupMergeRejectReason(tc.src, tc.dst, tc.candidates)
gotAllowed := reason == ""
if gotAllowed != tc.wantAllowed {
t.Fatalf("dedupMergeRejectReason(%q, %q) allowed=%v (reason=%q), want allowed=%v",
tc.src, tc.dst, gotAllowed, reason, tc.wantAllowed)
}
})
}
}
// helper: build a minimal entity/concept WikiPage.
func makePage(slug, title, typ string, aliases ...string) *types.WikiPage {
return &types.WikiPage{
Slug: slug,
Title: title,
PageType: typ,
Aliases: types.StringArray(aliases),
}
}
func pageSlugs(pages []*types.WikiPage) []string {
out := make([]string, 0, len(pages))
for _, p := range pages {
out = append(out, p.Slug)
}
return out
}
func containsSlug(pages []*types.WikiPage, slug string) bool {
for _, p := range pages {
if p.Slug == slug {
return true
}
}
return false
}
// Regression for the observed hallucination in llm_debug/20260422_171316.675.log:
// deepseek-v3.2 merged concept/chengzhen-dengji-shiye-renyuan into
// concept/zhong-hua-you-xiu-chuan-tong-wen-hua despite zero character overlap.
// With the prefilter in place, the traditional-culture page should not even
// be offered to the LLM as a candidate for that new item.
func TestSelectDedupCandidatePages_FiltersUnrelatedHallucinationTarget(t *testing.T) {
newItems := []extractedItem{
{Slug: "concept/chengzhen-dengji-shiye-renyuan", Name: "城镇登记失业人员", Aliases: []string{"登记失业人员"}},
{Slug: "entity/beijing-nongshang-yinxing", Name: "北京农商银行"},
}
// Build a corpus that mirrors the real log's shape: ~90 unrelated pages
// plus a few that share tokens with the new items (so the prefilter has
// plausible near-matches to keep).
pages := []*types.WikiPage{
makePage("concept/zhong-hua-you-xiu-chuan-tong-wen-hua", "中华优秀传统文化", "concept"),
makePage("concept/jiuye-jineng-peixun", "就业技能培训", "concept"),
makePage("concept/peixun-kaohe-pingjia", "培训考核评价", "concept"),
makePage("concept/ren-gong-zhi-neng-an-quan", "人工智能安全", "concept", "AI安全"),
makePage("entity/bei-jing-shi-jiao-yu-wei-yuan-hui", "北京市教育委员会", "entity", "北京市教委"),
makePage("entity/beijingshi-changping-zhiye-xuexiao", "北京市昌平职业学校", "entity"),
}
// Pad with filler so we exceed dedupSmallCorpusBypass.
for i := 0; i < 40; i++ {
pages = append(pages, makePage(
"concept/filler-"+strings.Repeat("x", i+1),
"填充概念"+strings.Repeat("占位", i+1),
"concept",
))
}
got := selectDedupCandidatePages(newItems, pages)
if containsSlug(got, "concept/zhong-hua-you-xiu-chuan-tong-wen-hua") {
t.Fatalf("expected unrelated page to be filtered out, but got it in candidates: %v",
pageSlugs(got))
}
if len(got) >= len(pages) {
t.Fatalf("expected prefilter to shrink the corpus (%d pages), but kept %d",
len(pages), len(got))
}
}
// A related page (shares tokens / characters with a new item) must survive
// the prefilter so the LLM can still evaluate the merge.
func TestSelectDedupCandidatePages_KeepsRelatedPages(t *testing.T) {
newItems := []extractedItem{
{Slug: "concept/chengzhen-dengji-shiye-renyuan", Name: "城镇登记失业人员", Aliases: []string{"登记失业人员"}},
}
// Build > dedupSmallCorpusBypass pages so the filter actually runs.
pages := []*types.WikiPage{
// Directly related: existing page whose title overlaps with the new
// item on "登记失业人员". Prefilter MUST keep this.
makePage("concept/deng-ji-shi-ye-ren-yuan", "登记失业人员", "concept", "城镇登记失业人员"),
// Unrelated.
makePage("concept/zhong-hua-you-xiu-chuan-tong-wen-hua", "中华优秀传统文化", "concept"),
}
for i := 0; i < 30; i++ {
pages = append(pages, makePage(
"entity/filler-"+strings.Repeat("x", i+1),
"填充实体"+strings.Repeat("占位", i+1),
"entity",
))
}
got := selectDedupCandidatePages(newItems, pages)
if !containsSlug(got, "concept/deng-ji-shi-ye-ren-yuan") {
t.Fatalf("expected strongly-related page to be kept, got: %v", pageSlugs(got))
}
}
// On small corpora the filter should be a no-op (minus page-type filtering):
// passing every page through is cheap and avoids cutting legitimate matches
// when the prompt is already small.
func TestSelectDedupCandidatePages_SmallCorpusBypass(t *testing.T) {
newItems := []extractedItem{
{Slug: "concept/a", Name: "A"},
}
pages := []*types.WikiPage{
makePage("concept/wholly-unrelated-1", "毫不相关一", "concept"),
makePage("concept/wholly-unrelated-2", "毫不相关二", "concept"),
makePage("concept/wholly-unrelated-3", "毫不相关三", "concept"),
}
got := selectDedupCandidatePages(newItems, pages)
if len(got) != len(pages) {
t.Fatalf("expected bypass on small corpus: got %d, want %d", len(got), len(pages))
}
}
// Non-entity/concept pages (summaries, logs, …) must be stripped regardless
// of corpus size — they are never valid merge targets.
func TestSelectDedupCandidatePages_DropsNonEntityConcept(t *testing.T) {
newItems := []extractedItem{
{Slug: "concept/foo", Name: "Foo"},
}
pages := []*types.WikiPage{
makePage("summary/some-doc", "Some Doc Summary", types.WikiPageTypeSummary),
makePage("log/batch-123", "Batch 123", types.WikiPageTypeLog),
makePage("concept/foo-related", "Foo Related", types.WikiPageTypeConcept),
}
got := selectDedupCandidatePages(newItems, pages)
for _, p := range got {
if p.PageType != types.WikiPageTypeEntity && p.PageType != types.WikiPageTypeConcept {
t.Fatalf("non-entity/concept page should have been filtered: %s (%s)",
p.Slug, p.PageType)
}
}
}
// surfaceGrams must yield empty intersection for the real hallucinated pair,
// confirming the underlying similarity signal is doing its job.
func TestSurfaceGrams_UnrelatedCJKPair(t *testing.T) {
a := surfaceGrams("城镇登记失业人员")
b := surfaceGrams("中华优秀传统文化")
for k := range a {
if _, ok := b[k]; ok {
t.Fatalf("expected zero bigram overlap, but shared %q", k)
}
}
}
// Latin abbreviation ↔ full-name pair must score highly (> floor) so the
// filter keeps legitimate merge candidates like "Acme Corp" ↔ "Acme Corporation".
func TestDedupPairScore_AcmeCorpVariant(t *testing.T) {
a := dedupSurface{
slugTokens: slugBaseTokens("entity/acme-corp"),
nameGramSets: gramsPerSurface([]string{"Acme Corp"}),
}
b := dedupSurface{
slugTokens: slugBaseTokens("entity/acme-corporation"),
nameGramSets: gramsPerSurface([]string{"Acme Corporation"}),
}
score := dedupPairScore(a, b)
if score < dedupCandidateScoreFloor {
t.Fatalf("expected Acme Corp ↔ Corporation score above floor %v, got %v",
dedupCandidateScoreFloor, score)
}
}
// Unrelated CJK pair (the exact case observed in production) must score 0.
func TestDedupPairScore_UnrelatedCJKPair(t *testing.T) {
a := dedupSurface{
slugTokens: slugBaseTokens("concept/chengzhen-dengji-shiye-renyuan"),
nameGramSets: gramsPerSurface([]string{"城镇登记失业人员", "登记失业人员"}),
}
b := dedupSurface{
slugTokens: slugBaseTokens("concept/zhong-hua-you-xiu-chuan-tong-wen-hua"),
nameGramSets: gramsPerSurface([]string{"中华优秀传统文化"}),
}
if s := dedupPairScore(a, b); s >= dedupCandidateScoreFloor {
t.Fatalf("expected unrelated CJK pair to score below floor %v, got %v",
dedupCandidateScoreFloor, s)
}
}