Files
Enterprise-Knowledge-Base/internal/application/service/wiki_ingest_dedup_test.go
T
wizardchen a8a3d9e694 feat: Enhance wiki extraction and configuration options
- Introduced extraction granularity settings in the wiki configuration, allowing users to control the level of detail in entity and concept extraction (focused, standard, exhaustive).
- Updated the UI to reflect these new options, including tooltips for better user guidance.
- Refactored the knowledge base editor to support the new extraction granularity feature, ensuring a seamless user experience.
- Improved the backend logic for wiki ingest tasks to utilize the new granularity settings, enhancing the accuracy of extracted content.
- Added tests to validate the new extraction granularity functionality and ensure robust performance across different configurations.
2026-04-22 21:18:32 +08:00

192 lines
7.1 KiB
Go

package service
import (
"strings"
"testing"
"github.com/Tencent/WeKnora/internal/types"
)
// helper: build a minimal entity/concept WikiPage.
func makePage(slug, title, typ string, aliases ...string) *types.WikiPage {
return &types.WikiPage{
Slug: slug,
Title: title,
PageType: typ,
Aliases: types.StringArray(aliases),
}
}
func pageSlugs(pages []*types.WikiPage) []string {
out := make([]string, 0, len(pages))
for _, p := range pages {
out = append(out, p.Slug)
}
return out
}
func containsSlug(pages []*types.WikiPage, slug string) bool {
for _, p := range pages {
if p.Slug == slug {
return true
}
}
return false
}
// Regression for the observed hallucination in llm_debug/20260422_171316.675.log:
// deepseek-v3.2 merged concept/chengzhen-dengji-shiye-renyuan into
// concept/zhong-hua-you-xiu-chuan-tong-wen-hua despite zero character overlap.
// With the prefilter in place, the traditional-culture page should not even
// be offered to the LLM as a candidate for that new item.
func TestSelectDedupCandidatePages_FiltersUnrelatedHallucinationTarget(t *testing.T) {
newItems := []extractedItem{
{Slug: "concept/chengzhen-dengji-shiye-renyuan", Name: "城镇登记失业人员", Aliases: []string{"登记失业人员"}},
{Slug: "entity/beijing-nongshang-yinxing", Name: "北京农商银行"},
}
// Build a corpus that mirrors the real log's shape: ~90 unrelated pages
// plus a few that share tokens with the new items (so the prefilter has
// plausible near-matches to keep).
pages := []*types.WikiPage{
makePage("concept/zhong-hua-you-xiu-chuan-tong-wen-hua", "中华优秀传统文化", "concept"),
makePage("concept/jiuye-jineng-peixun", "就业技能培训", "concept"),
makePage("concept/peixun-kaohe-pingjia", "培训考核评价", "concept"),
makePage("concept/ren-gong-zhi-neng-an-quan", "人工智能安全", "concept", "AI安全"),
makePage("entity/bei-jing-shi-jiao-yu-wei-yuan-hui", "北京市教育委员会", "entity", "北京市教委"),
makePage("entity/beijingshi-changping-zhiye-xuexiao", "北京市昌平职业学校", "entity"),
}
// Pad with filler so we exceed dedupSmallCorpusBypass.
for i := 0; i < 40; i++ {
pages = append(pages, makePage(
"concept/filler-"+strings.Repeat("x", i+1),
"填充概念"+strings.Repeat("占位", i+1),
"concept",
))
}
got := selectDedupCandidatePages(newItems, pages)
if containsSlug(got, "concept/zhong-hua-you-xiu-chuan-tong-wen-hua") {
t.Fatalf("expected unrelated page to be filtered out, but got it in candidates: %v",
pageSlugs(got))
}
if len(got) >= len(pages) {
t.Fatalf("expected prefilter to shrink the corpus (%d pages), but kept %d",
len(pages), len(got))
}
}
// A related page (shares tokens / characters with a new item) must survive
// the prefilter so the LLM can still evaluate the merge.
func TestSelectDedupCandidatePages_KeepsRelatedPages(t *testing.T) {
newItems := []extractedItem{
{Slug: "concept/chengzhen-dengji-shiye-renyuan", Name: "城镇登记失业人员", Aliases: []string{"登记失业人员"}},
}
// Build > dedupSmallCorpusBypass pages so the filter actually runs.
pages := []*types.WikiPage{
// Directly related: existing page whose title overlaps with the new
// item on "登记失业人员". Prefilter MUST keep this.
makePage("concept/deng-ji-shi-ye-ren-yuan", "登记失业人员", "concept", "城镇登记失业人员"),
// Unrelated.
makePage("concept/zhong-hua-you-xiu-chuan-tong-wen-hua", "中华优秀传统文化", "concept"),
}
for i := 0; i < 30; i++ {
pages = append(pages, makePage(
"entity/filler-"+strings.Repeat("x", i+1),
"填充实体"+strings.Repeat("占位", i+1),
"entity",
))
}
got := selectDedupCandidatePages(newItems, pages)
if !containsSlug(got, "concept/deng-ji-shi-ye-ren-yuan") {
t.Fatalf("expected strongly-related page to be kept, got: %v", pageSlugs(got))
}
}
// On small corpora the filter should be a no-op (minus page-type filtering):
// passing every page through is cheap and avoids cutting legitimate matches
// when the prompt is already small.
func TestSelectDedupCandidatePages_SmallCorpusBypass(t *testing.T) {
newItems := []extractedItem{
{Slug: "concept/a", Name: "A"},
}
pages := []*types.WikiPage{
makePage("concept/wholly-unrelated-1", "毫不相关一", "concept"),
makePage("concept/wholly-unrelated-2", "毫不相关二", "concept"),
makePage("concept/wholly-unrelated-3", "毫不相关三", "concept"),
}
got := selectDedupCandidatePages(newItems, pages)
if len(got) != len(pages) {
t.Fatalf("expected bypass on small corpus: got %d, want %d", len(got), len(pages))
}
}
// Non-entity/concept pages (summaries, logs, …) must be stripped regardless
// of corpus size — they are never valid merge targets.
func TestSelectDedupCandidatePages_DropsNonEntityConcept(t *testing.T) {
newItems := []extractedItem{
{Slug: "concept/foo", Name: "Foo"},
}
pages := []*types.WikiPage{
makePage("summary/some-doc", "Some Doc Summary", types.WikiPageTypeSummary),
makePage("log/batch-123", "Batch 123", types.WikiPageTypeLog),
makePage("concept/foo-related", "Foo Related", types.WikiPageTypeConcept),
}
got := selectDedupCandidatePages(newItems, pages)
for _, p := range got {
if p.PageType != types.WikiPageTypeEntity && p.PageType != types.WikiPageTypeConcept {
t.Fatalf("non-entity/concept page should have been filtered: %s (%s)",
p.Slug, p.PageType)
}
}
}
// surfaceGrams must yield empty intersection for the real hallucinated pair,
// confirming the underlying similarity signal is doing its job.
func TestSurfaceGrams_UnrelatedCJKPair(t *testing.T) {
a := surfaceGrams("城镇登记失业人员")
b := surfaceGrams("中华优秀传统文化")
for k := range a {
if _, ok := b[k]; ok {
t.Fatalf("expected zero bigram overlap, but shared %q", k)
}
}
}
// Latin abbreviation ↔ full-name pair must score highly (> floor) so the
// filter keeps legitimate merge candidates like "Acme Corp" ↔ "Acme Corporation".
func TestDedupPairScore_AcmeCorpVariant(t *testing.T) {
a := dedupSurface{
slugTokens: slugBaseTokens("entity/acme-corp"),
nameGramSets: gramsPerSurface([]string{"Acme Corp"}),
}
b := dedupSurface{
slugTokens: slugBaseTokens("entity/acme-corporation"),
nameGramSets: gramsPerSurface([]string{"Acme Corporation"}),
}
score := dedupPairScore(a, b)
if score < dedupCandidateScoreFloor {
t.Fatalf("expected Acme Corp ↔ Corporation score above floor %v, got %v",
dedupCandidateScoreFloor, score)
}
}
// Unrelated CJK pair (the exact case observed in production) must score 0.
func TestDedupPairScore_UnrelatedCJKPair(t *testing.T) {
a := dedupSurface{
slugTokens: slugBaseTokens("concept/chengzhen-dengji-shiye-renyuan"),
nameGramSets: gramsPerSurface([]string{"城镇登记失业人员", "登记失业人员"}),
}
b := dedupSurface{
slugTokens: slugBaseTokens("concept/zhong-hua-you-xiu-chuan-tong-wen-hua"),
nameGramSets: gramsPerSurface([]string{"中华优秀传统文化"}),
}
if s := dedupPairScore(a, b); s >= dedupCandidateScoreFloor {
t.Fatalf("expected unrelated CJK pair to score below floor %v, got %v",
dedupCandidateScoreFloor, s)
}
}