a8a3d9e694
- Introduced extraction granularity settings in the wiki configuration, allowing users to control the level of detail in entity and concept extraction (focused, standard, exhaustive). - Updated the UI to reflect these new options, including tooltips for better user guidance. - Refactored the knowledge base editor to support the new extraction granularity feature, ensuring a seamless user experience. - Improved the backend logic for wiki ingest tasks to utilize the new granularity settings, enhancing the accuracy of extracted content. - Added tests to validate the new extraction granularity functionality and ensure robust performance across different configurations.
192 lines
7.1 KiB
Go
192 lines
7.1 KiB
Go
package service
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
)
|
|
|
|
// helper: build a minimal entity/concept WikiPage.
|
|
func makePage(slug, title, typ string, aliases ...string) *types.WikiPage {
|
|
return &types.WikiPage{
|
|
Slug: slug,
|
|
Title: title,
|
|
PageType: typ,
|
|
Aliases: types.StringArray(aliases),
|
|
}
|
|
}
|
|
|
|
func pageSlugs(pages []*types.WikiPage) []string {
|
|
out := make([]string, 0, len(pages))
|
|
for _, p := range pages {
|
|
out = append(out, p.Slug)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func containsSlug(pages []*types.WikiPage, slug string) bool {
|
|
for _, p := range pages {
|
|
if p.Slug == slug {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// Regression for the observed hallucination in llm_debug/20260422_171316.675.log:
|
|
// deepseek-v3.2 merged concept/chengzhen-dengji-shiye-renyuan into
|
|
// concept/zhong-hua-you-xiu-chuan-tong-wen-hua despite zero character overlap.
|
|
// With the prefilter in place, the traditional-culture page should not even
|
|
// be offered to the LLM as a candidate for that new item.
|
|
func TestSelectDedupCandidatePages_FiltersUnrelatedHallucinationTarget(t *testing.T) {
|
|
newItems := []extractedItem{
|
|
{Slug: "concept/chengzhen-dengji-shiye-renyuan", Name: "城镇登记失业人员", Aliases: []string{"登记失业人员"}},
|
|
{Slug: "entity/beijing-nongshang-yinxing", Name: "北京农商银行"},
|
|
}
|
|
|
|
// Build a corpus that mirrors the real log's shape: ~90 unrelated pages
|
|
// plus a few that share tokens with the new items (so the prefilter has
|
|
// plausible near-matches to keep).
|
|
pages := []*types.WikiPage{
|
|
makePage("concept/zhong-hua-you-xiu-chuan-tong-wen-hua", "中华优秀传统文化", "concept"),
|
|
makePage("concept/jiuye-jineng-peixun", "就业技能培训", "concept"),
|
|
makePage("concept/peixun-kaohe-pingjia", "培训考核评价", "concept"),
|
|
makePage("concept/ren-gong-zhi-neng-an-quan", "人工智能安全", "concept", "AI安全"),
|
|
makePage("entity/bei-jing-shi-jiao-yu-wei-yuan-hui", "北京市教育委员会", "entity", "北京市教委"),
|
|
makePage("entity/beijingshi-changping-zhiye-xuexiao", "北京市昌平职业学校", "entity"),
|
|
}
|
|
// Pad with filler so we exceed dedupSmallCorpusBypass.
|
|
for i := 0; i < 40; i++ {
|
|
pages = append(pages, makePage(
|
|
"concept/filler-"+strings.Repeat("x", i+1),
|
|
"填充概念"+strings.Repeat("占位", i+1),
|
|
"concept",
|
|
))
|
|
}
|
|
|
|
got := selectDedupCandidatePages(newItems, pages)
|
|
|
|
if containsSlug(got, "concept/zhong-hua-you-xiu-chuan-tong-wen-hua") {
|
|
t.Fatalf("expected unrelated page to be filtered out, but got it in candidates: %v",
|
|
pageSlugs(got))
|
|
}
|
|
if len(got) >= len(pages) {
|
|
t.Fatalf("expected prefilter to shrink the corpus (%d pages), but kept %d",
|
|
len(pages), len(got))
|
|
}
|
|
}
|
|
|
|
// A related page (shares tokens / characters with a new item) must survive
|
|
// the prefilter so the LLM can still evaluate the merge.
|
|
func TestSelectDedupCandidatePages_KeepsRelatedPages(t *testing.T) {
|
|
newItems := []extractedItem{
|
|
{Slug: "concept/chengzhen-dengji-shiye-renyuan", Name: "城镇登记失业人员", Aliases: []string{"登记失业人员"}},
|
|
}
|
|
// Build > dedupSmallCorpusBypass pages so the filter actually runs.
|
|
pages := []*types.WikiPage{
|
|
// Directly related: existing page whose title overlaps with the new
|
|
// item on "登记失业人员". Prefilter MUST keep this.
|
|
makePage("concept/deng-ji-shi-ye-ren-yuan", "登记失业人员", "concept", "城镇登记失业人员"),
|
|
// Unrelated.
|
|
makePage("concept/zhong-hua-you-xiu-chuan-tong-wen-hua", "中华优秀传统文化", "concept"),
|
|
}
|
|
for i := 0; i < 30; i++ {
|
|
pages = append(pages, makePage(
|
|
"entity/filler-"+strings.Repeat("x", i+1),
|
|
"填充实体"+strings.Repeat("占位", i+1),
|
|
"entity",
|
|
))
|
|
}
|
|
|
|
got := selectDedupCandidatePages(newItems, pages)
|
|
|
|
if !containsSlug(got, "concept/deng-ji-shi-ye-ren-yuan") {
|
|
t.Fatalf("expected strongly-related page to be kept, got: %v", pageSlugs(got))
|
|
}
|
|
}
|
|
|
|
// On small corpora the filter should be a no-op (minus page-type filtering):
|
|
// passing every page through is cheap and avoids cutting legitimate matches
|
|
// when the prompt is already small.
|
|
func TestSelectDedupCandidatePages_SmallCorpusBypass(t *testing.T) {
|
|
newItems := []extractedItem{
|
|
{Slug: "concept/a", Name: "A"},
|
|
}
|
|
pages := []*types.WikiPage{
|
|
makePage("concept/wholly-unrelated-1", "毫不相关一", "concept"),
|
|
makePage("concept/wholly-unrelated-2", "毫不相关二", "concept"),
|
|
makePage("concept/wholly-unrelated-3", "毫不相关三", "concept"),
|
|
}
|
|
got := selectDedupCandidatePages(newItems, pages)
|
|
if len(got) != len(pages) {
|
|
t.Fatalf("expected bypass on small corpus: got %d, want %d", len(got), len(pages))
|
|
}
|
|
}
|
|
|
|
// Non-entity/concept pages (summaries, logs, …) must be stripped regardless
|
|
// of corpus size — they are never valid merge targets.
|
|
func TestSelectDedupCandidatePages_DropsNonEntityConcept(t *testing.T) {
|
|
newItems := []extractedItem{
|
|
{Slug: "concept/foo", Name: "Foo"},
|
|
}
|
|
pages := []*types.WikiPage{
|
|
makePage("summary/some-doc", "Some Doc Summary", types.WikiPageTypeSummary),
|
|
makePage("log/batch-123", "Batch 123", types.WikiPageTypeLog),
|
|
makePage("concept/foo-related", "Foo Related", types.WikiPageTypeConcept),
|
|
}
|
|
got := selectDedupCandidatePages(newItems, pages)
|
|
for _, p := range got {
|
|
if p.PageType != types.WikiPageTypeEntity && p.PageType != types.WikiPageTypeConcept {
|
|
t.Fatalf("non-entity/concept page should have been filtered: %s (%s)",
|
|
p.Slug, p.PageType)
|
|
}
|
|
}
|
|
}
|
|
|
|
// surfaceGrams must yield empty intersection for the real hallucinated pair,
|
|
// confirming the underlying similarity signal is doing its job.
|
|
func TestSurfaceGrams_UnrelatedCJKPair(t *testing.T) {
|
|
a := surfaceGrams("城镇登记失业人员")
|
|
b := surfaceGrams("中华优秀传统文化")
|
|
for k := range a {
|
|
if _, ok := b[k]; ok {
|
|
t.Fatalf("expected zero bigram overlap, but shared %q", k)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Latin abbreviation ↔ full-name pair must score highly (> floor) so the
|
|
// filter keeps legitimate merge candidates like "Acme Corp" ↔ "Acme Corporation".
|
|
func TestDedupPairScore_AcmeCorpVariant(t *testing.T) {
|
|
a := dedupSurface{
|
|
slugTokens: slugBaseTokens("entity/acme-corp"),
|
|
nameGramSets: gramsPerSurface([]string{"Acme Corp"}),
|
|
}
|
|
b := dedupSurface{
|
|
slugTokens: slugBaseTokens("entity/acme-corporation"),
|
|
nameGramSets: gramsPerSurface([]string{"Acme Corporation"}),
|
|
}
|
|
score := dedupPairScore(a, b)
|
|
if score < dedupCandidateScoreFloor {
|
|
t.Fatalf("expected Acme Corp ↔ Corporation score above floor %v, got %v",
|
|
dedupCandidateScoreFloor, score)
|
|
}
|
|
}
|
|
|
|
// Unrelated CJK pair (the exact case observed in production) must score 0.
|
|
func TestDedupPairScore_UnrelatedCJKPair(t *testing.T) {
|
|
a := dedupSurface{
|
|
slugTokens: slugBaseTokens("concept/chengzhen-dengji-shiye-renyuan"),
|
|
nameGramSets: gramsPerSurface([]string{"城镇登记失业人员", "登记失业人员"}),
|
|
}
|
|
b := dedupSurface{
|
|
slugTokens: slugBaseTokens("concept/zhong-hua-you-xiu-chuan-tong-wen-hua"),
|
|
nameGramSets: gramsPerSurface([]string{"中华优秀传统文化"}),
|
|
}
|
|
if s := dedupPairScore(a, b); s >= dedupCandidateScoreFloor {
|
|
t.Fatalf("expected unrelated CJK pair to score below floor %v, got %v",
|
|
dedupCandidateScoreFloor, s)
|
|
}
|
|
}
|