feat: Enhance wiki extraction and configuration options

- Introduced extraction granularity settings in the wiki configuration, allowing users to control the level of detail in entity and concept extraction (focused, standard, exhaustive).
- Updated the UI to reflect these new options, including tooltips for better user guidance.
- Refactored the knowledge base editor to support the new extraction granularity feature, ensuring a seamless user experience.
- Improved the backend logic for wiki ingest tasks to utilize the new granularity settings, enhancing the accuracy of extracted content.
- Added tests to validate the new extraction granularity functionality and ensure robust performance across different configurations.
This commit is contained in:
wizardchen
2026-04-22 20:27:19 +08:00
parent 623add89f1
commit a8a3d9e694
23 changed files with 2110 additions and 174 deletions
@@ -123,8 +123,8 @@ func (s *KnowledgePostProcessService) Handle(ctx context.Context, task *asynq.Ta
}
}
// 6. Spawn Wiki Ingest Task if wiki indexing is enabled in IndexingStrategy and auto ingest is on
if kb.IndexingStrategy.WikiEnabled && kb.WikiConfig != nil && kb.WikiConfig.AutoIngest && len(textChunks) > 0 {
// 6. Spawn Wiki Ingest Task if wiki indexing is enabled in IndexingStrategy
if kb.IndexingStrategy.WikiEnabled && len(textChunks) > 0 {
EnqueueWikiIngest(ctx, s.taskEnqueuer, s.redisClient, payload.TenantID, payload.KnowledgeBaseID, payload.KnowledgeID)
logger.Infof(ctx, "[KnowledgePostProcess] Enqueued wiki ingest task for %s", payload.KnowledgeID)
}
+77 -10
View File
@@ -3,6 +3,7 @@ package service
import (
"context"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
@@ -19,6 +20,15 @@ import (
"github.com/redis/go-redis/v9"
)
// ErrWikiIngestConcurrent is returned by the wiki ingest handler when another
// batch is already running for the same KB (i.e. the `wiki:active:<kbID>`
// Redis lock is held). The asynq server's RetryDelayFunc uses errors.Is on
// this sentinel to apply a short, fixed retry delay instead of asynq's default
// exponential backoff — otherwise a freshly orphaned lock (e.g. from a crash
// or restart) would force newcomers to wait minutes even after the lock
// naturally expires.
var ErrWikiIngestConcurrent = errors.New("concurrent wiki task active")
const (
// maxContentForWiki limits the document content sent to LLM for wiki generation
maxContentForWiki = 32768
@@ -54,6 +64,20 @@ const (
// wikiDeletedTTL bounds how long we remember a deletion. Must comfortably
// exceed the longest plausible ingest run (LLM extraction + reduce).
wikiDeletedTTL = 1 * time.Hour
// wikiActiveLockTTL is the TTL for the per-KB "batch in progress" flag.
// Kept short (relative to total batch runtime) so that if the owning
// process crashes without running its `defer Del`, the orphaned lock
// expires quickly and newcomers aren't blocked. A periodic renew
// (wikiActiveLockRenew) keeps the lock alive while the handler is
// genuinely still running.
wikiActiveLockTTL = 60 * time.Second
// wikiActiveLockRenew is how often the in-flight handler bumps the TTL.
// Must be comfortably shorter than wikiActiveLockTTL so a single missed
// tick (GC pause, Redis blip) doesn't let the lock slip out from under a
// live handler.
wikiActiveLockRenew = 20 * time.Second
)
// WikiDeletedTombstoneKey returns the Redis key used to mark a knowledge as
@@ -318,6 +342,12 @@ type WikiBatchContext struct {
AllPages []*types.WikiPage
SlugTitleMap map[string]string
SummaryContentByKnowledgeID map[string]string
// ExtractionGranularity drives Pass 0 (candidate slug extraction)
// aggressiveness. Resolved once per batch from the KnowledgeBase's
// WikiConfig so every doc in the batch sees the same scope rules.
// Already Normalize()'d — consumers can assume it is one of the
// three valid values.
ExtractionGranularity types.WikiExtractionGranularity
}
// SlugUpdate represents a single update operation for a specific slug
@@ -332,6 +362,19 @@ type SlugUpdate struct {
SummaryBody string // For summary
SummaryLine string // For summary
RetractDocContent string // For retract / retractStale
// SourceChunks lists the chunk IDs (within KnowledgeID) that substantively
// support this update. Mirrors Item.SourceChunks for convenience — the
// Reduce phase reads from here to avoid an extra field hop.
SourceChunks []string
// DocSummary is the document-level summary body produced by
// WikiSummaryPrompt (everything after the SUMMARY: ... headline, falling
// back to the raw output if no headline could be parsed out). Carried
// here so the Reduce phase can frame cited chunks with a rich
// <source_context> block that tells the editor model what the document
// is about AND what kind of document it is (resume vs announcement vs
// product page). The one-line headline alone was too terse to keep the
// editor grounded on longer / multi-topic source documents.
DocSummary string
}
func previewText(s string, maxRunes int) string {
@@ -537,13 +580,19 @@ func (s *wikiIngestService) getExistingPageSlugsForKnowledge(ctx context.Context
// Multi-source → remove ref, queue retract
// extractedItem represents a single extracted entity or concept
// extractedItem represents a single extracted entity or concept.
//
// SourceChunks holds the stable chunk IDs (from the source document) that
// substantively discuss this item. Populated by the chunk-citation pass; when
// non-empty the Reduce phase uses these chunks verbatim as the item's
// evidence instead of the shorter Description/Details fields.
type extractedItem struct {
Name string `json:"name"`
Slug string `json:"slug"`
Aliases []string `json:"aliases"`
Description string `json:"description"`
Details string `json:"details"`
Name string `json:"name"`
Slug string `json:"slug"`
Aliases []string `json:"aliases"`
Description string `json:"description"`
Details string `json:"details"`
SourceChunks []string `json:"source_chunks,omitempty"`
}
// combinedExtraction represents the parsed result of the combined entity+concept extraction
@@ -779,11 +828,29 @@ func (s *wikiIngestService) deduplicateExtractedBatch(
return entities, concepts
}
if len(allPages) == 0 {
return entities, concepts
}
// Pre-filter the candidate existing-pages set. Passing the full corpus
// to the LLM on large KBs bloats the prompt and empirically lets the
// model hallucinate merges between totally unrelated slugs. The filter
// is conservative (high recall) and the downstream validMerge check
// remains in place for defense in depth.
newItems := make([]extractedItem, 0, len(entities)+len(concepts))
newItems = append(newItems, entities...)
newItems = append(newItems, concepts...)
candidatePages := selectDedupCandidatePages(newItems, allPages)
if len(candidatePages) == 0 {
return entities, concepts
}
if origCount := countEntityConceptPages(allPages); origCount > len(candidatePages) {
logger.Infof(ctx, "wiki ingest: dedup candidate filter kept %d/%d existing pages for %d new items",
len(candidatePages), origCount, len(newItems))
}
var existingBuf strings.Builder
for _, p := range allPages {
if p.PageType != types.WikiPageTypeEntity && p.PageType != types.WikiPageTypeConcept {
continue
}
for _, p := range candidatePages {
writeDedupItemXML(&existingBuf, p.Slug, p.Title, string(p.PageType), []string(p.Aliases))
}
if existingBuf.Len() == 0 {
+227 -49
View File
@@ -101,16 +101,22 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
ctx = context.WithValue(ctx, types.LanguageContextKey, payload.Language)
}
// Try to acquire the "active batch" flag (non-blocking)
// Try to acquire the "active batch" flag (non-blocking).
//
// TTL is intentionally short (wikiActiveLockTTL ≈ 60s) so that if the
// owning process dies without releasing the lock (crash, kill -9,
// container restart), the orphaned key expires within ~1 minute and new
// tasks aren't starved. A renew goroutine keeps the lock alive while
// the handler is genuinely running.
if s.redisClient != nil {
activeKey := wikiActiveKeyPrefix + payload.KnowledgeBaseID
acquired, err := s.redisClient.SetNX(ctx, activeKey, "1", 5*time.Minute).Result()
acquired, err := s.redisClient.SetNX(ctx, activeKey, "1", wikiActiveLockTTL).Result()
if err != nil {
logger.Warnf(ctx, "wiki ingest: redis SetNX failed: %v", err)
} else if !acquired {
exitStatus = "active_lock_conflict"
logger.Infof(ctx, "wiki ingest: another batch active for KB %s, deferring to asynq retry", payload.KnowledgeBaseID)
return fmt.Errorf("concurrent wiki task active, please retry")
return ErrWikiIngestConcurrent
}
lockAcquired = acquired
@@ -121,14 +127,14 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
}()
go func() {
ticker := time.NewTicker(2 * time.Minute)
ticker := time.NewTicker(wikiActiveLockRenew)
defer ticker.Stop()
for {
select {
case <-lockCtx.Done():
return
case <-ticker.C:
s.redisClient.Expire(context.Background(), activeKey, 5*time.Minute)
s.redisClient.Expire(context.Background(), activeKey, wikiActiveLockTTL)
}
}
}()
@@ -145,13 +151,11 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
exitStatus = "kb_not_wiki_enabled"
return fmt.Errorf("wiki ingest: KB %s is not wiki type", kb.ID)
}
if kb.WikiConfig == nil || !kb.WikiConfig.AutoIngest {
exitStatus = "auto_ingest_disabled"
logger.Infof(ctx, "wiki ingest: auto_ingest disabled for KB %s, skipping", kb.ID)
return nil
}
synthesisModelID := kb.WikiConfig.SynthesisModelID
var synthesisModelID string
if kb.WikiConfig != nil {
synthesisModelID = kb.WikiConfig.SynthesisModelID
}
if synthesisModelID == "" {
synthesisModelID = kb.SummaryModelID
}
@@ -189,10 +193,21 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
// Fetch all existing pages once, shared across Map and Reduce phases
allPages, _ := s.wikiService.ListAllPages(ctx, payload.KnowledgeBaseID)
// Resolve extraction granularity once per batch. Historical rows with
// empty/unknown values fall back to Standard via Normalize(). Failures
// to load the KB (unlikely since we're already acting on it) also
// degrade gracefully to Standard.
granularity := types.WikiExtractionStandard
if kb, kbErr := s.kbService.GetKnowledgeBaseByID(ctx, payload.KnowledgeBaseID); kbErr == nil && kb != nil && kb.WikiConfig != nil {
granularity = kb.WikiConfig.ExtractionGranularity.Normalize()
}
batchCtx := &WikiBatchContext{
AllPages: allPages,
SlugTitleMap: make(map[string]string),
SummaryContentByKnowledgeID: make(map[string]string),
ExtractionGranularity: granularity,
}
for _, p := range allPages {
if p.PageType != types.WikiPageTypeIndex && p.PageType != types.WikiPageTypeLog && p.Status != types.WikiPageStatusArchived {
@@ -457,22 +472,35 @@ func (s *wikiIngestService) mapOneDocument(
sourceRef := fmt.Sprintf("%s|%s", knowledgeID, docTitle)
oldPageSlugs := s.getExistingPageSlugsForKnowledge(ctx, payload.KnowledgeBaseID, knowledgeID)
logger.Infof(ctx, "wiki ingest: extracting entities and concepts for %s", knowledgeID)
extractedEntities, extractedConcepts, slugItems, err := s.extractEntitiesAndConceptsNoUpsert(ctx, chatModel, content, docTitle, lang, oldPageSlugs, batchCtx)
// Pass 0: lightweight candidate slug extraction (skeleton only).
// On failure we fall back to the legacy single-shot extractor so the doc
// still gets ingested, just without chunk-level citations.
var (
extractedEntities []extractedItem
extractedConcepts []extractedItem
slugItems map[string]extractedItem
pass0Failed bool
)
logger.Infof(ctx, "wiki ingest: pass 0 — extracting candidate slugs for %s", knowledgeID)
extractedEntities, extractedConcepts, slugItems, err = s.extractCandidateSlugs(ctx, chatModel, content, docTitle, lang, oldPageSlugs, batchCtx)
if err != nil {
logger.Warnf(ctx, "wiki ingest: knowledge extraction failed for %s: %v", knowledgeID, err)
return nil, nil, err
logger.Warnf(ctx, "wiki ingest: pass 0 failed for %s (%v) — falling back to legacy extractor", knowledgeID, err)
pass0Failed = true
extractedEntities, extractedConcepts, slugItems, err = s.extractEntitiesAndConceptsNoUpsert(ctx, chatModel, content, docTitle, lang, oldPageSlugs, batchCtx)
if err != nil {
logger.Warnf(ctx, "wiki ingest: legacy fallback also failed for %s: %v", knowledgeID, err)
return nil, nil, err
}
}
var extractedPages []string
// Build slug listing for Summary's wiki-link input.
var summaryExtractedPages []string
for slug := range slugItems {
extractedPages = append(extractedPages, slug)
summaryExtractedPages = append(summaryExtractedPages, slug)
}
// Summary
summarySlug := fmt.Sprintf("summary/%s", slugify(docTitle))
var slugListing string
for _, slug := range extractedPages {
for _, slug := range summaryExtractedPages {
if item, ok := slugItems[slug]; ok {
aliases := ""
if len(item.Aliases) > 0 {
@@ -484,20 +512,87 @@ func (s *wikiIngestService) mapOneDocument(
}
}
var docSummaryLine string
summaryContent, err := s.generateWithTemplate(ctx, chatModel, agent.WikiSummaryPrompt, map[string]string{
"Title": docTitle,
"FileName": docTitle,
"FileType": "document",
"Content": content,
"Language": lang,
"ExtractedSlugs": slugListing,
})
// Summary and chunk classification are independent given Pass 0 output —
// run them in parallel. Summary handles wiki-link injection; classification
// attaches concrete chunk IDs to each candidate slug.
var (
summaryContent string
summaryErr error
citations map[string][]string
newSlugs []newSlugFromCitation
batchCount int
)
var wg sync.WaitGroup
wg.Add(2)
go func() {
defer wg.Done()
summaryContent, summaryErr = s.generateWithTemplate(ctx, chatModel, agent.WikiSummaryPrompt, map[string]string{
"Title": docTitle,
"FileName": docTitle,
"FileType": "document",
"Content": content,
"Language": lang,
"ExtractedSlugs": slugListing,
})
}()
go func() {
defer wg.Done()
// Skip citation pass when Pass 0 has fallen back to the legacy path —
// the legacy output already contains paraphrased Details, so chunk
// citations would be redundant and we'd spend LLM calls for nothing.
if pass0Failed {
citations = map[string][]string{}
return
}
candidatesXML := renderCandidateSlugsXML(extractedEntities, extractedConcepts)
citations, newSlugs, batchCount = s.classifyChunkCitations(ctx, chatModel, candidatesXML, docTitle, chunks, lang)
}()
wg.Wait()
// Merge citations back into the item structs (non-failing; items without
// citations simply keep their Description+Details fallback).
var uncited int
extractedEntities, extractedConcepts, uncited = mergeCitationsIntoItems(extractedEntities, extractedConcepts, citations, newSlugs)
// Rebuild slugItems so stale entries (for slugs that did not survive the
// merge) and brand-new slugs discovered by the citation pass are both
// reflected in summaryExtractedPages tracking.
slugItems = make(map[string]extractedItem, len(extractedEntities)+len(extractedConcepts))
for _, item := range extractedEntities {
if item.Slug != "" && item.Name != "" {
slugItems[item.Slug] = item
}
}
for _, item := range extractedConcepts {
if item.Slug != "" && item.Name != "" {
slugItems[item.Slug] = item
}
}
extractedPages := make([]string, 0, len(slugItems)+1)
for slug := range slugItems {
extractedPages = append(extractedPages, slug)
}
// Count total distinct chunks cited across all slugs for logging.
citedChunkSet := make(map[string]bool)
for _, ids := range citations {
for _, id := range ids {
citedChunkSet[id] = true
}
}
var updates []SlugUpdate
// docSummaryLine is the one-sentence headline used for terse log/audit
// previews and for <document_added> blocks in retract prompts.
// docSummary is the full summary body attached to each entity/concept
// update so the editor model gets rich framing in <source_context>.
var docSummaryLine string
var docSummary string
if err != nil {
logger.Errorf(ctx, "wiki ingest: generate summary failed for %s: %v", knowledgeID, err)
if summaryErr != nil {
logger.Errorf(ctx, "wiki ingest: generate summary failed for %s: %v", knowledgeID, summaryErr)
} else {
sumLine, sumBody := splitSummaryLine(summaryContent)
if sumBody == "" {
@@ -507,6 +602,10 @@ func (s *wikiIngestService) mapOneDocument(
sumLine = docTitle
}
docSummaryLine = sumLine
docSummary = sumBody
if strings.TrimSpace(docSummary) == "" {
docSummary = sumLine
}
updates = append(updates, SlugUpdate{
Slug: summarySlug,
Type: types.WikiPageTypeSummary,
@@ -524,13 +623,15 @@ func (s *wikiIngestService) mapOneDocument(
for _, item := range extractedEntities {
if item.Slug != "" {
updates = append(updates, SlugUpdate{
Slug: item.Slug,
Type: types.WikiPageTypeEntity,
Item: item,
DocTitle: docTitle,
KnowledgeID: knowledgeID,
SourceRef: sourceRef,
Language: lang,
Slug: item.Slug,
Type: types.WikiPageTypeEntity,
Item: item,
DocTitle: docTitle,
KnowledgeID: knowledgeID,
SourceRef: sourceRef,
Language: lang,
SourceChunks: item.SourceChunks,
DocSummary: docSummary,
})
}
}
@@ -539,13 +640,15 @@ func (s *wikiIngestService) mapOneDocument(
for _, item := range extractedConcepts {
if item.Slug != "" {
updates = append(updates, SlugUpdate{
Slug: item.Slug,
Type: types.WikiPageTypeConcept,
Item: item,
DocTitle: docTitle,
KnowledgeID: knowledgeID,
SourceRef: sourceRef,
Language: lang,
Slug: item.Slug,
Type: types.WikiPageTypeConcept,
Item: item,
DocTitle: docTitle,
KnowledgeID: knowledgeID,
SourceRef: sourceRef,
Language: lang,
SourceChunks: item.SourceChunks,
DocSummary: docSummary,
})
}
}
@@ -571,8 +674,13 @@ func (s *wikiIngestService) mapOneDocument(
}
}
logger.Infof(ctx, "wiki ingest: mapped knowledge %s title=%q generated_updates=%d elapsed=%s",
knowledgeID, previewText(docTitle, 80), len(updates), time.Since(docStartedAt).Round(time.Millisecond))
logger.Infof(ctx,
"wiki ingest: mapped knowledge %s title=%q candidates=%d chunks=%d batches=%d cited_chunks=%d uncited_slugs=%d new_slugs=%d updates=%d pass0_fallback=%v elapsed=%s",
knowledgeID, previewText(docTitle, 80),
len(slugItems), len(chunks), batchCount, len(citedChunkSet), uncited, len(newSlugs),
len(updates), pass0Failed,
time.Since(docStartedAt).Round(time.Millisecond),
)
return &docIngestResult{
KnowledgeID: knowledgeID,
@@ -716,6 +824,11 @@ func (s *wikiIngestService) reduceSlugUpdates(
page.Summary = summaryUpdate.SummaryLine
page.PageType = types.WikiPageTypeSummary
page.SourceRefs = appendUnique(page.SourceRefs, summaryUpdate.SourceRef)
// Summary pages don't carry chunk-level citations (they are document-
// level synopses generated from the whole content). Clear any stale
// chunk refs that may remain if this slug was once an entity page
// and got converted to a summary page.
page.ChunkRefs = types.StringArray{}
changed = true
if exists {
@@ -786,9 +899,38 @@ func (s *wikiIngestService) reduceSlugUpdates(
if len(additions) > 0 {
language = additions[0].Language
// Resolve SourceChunks → chunk contents in a single batched query per
// knowledge ID, so the <new_information> block can quote the chunks
// verbatim instead of relying on the short Details paraphrase.
chunkContentByID := s.resolveCitedChunks(ctx, tenantID, additions)
for _, add := range additions {
fmt.Fprintf(&newContentBuilder, "<document>\n<title>%s</title>\n<content>\n**%s**: %s\n\n%s\n</content>\n</document>\n\n",
add.DocTitle, add.Item.Name, add.Item.Description, add.Item.Details)
cited := collectCitedChunkContent(add.SourceChunks, chunkContentByID)
// Frame the chunks with the document-level summary body so the
// editor model knows BOTH what the document is about AND what
// kind of document it is (resume vs announcement vs product
// page vs schedule). The one-sentence headline alone was too
// terse to keep the editor grounded on longer or multi-topic
// source documents, and calibrating tone (self-reported vs
// third-party authoritative) benefits from the richer context.
sourceCtx := strings.TrimSpace(add.DocSummary)
sourceCtxBlock := ""
if sourceCtx != "" {
sourceCtxBlock = fmt.Sprintf("<source_context>\n%s\n</source_context>\n", sourceCtx)
}
if cited != "" {
fmt.Fprintf(&newContentBuilder,
"<document>\n<title>%s</title>\n%s<content>\n**%s**: %s\n\n%s\n</content>\n</document>\n\n",
add.DocTitle, sourceCtxBlock, add.Item.Name, add.Item.Description, cited)
} else {
// Fallback: no citations available (legacy path, citation pass
// failed, or bad chunk IDs were filtered out) — stick with
// the short Details summary so the page still gets real text.
fmt.Fprintf(&newContentBuilder,
"<document>\n<title>%s</title>\n%s<content>\n**%s**: %s\n\n%s\n</content>\n</document>\n\n",
add.DocTitle, sourceCtxBlock, add.Item.Name, add.Item.Description, add.Item.Details)
}
docTitles = appendUnique(docTitles, add.DocTitle)
for _, alias := range add.Item.Aliases {
@@ -872,6 +1014,11 @@ func (s *wikiIngestService) reduceSlugUpdates(
}
if changed {
// Refresh chunk refs in-place on the page so they persist alongside
// the rest of the row. Retract-only updates (no additions) preserve
// the existing refs; addition rounds append the newly-cited chunks
// on top of what was already there, deduplicated.
page.ChunkRefs = mergeChunkRefs(page.ChunkRefs, additions)
if exists {
_, err = s.wikiService.UpdatePage(ctx, page)
} else {
@@ -882,3 +1029,34 @@ func (s *wikiIngestService) reduceSlugUpdates(
return false, "", nil
}
// mergeChunkRefs unions the chunk IDs currently on the page with the ones
// cited by this batch's additions, preserving insertion order and dropping
// duplicates. Empty strings are filtered out so a malformed source_chunks
// array can't leave junk in the column.
//
// A retract round with no additions leaves the current refs untouched —
// retract-only paths don't carry chunk IDs (only knowledge IDs), and we
// can't surgically filter without that info. The next time the slug is
// re-materialized via additions the fresh chunks will overlay on top.
func mergeChunkRefs(current types.StringArray, additions []SlugUpdate) types.StringArray {
seen := make(map[string]bool, len(current))
out := make(types.StringArray, 0, len(current))
for _, id := range current {
if id == "" || seen[id] {
continue
}
seen[id] = true
out = append(out, id)
}
for _, add := range additions {
for _, chunkID := range add.SourceChunks {
if chunkID == "" || seen[chunkID] {
continue
}
seen[chunkID] = true
out = append(out, chunkID)
}
}
return out
}
@@ -0,0 +1,566 @@
package service
import (
"context"
"encoding/json"
"fmt"
"sort"
"strings"
"sync"
"github.com/Tencent/WeKnora/internal/agent"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/types"
"golang.org/x/sync/errgroup"
)
const (
// maxRunesPerCitationBatch bounds the size of a single chunk-citation
// batch by rune count (a fast approximation of token count). Small enough
// that batches stay comfortably inside the LLM's context and output
// budgets, large enough that most short/medium documents classify in a
// single batch. Tune via benchmarks if needed.
maxRunesPerCitationBatch = 12000
// maxCitationBatchConcurrency limits parallelism across chunk-citation
// batches so a single long document can't saturate the synthesis model.
maxCitationBatchConcurrency = 4
)
// citationBatchResult is the JSON shape we expect back from one invocation of
// WikiChunkCitationPrompt.
type citationBatchResult struct {
Citations map[string][]string `json:"citations"`
NewSlugs []newSlugFromCitation `json:"new_slugs"`
}
// newSlugFromCitation is the shape of an entry in the "new_slugs" array of
// WikiChunkCitationPrompt. Mirrors extractedItem but also carries a "type"
// tag because this prompt emits entities and concepts in a single array.
type newSlugFromCitation struct {
Type string `json:"type"`
Name string `json:"name"`
Slug string `json:"slug"`
Aliases []string `json:"aliases"`
Description string `json:"description"`
Details string `json:"details"`
SourceChunks []string `json:"source_chunks"`
}
// citationPipelineOutcome carries the raw numbers produced by the Pass
// 0 + classification flow so callers (mapOneDocument) can log a single
// unified stat line.
type citationPipelineOutcome struct {
CandidateCount int
ChunkCount int
BatchCount int
CitedChunks int
UncitedSlugs int
NewSlugCount int
}
// extractCandidateSlugs runs Pass 0 of the chunk-cited pipeline: it scans the
// full (reconstructed) document text and returns a lightweight skeleton of
// every significant entity/concept. Unlike the legacy single-shot extraction,
// this pass explicitly does NOT ask the LLM to paraphrase full facts per item;
// those will come from the chunk-citation pass instead.
//
// Returns (entities, concepts, slugItems, error). On LLM or parse failure it
// returns an error — the caller can then fall back to the legacy extractor.
func (s *wikiIngestService) extractCandidateSlugs(
ctx context.Context,
chatModel chat.Chat,
content, docTitle, lang string,
oldPageSlugs map[string]bool,
batchCtx *WikiBatchContext,
) ([]extractedItem, []extractedItem, map[string]extractedItem, error) {
var prevSlugsText string
if len(oldPageSlugs) > 0 {
var sb strings.Builder
for slug := range oldPageSlugs {
if !strings.HasPrefix(slug, "entity/") && !strings.HasPrefix(slug, "concept/") {
continue
}
fmt.Fprintf(&sb, "- %s\n", slug)
}
prevSlugsText = sb.String()
}
if prevSlugsText == "" {
prevSlugsText = "(none — this is a new document)"
}
granularity := batchCtx.ExtractionGranularity.Normalize()
raw, err := s.generateWithTemplate(ctx, chatModel, agent.WikiCandidateSlugPrompt, map[string]string{
"Title": docTitle,
"Content": content,
"Language": lang,
"PreviousSlugs": prevSlugsText,
"Granularity": string(granularity),
"GranularityGuidance": agent.WikiGranularityGuidance(string(granularity)),
})
if err != nil {
return nil, nil, nil, fmt.Errorf("candidate slug extraction failed: %w", err)
}
raw = cleanLLMJSON(raw)
var result combinedExtraction
if err := json.Unmarshal([]byte(raw), &result); err != nil {
logger.Warnf(ctx, "wiki ingest: failed to parse candidate slug JSON: %v\nRaw: %s", err, raw)
return nil, nil, nil, fmt.Errorf("parse candidate slug JSON: %w", err)
}
result.Entities, result.Concepts = s.deduplicateExtractedBatch(
ctx, chatModel, result.Entities, result.Concepts, batchCtx.AllPages,
)
slugItems := make(map[string]extractedItem, len(result.Entities)+len(result.Concepts))
for _, item := range result.Entities {
if item.Slug != "" && item.Name != "" {
slugItems[item.Slug] = item
}
}
for _, item := range result.Concepts {
if item.Slug != "" && item.Name != "" {
slugItems[item.Slug] = item
}
}
return result.Entities, result.Concepts, slugItems, nil
}
// chunkBatch groups chunks that will be sent in a single WikiChunkCitationPrompt call.
type chunkBatch struct {
chunks []*types.Chunk
aliasToID map[string]string
totalRuneLen int
}
// splitChunksIntoCitationBatches partitions chunks into batches whose total
// rune count stays under maxRunesPerCitationBatch. Chunk order (by ChunkIndex)
// is preserved, and a chunk that by itself exceeds the budget occupies its
// own batch so we never silently drop content. Each chunk is assigned a short
// alias ("c000", "c001", ...) that the prompt uses in place of the raw UUID;
// the alias → real ID map is kept on the batch so the caller can translate
// LLM responses back to stable chunk IDs.
func splitChunksIntoCitationBatches(chunks []*types.Chunk) []chunkBatch {
// Only cite text chunks — image/ocr chunks are already merged into the
// text content via reconstructEnrichedContent and the LLM doesn't see
// them as standalone units.
filtered := make([]*types.Chunk, 0, len(chunks))
for _, c := range chunks {
if c == nil || c.Content == "" {
continue
}
if c.ChunkType != types.ChunkTypeText && c.ChunkType != "" {
continue
}
filtered = append(filtered, c)
}
if len(filtered) == 0 {
return nil
}
// Preserve document order for human-readable citation
sort.Slice(filtered, func(i, j int) bool {
if filtered[i].ChunkIndex == filtered[j].ChunkIndex {
return filtered[i].StartAt < filtered[j].StartAt
}
return filtered[i].ChunkIndex < filtered[j].ChunkIndex
})
var batches []chunkBatch
current := chunkBatch{aliasToID: make(map[string]string)}
aliasCounter := 0
flush := func() {
if len(current.chunks) == 0 {
return
}
batches = append(batches, current)
current = chunkBatch{aliasToID: make(map[string]string)}
}
for _, c := range filtered {
runeLen := len([]rune(c.Content))
// If adding this chunk would exceed the budget and the current batch
// isn't empty, flush first so the oversized chunk is not merged with
// already-queued ones. Oversized chunks still go through — they just
// get their own batch.
if len(current.chunks) > 0 && current.totalRuneLen+runeLen > maxRunesPerCitationBatch {
flush()
}
alias := fmt.Sprintf("c%03d", aliasCounter)
aliasCounter++
current.aliasToID[alias] = c.ID
current.chunks = append(current.chunks, c)
current.totalRuneLen += runeLen
}
flush()
// Restart alias numbering per batch so the LLM sees a small, local ID
// space (helps it follow exact-match citation rules). We do this by
// rebuilding the alias map in a second pass.
for bi := range batches {
newAliasToID := make(map[string]string, len(batches[bi].chunks))
for idx, c := range batches[bi].chunks {
alias := fmt.Sprintf("c%03d", idx)
newAliasToID[alias] = c.ID
}
batches[bi].aliasToID = newAliasToID
}
return batches
}
// renderCandidateSlugsXML renders candidate slugs as a compact list suitable
// for the prompt's <candidate_slugs> block.
func renderCandidateSlugsXML(entities, concepts []extractedItem) string {
var sb strings.Builder
write := func(item extractedItem, kind string) {
aliases := ""
if len(item.Aliases) > 0 {
aliases = fmt.Sprintf(" aliases=%q", strings.Join(item.Aliases, ", "))
}
fmt.Fprintf(&sb, "- slug: %s, type: %s, name: %q%s, description: %s\n",
item.Slug, kind, item.Name, aliases, item.Description)
}
for _, item := range entities {
if item.Slug == "" || item.Name == "" {
continue
}
write(item, "entity")
}
for _, item := range concepts {
if item.Slug == "" || item.Name == "" {
continue
}
write(item, "concept")
}
return sb.String()
}
// renderChunksXML formats one batch's chunks into the <chunks> block, using
// per-batch aliases (c000, c001, ...) instead of raw UUIDs.
func renderChunksXML(batch chunkBatch) string {
aliases := make([]string, 0, len(batch.aliasToID))
for a := range batch.aliasToID {
aliases = append(aliases, a)
}
sort.Strings(aliases)
idToAlias := make(map[string]string, len(batch.aliasToID))
for a, id := range batch.aliasToID {
idToAlias[id] = a
}
var sb strings.Builder
for _, c := range batch.chunks {
alias := idToAlias[c.ID]
fmt.Fprintf(&sb, "<c id=%q index=\"%d\">\n%s\n</c>\n", alias, c.ChunkIndex, c.Content)
}
return sb.String()
}
// classifyChunkCitations runs Pass 1..N of the chunk-cited pipeline: given a
// set of candidate slugs (from Pass 0) and the document's chunks, it asks the
// LLM which chunks substantively discuss each candidate. Results across
// batches are merged into a single slug → union(chunk_id) map, and any
// "new_slugs" that Pass 0 missed are collected separately.
//
// Returns (citations, newSlugs, batchCount). citations is keyed by slug and
// contains real chunk UUIDs (already translated from batch aliases). newSlugs
// likewise carry real chunk UUIDs in SourceChunks.
func (s *wikiIngestService) classifyChunkCitations(
ctx context.Context,
chatModel chat.Chat,
candidatesXML string,
docTitle string,
chunks []*types.Chunk,
lang string,
) (map[string][]string, []newSlugFromCitation, int) {
batches := splitChunksIntoCitationBatches(chunks)
if len(batches) == 0 || strings.TrimSpace(candidatesXML) == "" {
return map[string][]string{}, nil, 0
}
// Merge state. Using sets keyed by (slug, chunkID) to dedup across
// batches; order is re-imposed from chunk ChunkIndex at the end.
var mu sync.Mutex
citationSet := make(map[string]map[string]bool) // slug → set of real chunk IDs
var newSlugsAll []newSlugFromCitation
eg, ectx := errgroup.WithContext(ctx)
eg.SetLimit(maxCitationBatchConcurrency)
for bi := range batches {
batch := batches[bi]
batchIdx := bi
eg.Go(func() error {
chunksXML := renderChunksXML(batch)
raw, err := s.generateWithTemplate(ectx, chatModel, agent.WikiChunkCitationPrompt, map[string]string{
"DocTitle": docTitle,
"CandidateSlugs": candidatesXML,
"ChunksXML": chunksXML,
"Language": lang,
})
if err != nil {
logger.Warnf(ectx, "wiki ingest: citation batch %d failed: %v", batchIdx, err)
return nil // don't abort peer batches
}
raw = cleanLLMJSON(raw)
var parsed citationBatchResult
if jerr := json.Unmarshal([]byte(raw), &parsed); jerr != nil {
logger.Warnf(ectx, "wiki ingest: citation batch %d parse failed: %v\nRaw: %s", batchIdx, jerr, raw)
return nil
}
// Translate aliases → real chunk UUIDs; drop unknown aliases.
mu.Lock()
defer mu.Unlock()
for slug, aliasList := range parsed.Citations {
if slug == "" {
continue
}
set, ok := citationSet[slug]
if !ok {
set = make(map[string]bool)
citationSet[slug] = set
}
for _, alias := range aliasList {
realID, known := batch.aliasToID[alias]
if !known {
logger.Warnf(ectx, "wiki ingest: citation batch %d referenced unknown chunk alias %q for slug %s", batchIdx, alias, slug)
continue
}
set[realID] = true
}
}
for _, ns := range parsed.NewSlugs {
if ns.Slug == "" || ns.Name == "" {
continue
}
real := make([]string, 0, len(ns.SourceChunks))
for _, alias := range ns.SourceChunks {
if id, ok := batch.aliasToID[alias]; ok {
real = append(real, id)
}
}
ns.SourceChunks = real
newSlugsAll = append(newSlugsAll, ns)
}
return nil
})
}
_ = eg.Wait()
// Build a stable chunk-order so the final citations come out in document order.
chunkOrder := make(map[string]int, len(chunks))
for _, c := range chunks {
chunkOrder[c.ID] = c.ChunkIndex
}
out := make(map[string][]string, len(citationSet))
for slug, set := range citationSet {
ids := make([]string, 0, len(set))
for id := range set {
ids = append(ids, id)
}
sort.SliceStable(ids, func(i, j int) bool {
return chunkOrder[ids[i]] < chunkOrder[ids[j]]
})
out[slug] = ids
}
return out, newSlugsAll, len(batches)
}
// resolveCitedChunks loads the content of every chunk referenced by the
// given additions in a single batched query per knowledge ID, returning a
// map[chunkID]content. Missing / out-of-tenant chunk IDs are silently skipped
// (logged at warn level) so the Reduce phase can gracefully fall back to the
// Details paraphrase.
func (s *wikiIngestService) resolveCitedChunks(
ctx context.Context,
tenantID uint64,
additions []SlugUpdate,
) map[string]string {
// Group chunk IDs by knowledge ID — chunk repo queries are scoped by
// tenant but not by knowledge, so we could fetch everything in one go.
// We batch per knowledge anyway to keep the IN(...) list bounded for
// large multi-knowledge reduces.
byKnowledge := make(map[string]map[string]bool)
for _, add := range additions {
if add.KnowledgeID == "" {
continue
}
for _, chunkID := range add.SourceChunks {
if chunkID == "" {
continue
}
set, ok := byKnowledge[add.KnowledgeID]
if !ok {
set = make(map[string]bool)
byKnowledge[add.KnowledgeID] = set
}
set[chunkID] = true
}
}
if len(byKnowledge) == 0 {
return nil
}
out := make(map[string]string)
for _, idSet := range byKnowledge {
ids := make([]string, 0, len(idSet))
for id := range idSet {
ids = append(ids, id)
}
if len(ids) == 0 {
continue
}
chunks, err := s.chunkRepo.ListChunksByID(ctx, tenantID, ids)
if err != nil {
logger.Warnf(ctx, "wiki ingest: failed to resolve cited chunks: %v", err)
continue
}
for _, c := range chunks {
if c == nil || c.Content == "" {
continue
}
out[c.ID] = c.Content
}
}
return out
}
// collectCitedChunkContent materializes the verbatim content of every
// referenced chunk, concatenated in the order provided. Chunk IDs that can't
// be resolved are silently dropped (logged upstream).
func collectCitedChunkContent(chunkIDs []string, contentByID map[string]string) string {
if len(chunkIDs) == 0 || len(contentByID) == 0 {
return ""
}
var sb strings.Builder
for _, id := range chunkIDs {
content, ok := contentByID[id]
if !ok || strings.TrimSpace(content) == "" {
continue
}
if sb.Len() > 0 {
sb.WriteString("\n\n")
}
sb.WriteString(content)
}
return sb.String()
}
// mergeCitationsIntoItems backfills SourceChunks on every extractedItem from
// the citation map, and augments the item slices with any genuinely new slugs
// the citation pass discovered. Items whose slug is not in citations are left
// untouched — Reduce will fall back to Description/Details for them.
//
// Returns the updated entity and concept slices plus a count of candidates
// that ended with zero citations (for observability).
func mergeCitationsIntoItems(
entities, concepts []extractedItem,
citations map[string][]string,
newSlugs []newSlugFromCitation,
) ([]extractedItem, []extractedItem, int) {
uncited := 0
for i := range entities {
ids := citations[entities[i].Slug]
entities[i].SourceChunks = ids
if len(ids) == 0 {
uncited++
}
}
for i := range concepts {
ids := citations[concepts[i].Slug]
concepts[i].SourceChunks = ids
if len(ids) == 0 {
uncited++
}
}
// Append new_slugs discovered by the citation pass, avoiding duplicates.
existingSlugs := make(map[string]bool, len(entities)+len(concepts))
for _, e := range entities {
existingSlugs[e.Slug] = true
}
for _, c := range concepts {
existingSlugs[c.Slug] = true
}
// Aggregate per-slug across batches so the same "newly discovered" slug
// surfacing in multiple batches merges into a single item.
type mergedNew struct {
item extractedItem
typ string
}
merged := make(map[string]*mergedNew)
slugOrder := []string{}
for _, ns := range newSlugs {
if ns.Slug == "" || ns.Name == "" {
continue
}
if existingSlugs[ns.Slug] {
// Treat as citation for existing candidate
continue
}
kind := strings.TrimSpace(strings.ToLower(ns.Type))
if kind == "" {
if strings.HasPrefix(ns.Slug, "concept/") {
kind = "concept"
} else {
kind = "entity"
}
}
existing, ok := merged[ns.Slug]
if !ok {
existing = &mergedNew{
item: extractedItem{
Name: ns.Name,
Slug: ns.Slug,
Aliases: append([]string(nil), ns.Aliases...),
Description: ns.Description,
Details: ns.Details,
SourceChunks: append([]string(nil), ns.SourceChunks...),
},
typ: kind,
}
merged[ns.Slug] = existing
slugOrder = append(slugOrder, ns.Slug)
continue
}
// Union source chunks
seen := make(map[string]bool, len(existing.item.SourceChunks))
for _, id := range existing.item.SourceChunks {
seen[id] = true
}
for _, id := range ns.SourceChunks {
if !seen[id] {
existing.item.SourceChunks = append(existing.item.SourceChunks, id)
seen[id] = true
}
}
}
for _, slug := range slugOrder {
m := merged[slug]
if m.typ == "concept" {
concepts = append(concepts, m.item)
} else {
entities = append(entities, m.item)
}
}
return entities, concepts, uncited
}
@@ -0,0 +1,196 @@
package service
import (
"sort"
"testing"
"github.com/Tencent/WeKnora/internal/types"
)
// TestMergeCitationsIntoItems_PopulatesSourceChunksOnCandidates verifies that
// citations returned by the chunk-classification pass are attached back onto
// the matching candidate items while non-cited candidates are left untouched.
func TestMergeCitationsIntoItems_PopulatesSourceChunksOnCandidates(t *testing.T) {
entities := []extractedItem{
{Name: "Acme", Slug: "entity/acme"},
{Name: "Beta", Slug: "entity/beta"},
}
concepts := []extractedItem{
{Name: "RAG", Slug: "concept/rag"},
}
citations := map[string][]string{
"entity/acme": {"chunk-1", "chunk-3"},
"concept/rag": {"chunk-2"},
}
gotE, gotC, uncited := mergeCitationsIntoItems(entities, concepts, citations, nil)
if len(gotE) != 2 || len(gotC) != 1 {
t.Fatalf("expected 2 entities + 1 concept, got %d + %d", len(gotE), len(gotC))
}
acme := findBySlug(gotE, "entity/acme")
if acme == nil {
t.Fatalf("entity/acme missing from result")
}
if !equalStrings(acme.SourceChunks, []string{"chunk-1", "chunk-3"}) {
t.Errorf("entity/acme source_chunks = %v, want [chunk-1 chunk-3]", acme.SourceChunks)
}
beta := findBySlug(gotE, "entity/beta")
if beta == nil {
t.Fatalf("entity/beta missing from result")
}
if len(beta.SourceChunks) != 0 {
t.Errorf("entity/beta should have no citations, got %v", beta.SourceChunks)
}
rag := findBySlug(gotC, "concept/rag")
if rag == nil {
t.Fatalf("concept/rag missing")
}
if !equalStrings(rag.SourceChunks, []string{"chunk-2"}) {
t.Errorf("concept/rag source_chunks = %v, want [chunk-2]", rag.SourceChunks)
}
if uncited != 1 {
t.Errorf("uncited = %d, want 1", uncited)
}
}
// TestMergeCitationsIntoItems_AddsNewSlugsAndUnionsChunksAcrossBatches checks
// that genuinely new slugs (ones Pass 0 missed) are appended to the right
// type slice, and that a slug surfacing in two batches ends up with the union
// of its source chunks.
func TestMergeCitationsIntoItems_AddsNewSlugsAndUnionsChunksAcrossBatches(t *testing.T) {
entities := []extractedItem{
{Name: "Known", Slug: "entity/known"},
}
concepts := []extractedItem{}
newSlugs := []newSlugFromCitation{
{
Type: "entity",
Name: "Fresh Entity",
Slug: "entity/fresh",
Description: "desc",
Details: "details",
SourceChunks: []string{"c001", "c002"},
},
{
// Same slug as above, appears in another batch — must union.
Type: "entity",
Name: "Fresh Entity",
Slug: "entity/fresh",
SourceChunks: []string{"c002", "c003"},
},
{
Type: "concept",
Name: "New Concept",
Slug: "concept/new-concept",
SourceChunks: []string{"c010"},
},
{
// Duplicate of an existing candidate — should NOT produce a
// duplicate entry (Known already exists in `entities`).
Type: "entity",
Name: "Known",
Slug: "entity/known",
SourceChunks: []string{"c020"},
},
}
gotE, gotC, _ := mergeCitationsIntoItems(entities, concepts, nil, newSlugs)
if len(gotE) != 2 {
t.Fatalf("expected 2 entities, got %d (%+v)", len(gotE), gotE)
}
if len(gotC) != 1 {
t.Fatalf("expected 1 concept, got %d (%+v)", len(gotC), gotC)
}
fresh := findBySlug(gotE, "entity/fresh")
if fresh == nil {
t.Fatalf("entity/fresh missing")
}
sort.Strings(fresh.SourceChunks)
if !equalStrings(fresh.SourceChunks, []string{"c001", "c002", "c003"}) {
t.Errorf("entity/fresh source_chunks = %v, want union [c001 c002 c003]", fresh.SourceChunks)
}
newC := findBySlug(gotC, "concept/new-concept")
if newC == nil || !equalStrings(newC.SourceChunks, []string{"c010"}) {
t.Errorf("concept/new-concept missing or wrong chunks: %+v", newC)
}
}
// TestSplitChunksIntoCitationBatches_RespectsBudgetAndOrder verifies that the
// batcher never puts too many runes in one batch, preserves document order,
// and that an oversized chunk gets its own batch.
func TestSplitChunksIntoCitationBatches_RespectsBudgetAndOrder(t *testing.T) {
// Each small chunk is 5k runes → 3 of them should fit in one batch
// (15k > 12k limit would spill to a second batch).
mk := func(idx int, runes int, id string) *types.Chunk {
return &types.Chunk{
ID: id,
ChunkIndex: idx,
Content: repeatRune('a', runes),
ChunkType: types.ChunkTypeText,
}
}
chunks := []*types.Chunk{
mk(0, 5000, "id-0"),
mk(1, 5000, "id-1"),
mk(2, 5000, "id-2"), // this should start a new batch (15k > 12k)
// An oversized chunk gets a dedicated batch.
mk(3, 20000, "id-big"),
mk(4, 1000, "id-small"),
}
batches := splitChunksIntoCitationBatches(chunks)
if len(batches) < 3 {
t.Fatalf("expected at least 3 batches, got %d", len(batches))
}
// All input IDs should show up in some batch, exactly once, in order.
seen := []string{}
for _, b := range batches {
for _, c := range b.chunks {
seen = append(seen, c.ID)
}
}
wantOrder := []string{"id-0", "id-1", "id-2", "id-big", "id-small"}
if !equalStrings(seen, wantOrder) {
t.Errorf("batch order = %v, want %v", seen, wantOrder)
}
// Verify alias → id map is populated per batch with unique aliases.
for bi, b := range batches {
if len(b.aliasToID) != len(b.chunks) {
t.Errorf("batch %d alias count %d != chunk count %d", bi, len(b.aliasToID), len(b.chunks))
}
}
}
// --- helpers ---
func findBySlug(items []extractedItem, slug string) *extractedItem {
for i := range items {
if items[i].Slug == slug {
return &items[i]
}
}
return nil
}
func equalStrings(a, b []string) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}
func repeatRune(r rune, n int) string {
out := make([]rune, n)
for i := range out {
out[i] = r
}
return string(out)
}
@@ -0,0 +1,257 @@
package service
import (
"sort"
"strings"
"unicode"
"github.com/Tencent/WeKnora/internal/searchutil"
"github.com/Tencent/WeKnora/internal/types"
)
// Pre-filtering candidate existing pages before the dedup LLM call.
//
// Without pre-filtering, the dedup prompt packs the entire entity+concept
// page corpus into <existing_pages>. On knowledge bases with 100+ pages
// this inflates input tokens and — more importantly — gives weaker LLMs
// enough rope to hallucinate merges between totally unrelated slugs just
// because the output looked plausible. Observed cases include
// "城镇登记失业人员" → "中华优秀传统文化" (zero shared characters).
//
// The filter below keeps only pages that share at least some cheap
// surface-level signal with one of the new items. Fast to compute, no
// external calls, and it only ever *removes* candidates from the prompt —
// the downstream validMerge check still guards the final write.
const (
// dedupCandidateTopK is how many existing pages we keep per new item
// even if none pass the score floor. Guarantees the LLM sees a small
// ordered shortlist rather than an empty list when nothing matches
// strongly.
dedupCandidateTopK = 20
// dedupCandidateScoreFloor is the Jaccard floor. Pairs at or above
// this similarity are always included regardless of the top-K cap.
// Tuned so that "城镇登记失业人员" vs "中华优秀传统文化" (Jaccard 0)
// is excluded while "Acme Corp" vs "Acme Corporation" (Jaccard ≈ 0.5)
// clearly passes.
dedupCandidateScoreFloor = 0.08
// dedupSmallCorpusBypass skips pre-filtering entirely when the
// existing-page corpus is already small enough to fit in the prompt
// without degrading the LLM. The filter only earns its keep on large
// KBs; on small ones it risks cutting legitimate matches with no
// real token savings.
dedupSmallCorpusBypass = 25
)
// dedupSurface is the pre-computed similarity feature set for one side of
// a (new item, existing page) comparison.
type dedupSurface struct {
// slugTokens are the kebab-case tokens from the slug base (after "/").
// Slugs are an orthogonal signal to the surface names — Chinese pages
// carry their pinyin here, which keeps the filter useful on purely
// Latin-script new items too.
slugTokens map[string]struct{}
// nameGramSets holds one char-bigram set per surface form (name and
// each alias). We keep them separate so the pair score is max-over-
// surfaces — a rare alias match shouldn't be diluted by the primary
// name disagreeing.
nameGramSets []map[string]struct{}
}
// countEntityConceptPages returns how many of the given pages are
// entity- or concept-typed. Used only for logging the prefilter's
// reduction ratio.
func countEntityConceptPages(pages []*types.WikiPage) int {
n := 0
for _, p := range pages {
if p == nil {
continue
}
if p.PageType == types.WikiPageTypeEntity || p.PageType == types.WikiPageTypeConcept {
n++
}
}
return n
}
// selectDedupCandidatePages returns the subset of allPages plausibly
// related to at least one of newItems. Non-entity/concept pages are
// dropped unconditionally. The returned slice preserves the input order
// so the downstream prompt stays stable across runs.
//
// On small corpora (<= dedupSmallCorpusBypass entries) this is a no-op
// aside from the page-type filter.
func selectDedupCandidatePages(
newItems []extractedItem,
allPages []*types.WikiPage,
) []*types.WikiPage {
pages := make([]*types.WikiPage, 0, len(allPages))
for _, p := range allPages {
if p == nil {
continue
}
if p.PageType != types.WikiPageTypeEntity && p.PageType != types.WikiPageTypeConcept {
continue
}
pages = append(pages, p)
}
if len(pages) == 0 {
return pages
}
if len(newItems) == 0 || len(pages) <= dedupSmallCorpusBypass {
return pages
}
pageFeats := make([]dedupSurface, len(pages))
for i, p := range pages {
surfaces := make([]string, 0, 1+len(p.Aliases))
surfaces = append(surfaces, p.Title)
surfaces = append(surfaces, []string(p.Aliases)...)
pageFeats[i] = dedupSurface{
slugTokens: slugBaseTokens(p.Slug),
nameGramSets: gramsPerSurface(surfaces),
}
}
selected := make(map[int]bool, len(pages))
for _, it := range newItems {
surfaces := make([]string, 0, 1+len(it.Aliases))
surfaces = append(surfaces, it.Name)
surfaces = append(surfaces, it.Aliases...)
itemFeat := dedupSurface{
slugTokens: slugBaseTokens(it.Slug),
nameGramSets: gramsPerSurface(surfaces),
}
if len(itemFeat.slugTokens) == 0 && len(itemFeat.nameGramSets) == 0 {
continue
}
scores := make([]struct {
idx int
score float64
}, len(pageFeats))
for i := range pageFeats {
scores[i].idx = i
scores[i].score = dedupPairScore(itemFeat, pageFeats[i])
}
// Stable sort so ties break deterministically by original index.
sort.SliceStable(scores, func(i, j int) bool {
return scores[i].score > scores[j].score
})
topKRemaining := dedupCandidateTopK
for _, s := range scores {
if s.score >= dedupCandidateScoreFloor {
selected[s.idx] = true
continue
}
// Below the floor but we still owe the LLM some candidates so
// it can decline cleanly — fill the top-K budget with the
// highest-scoring remaining pages, as long as the score isn't
// flatly zero (a zero score means we have nothing in common
// with this page and including it just invites hallucination).
if topKRemaining > 0 && s.score > 0 {
selected[s.idx] = true
topKRemaining--
continue
}
break
}
}
out := make([]*types.WikiPage, 0, len(selected))
for i, p := range pages {
if selected[i] {
out = append(out, p)
}
}
return out
}
// dedupPairScore is the max similarity between any surface form of a and
// b (plus the slug-token similarity). Slug and name signals live in
// different symbol spaces (ASCII pinyin vs raw surface form) so we take
// the max rather than e.g. their average.
func dedupPairScore(a, b dedupSurface) float64 {
best := searchutil.Jaccard(a.slugTokens, b.slugTokens)
for _, ag := range a.nameGramSets {
for _, bg := range b.nameGramSets {
if v := searchutil.Jaccard(ag, bg); v > best {
best = v
}
}
}
return best
}
// slugBaseTokens returns the kebab-case tokens of a slug's base component.
// "entity/beijing-nongshang-yinxing" → {"beijing", "nongshang", "yinxing"}.
func slugBaseTokens(slug string) map[string]struct{} {
if slug == "" {
return nil
}
base := slug
if i := strings.Index(slug, "/"); i >= 0 {
base = slug[i+1:]
}
base = strings.ToLower(base)
fields := strings.FieldsFunc(base, func(r rune) bool {
return r == '-' || r == '_' || r == '.' || unicode.IsSpace(r)
})
if len(fields) == 0 {
return nil
}
out := make(map[string]struct{}, len(fields))
for _, tok := range fields {
if tok == "" {
continue
}
out[tok] = struct{}{}
}
return out
}
// gramsPerSurface computes a gram set per non-empty surface form.
func gramsPerSurface(surfaces []string) []map[string]struct{} {
out := make([]map[string]struct{}, 0, len(surfaces))
for _, s := range surfaces {
g := surfaceGrams(s)
if len(g) > 0 {
out = append(out, g)
}
}
return out
}
// surfaceGrams returns a character-bigram set for a surface form after
// lowercasing and stripping non-letter/digit runes. Bigrams work well
// across both CJK (where each bigram approximates a word) and Latin
// (where they catch stem overlap like "corporation" ↔ "corp"). Single-
// rune strings fall back to a 1-gram so they still contribute a signal.
func surfaceGrams(s string) map[string]struct{} {
if s == "" {
return nil
}
var b strings.Builder
b.Grow(len(s))
for _, r := range strings.ToLower(s) {
if unicode.IsLetter(r) || unicode.IsDigit(r) {
b.WriteRune(r)
}
}
runes := []rune(b.String())
if len(runes) == 0 {
return nil
}
out := make(map[string]struct{}, len(runes))
if len(runes) == 1 {
out[string(runes)] = struct{}{}
return out
}
for i := 0; i < len(runes)-1; i++ {
out[string(runes[i:i+2])] = struct{}{}
}
return out
}
@@ -0,0 +1,191 @@
package service
import (
"strings"
"testing"
"github.com/Tencent/WeKnora/internal/types"
)
// helper: build a minimal entity/concept WikiPage.
func makePage(slug, title, typ string, aliases ...string) *types.WikiPage {
return &types.WikiPage{
Slug: slug,
Title: title,
PageType: typ,
Aliases: types.StringArray(aliases),
}
}
func pageSlugs(pages []*types.WikiPage) []string {
out := make([]string, 0, len(pages))
for _, p := range pages {
out = append(out, p.Slug)
}
return out
}
func containsSlug(pages []*types.WikiPage, slug string) bool {
for _, p := range pages {
if p.Slug == slug {
return true
}
}
return false
}
// Regression for the observed hallucination in llm_debug/20260422_171316.675.log:
// deepseek-v3.2 merged concept/chengzhen-dengji-shiye-renyuan into
// concept/zhong-hua-you-xiu-chuan-tong-wen-hua despite zero character overlap.
// With the prefilter in place, the traditional-culture page should not even
// be offered to the LLM as a candidate for that new item.
func TestSelectDedupCandidatePages_FiltersUnrelatedHallucinationTarget(t *testing.T) {
newItems := []extractedItem{
{Slug: "concept/chengzhen-dengji-shiye-renyuan", Name: "城镇登记失业人员", Aliases: []string{"登记失业人员"}},
{Slug: "entity/beijing-nongshang-yinxing", Name: "北京农商银行"},
}
// Build a corpus that mirrors the real log's shape: ~90 unrelated pages
// plus a few that share tokens with the new items (so the prefilter has
// plausible near-matches to keep).
pages := []*types.WikiPage{
makePage("concept/zhong-hua-you-xiu-chuan-tong-wen-hua", "中华优秀传统文化", "concept"),
makePage("concept/jiuye-jineng-peixun", "就业技能培训", "concept"),
makePage("concept/peixun-kaohe-pingjia", "培训考核评价", "concept"),
makePage("concept/ren-gong-zhi-neng-an-quan", "人工智能安全", "concept", "AI安全"),
makePage("entity/bei-jing-shi-jiao-yu-wei-yuan-hui", "北京市教育委员会", "entity", "北京市教委"),
makePage("entity/beijingshi-changping-zhiye-xuexiao", "北京市昌平职业学校", "entity"),
}
// Pad with filler so we exceed dedupSmallCorpusBypass.
for i := 0; i < 40; i++ {
pages = append(pages, makePage(
"concept/filler-"+strings.Repeat("x", i+1),
"填充概念"+strings.Repeat("占位", i+1),
"concept",
))
}
got := selectDedupCandidatePages(newItems, pages)
if containsSlug(got, "concept/zhong-hua-you-xiu-chuan-tong-wen-hua") {
t.Fatalf("expected unrelated page to be filtered out, but got it in candidates: %v",
pageSlugs(got))
}
if len(got) >= len(pages) {
t.Fatalf("expected prefilter to shrink the corpus (%d pages), but kept %d",
len(pages), len(got))
}
}
// A related page (shares tokens / characters with a new item) must survive
// the prefilter so the LLM can still evaluate the merge.
func TestSelectDedupCandidatePages_KeepsRelatedPages(t *testing.T) {
newItems := []extractedItem{
{Slug: "concept/chengzhen-dengji-shiye-renyuan", Name: "城镇登记失业人员", Aliases: []string{"登记失业人员"}},
}
// Build > dedupSmallCorpusBypass pages so the filter actually runs.
pages := []*types.WikiPage{
// Directly related: existing page whose title overlaps with the new
// item on "登记失业人员". Prefilter MUST keep this.
makePage("concept/deng-ji-shi-ye-ren-yuan", "登记失业人员", "concept", "城镇登记失业人员"),
// Unrelated.
makePage("concept/zhong-hua-you-xiu-chuan-tong-wen-hua", "中华优秀传统文化", "concept"),
}
for i := 0; i < 30; i++ {
pages = append(pages, makePage(
"entity/filler-"+strings.Repeat("x", i+1),
"填充实体"+strings.Repeat("占位", i+1),
"entity",
))
}
got := selectDedupCandidatePages(newItems, pages)
if !containsSlug(got, "concept/deng-ji-shi-ye-ren-yuan") {
t.Fatalf("expected strongly-related page to be kept, got: %v", pageSlugs(got))
}
}
// On small corpora the filter should be a no-op (minus page-type filtering):
// passing every page through is cheap and avoids cutting legitimate matches
// when the prompt is already small.
func TestSelectDedupCandidatePages_SmallCorpusBypass(t *testing.T) {
newItems := []extractedItem{
{Slug: "concept/a", Name: "A"},
}
pages := []*types.WikiPage{
makePage("concept/wholly-unrelated-1", "毫不相关一", "concept"),
makePage("concept/wholly-unrelated-2", "毫不相关二", "concept"),
makePage("concept/wholly-unrelated-3", "毫不相关三", "concept"),
}
got := selectDedupCandidatePages(newItems, pages)
if len(got) != len(pages) {
t.Fatalf("expected bypass on small corpus: got %d, want %d", len(got), len(pages))
}
}
// Non-entity/concept pages (summaries, logs, …) must be stripped regardless
// of corpus size — they are never valid merge targets.
func TestSelectDedupCandidatePages_DropsNonEntityConcept(t *testing.T) {
newItems := []extractedItem{
{Slug: "concept/foo", Name: "Foo"},
}
pages := []*types.WikiPage{
makePage("summary/some-doc", "Some Doc Summary", types.WikiPageTypeSummary),
makePage("log/batch-123", "Batch 123", types.WikiPageTypeLog),
makePage("concept/foo-related", "Foo Related", types.WikiPageTypeConcept),
}
got := selectDedupCandidatePages(newItems, pages)
for _, p := range got {
if p.PageType != types.WikiPageTypeEntity && p.PageType != types.WikiPageTypeConcept {
t.Fatalf("non-entity/concept page should have been filtered: %s (%s)",
p.Slug, p.PageType)
}
}
}
// surfaceGrams must yield empty intersection for the real hallucinated pair,
// confirming the underlying similarity signal is doing its job.
func TestSurfaceGrams_UnrelatedCJKPair(t *testing.T) {
a := surfaceGrams("城镇登记失业人员")
b := surfaceGrams("中华优秀传统文化")
for k := range a {
if _, ok := b[k]; ok {
t.Fatalf("expected zero bigram overlap, but shared %q", k)
}
}
}
// Latin abbreviation ↔ full-name pair must score highly (> floor) so the
// filter keeps legitimate merge candidates like "Acme Corp" ↔ "Acme Corporation".
func TestDedupPairScore_AcmeCorpVariant(t *testing.T) {
a := dedupSurface{
slugTokens: slugBaseTokens("entity/acme-corp"),
nameGramSets: gramsPerSurface([]string{"Acme Corp"}),
}
b := dedupSurface{
slugTokens: slugBaseTokens("entity/acme-corporation"),
nameGramSets: gramsPerSurface([]string{"Acme Corporation"}),
}
score := dedupPairScore(a, b)
if score < dedupCandidateScoreFloor {
t.Fatalf("expected Acme Corp ↔ Corporation score above floor %v, got %v",
dedupCandidateScoreFloor, score)
}
}
// Unrelated CJK pair (the exact case observed in production) must score 0.
func TestDedupPairScore_UnrelatedCJKPair(t *testing.T) {
a := dedupSurface{
slugTokens: slugBaseTokens("concept/chengzhen-dengji-shiye-renyuan"),
nameGramSets: gramsPerSurface([]string{"城镇登记失业人员", "登记失业人员"}),
}
b := dedupSurface{
slugTokens: slugBaseTokens("concept/zhong-hua-you-xiu-chuan-tong-wen-hua"),
nameGramSets: gramsPerSurface([]string{"中华优秀传统文化"}),
}
if s := dedupPairScore(a, b); s >= dedupCandidateScoreFloor {
t.Fatalf("expected unrelated CJK pair to score below floor %v, got %v",
dedupCandidateScoreFloor, s)
}
}
@@ -92,6 +92,7 @@ func (s *wikiPageService) UpdatePage(ctx context.Context, page *types.WikiPage)
existing.Summary = page.Summary
existing.PageType = page.PageType
existing.SourceRefs = page.SourceRefs
existing.ChunkRefs = page.ChunkRefs
existing.PageMetadata = page.PageMetadata
existing.Status = page.Status
existing.UpdatedAt = time.Now()