fix(wiki): harden slug integrity across ingest and agent write paths

Weak LLMs routinely mangle high-entropy wiki slugs (especially UUID-based
summary/<id> slugs), producing dead cross-links and cross-item dedup
hallucinations. This tightens integrity end to end:

- Ingest dedup now groups each new item with only its own trigram
  candidates and validates merges against that per-item scope, rejecting
  cross-item mispairings; candidate top-K narrowed 20 -> 5.
- Editor path aliases out-link slugs behind low-entropy ref-N tokens and
  translates them back after generation, removing the chance to mangle.
- Agent write tool normalizes/validates slugs, blocks manual creation of
  system-owned summary pages, and auto-repairs dead [[slug]] links
  (rewrite-only, never strips) via bigram/display-text resolution.
- llmresource registry aliases summary/<uuid> slugs the same way as
  storage handles so they round-trip safely through chat.

Adds unit tests for slug validation, aliasing round-trip, dead-link
repair, and dedup rejection rules.
This commit is contained in:
wizardchen
2026-07-24 15:06:49 +08:00
committed by lyingbug
parent d283fabc99
commit 5d0326f987
14 changed files with 750 additions and 68 deletions
+12 -10
View File
@@ -453,21 +453,23 @@ const WikiLogEntryTemplate = `## [{{.Date}}] {{.Operation}} | {{.Title}}
// WikiDeduplicationPrompt asks the LLM to identify duplicate entities/concepts
// between newly extracted items and existing wiki pages.
const WikiDeduplicationPrompt = `You are a strict deduplication system. Given a list of newly extracted items and a list of existing wiki pages, determine which new items refer to the **exact same** real-world entity or concept as an existing page.
const WikiDeduplicationPrompt = `You are a strict deduplication system. You are given a list of newly extracted items. Each item carries its OWN short list of existing wiki pages that are surface-similar to it (its <candidates>). For each item, decide whether it refers to the **exact same** real-world entity or concept as ONE of its own candidates.
<new_items>
{{.NewItems}}
</new_items>
<existing_pages>
{{.ExistingPages}}
</existing_pages>
<items>
{{.Candidates}}
</items>
<instructions>
### How to read the input
Each <item> is a newly extracted entity/concept. The <candidates> nested inside it are the ONLY existing pages you may merge that item into — they were pre-selected as similar to that specific item. A page listed under one item tells you NOTHING about any other item.
### Hard constraints — a merge is only valid when ALL hold:
- The target slug is one of the candidate <page> slugs listed **inside that same item**. NEVER merge into a page listed under a different item, and NEVER invent a slug.
- The types are compatible: entities merge with entities, concepts merge with concepts. **Never merge an entity into a concept or vice versa.**
### Merge criteria — ALL must be true:
1. The new item and the existing page refer to the **same real-world thing** (same person, same organization, same specific concept).
1. The new item and the candidate page refer to the **same real-world thing** (same person, same organization, same specific concept).
2. The match is a **name variation**: abbreviation ↔ full name, translation, or minor spelling difference.
3. The types are compatible: entities merge with entities, concepts merge with concepts. **Never merge an entity into a concept or vice versa.**
### Examples of CORRECT merges:
- "Acme Corp" → "Acme Corporation" (same company, abbreviation)
+67
View File
@@ -92,12 +92,45 @@ func (t *wikiWritePageTool) Execute(ctx context.Context, args json.RawMessage) (
return &types.ToolResult{Success: false, Error: "title, summary, content, and page_type are required for write action"}, nil
}
// Validate + normalize the slug up front. The model routinely emits
// malformed slugs (stray characters, mangled UUIDs); persisting them
// verbatim creates unreachable pages and dead cross-links.
normalizedSlug, slugErr := normalizeAndValidateWikiSlug(params.Slug)
if slugErr != nil {
return &types.ToolResult{Success: false, Error: slugErr.Error()}, nil
}
params.Slug = normalizedSlug
// Try to get the existing page
existingPage, err := t.wikiPageService.GetPageBySlug(ctx, kbID, params.Slug)
if err != nil && !errors.Is(err, repository.ErrWikiPageNotFound) {
return &types.ToolResult{Success: false, Error: "Failed to check existing page: " + err.Error()}, nil
}
// Summary pages are system-owned: they are generated deterministically
// from a source document and keyed by its knowledge ID
// (summary/<knowledgeID>). Letting the agent CREATE one with an
// arbitrary/hand-typed slug is precisely how mangled-UUID ghost summary
// pages (e.g. summary/…b171c…) get into the KB. Allow updating an
// existing summary page, but never fabricating a new one.
if existingPage == nil &&
(isSummaryNamespace(params.Slug) || strings.EqualFold(params.PageType, types.WikiPageTypeSummary)) {
return &types.ToolResult{
Success: false,
Error: "summary pages are generated automatically from source documents and cannot be created manually. " +
"Use page_type 'synthesis'/'comparison'/'entity'/'concept' for authored pages, " +
"or target an existing summary page to update it.",
}, nil
}
// Auto-repair dead [[slug]] references in the body before persisting.
// This rewrites LLM-mangled links (most importantly UUID-based summary
// slugs) back to their real target when a confident match exists, and
// leaves everything else untouched. Best-effort — never block the write.
if repaired, changed, rerr := t.wikiPageService.RepairContentLinks(ctx, kbID, params.Slug, params.Content); rerr == nil && changed {
params.Content = repaired
}
resolvedRefs := resolveSourceRefs(ctx, t.knowledgeService, params.SourceRefs)
var action string
@@ -164,3 +197,37 @@ func (t *wikiWritePageTool) Execute(ctx context.Context, args json.RawMessage) (
},
}, nil
}
// normalizeAndValidateWikiSlug lowercases + trims a model-supplied slug and
// rejects malformed ones. A valid slug contains only lowercase ASCII letters,
// digits, '-', '/', or CJK characters, has no leading/trailing/duplicate '/',
// and is non-empty. Keeping this strict stops the agent from persisting
// unreachable pages built from stray characters or garbled identifiers.
func normalizeAndValidateWikiSlug(raw string) (string, error) {
s := strings.ToLower(strings.TrimSpace(raw))
s = strings.ReplaceAll(s, " ", "-")
if s == "" {
return "", errors.New("slug is required and must be non-empty")
}
if strings.Contains(s, "//") || strings.HasPrefix(s, "/") || strings.HasSuffix(s, "/") {
return "", fmt.Errorf("invalid slug %q: '/' separators are malformed", raw)
}
for _, r := range s {
switch {
case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '-', r == '/':
case r >= 0x4E00 && r <= 0x9FFF: // keep CJK characters
default:
return "", fmt.Errorf(
"invalid slug %q: character %q is not allowed (use lowercase letters, digits, '-', '/', or CJK)",
raw, string(r),
)
}
}
return s, nil
}
// isSummaryNamespace reports whether a slug lives in the system-owned summary
// namespace (summary/…).
func isSummaryNamespace(slug string) bool {
return strings.HasPrefix(slug, types.WikiPageTypeSummary+"/")
}
@@ -0,0 +1,53 @@
package tools
import "testing"
func TestNormalizeAndValidateWikiSlug(t *testing.T) {
tests := []struct {
name string
in string
want string
wantErr bool
}{
{name: "already valid", in: "entity/acme-corp", want: "entity/acme-corp"},
{name: "lowercased and spaces", in: "Entity/Acme Corp", want: "entity/acme-corp"},
{name: "trimmed", in: " concept/rag ", want: "concept/rag"},
{name: "cjk kept", in: "entity/上海中心大厦", want: "entity/上海中心大厦"},
{name: "uuid summary", in: "summary/07a20bb1-a662-47cf-9929-06fb5d5b5b5e", want: "summary/07a20bb1-a662-47cf-9929-06fb5d5b5b5e"},
{name: "empty", in: " ", wantErr: true},
{name: "leading slash", in: "/entity/x", wantErr: true},
{name: "trailing slash", in: "entity/x/", wantErr: true},
{name: "double slash", in: "entity//x", wantErr: true},
{name: "invalid char", in: "entity/x!y", wantErr: true},
{name: "invalid space-only becomes empty", in: " ", wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := normalizeAndValidateWikiSlug(tt.in)
if tt.wantErr {
if err == nil {
t.Fatalf("expected error for %q, got slug %q", tt.in, got)
}
return
}
if err != nil {
t.Fatalf("unexpected error for %q: %v", tt.in, err)
}
if got != tt.want {
t.Fatalf("normalizeAndValidateWikiSlug(%q) = %q, want %q", tt.in, got, tt.want)
}
})
}
}
func TestIsSummaryNamespace(t *testing.T) {
if !isSummaryNamespace("summary/abc") {
t.Fatal("summary/abc must be in the summary namespace")
}
if isSummaryNamespace("summary") {
t.Fatal("bare 'summary' (no slash) must not count as the summary namespace")
}
if isSummaryNamespace("entity/summary-of-x") {
t.Fatal("entity/summary-of-x must not count as the summary namespace")
}
}
+93 -51
View File
@@ -2170,19 +2170,39 @@ func (s *wikiIngestService) publishDraftPages(ctx context.Context, kbID string,
}
}
// writeDedupItemXML renders a single entity/concept entry as a structured XML
// block for the deduplication prompt. Structured form (versus a single
// pipe-separated line) helps the LLM reliably tell name / aliases / type apart
// and reduces nonsensical merges like "居民身份证" → "工作居住证".
func writeDedupItemXML(buf *strings.Builder, slug, name, itemType string, aliases []string) {
fmt.Fprintf(buf, " <item slug=%q type=%q>\n", slug, itemType)
fmt.Fprintf(buf, " <name>%s</name>\n", xmlEscape(name))
for _, alias := range aliases {
// writeDedupCandidateGroup renders one new item together with its own
// similarity-candidate existing pages, nested under a <candidates> element.
// This per-item grouping is what constrains the dedup model to local
// decisions (see the grouping rationale in deduplicateExtractedBatch). The
// candidate pages keep their aliases so the model still has the acronym /
// translation signal it needs to accept a legitimate merge.
func writeDedupCandidateGroup(
buf *strings.Builder, item extractedItem, itemType string, candidates []*types.WikiPageLite,
) {
fmt.Fprintf(buf, " <item slug=%q type=%q>\n", item.Slug, itemType)
fmt.Fprintf(buf, " <name>%s</name>\n", xmlEscape(item.Name))
for _, alias := range item.Aliases {
if alias == "" {
continue
}
fmt.Fprintf(buf, " <alias>%s</alias>\n", xmlEscape(alias))
}
buf.WriteString(" <candidates>\n")
for _, p := range candidates {
if p == nil {
continue
}
fmt.Fprintf(buf, " <page slug=%q type=%q>\n", p.Slug, p.PageType)
fmt.Fprintf(buf, " <name>%s</name>\n", xmlEscape(p.Title))
for _, alias := range []string(p.Aliases) {
if alias == "" {
continue
}
fmt.Fprintf(buf, " <alias>%s</alias>\n", xmlEscape(alias))
}
buf.WriteString(" </page>\n")
}
buf.WriteString(" </candidates>\n")
buf.WriteString(" </item>\n")
}
@@ -2226,7 +2246,16 @@ func (s *wikiIngestService) deduplicateExtractedBatch(
// Build the candidate set: for each new item, ask the repo for
// the top-K trigram-similar pages and union the results. Dedup by
// slug as we go so the prompt only carries each candidate once.
//
// itemCandidates additionally records, per new item, the slugs that
// surfaced for THAT item specifically. The prompt only ever sees the
// flattened union, so validMerge below uses this per-item scoping to
// reject a merge whose target was pulled in for a *different* item —
// the class of hallucination the union otherwise enables (e.g. weak
// models emitting entity/tencent-open → entity/hiring-agent, which
// share no trigram signal and were never candidates for each other).
candidatePages := make(map[string]*types.WikiPageLite)
itemCandidates := make(map[string]map[string]bool)
probe := func(item extractedItem) {
queries := make([]string, 0, 1+len(item.Aliases))
if item.Name != "" {
@@ -2237,6 +2266,11 @@ func (s *wikiIngestService) deduplicateExtractedBatch(
queries = append(queries, alias)
}
}
own := itemCandidates[item.Slug]
if own == nil {
own = make(map[string]bool)
itemCandidates[item.Slug] = own
}
for _, q := range queries {
pages, err := s.wikiService.FindSimilarPages(ctx, kbID, q,
[]string{types.WikiPageTypeEntity, types.WikiPageTypeConcept},
@@ -2252,6 +2286,7 @@ func (s *wikiIngestService) deduplicateExtractedBatch(
if _, ok := candidatePages[p.Slug]; !ok {
candidatePages[p.Slug] = p
}
own[p.Slug] = true
}
}
}
@@ -2270,25 +2305,59 @@ func (s *wikiIngestService) deduplicateExtractedBatch(
logger.Infof(ctx, "wiki ingest: %d similar existing pages selected for %d new items",
len(candidatePages), len(entities)+len(concepts))
var existingBuf strings.Builder
for _, p := range candidatePages {
writeDedupItemXML(&existingBuf, p.Slug, p.Title, p.PageType, []string(p.Aliases))
// Group each new item with ONLY the existing pages that surfaced for
// its own similarity probe. Presenting the model two flat lists (all
// new items × all candidates) invites cross-item mispairings — it has
// no way to tell which candidate is relevant to which item, so a weak
// model pairs unrelated slugs that merely coexist in the prompt. A
// per-item shortlist turns dedup into a local yes/no decision against
// a handful of genuinely-similar pages and makes cross-item pairings
// structurally unnatural to express. Items with no candidate are
// omitted entirely (they cannot merge and only add hallucination
// surface + tokens).
var candBuf strings.Builder
groups := 0
renderGroup := func(item extractedItem, itemType string) {
cset := itemCandidates[item.Slug]
if len(cset) == 0 {
return
}
slugs := make([]string, 0, len(cset))
for slug := range cset {
// Skip the item's own slug: an identically-slugged existing
// page is a re-ingest/update, not a merge target.
if slug == item.Slug {
continue
}
if _, ok := candidatePages[slug]; ok {
slugs = append(slugs, slug)
}
}
if len(slugs) == 0 {
return
}
sort.Strings(slugs)
pages := make([]*types.WikiPageLite, 0, len(slugs))
for _, slug := range slugs {
pages = append(pages, candidatePages[slug])
}
writeDedupCandidateGroup(&candBuf, item, itemType, pages)
groups++
}
if existingBuf.Len() == 0 {
for _, item := range entities {
renderGroup(item, "entity")
}
for _, item := range concepts {
renderGroup(item, "concept")
}
if groups == 0 {
// Every new item's candidate list is empty after scoping —
// nothing the model could safely merge.
return entities, concepts
}
var newBuf strings.Builder
for _, item := range entities {
writeDedupItemXML(&newBuf, item.Slug, item.Name, "entity", item.Aliases)
}
for _, item := range concepts {
writeDedupItemXML(&newBuf, item.Slug, item.Name, "concept", item.Aliases)
}
dedupeJSON, err := s.generateWithTemplate(ctx, chatModel, agent.WikiDeduplicationPrompt, map[string]string{
"NewItems": newBuf.String(),
"ExistingPages": existingBuf.String(),
"Candidates": candBuf.String(),
})
if err != nil {
logger.Warnf(ctx, "wiki ingest: deduplication LLM call failed: %v", err)
@@ -2309,36 +2378,9 @@ func (s *wikiIngestService) deduplicateExtractedBatch(
return entities, concepts
}
// Build the existing-slug set from the candidate map: anything not
// in candidates is rejected as an LLM hallucination, since by
// construction the model only ever saw those slugs as merge
// targets. Compare with the legacy "look up against allPages"
// path which had a wider acceptance window.
existingSlugs := make(map[string]bool, len(candidatePages))
for slug := range candidatePages {
existingSlugs[slug] = true
}
validMerge := func(srcSlug, dstSlug string) bool {
if !existingSlugs[dstSlug] {
logger.Warnf(ctx, "wiki ingest: dedup rejected %s → %s (target slug does not exist in candidate set)", srcSlug, dstSlug)
return false
}
srcSlash := strings.Index(srcSlug, "/")
dstSlash := strings.Index(dstSlug, "/")
if srcSlash <= 0 || dstSlash <= 0 {
// A type-prefixed slug must look like "entity/foo" or
// "concept/bar". An LLM that emits an un-prefixed slug
// here is hallucinating; reject rather than fall through
// the prefix-equality check (which would treat both empty
// prefixes as a match).
logger.Warnf(ctx, "wiki ingest: dedup rejected %s → %s (missing type prefix)", srcSlug, dstSlug)
return false
}
srcPrefix := srcSlug[:srcSlash+1]
dstPrefix := dstSlug[:dstSlash+1]
if srcPrefix != dstPrefix {
logger.Warnf(ctx, "wiki ingest: dedup rejected %s → %s (type mismatch: %s vs %s)", srcSlug, dstSlug, srcPrefix, dstPrefix)
if reason := dedupMergeRejectReason(srcSlug, dstSlug, itemCandidates[srcSlug]); reason != "" {
logger.Warnf(ctx, "wiki ingest: dedup rejected %s → %s (%s)", srcSlug, dstSlug, reason)
return false
}
return true
@@ -2003,9 +2003,27 @@ func (s *wikiIngestService) reduceSlugUpdates(
if len(additions) > 0 || len(retracts) > 0 {
titles := batchCtx.SlugTitleMany(ctx, []string(page.OutLinks))
// aliaser escapes high-entropy slugs behind short reference tokens
// (ref-1, ref-2, …) for the editor LLM, then translates them back to
// real slugs after generation (see dealiasContent below).
aliaser := newSlugAliaser()
// Escape every out-link slug behind a reference token so the editor
// never has to retype a real slug — this is where UUID-based summary
// slugs (summary/<uuid>) got mangled into 404-ing links. Both the
// <valid_wiki_links> listing AND the [[...]] refs already present in
// <existing_page_content> are aliased with the SAME token map, so the
// model sees one consistent, copy-safe identifier space; we translate
// the tokens back to real slugs after generation.
known := make(map[string]struct{}, len(page.OutLinks))
for _, outSlug := range page.OutLinks {
known[outSlug] = struct{}{}
aliaser.token(outSlug) // pre-assign for stable ordering
}
for _, outSlug := range page.OutLinks {
if title := titles[outSlug]; title != "" {
fmt.Fprintf(&relatedSlugs, "- %s (%s)\n", outSlug, title)
fmt.Fprintf(&relatedSlugs, "- %s (%s)\n", aliaser.token(outSlug), title)
}
}
@@ -2013,6 +2031,7 @@ func (s *wikiIngestService) reduceSlugUpdates(
// [c003]. They are internal ingest metadata; keep the editor context
// clean so a subsequent update cannot copy them into rewritten prose.
existingContent := stripWikiInlineChunkCitations(page.Content)
existingContent = aliaser.aliasContent(existingContent, known)
if !exists || existingContent == "" {
existingContent = "(New page)"
}
@@ -2060,6 +2079,10 @@ func (s *wikiIngestService) reduceSlugUpdates(
})
if err == nil && updatedContent != "" {
// Translate reference tokens (ref-N) the model copied from the
// aliased context back to their real slugs BEFORE the content is
// parsed/stored, so out_links reflect real pages again.
updatedContent = aliaser.dealiasContent(updatedContent)
updatedSummary, updatedBody := splitSummaryLine(updatedContent)
if updatedBody != "" {
page.Content = updatedBody
@@ -23,11 +23,15 @@ import (
// external calls, and it only ever *removes* candidates from the prompt —
// the downstream validMerge check still guards the final write.
const (
// dedupCandidateTopK is how many existing pages we keep per new item
// even if none pass the score floor. Guarantees the LLM sees a small
// ordered shortlist rather than an empty list when nothing matches
// strongly.
dedupCandidateTopK = 20
// dedupCandidateTopK bounds how many trigram-similar existing pages
// each new item's similarity probe returns (the LIMIT passed to
// FindSimilarPages, applied per query term = name + each alias). Kept
// deliberately small: the DB already orders by similarity desc behind a
// pg_trgm threshold, so a genuine same-entity target is essentially
// always the top hit. A tight K keeps each item's <candidates> list in
// the dedup prompt short, which improves the model's precision and cuts
// tokens, at negligible recall cost.
dedupCandidateTopK = 5
// dedupCandidateScoreFloor is the Jaccard floor. Pairs at or above
// this similarity are always included regardless of the top-K cap.
@@ -170,6 +174,44 @@ func selectDedupCandidatePages(
return out
}
// dedupMergeRejectReason validates a single LLM-proposed merge (srcSlug →
// dstSlug) against deterministic, model-independent rules. It returns an
// empty string when the merge is allowed, or a short human-readable reason
// when it must be rejected. srcCandidates is the set of existing-page slugs
// that surfaced for srcSlug's OWN similarity probe (see itemCandidates in
// deduplicateExtractedBatch).
//
// The per-item scoping check is the key guard: the dedup prompt shows the
// model a flattened union of candidates across every new item, so nothing
// stops a weak model from pairing an item with a page that was only similar
// to a *different* item (observed: entity/tencent-open → entity/hiring-agent,
// which share no trigram signal). Requiring dstSlug to be one of srcSlug's
// own candidates rejects that entire class without depending on the model
// getting the semantic judgment right.
func dedupMergeRejectReason(srcSlug, dstSlug string, srcCandidates map[string]bool) string {
if !srcCandidates[dstSlug] {
// Covers both an outright hallucinated target (in no candidate
// set at all) and a real page that was only similar to another
// item. Either way the pair lacks a similarity signal for THIS
// item, so it is not a safe merge.
return "target is not a similarity candidate for this item"
}
srcSlash := strings.Index(srcSlug, "/")
dstSlash := strings.Index(dstSlug, "/")
if srcSlash <= 0 || dstSlash <= 0 {
// A type-prefixed slug must look like "entity/foo" or
// "concept/bar". An LLM that emits an un-prefixed slug here is
// hallucinating; reject rather than fall through the prefix-
// equality check (which would treat both empty prefixes as a
// match).
return "missing type prefix"
}
if srcSlug[:srcSlash+1] != dstSlug[:dstSlash+1] {
return "type mismatch: " + srcSlug[:srcSlash+1] + " vs " + dstSlug[:dstSlash+1]
}
return ""
}
// dedupPairScore is the max similarity between any surface form of a and
// b (plus the slug-token similarity). Slug and name signals live in
// different symbol spaces (ASCII pinyin vs raw surface form) so we take
@@ -7,6 +7,61 @@ import (
"github.com/Tencent/WeKnora/internal/types"
)
func TestDedupMergeRejectReason(t *testing.T) {
cases := []struct {
name string
src, dst string
candidates map[string]bool
wantAllowed bool
}{
{
name: "allowed: dst is a candidate for this item, same type prefix",
src: "entity/acme-corp",
dst: "entity/acme-corporation",
candidates: map[string]bool{"entity/acme-corporation": true},
wantAllowed: true,
},
{
name: "rejected: dst similar to a DIFFERENT item (union hallucination)",
src: "entity/tencent-open",
dst: "entity/hiring-agent",
candidates: map[string]bool{"entity/tencent-ur": true}, // hiring-agent not here
wantAllowed: false,
},
{
name: "rejected: dst is not a candidate at all (pure hallucination)",
src: "entity/hy3-preview",
dst: "entity/llm-cli-tool",
candidates: nil,
wantAllowed: false,
},
{
name: "rejected: type mismatch even when dst is a candidate",
src: "entity/foo",
dst: "concept/foo",
candidates: map[string]bool{"concept/foo": true},
wantAllowed: false,
},
{
name: "rejected: missing type prefix",
src: "foo",
dst: "entity/foo",
candidates: map[string]bool{"entity/foo": true},
wantAllowed: false,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
reason := dedupMergeRejectReason(tc.src, tc.dst, tc.candidates)
gotAllowed := reason == ""
if gotAllowed != tc.wantAllowed {
t.Fatalf("dedupMergeRejectReason(%q, %q) allowed=%v (reason=%q), want allowed=%v",
tc.src, tc.dst, gotAllowed, reason, tc.wantAllowed)
}
})
}
}
// helper: build a minimal entity/concept WikiPage.
func makePage(slug, title, typ string, aliases ...string) *types.WikiPage {
return &types.WikiPage{
+145
View File
@@ -918,6 +918,151 @@ func normalizeSlug(slug string) string {
return slug
}
// slugNamespace returns the prefix of a slug up to (but excluding) the first
// '/', e.g. "summary/abc" -> "summary". Slugs without a '/' map to "".
func slugNamespace(slug string) string {
if i := strings.IndexByte(slug, '/'); i >= 0 {
return slug[:i]
}
return ""
}
// rewriteDeadWikiLinks walks every [[slug]] / [[slug|display]] occurrence in
// content and lets `resolve` decide, per link, whether to rewrite the slug.
// `resolve` receives the normalized slug and its display text and returns the
// replacement slug plus true to rewrite, or ("", false) to leave the link
// untouched. Display text is preserved verbatim. This is a pure helper — the
// resolution policy lives entirely in the callback.
func rewriteDeadWikiLinks(content string, resolve func(normSlug, display string) (string, bool)) (string, bool) {
changed := false
out := wikiLinkRegex.ReplaceAllStringFunc(content, func(match string) string {
inner := match[2 : len(match)-2]
rawSlug := inner
display := ""
if parts := strings.SplitN(inner, "|", 2); len(parts) == 2 {
rawSlug = parts[0]
display = strings.TrimSpace(parts[1])
}
norm := normalizeSlug(rawSlug)
if norm == "" {
return match
}
newSlug, ok := resolve(norm, display)
if !ok || newSlug == "" || newSlug == norm {
return match
}
changed = true
if display != "" {
return "[[" + newSlug + "|" + display + "]]"
}
return "[[" + newSlug + "]]"
})
return out, changed
}
// RepairContentLinks rewrites [[slug]] / [[slug|display]] references in
// `content` whose target does not exist in the KB but is almost certainly a
// mangled form of a real page. The canonical case: an LLM re-typed a summary
// page's UUID-based slug and inserted/dropped a hex digit
// (summary/…06fb5d5b5b5e → summary/…06fb14d5b14b14e), producing a link that
// 404s and can never be recovered by exact lookup.
//
// Unlike stripDeadWikiLinks (the ingest cleanup pass), this method is
// REWRITE-ONLY: a dead link is corrected only when a confident live candidate
// exists; otherwise it is left completely untouched. It NEVER strips a link to
// plain text, so it is safe on any write path — including writes whose targets
// legitimately do not exist yet (those simply stay as-is until they do).
//
// The candidate pool for each dead link is scoped to live slugs that share the
// same namespace prefix (a dead `summary/<uuid>` only resolves against live
// `summary/*` slugs). Scoping keeps the bigram-similarity lever safe: distinct
// high-entropy UUIDs in one namespace don't collide, while a one-character
// mangle stays comfortably above threshold against its true source.
//
// Returns the possibly-updated content and whether any rewrite happened.
// Errors are only returned for hard repo failures; callers may treat repair as
// best-effort and ignore them.
func (s *wikiPageService) RepairContentLinks(
ctx context.Context, kbID, selfSlug, content string,
) (string, bool, error) {
if strings.TrimSpace(content) == "" {
return content, false, nil
}
outLinks := s.parseOutLinks(content)
if len(outLinks) == 0 {
return content, false, nil
}
existMap, err := s.repo.ExistsSlugs(ctx, kbID, outLinks)
if err != nil {
return content, false, err
}
deadPrefixes := make(map[string]struct{})
for _, l := range outLinks {
if l == selfSlug || existMap[l] {
continue
}
deadPrefixes[slugNamespace(l)] = struct{}{}
}
if len(deadPrefixes) == 0 {
return content, false, nil
}
allSlugs, err := s.repo.ListAllSlugs(ctx, kbID)
if err != nil {
return content, false, err
}
liveByPrefix := make(map[string]map[string]struct{})
var candidateSlugs []string
for _, sl := range allSlugs {
ns := slugNamespace(sl)
if _, want := deadPrefixes[ns]; !want {
continue
}
set, ok := liveByPrefix[ns]
if !ok {
set = make(map[string]struct{})
liveByPrefix[ns] = set
}
set[sl] = struct{}{}
candidateSlugs = append(candidateSlugs, sl)
}
if len(candidateSlugs) == 0 {
return content, false, nil
}
// Build a title -> slug reverse lookup for candidate pages so the
// display-text lever (the safest, most precise one) can fire. Bounded to
// the relevant namespaces so this stays cheap even on large KBs.
titleToSlug := make(map[string]string)
if lites, lerr := s.repo.ListBySlugs(ctx, kbID, candidateSlugs); lerr == nil {
for _, lp := range lites {
if lp != nil && lp.Title != "" {
titleToSlug[lp.Title] = lp.Slug
}
}
}
resolveCache := make(map[string]string)
newContent, changed := rewriteDeadWikiLinks(content, func(norm, display string) (string, bool) {
if norm == selfSlug || existMap[norm] {
return "", false
}
key := norm + "\x00" + display
if cached, ok := resolveCache[key]; ok {
return cached, cached != ""
}
resolved, ok := resolveDeadSlug(norm, display, liveByPrefix[slugNamespace(norm)], titleToSlug)
if !ok || resolved == norm {
resolveCache[key] = ""
return "", false
}
resolveCache[key] = resolved
return resolved, true
})
return newContent, changed, nil
}
// updateInLinks adds the source slug to the in_links of target pages
func (s *wikiPageService) updateInLinks(ctx context.Context, kbID string, sourceSlug string, targets types.StringArray) {
for _, targetSlug := range targets {
@@ -8,6 +8,7 @@ import (
"github.com/Tencent/WeKnora/internal/application/repository"
"github.com/Tencent/WeKnora/internal/types"
"github.com/google/uuid"
"github.com/stretchr/testify/require"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
@@ -140,6 +141,60 @@ func TestParseOutLinks(t *testing.T) {
}
}
func TestRepairContentLinks(t *testing.T) {
db, err := gorm.Open(sqlite.Open(fmt.Sprintf("file:%s?mode=memory&cache=shared", t.Name())), &gorm.Config{})
require.NoError(t, err)
require.NoError(t, db.AutoMigrate(&types.WikiFolder{}, &types.WikiPage{}))
ctx := context.Background()
repo := repository.NewWikiPageRepository(db)
svc := NewWikiPageService(repo, nil, nil, nil, nil)
const kbID = "kb-repair"
now := time.Now()
seed := func(slug, title, pageType string) {
require.NoError(t, repo.Create(ctx, &types.WikiPage{
ID: uuid.New().String(), TenantID: 1, KnowledgeBaseID: kbID,
Slug: slug, Title: title, PageType: pageType,
Status: types.WikiPageStatusPublished, Version: 1,
CreatedAt: now, UpdatedAt: now,
}))
}
// Real summary page (clean UUID slug) + a distractor summary + an entity.
realSummary := "summary/07a20bb1-a662-47cf-9929-06fb5d5b5b5e"
seed(realSummary, "Weknora 试错记录.md - Summary", types.WikiPageTypeSummary)
seed("summary/fcadcaab-e094-4037-b71c-9edfdcdba058", "Another Doc - Summary", types.WikiPageTypeSummary)
seed("entity/mongodb", "MongoDB", types.WikiPageTypeEntity)
t.Run("mangled uuid summary link repaired via bigram", func(t *testing.T) {
// One hex digit inserted into the last group — 404s on exact lookup.
content := "See [[summary/07a20bb1-a662-47cf-9929-06fb14d5b14b14e|Weknora 试错记录.md - Summary]] for context."
got, changed, err := svc.RepairContentLinks(ctx, kbID, "synthesis/notes", content)
require.NoError(t, err)
require.True(t, changed, "expected the mangled summary link to be rewritten")
require.Contains(t, got, "[["+realSummary+"|Weknora 试错记录.md - Summary]]")
require.NotContains(t, got, "06fb14d5b14b14e")
})
t.Run("live links untouched", func(t *testing.T) {
content := "Fine: [[entity/mongodb]] and [[" + realSummary + "]]."
got, changed, err := svc.RepairContentLinks(ctx, kbID, "synthesis/notes", content)
require.NoError(t, err)
require.False(t, changed)
require.Equal(t, content, got)
})
t.Run("unresolvable dead link left as-is, never stripped", func(t *testing.T) {
// A future entity page that doesn't exist and isn't similar to anything.
content := "Pending: [[entity/some-brand-new-topic|New Topic]]."
got, changed, err := svc.RepairContentLinks(ctx, kbID, "synthesis/notes", content)
require.NoError(t, err)
require.False(t, changed)
require.Equal(t, content, got, "unresolvable links must be preserved verbatim, not stripped")
})
}
func TestNormalizeSlug(t *testing.T) {
tests := []struct {
input string
@@ -0,0 +1,88 @@
package service
import (
"fmt"
"strings"
)
// slugAliaser maps real wiki slugs to short, low-entropy reference tokens
// (ref-1, ref-2, …) so the ingest editor LLM never has to reproduce a
// high-entropy slug verbatim — most importantly the UUID-based summary slugs
// (summary/<knowledgeID>) that models routinely mangle by inserting or
// dropping hex digits. The model copies the tiny token; we translate tokens
// back to real slugs on output.
//
// This is the generation-stage counterpart to wiki_write_page's slug
// validation: rather than repairing a mangled slug after the fact, it removes
// the opportunity to mangle at the source. It mirrors the chunk-alias
// (c000/c001) indirection already used by the chunk-citation pass in
// wiki_ingest_cite.go.
//
// A slugAliaser is single-use and NOT safe for concurrent use; construct one
// per LLM call.
type slugAliaser struct {
toToken map[string]string // real slug -> token
toSlug map[string]string // token -> real slug
}
func newSlugAliaser() *slugAliaser {
return &slugAliaser{
toToken: make(map[string]string),
toSlug: make(map[string]string),
}
}
// token returns the stable token for a real slug, assigning a fresh one on
// first use. An empty slug returns "". Tokens have no '/' so they can never
// collide with a namespaced real slug (entity/…, summary/…).
func (a *slugAliaser) token(realSlug string) string {
realSlug = strings.TrimSpace(realSlug)
if realSlug == "" {
return ""
}
if t, ok := a.toToken[realSlug]; ok {
return t
}
t := fmt.Sprintf("ref-%d", len(a.toToken)+1)
a.toToken[realSlug] = t
a.toSlug[t] = realSlug
return t
}
// empty reports whether no tokens have been assigned yet (nothing to alias).
func (a *slugAliaser) empty() bool { return len(a.toToken) == 0 }
// aliasContent rewrites [[realSlug|disp]] / [[realSlug]] occurrences to their
// token form, but only for slugs present in `known`. Unknown links are left
// untouched. Assigns tokens on demand so a slug that appears only in the body
// (not in the listing) still gets a consistent token.
func (a *slugAliaser) aliasContent(content string, known map[string]struct{}) string {
if content == "" || len(known) == 0 {
return content
}
out, _ := rewriteDeadWikiLinks(content, func(norm, _ string) (string, bool) {
if _, ok := known[norm]; !ok {
return "", false
}
return a.token(norm), true
})
return out
}
// dealiasContent rewrites [[token|disp]] / [[token]] occurrences back to their
// real slug. Tokens with no mapping are left untouched — they fall through to
// the ordinary parse/dead-link-cleanup path, which is the correct behavior for
// anything the model invented that isn't a known reference.
func (a *slugAliaser) dealiasContent(content string) string {
if content == "" || a.empty() {
return content
}
out, _ := rewriteDeadWikiLinks(content, func(norm, _ string) (string, bool) {
real, ok := a.toSlug[norm]
if !ok {
return "", false
}
return real, true
})
return out
}
@@ -0,0 +1,72 @@
package service
import (
"strings"
"testing"
)
func TestSlugAliaser_RoundTripEscapesUUIDSlugs(t *testing.T) {
summarySlug := "summary/07a20bb1-a662-47cf-9929-06fb5d5b5b5e"
entitySlug := "entity/mongodb"
known := map[string]struct{}{summarySlug: {}, entitySlug: {}}
a := newSlugAliaser()
// Pre-assign tokens the way the ingest listing loop does.
tSummary := a.token(summarySlug)
tEntity := a.token(entitySlug)
if tSummary == summarySlug || tEntity == entitySlug {
t.Fatalf("tokens must differ from real slugs: %q %q", tSummary, tEntity)
}
if strings.Contains(tSummary, "/") {
t.Fatalf("token %q must not contain '/'", tSummary)
}
body := "See [[" + summarySlug + "|Weknora 试错记录.md - Summary]] and [[" + entitySlug + "]]."
aliased := a.aliasContent(body, known)
// The high-entropy UUID must no longer be present in what the model sees.
if strings.Contains(aliased, "06fb5d5b5b5e") {
t.Fatalf("aliased content still exposes the UUID: %q", aliased)
}
if !strings.Contains(aliased, "[["+tSummary+"|Weknora 试错记录.md - Summary]]") {
t.Fatalf("summary link not aliased: %q", aliased)
}
if !strings.Contains(aliased, "[["+tEntity+"]]") {
t.Fatalf("entity link not aliased: %q", aliased)
}
// Round-trip must restore the original body exactly.
if got := a.dealiasContent(aliased); got != body {
t.Fatalf("dealias round-trip mismatch:\n got %q\n want %q", got, body)
}
}
func TestSlugAliaser_UnknownTokensAndSlugsUntouched(t *testing.T) {
a := newSlugAliaser()
a.token("summary/real-uuid")
// A slug not in `known` must not be aliased.
known := map[string]struct{}{"summary/real-uuid": {}}
body := "keep [[entity/invented-by-model]] as-is"
if got := a.aliasContent(body, known); got != body {
t.Fatalf("unknown slug should be left untouched, got %q", got)
}
// An unmapped token in output must be left untouched (falls through to the
// normal dead-link cleanup path).
out := "stray [[ref-999|X]] token"
if got := a.dealiasContent(out); got != out {
t.Fatalf("unmapped token should be left untouched, got %q", got)
}
}
func TestSlugAliaser_EmptyIsNoop(t *testing.T) {
a := newSlugAliaser()
if !a.empty() {
t.Fatal("fresh aliaser must be empty")
}
body := "[[summary/x]] unchanged"
if got := a.dealiasContent(body); got != body {
t.Fatalf("empty aliaser dealias must be a no-op, got %q", got)
}
}
+13 -1
View File
@@ -17,10 +17,22 @@ import (
// resource:// handles, but old chunks and message history can still contain a
// provider URL. Giving both forms the same request-local alias makes rollout
// safe without a blocking full-table content rewrite.
//
// The final alternative aliases wiki summary-page slugs (summary/<uuid>). They
// are not storage handles, but they share the exact failure mode this registry
// exists to prevent: a high-entropy identifier the model must reproduce
// verbatim (inside [[slug|display]] links and as wiki-tool slug arguments).
// Models routinely mangle the UUID by inserting or dropping hex digits, which
// yields dead cross-links. Aliasing the slug to a low-entropy res:// token
// removes the opportunity to mangle at the source; the token round-trips back
// to the real slug on stream output and in decoded tool-call arguments before
// anything is persisted. Entity slugs (entity/<readable-title>) are low-entropy
// and semantically meaningful, so they are deliberately left untouched.
var storedRefRE = regexp.MustCompile(
`resource://[0-9A-Za-z_-]{22}|` +
`(?:storage://[0-9A-Za-z_-]+/)?` +
`(?:local|minio|cos|tos|s3|oss|ks3|obs)://[^\s)\]>"']+`,
`(?:local|minio|cos|tos|s3|oss|ks3|obs)://[^\s)\]>"']+|` +
`summary/[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}`,
)
// aliasShapeRE matches the alias syntax produced by EncodeText. It is used only
+19
View File
@@ -23,6 +23,25 @@ func TestRegistryAliasesLegacyPhysicalReferencesDuringRollout(t *testing.T) {
require.Equal(t, "![image]("+ref+")", r.DecodeText(encoded))
}
func TestRegistryAliasesWikiSummarySlug(t *testing.T) {
r := NewRegistry()
slug := "summary/07a20bb1-a662-47cf-9929-06fb5d5b5b5e"
// Inside a [[slug|display]] link the model must copy verbatim.
encoded := r.EncodeText("see [[" + slug + "|Foo.md - Summary]]")
require.Equal(t, "see [[res://0001|Foo.md - Summary]]", encoded)
require.Equal(t, "see [["+slug+"|Foo.md - Summary]]", r.DecodeText(encoded))
// The same slug as a tool-call argument value resolves to one alias.
require.Equal(t, "res://0001", r.EncodeText(slug))
}
func TestRegistryLeavesEntitySlugUntouched(t *testing.T) {
r := NewRegistry()
// Entity slugs are low-entropy and semantically meaningful; not aliased.
entity := "[[entity/weknora-error-log|WeKnora 试错记录]]"
require.Equal(t, entity, r.EncodeText(entity))
}
func TestRegistryEncodesMessageCopies(t *testing.T) {
r := NewRegistry()
ref := "resource://AbCdEfGhIjKlMnOpQrStUv"
+7
View File
@@ -39,6 +39,13 @@ type WikiPageService interface {
// GetPageBySlug retrieves a wiki page by its slug within a knowledge base.
GetPageBySlug(ctx context.Context, kbID string, slug string) (*types.WikiPage, error)
// RepairContentLinks rewrites dead [[slug]] references in content to their
// most likely live target (rewrite-only — never strips). Used by the agent
// write path to auto-correct LLM-mangled slugs (especially UUID-based
// summary slugs) before persistence. Returns the possibly-updated content
// and whether any rewrite happened. Best-effort: callers may ignore errors.
RepairContentLinks(ctx context.Context, kbID, selfSlug, content string) (string, bool, error)
// GetPageByID retrieves a wiki page by its unique ID.
GetPageByID(ctx context.Context, id string) (*types.WikiPage, error)