fix(wiki): harden slug integrity across ingest and agent write paths
Weak LLMs routinely mangle high-entropy wiki slugs (especially UUID-based summary/<id> slugs), producing dead cross-links and cross-item dedup hallucinations. This tightens integrity end to end: - Ingest dedup now groups each new item with only its own trigram candidates and validates merges against that per-item scope, rejecting cross-item mispairings; candidate top-K narrowed 20 -> 5. - Editor path aliases out-link slugs behind low-entropy ref-N tokens and translates them back after generation, removing the chance to mangle. - Agent write tool normalizes/validates slugs, blocks manual creation of system-owned summary pages, and auto-repairs dead [[slug]] links (rewrite-only, never strips) via bigram/display-text resolution. - llmresource registry aliases summary/<uuid> slugs the same way as storage handles so they round-trip safely through chat. Adds unit tests for slug validation, aliasing round-trip, dead-link repair, and dedup rejection rules.
This commit is contained in:
@@ -453,21 +453,23 @@ const WikiLogEntryTemplate = `## [{{.Date}}] {{.Operation}} | {{.Title}}
|
||||
|
||||
// WikiDeduplicationPrompt asks the LLM to identify duplicate entities/concepts
|
||||
// between newly extracted items and existing wiki pages.
|
||||
const WikiDeduplicationPrompt = `You are a strict deduplication system. Given a list of newly extracted items and a list of existing wiki pages, determine which new items refer to the **exact same** real-world entity or concept as an existing page.
|
||||
const WikiDeduplicationPrompt = `You are a strict deduplication system. You are given a list of newly extracted items. Each item carries its OWN short list of existing wiki pages that are surface-similar to it (its <candidates>). For each item, decide whether it refers to the **exact same** real-world entity or concept as ONE of its own candidates.
|
||||
|
||||
<new_items>
|
||||
{{.NewItems}}
|
||||
</new_items>
|
||||
|
||||
<existing_pages>
|
||||
{{.ExistingPages}}
|
||||
</existing_pages>
|
||||
<items>
|
||||
{{.Candidates}}
|
||||
</items>
|
||||
|
||||
<instructions>
|
||||
### How to read the input
|
||||
Each <item> is a newly extracted entity/concept. The <candidates> nested inside it are the ONLY existing pages you may merge that item into — they were pre-selected as similar to that specific item. A page listed under one item tells you NOTHING about any other item.
|
||||
|
||||
### Hard constraints — a merge is only valid when ALL hold:
|
||||
- The target slug is one of the candidate <page> slugs listed **inside that same item**. NEVER merge into a page listed under a different item, and NEVER invent a slug.
|
||||
- The types are compatible: entities merge with entities, concepts merge with concepts. **Never merge an entity into a concept or vice versa.**
|
||||
|
||||
### Merge criteria — ALL must be true:
|
||||
1. The new item and the existing page refer to the **same real-world thing** (same person, same organization, same specific concept).
|
||||
1. The new item and the candidate page refer to the **same real-world thing** (same person, same organization, same specific concept).
|
||||
2. The match is a **name variation**: abbreviation ↔ full name, translation, or minor spelling difference.
|
||||
3. The types are compatible: entities merge with entities, concepts merge with concepts. **Never merge an entity into a concept or vice versa.**
|
||||
|
||||
### Examples of CORRECT merges:
|
||||
- "Acme Corp" → "Acme Corporation" (same company, abbreviation)
|
||||
|
||||
@@ -92,12 +92,45 @@ func (t *wikiWritePageTool) Execute(ctx context.Context, args json.RawMessage) (
|
||||
return &types.ToolResult{Success: false, Error: "title, summary, content, and page_type are required for write action"}, nil
|
||||
}
|
||||
|
||||
// Validate + normalize the slug up front. The model routinely emits
|
||||
// malformed slugs (stray characters, mangled UUIDs); persisting them
|
||||
// verbatim creates unreachable pages and dead cross-links.
|
||||
normalizedSlug, slugErr := normalizeAndValidateWikiSlug(params.Slug)
|
||||
if slugErr != nil {
|
||||
return &types.ToolResult{Success: false, Error: slugErr.Error()}, nil
|
||||
}
|
||||
params.Slug = normalizedSlug
|
||||
|
||||
// Try to get the existing page
|
||||
existingPage, err := t.wikiPageService.GetPageBySlug(ctx, kbID, params.Slug)
|
||||
if err != nil && !errors.Is(err, repository.ErrWikiPageNotFound) {
|
||||
return &types.ToolResult{Success: false, Error: "Failed to check existing page: " + err.Error()}, nil
|
||||
}
|
||||
|
||||
// Summary pages are system-owned: they are generated deterministically
|
||||
// from a source document and keyed by its knowledge ID
|
||||
// (summary/<knowledgeID>). Letting the agent CREATE one with an
|
||||
// arbitrary/hand-typed slug is precisely how mangled-UUID ghost summary
|
||||
// pages (e.g. summary/…b171c…) get into the KB. Allow updating an
|
||||
// existing summary page, but never fabricating a new one.
|
||||
if existingPage == nil &&
|
||||
(isSummaryNamespace(params.Slug) || strings.EqualFold(params.PageType, types.WikiPageTypeSummary)) {
|
||||
return &types.ToolResult{
|
||||
Success: false,
|
||||
Error: "summary pages are generated automatically from source documents and cannot be created manually. " +
|
||||
"Use page_type 'synthesis'/'comparison'/'entity'/'concept' for authored pages, " +
|
||||
"or target an existing summary page to update it.",
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Auto-repair dead [[slug]] references in the body before persisting.
|
||||
// This rewrites LLM-mangled links (most importantly UUID-based summary
|
||||
// slugs) back to their real target when a confident match exists, and
|
||||
// leaves everything else untouched. Best-effort — never block the write.
|
||||
if repaired, changed, rerr := t.wikiPageService.RepairContentLinks(ctx, kbID, params.Slug, params.Content); rerr == nil && changed {
|
||||
params.Content = repaired
|
||||
}
|
||||
|
||||
resolvedRefs := resolveSourceRefs(ctx, t.knowledgeService, params.SourceRefs)
|
||||
|
||||
var action string
|
||||
@@ -164,3 +197,37 @@ func (t *wikiWritePageTool) Execute(ctx context.Context, args json.RawMessage) (
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// normalizeAndValidateWikiSlug lowercases + trims a model-supplied slug and
|
||||
// rejects malformed ones. A valid slug contains only lowercase ASCII letters,
|
||||
// digits, '-', '/', or CJK characters, has no leading/trailing/duplicate '/',
|
||||
// and is non-empty. Keeping this strict stops the agent from persisting
|
||||
// unreachable pages built from stray characters or garbled identifiers.
|
||||
func normalizeAndValidateWikiSlug(raw string) (string, error) {
|
||||
s := strings.ToLower(strings.TrimSpace(raw))
|
||||
s = strings.ReplaceAll(s, " ", "-")
|
||||
if s == "" {
|
||||
return "", errors.New("slug is required and must be non-empty")
|
||||
}
|
||||
if strings.Contains(s, "//") || strings.HasPrefix(s, "/") || strings.HasSuffix(s, "/") {
|
||||
return "", fmt.Errorf("invalid slug %q: '/' separators are malformed", raw)
|
||||
}
|
||||
for _, r := range s {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '-', r == '/':
|
||||
case r >= 0x4E00 && r <= 0x9FFF: // keep CJK characters
|
||||
default:
|
||||
return "", fmt.Errorf(
|
||||
"invalid slug %q: character %q is not allowed (use lowercase letters, digits, '-', '/', or CJK)",
|
||||
raw, string(r),
|
||||
)
|
||||
}
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// isSummaryNamespace reports whether a slug lives in the system-owned summary
|
||||
// namespace (summary/…).
|
||||
func isSummaryNamespace(slug string) bool {
|
||||
return strings.HasPrefix(slug, types.WikiPageTypeSummary+"/")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
package tools
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestNormalizeAndValidateWikiSlug(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
in string
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "already valid", in: "entity/acme-corp", want: "entity/acme-corp"},
|
||||
{name: "lowercased and spaces", in: "Entity/Acme Corp", want: "entity/acme-corp"},
|
||||
{name: "trimmed", in: " concept/rag ", want: "concept/rag"},
|
||||
{name: "cjk kept", in: "entity/上海中心大厦", want: "entity/上海中心大厦"},
|
||||
{name: "uuid summary", in: "summary/07a20bb1-a662-47cf-9929-06fb5d5b5b5e", want: "summary/07a20bb1-a662-47cf-9929-06fb5d5b5b5e"},
|
||||
{name: "empty", in: " ", wantErr: true},
|
||||
{name: "leading slash", in: "/entity/x", wantErr: true},
|
||||
{name: "trailing slash", in: "entity/x/", wantErr: true},
|
||||
{name: "double slash", in: "entity//x", wantErr: true},
|
||||
{name: "invalid char", in: "entity/x!y", wantErr: true},
|
||||
{name: "invalid space-only becomes empty", in: " ", wantErr: true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := normalizeAndValidateWikiSlug(tt.in)
|
||||
if tt.wantErr {
|
||||
if err == nil {
|
||||
t.Fatalf("expected error for %q, got slug %q", tt.in, got)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error for %q: %v", tt.in, err)
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Fatalf("normalizeAndValidateWikiSlug(%q) = %q, want %q", tt.in, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsSummaryNamespace(t *testing.T) {
|
||||
if !isSummaryNamespace("summary/abc") {
|
||||
t.Fatal("summary/abc must be in the summary namespace")
|
||||
}
|
||||
if isSummaryNamespace("summary") {
|
||||
t.Fatal("bare 'summary' (no slash) must not count as the summary namespace")
|
||||
}
|
||||
if isSummaryNamespace("entity/summary-of-x") {
|
||||
t.Fatal("entity/summary-of-x must not count as the summary namespace")
|
||||
}
|
||||
}
|
||||
@@ -2170,19 +2170,39 @@ func (s *wikiIngestService) publishDraftPages(ctx context.Context, kbID string,
|
||||
}
|
||||
}
|
||||
|
||||
// writeDedupItemXML renders a single entity/concept entry as a structured XML
|
||||
// block for the deduplication prompt. Structured form (versus a single
|
||||
// pipe-separated line) helps the LLM reliably tell name / aliases / type apart
|
||||
// and reduces nonsensical merges like "居民身份证" → "工作居住证".
|
||||
func writeDedupItemXML(buf *strings.Builder, slug, name, itemType string, aliases []string) {
|
||||
fmt.Fprintf(buf, " <item slug=%q type=%q>\n", slug, itemType)
|
||||
fmt.Fprintf(buf, " <name>%s</name>\n", xmlEscape(name))
|
||||
for _, alias := range aliases {
|
||||
// writeDedupCandidateGroup renders one new item together with its own
|
||||
// similarity-candidate existing pages, nested under a <candidates> element.
|
||||
// This per-item grouping is what constrains the dedup model to local
|
||||
// decisions (see the grouping rationale in deduplicateExtractedBatch). The
|
||||
// candidate pages keep their aliases so the model still has the acronym /
|
||||
// translation signal it needs to accept a legitimate merge.
|
||||
func writeDedupCandidateGroup(
|
||||
buf *strings.Builder, item extractedItem, itemType string, candidates []*types.WikiPageLite,
|
||||
) {
|
||||
fmt.Fprintf(buf, " <item slug=%q type=%q>\n", item.Slug, itemType)
|
||||
fmt.Fprintf(buf, " <name>%s</name>\n", xmlEscape(item.Name))
|
||||
for _, alias := range item.Aliases {
|
||||
if alias == "" {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(buf, " <alias>%s</alias>\n", xmlEscape(alias))
|
||||
}
|
||||
buf.WriteString(" <candidates>\n")
|
||||
for _, p := range candidates {
|
||||
if p == nil {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(buf, " <page slug=%q type=%q>\n", p.Slug, p.PageType)
|
||||
fmt.Fprintf(buf, " <name>%s</name>\n", xmlEscape(p.Title))
|
||||
for _, alias := range []string(p.Aliases) {
|
||||
if alias == "" {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(buf, " <alias>%s</alias>\n", xmlEscape(alias))
|
||||
}
|
||||
buf.WriteString(" </page>\n")
|
||||
}
|
||||
buf.WriteString(" </candidates>\n")
|
||||
buf.WriteString(" </item>\n")
|
||||
}
|
||||
|
||||
@@ -2226,7 +2246,16 @@ func (s *wikiIngestService) deduplicateExtractedBatch(
|
||||
// Build the candidate set: for each new item, ask the repo for
|
||||
// the top-K trigram-similar pages and union the results. Dedup by
|
||||
// slug as we go so the prompt only carries each candidate once.
|
||||
//
|
||||
// itemCandidates additionally records, per new item, the slugs that
|
||||
// surfaced for THAT item specifically. The prompt only ever sees the
|
||||
// flattened union, so validMerge below uses this per-item scoping to
|
||||
// reject a merge whose target was pulled in for a *different* item —
|
||||
// the class of hallucination the union otherwise enables (e.g. weak
|
||||
// models emitting entity/tencent-open → entity/hiring-agent, which
|
||||
// share no trigram signal and were never candidates for each other).
|
||||
candidatePages := make(map[string]*types.WikiPageLite)
|
||||
itemCandidates := make(map[string]map[string]bool)
|
||||
probe := func(item extractedItem) {
|
||||
queries := make([]string, 0, 1+len(item.Aliases))
|
||||
if item.Name != "" {
|
||||
@@ -2237,6 +2266,11 @@ func (s *wikiIngestService) deduplicateExtractedBatch(
|
||||
queries = append(queries, alias)
|
||||
}
|
||||
}
|
||||
own := itemCandidates[item.Slug]
|
||||
if own == nil {
|
||||
own = make(map[string]bool)
|
||||
itemCandidates[item.Slug] = own
|
||||
}
|
||||
for _, q := range queries {
|
||||
pages, err := s.wikiService.FindSimilarPages(ctx, kbID, q,
|
||||
[]string{types.WikiPageTypeEntity, types.WikiPageTypeConcept},
|
||||
@@ -2252,6 +2286,7 @@ func (s *wikiIngestService) deduplicateExtractedBatch(
|
||||
if _, ok := candidatePages[p.Slug]; !ok {
|
||||
candidatePages[p.Slug] = p
|
||||
}
|
||||
own[p.Slug] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2270,25 +2305,59 @@ func (s *wikiIngestService) deduplicateExtractedBatch(
|
||||
logger.Infof(ctx, "wiki ingest: %d similar existing pages selected for %d new items",
|
||||
len(candidatePages), len(entities)+len(concepts))
|
||||
|
||||
var existingBuf strings.Builder
|
||||
for _, p := range candidatePages {
|
||||
writeDedupItemXML(&existingBuf, p.Slug, p.Title, p.PageType, []string(p.Aliases))
|
||||
// Group each new item with ONLY the existing pages that surfaced for
|
||||
// its own similarity probe. Presenting the model two flat lists (all
|
||||
// new items × all candidates) invites cross-item mispairings — it has
|
||||
// no way to tell which candidate is relevant to which item, so a weak
|
||||
// model pairs unrelated slugs that merely coexist in the prompt. A
|
||||
// per-item shortlist turns dedup into a local yes/no decision against
|
||||
// a handful of genuinely-similar pages and makes cross-item pairings
|
||||
// structurally unnatural to express. Items with no candidate are
|
||||
// omitted entirely (they cannot merge and only add hallucination
|
||||
// surface + tokens).
|
||||
var candBuf strings.Builder
|
||||
groups := 0
|
||||
renderGroup := func(item extractedItem, itemType string) {
|
||||
cset := itemCandidates[item.Slug]
|
||||
if len(cset) == 0 {
|
||||
return
|
||||
}
|
||||
slugs := make([]string, 0, len(cset))
|
||||
for slug := range cset {
|
||||
// Skip the item's own slug: an identically-slugged existing
|
||||
// page is a re-ingest/update, not a merge target.
|
||||
if slug == item.Slug {
|
||||
continue
|
||||
}
|
||||
if _, ok := candidatePages[slug]; ok {
|
||||
slugs = append(slugs, slug)
|
||||
}
|
||||
}
|
||||
if len(slugs) == 0 {
|
||||
return
|
||||
}
|
||||
sort.Strings(slugs)
|
||||
pages := make([]*types.WikiPageLite, 0, len(slugs))
|
||||
for _, slug := range slugs {
|
||||
pages = append(pages, candidatePages[slug])
|
||||
}
|
||||
writeDedupCandidateGroup(&candBuf, item, itemType, pages)
|
||||
groups++
|
||||
}
|
||||
if existingBuf.Len() == 0 {
|
||||
for _, item := range entities {
|
||||
renderGroup(item, "entity")
|
||||
}
|
||||
for _, item := range concepts {
|
||||
renderGroup(item, "concept")
|
||||
}
|
||||
if groups == 0 {
|
||||
// Every new item's candidate list is empty after scoping —
|
||||
// nothing the model could safely merge.
|
||||
return entities, concepts
|
||||
}
|
||||
|
||||
var newBuf strings.Builder
|
||||
for _, item := range entities {
|
||||
writeDedupItemXML(&newBuf, item.Slug, item.Name, "entity", item.Aliases)
|
||||
}
|
||||
for _, item := range concepts {
|
||||
writeDedupItemXML(&newBuf, item.Slug, item.Name, "concept", item.Aliases)
|
||||
}
|
||||
|
||||
dedupeJSON, err := s.generateWithTemplate(ctx, chatModel, agent.WikiDeduplicationPrompt, map[string]string{
|
||||
"NewItems": newBuf.String(),
|
||||
"ExistingPages": existingBuf.String(),
|
||||
"Candidates": candBuf.String(),
|
||||
})
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: deduplication LLM call failed: %v", err)
|
||||
@@ -2309,36 +2378,9 @@ func (s *wikiIngestService) deduplicateExtractedBatch(
|
||||
return entities, concepts
|
||||
}
|
||||
|
||||
// Build the existing-slug set from the candidate map: anything not
|
||||
// in candidates is rejected as an LLM hallucination, since by
|
||||
// construction the model only ever saw those slugs as merge
|
||||
// targets. Compare with the legacy "look up against allPages"
|
||||
// path which had a wider acceptance window.
|
||||
existingSlugs := make(map[string]bool, len(candidatePages))
|
||||
for slug := range candidatePages {
|
||||
existingSlugs[slug] = true
|
||||
}
|
||||
|
||||
validMerge := func(srcSlug, dstSlug string) bool {
|
||||
if !existingSlugs[dstSlug] {
|
||||
logger.Warnf(ctx, "wiki ingest: dedup rejected %s → %s (target slug does not exist in candidate set)", srcSlug, dstSlug)
|
||||
return false
|
||||
}
|
||||
srcSlash := strings.Index(srcSlug, "/")
|
||||
dstSlash := strings.Index(dstSlug, "/")
|
||||
if srcSlash <= 0 || dstSlash <= 0 {
|
||||
// A type-prefixed slug must look like "entity/foo" or
|
||||
// "concept/bar". An LLM that emits an un-prefixed slug
|
||||
// here is hallucinating; reject rather than fall through
|
||||
// the prefix-equality check (which would treat both empty
|
||||
// prefixes as a match).
|
||||
logger.Warnf(ctx, "wiki ingest: dedup rejected %s → %s (missing type prefix)", srcSlug, dstSlug)
|
||||
return false
|
||||
}
|
||||
srcPrefix := srcSlug[:srcSlash+1]
|
||||
dstPrefix := dstSlug[:dstSlash+1]
|
||||
if srcPrefix != dstPrefix {
|
||||
logger.Warnf(ctx, "wiki ingest: dedup rejected %s → %s (type mismatch: %s vs %s)", srcSlug, dstSlug, srcPrefix, dstPrefix)
|
||||
if reason := dedupMergeRejectReason(srcSlug, dstSlug, itemCandidates[srcSlug]); reason != "" {
|
||||
logger.Warnf(ctx, "wiki ingest: dedup rejected %s → %s (%s)", srcSlug, dstSlug, reason)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
|
||||
@@ -2003,9 +2003,27 @@ func (s *wikiIngestService) reduceSlugUpdates(
|
||||
|
||||
if len(additions) > 0 || len(retracts) > 0 {
|
||||
titles := batchCtx.SlugTitleMany(ctx, []string(page.OutLinks))
|
||||
|
||||
// aliaser escapes high-entropy slugs behind short reference tokens
|
||||
// (ref-1, ref-2, …) for the editor LLM, then translates them back to
|
||||
// real slugs after generation (see dealiasContent below).
|
||||
aliaser := newSlugAliaser()
|
||||
|
||||
// Escape every out-link slug behind a reference token so the editor
|
||||
// never has to retype a real slug — this is where UUID-based summary
|
||||
// slugs (summary/<uuid>) got mangled into 404-ing links. Both the
|
||||
// <valid_wiki_links> listing AND the [[...]] refs already present in
|
||||
// <existing_page_content> are aliased with the SAME token map, so the
|
||||
// model sees one consistent, copy-safe identifier space; we translate
|
||||
// the tokens back to real slugs after generation.
|
||||
known := make(map[string]struct{}, len(page.OutLinks))
|
||||
for _, outSlug := range page.OutLinks {
|
||||
known[outSlug] = struct{}{}
|
||||
aliaser.token(outSlug) // pre-assign for stable ordering
|
||||
}
|
||||
for _, outSlug := range page.OutLinks {
|
||||
if title := titles[outSlug]; title != "" {
|
||||
fmt.Fprintf(&relatedSlugs, "- %s (%s)\n", outSlug, title)
|
||||
fmt.Fprintf(&relatedSlugs, "- %s (%s)\n", aliaser.token(outSlug), title)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2013,6 +2031,7 @@ func (s *wikiIngestService) reduceSlugUpdates(
|
||||
// [c003]. They are internal ingest metadata; keep the editor context
|
||||
// clean so a subsequent update cannot copy them into rewritten prose.
|
||||
existingContent := stripWikiInlineChunkCitations(page.Content)
|
||||
existingContent = aliaser.aliasContent(existingContent, known)
|
||||
if !exists || existingContent == "" {
|
||||
existingContent = "(New page)"
|
||||
}
|
||||
@@ -2060,6 +2079,10 @@ func (s *wikiIngestService) reduceSlugUpdates(
|
||||
})
|
||||
|
||||
if err == nil && updatedContent != "" {
|
||||
// Translate reference tokens (ref-N) the model copied from the
|
||||
// aliased context back to their real slugs BEFORE the content is
|
||||
// parsed/stored, so out_links reflect real pages again.
|
||||
updatedContent = aliaser.dealiasContent(updatedContent)
|
||||
updatedSummary, updatedBody := splitSummaryLine(updatedContent)
|
||||
if updatedBody != "" {
|
||||
page.Content = updatedBody
|
||||
|
||||
@@ -23,11 +23,15 @@ import (
|
||||
// external calls, and it only ever *removes* candidates from the prompt —
|
||||
// the downstream validMerge check still guards the final write.
|
||||
const (
|
||||
// dedupCandidateTopK is how many existing pages we keep per new item
|
||||
// even if none pass the score floor. Guarantees the LLM sees a small
|
||||
// ordered shortlist rather than an empty list when nothing matches
|
||||
// strongly.
|
||||
dedupCandidateTopK = 20
|
||||
// dedupCandidateTopK bounds how many trigram-similar existing pages
|
||||
// each new item's similarity probe returns (the LIMIT passed to
|
||||
// FindSimilarPages, applied per query term = name + each alias). Kept
|
||||
// deliberately small: the DB already orders by similarity desc behind a
|
||||
// pg_trgm threshold, so a genuine same-entity target is essentially
|
||||
// always the top hit. A tight K keeps each item's <candidates> list in
|
||||
// the dedup prompt short, which improves the model's precision and cuts
|
||||
// tokens, at negligible recall cost.
|
||||
dedupCandidateTopK = 5
|
||||
|
||||
// dedupCandidateScoreFloor is the Jaccard floor. Pairs at or above
|
||||
// this similarity are always included regardless of the top-K cap.
|
||||
@@ -170,6 +174,44 @@ func selectDedupCandidatePages(
|
||||
return out
|
||||
}
|
||||
|
||||
// dedupMergeRejectReason validates a single LLM-proposed merge (srcSlug →
|
||||
// dstSlug) against deterministic, model-independent rules. It returns an
|
||||
// empty string when the merge is allowed, or a short human-readable reason
|
||||
// when it must be rejected. srcCandidates is the set of existing-page slugs
|
||||
// that surfaced for srcSlug's OWN similarity probe (see itemCandidates in
|
||||
// deduplicateExtractedBatch).
|
||||
//
|
||||
// The per-item scoping check is the key guard: the dedup prompt shows the
|
||||
// model a flattened union of candidates across every new item, so nothing
|
||||
// stops a weak model from pairing an item with a page that was only similar
|
||||
// to a *different* item (observed: entity/tencent-open → entity/hiring-agent,
|
||||
// which share no trigram signal). Requiring dstSlug to be one of srcSlug's
|
||||
// own candidates rejects that entire class without depending on the model
|
||||
// getting the semantic judgment right.
|
||||
func dedupMergeRejectReason(srcSlug, dstSlug string, srcCandidates map[string]bool) string {
|
||||
if !srcCandidates[dstSlug] {
|
||||
// Covers both an outright hallucinated target (in no candidate
|
||||
// set at all) and a real page that was only similar to another
|
||||
// item. Either way the pair lacks a similarity signal for THIS
|
||||
// item, so it is not a safe merge.
|
||||
return "target is not a similarity candidate for this item"
|
||||
}
|
||||
srcSlash := strings.Index(srcSlug, "/")
|
||||
dstSlash := strings.Index(dstSlug, "/")
|
||||
if srcSlash <= 0 || dstSlash <= 0 {
|
||||
// A type-prefixed slug must look like "entity/foo" or
|
||||
// "concept/bar". An LLM that emits an un-prefixed slug here is
|
||||
// hallucinating; reject rather than fall through the prefix-
|
||||
// equality check (which would treat both empty prefixes as a
|
||||
// match).
|
||||
return "missing type prefix"
|
||||
}
|
||||
if srcSlug[:srcSlash+1] != dstSlug[:dstSlash+1] {
|
||||
return "type mismatch: " + srcSlug[:srcSlash+1] + " vs " + dstSlug[:dstSlash+1]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// dedupPairScore is the max similarity between any surface form of a and
|
||||
// b (plus the slug-token similarity). Slug and name signals live in
|
||||
// different symbol spaces (ASCII pinyin vs raw surface form) so we take
|
||||
|
||||
@@ -7,6 +7,61 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
func TestDedupMergeRejectReason(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
src, dst string
|
||||
candidates map[string]bool
|
||||
wantAllowed bool
|
||||
}{
|
||||
{
|
||||
name: "allowed: dst is a candidate for this item, same type prefix",
|
||||
src: "entity/acme-corp",
|
||||
dst: "entity/acme-corporation",
|
||||
candidates: map[string]bool{"entity/acme-corporation": true},
|
||||
wantAllowed: true,
|
||||
},
|
||||
{
|
||||
name: "rejected: dst similar to a DIFFERENT item (union hallucination)",
|
||||
src: "entity/tencent-open",
|
||||
dst: "entity/hiring-agent",
|
||||
candidates: map[string]bool{"entity/tencent-ur": true}, // hiring-agent not here
|
||||
wantAllowed: false,
|
||||
},
|
||||
{
|
||||
name: "rejected: dst is not a candidate at all (pure hallucination)",
|
||||
src: "entity/hy3-preview",
|
||||
dst: "entity/llm-cli-tool",
|
||||
candidates: nil,
|
||||
wantAllowed: false,
|
||||
},
|
||||
{
|
||||
name: "rejected: type mismatch even when dst is a candidate",
|
||||
src: "entity/foo",
|
||||
dst: "concept/foo",
|
||||
candidates: map[string]bool{"concept/foo": true},
|
||||
wantAllowed: false,
|
||||
},
|
||||
{
|
||||
name: "rejected: missing type prefix",
|
||||
src: "foo",
|
||||
dst: "entity/foo",
|
||||
candidates: map[string]bool{"entity/foo": true},
|
||||
wantAllowed: false,
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
reason := dedupMergeRejectReason(tc.src, tc.dst, tc.candidates)
|
||||
gotAllowed := reason == ""
|
||||
if gotAllowed != tc.wantAllowed {
|
||||
t.Fatalf("dedupMergeRejectReason(%q, %q) allowed=%v (reason=%q), want allowed=%v",
|
||||
tc.src, tc.dst, gotAllowed, reason, tc.wantAllowed)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// helper: build a minimal entity/concept WikiPage.
|
||||
func makePage(slug, title, typ string, aliases ...string) *types.WikiPage {
|
||||
return &types.WikiPage{
|
||||
|
||||
@@ -918,6 +918,151 @@ func normalizeSlug(slug string) string {
|
||||
return slug
|
||||
}
|
||||
|
||||
// slugNamespace returns the prefix of a slug up to (but excluding) the first
|
||||
// '/', e.g. "summary/abc" -> "summary". Slugs without a '/' map to "".
|
||||
func slugNamespace(slug string) string {
|
||||
if i := strings.IndexByte(slug, '/'); i >= 0 {
|
||||
return slug[:i]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// rewriteDeadWikiLinks walks every [[slug]] / [[slug|display]] occurrence in
|
||||
// content and lets `resolve` decide, per link, whether to rewrite the slug.
|
||||
// `resolve` receives the normalized slug and its display text and returns the
|
||||
// replacement slug plus true to rewrite, or ("", false) to leave the link
|
||||
// untouched. Display text is preserved verbatim. This is a pure helper — the
|
||||
// resolution policy lives entirely in the callback.
|
||||
func rewriteDeadWikiLinks(content string, resolve func(normSlug, display string) (string, bool)) (string, bool) {
|
||||
changed := false
|
||||
out := wikiLinkRegex.ReplaceAllStringFunc(content, func(match string) string {
|
||||
inner := match[2 : len(match)-2]
|
||||
rawSlug := inner
|
||||
display := ""
|
||||
if parts := strings.SplitN(inner, "|", 2); len(parts) == 2 {
|
||||
rawSlug = parts[0]
|
||||
display = strings.TrimSpace(parts[1])
|
||||
}
|
||||
norm := normalizeSlug(rawSlug)
|
||||
if norm == "" {
|
||||
return match
|
||||
}
|
||||
newSlug, ok := resolve(norm, display)
|
||||
if !ok || newSlug == "" || newSlug == norm {
|
||||
return match
|
||||
}
|
||||
changed = true
|
||||
if display != "" {
|
||||
return "[[" + newSlug + "|" + display + "]]"
|
||||
}
|
||||
return "[[" + newSlug + "]]"
|
||||
})
|
||||
return out, changed
|
||||
}
|
||||
|
||||
// RepairContentLinks rewrites [[slug]] / [[slug|display]] references in
|
||||
// `content` whose target does not exist in the KB but is almost certainly a
|
||||
// mangled form of a real page. The canonical case: an LLM re-typed a summary
|
||||
// page's UUID-based slug and inserted/dropped a hex digit
|
||||
// (summary/…06fb5d5b5b5e → summary/…06fb14d5b14b14e), producing a link that
|
||||
// 404s and can never be recovered by exact lookup.
|
||||
//
|
||||
// Unlike stripDeadWikiLinks (the ingest cleanup pass), this method is
|
||||
// REWRITE-ONLY: a dead link is corrected only when a confident live candidate
|
||||
// exists; otherwise it is left completely untouched. It NEVER strips a link to
|
||||
// plain text, so it is safe on any write path — including writes whose targets
|
||||
// legitimately do not exist yet (those simply stay as-is until they do).
|
||||
//
|
||||
// The candidate pool for each dead link is scoped to live slugs that share the
|
||||
// same namespace prefix (a dead `summary/<uuid>` only resolves against live
|
||||
// `summary/*` slugs). Scoping keeps the bigram-similarity lever safe: distinct
|
||||
// high-entropy UUIDs in one namespace don't collide, while a one-character
|
||||
// mangle stays comfortably above threshold against its true source.
|
||||
//
|
||||
// Returns the possibly-updated content and whether any rewrite happened.
|
||||
// Errors are only returned for hard repo failures; callers may treat repair as
|
||||
// best-effort and ignore them.
|
||||
func (s *wikiPageService) RepairContentLinks(
|
||||
ctx context.Context, kbID, selfSlug, content string,
|
||||
) (string, bool, error) {
|
||||
if strings.TrimSpace(content) == "" {
|
||||
return content, false, nil
|
||||
}
|
||||
outLinks := s.parseOutLinks(content)
|
||||
if len(outLinks) == 0 {
|
||||
return content, false, nil
|
||||
}
|
||||
|
||||
existMap, err := s.repo.ExistsSlugs(ctx, kbID, outLinks)
|
||||
if err != nil {
|
||||
return content, false, err
|
||||
}
|
||||
deadPrefixes := make(map[string]struct{})
|
||||
for _, l := range outLinks {
|
||||
if l == selfSlug || existMap[l] {
|
||||
continue
|
||||
}
|
||||
deadPrefixes[slugNamespace(l)] = struct{}{}
|
||||
}
|
||||
if len(deadPrefixes) == 0 {
|
||||
return content, false, nil
|
||||
}
|
||||
|
||||
allSlugs, err := s.repo.ListAllSlugs(ctx, kbID)
|
||||
if err != nil {
|
||||
return content, false, err
|
||||
}
|
||||
liveByPrefix := make(map[string]map[string]struct{})
|
||||
var candidateSlugs []string
|
||||
for _, sl := range allSlugs {
|
||||
ns := slugNamespace(sl)
|
||||
if _, want := deadPrefixes[ns]; !want {
|
||||
continue
|
||||
}
|
||||
set, ok := liveByPrefix[ns]
|
||||
if !ok {
|
||||
set = make(map[string]struct{})
|
||||
liveByPrefix[ns] = set
|
||||
}
|
||||
set[sl] = struct{}{}
|
||||
candidateSlugs = append(candidateSlugs, sl)
|
||||
}
|
||||
if len(candidateSlugs) == 0 {
|
||||
return content, false, nil
|
||||
}
|
||||
|
||||
// Build a title -> slug reverse lookup for candidate pages so the
|
||||
// display-text lever (the safest, most precise one) can fire. Bounded to
|
||||
// the relevant namespaces so this stays cheap even on large KBs.
|
||||
titleToSlug := make(map[string]string)
|
||||
if lites, lerr := s.repo.ListBySlugs(ctx, kbID, candidateSlugs); lerr == nil {
|
||||
for _, lp := range lites {
|
||||
if lp != nil && lp.Title != "" {
|
||||
titleToSlug[lp.Title] = lp.Slug
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resolveCache := make(map[string]string)
|
||||
newContent, changed := rewriteDeadWikiLinks(content, func(norm, display string) (string, bool) {
|
||||
if norm == selfSlug || existMap[norm] {
|
||||
return "", false
|
||||
}
|
||||
key := norm + "\x00" + display
|
||||
if cached, ok := resolveCache[key]; ok {
|
||||
return cached, cached != ""
|
||||
}
|
||||
resolved, ok := resolveDeadSlug(norm, display, liveByPrefix[slugNamespace(norm)], titleToSlug)
|
||||
if !ok || resolved == norm {
|
||||
resolveCache[key] = ""
|
||||
return "", false
|
||||
}
|
||||
resolveCache[key] = resolved
|
||||
return resolved, true
|
||||
})
|
||||
return newContent, changed, nil
|
||||
}
|
||||
|
||||
// updateInLinks adds the source slug to the in_links of target pages
|
||||
func (s *wikiPageService) updateInLinks(ctx context.Context, kbID string, sourceSlug string, targets types.StringArray) {
|
||||
for _, targetSlug := range targets {
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/application/repository"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
@@ -140,6 +141,60 @@ func TestParseOutLinks(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepairContentLinks(t *testing.T) {
|
||||
db, err := gorm.Open(sqlite.Open(fmt.Sprintf("file:%s?mode=memory&cache=shared", t.Name())), &gorm.Config{})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, db.AutoMigrate(&types.WikiFolder{}, &types.WikiPage{}))
|
||||
|
||||
ctx := context.Background()
|
||||
repo := repository.NewWikiPageRepository(db)
|
||||
svc := NewWikiPageService(repo, nil, nil, nil, nil)
|
||||
const kbID = "kb-repair"
|
||||
now := time.Now()
|
||||
|
||||
seed := func(slug, title, pageType string) {
|
||||
require.NoError(t, repo.Create(ctx, &types.WikiPage{
|
||||
ID: uuid.New().String(), TenantID: 1, KnowledgeBaseID: kbID,
|
||||
Slug: slug, Title: title, PageType: pageType,
|
||||
Status: types.WikiPageStatusPublished, Version: 1,
|
||||
CreatedAt: now, UpdatedAt: now,
|
||||
}))
|
||||
}
|
||||
|
||||
// Real summary page (clean UUID slug) + a distractor summary + an entity.
|
||||
realSummary := "summary/07a20bb1-a662-47cf-9929-06fb5d5b5b5e"
|
||||
seed(realSummary, "Weknora 试错记录.md - Summary", types.WikiPageTypeSummary)
|
||||
seed("summary/fcadcaab-e094-4037-b71c-9edfdcdba058", "Another Doc - Summary", types.WikiPageTypeSummary)
|
||||
seed("entity/mongodb", "MongoDB", types.WikiPageTypeEntity)
|
||||
|
||||
t.Run("mangled uuid summary link repaired via bigram", func(t *testing.T) {
|
||||
// One hex digit inserted into the last group — 404s on exact lookup.
|
||||
content := "See [[summary/07a20bb1-a662-47cf-9929-06fb14d5b14b14e|Weknora 试错记录.md - Summary]] for context."
|
||||
got, changed, err := svc.RepairContentLinks(ctx, kbID, "synthesis/notes", content)
|
||||
require.NoError(t, err)
|
||||
require.True(t, changed, "expected the mangled summary link to be rewritten")
|
||||
require.Contains(t, got, "[["+realSummary+"|Weknora 试错记录.md - Summary]]")
|
||||
require.NotContains(t, got, "06fb14d5b14b14e")
|
||||
})
|
||||
|
||||
t.Run("live links untouched", func(t *testing.T) {
|
||||
content := "Fine: [[entity/mongodb]] and [[" + realSummary + "]]."
|
||||
got, changed, err := svc.RepairContentLinks(ctx, kbID, "synthesis/notes", content)
|
||||
require.NoError(t, err)
|
||||
require.False(t, changed)
|
||||
require.Equal(t, content, got)
|
||||
})
|
||||
|
||||
t.Run("unresolvable dead link left as-is, never stripped", func(t *testing.T) {
|
||||
// A future entity page that doesn't exist and isn't similar to anything.
|
||||
content := "Pending: [[entity/some-brand-new-topic|New Topic]]."
|
||||
got, changed, err := svc.RepairContentLinks(ctx, kbID, "synthesis/notes", content)
|
||||
require.NoError(t, err)
|
||||
require.False(t, changed)
|
||||
require.Equal(t, content, got, "unresolvable links must be preserved verbatim, not stripped")
|
||||
})
|
||||
}
|
||||
|
||||
func TestNormalizeSlug(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// slugAliaser maps real wiki slugs to short, low-entropy reference tokens
|
||||
// (ref-1, ref-2, …) so the ingest editor LLM never has to reproduce a
|
||||
// high-entropy slug verbatim — most importantly the UUID-based summary slugs
|
||||
// (summary/<knowledgeID>) that models routinely mangle by inserting or
|
||||
// dropping hex digits. The model copies the tiny token; we translate tokens
|
||||
// back to real slugs on output.
|
||||
//
|
||||
// This is the generation-stage counterpart to wiki_write_page's slug
|
||||
// validation: rather than repairing a mangled slug after the fact, it removes
|
||||
// the opportunity to mangle at the source. It mirrors the chunk-alias
|
||||
// (c000/c001) indirection already used by the chunk-citation pass in
|
||||
// wiki_ingest_cite.go.
|
||||
//
|
||||
// A slugAliaser is single-use and NOT safe for concurrent use; construct one
|
||||
// per LLM call.
|
||||
type slugAliaser struct {
|
||||
toToken map[string]string // real slug -> token
|
||||
toSlug map[string]string // token -> real slug
|
||||
}
|
||||
|
||||
func newSlugAliaser() *slugAliaser {
|
||||
return &slugAliaser{
|
||||
toToken: make(map[string]string),
|
||||
toSlug: make(map[string]string),
|
||||
}
|
||||
}
|
||||
|
||||
// token returns the stable token for a real slug, assigning a fresh one on
|
||||
// first use. An empty slug returns "". Tokens have no '/' so they can never
|
||||
// collide with a namespaced real slug (entity/…, summary/…).
|
||||
func (a *slugAliaser) token(realSlug string) string {
|
||||
realSlug = strings.TrimSpace(realSlug)
|
||||
if realSlug == "" {
|
||||
return ""
|
||||
}
|
||||
if t, ok := a.toToken[realSlug]; ok {
|
||||
return t
|
||||
}
|
||||
t := fmt.Sprintf("ref-%d", len(a.toToken)+1)
|
||||
a.toToken[realSlug] = t
|
||||
a.toSlug[t] = realSlug
|
||||
return t
|
||||
}
|
||||
|
||||
// empty reports whether no tokens have been assigned yet (nothing to alias).
|
||||
func (a *slugAliaser) empty() bool { return len(a.toToken) == 0 }
|
||||
|
||||
// aliasContent rewrites [[realSlug|disp]] / [[realSlug]] occurrences to their
|
||||
// token form, but only for slugs present in `known`. Unknown links are left
|
||||
// untouched. Assigns tokens on demand so a slug that appears only in the body
|
||||
// (not in the listing) still gets a consistent token.
|
||||
func (a *slugAliaser) aliasContent(content string, known map[string]struct{}) string {
|
||||
if content == "" || len(known) == 0 {
|
||||
return content
|
||||
}
|
||||
out, _ := rewriteDeadWikiLinks(content, func(norm, _ string) (string, bool) {
|
||||
if _, ok := known[norm]; !ok {
|
||||
return "", false
|
||||
}
|
||||
return a.token(norm), true
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// dealiasContent rewrites [[token|disp]] / [[token]] occurrences back to their
|
||||
// real slug. Tokens with no mapping are left untouched — they fall through to
|
||||
// the ordinary parse/dead-link-cleanup path, which is the correct behavior for
|
||||
// anything the model invented that isn't a known reference.
|
||||
func (a *slugAliaser) dealiasContent(content string) string {
|
||||
if content == "" || a.empty() {
|
||||
return content
|
||||
}
|
||||
out, _ := rewriteDeadWikiLinks(content, func(norm, _ string) (string, bool) {
|
||||
real, ok := a.toSlug[norm]
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
return real, true
|
||||
})
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSlugAliaser_RoundTripEscapesUUIDSlugs(t *testing.T) {
|
||||
summarySlug := "summary/07a20bb1-a662-47cf-9929-06fb5d5b5b5e"
|
||||
entitySlug := "entity/mongodb"
|
||||
known := map[string]struct{}{summarySlug: {}, entitySlug: {}}
|
||||
|
||||
a := newSlugAliaser()
|
||||
// Pre-assign tokens the way the ingest listing loop does.
|
||||
tSummary := a.token(summarySlug)
|
||||
tEntity := a.token(entitySlug)
|
||||
if tSummary == summarySlug || tEntity == entitySlug {
|
||||
t.Fatalf("tokens must differ from real slugs: %q %q", tSummary, tEntity)
|
||||
}
|
||||
if strings.Contains(tSummary, "/") {
|
||||
t.Fatalf("token %q must not contain '/'", tSummary)
|
||||
}
|
||||
|
||||
body := "See [[" + summarySlug + "|Weknora 试错记录.md - Summary]] and [[" + entitySlug + "]]."
|
||||
aliased := a.aliasContent(body, known)
|
||||
|
||||
// The high-entropy UUID must no longer be present in what the model sees.
|
||||
if strings.Contains(aliased, "06fb5d5b5b5e") {
|
||||
t.Fatalf("aliased content still exposes the UUID: %q", aliased)
|
||||
}
|
||||
if !strings.Contains(aliased, "[["+tSummary+"|Weknora 试错记录.md - Summary]]") {
|
||||
t.Fatalf("summary link not aliased: %q", aliased)
|
||||
}
|
||||
if !strings.Contains(aliased, "[["+tEntity+"]]") {
|
||||
t.Fatalf("entity link not aliased: %q", aliased)
|
||||
}
|
||||
|
||||
// Round-trip must restore the original body exactly.
|
||||
if got := a.dealiasContent(aliased); got != body {
|
||||
t.Fatalf("dealias round-trip mismatch:\n got %q\n want %q", got, body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSlugAliaser_UnknownTokensAndSlugsUntouched(t *testing.T) {
|
||||
a := newSlugAliaser()
|
||||
a.token("summary/real-uuid")
|
||||
|
||||
// A slug not in `known` must not be aliased.
|
||||
known := map[string]struct{}{"summary/real-uuid": {}}
|
||||
body := "keep [[entity/invented-by-model]] as-is"
|
||||
if got := a.aliasContent(body, known); got != body {
|
||||
t.Fatalf("unknown slug should be left untouched, got %q", got)
|
||||
}
|
||||
|
||||
// An unmapped token in output must be left untouched (falls through to the
|
||||
// normal dead-link cleanup path).
|
||||
out := "stray [[ref-999|X]] token"
|
||||
if got := a.dealiasContent(out); got != out {
|
||||
t.Fatalf("unmapped token should be left untouched, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSlugAliaser_EmptyIsNoop(t *testing.T) {
|
||||
a := newSlugAliaser()
|
||||
if !a.empty() {
|
||||
t.Fatal("fresh aliaser must be empty")
|
||||
}
|
||||
body := "[[summary/x]] unchanged"
|
||||
if got := a.dealiasContent(body); got != body {
|
||||
t.Fatalf("empty aliaser dealias must be a no-op, got %q", got)
|
||||
}
|
||||
}
|
||||
@@ -17,10 +17,22 @@ import (
|
||||
// resource:// handles, but old chunks and message history can still contain a
|
||||
// provider URL. Giving both forms the same request-local alias makes rollout
|
||||
// safe without a blocking full-table content rewrite.
|
||||
//
|
||||
// The final alternative aliases wiki summary-page slugs (summary/<uuid>). They
|
||||
// are not storage handles, but they share the exact failure mode this registry
|
||||
// exists to prevent: a high-entropy identifier the model must reproduce
|
||||
// verbatim (inside [[slug|display]] links and as wiki-tool slug arguments).
|
||||
// Models routinely mangle the UUID by inserting or dropping hex digits, which
|
||||
// yields dead cross-links. Aliasing the slug to a low-entropy res:// token
|
||||
// removes the opportunity to mangle at the source; the token round-trips back
|
||||
// to the real slug on stream output and in decoded tool-call arguments before
|
||||
// anything is persisted. Entity slugs (entity/<readable-title>) are low-entropy
|
||||
// and semantically meaningful, so they are deliberately left untouched.
|
||||
var storedRefRE = regexp.MustCompile(
|
||||
`resource://[0-9A-Za-z_-]{22}|` +
|
||||
`(?:storage://[0-9A-Za-z_-]+/)?` +
|
||||
`(?:local|minio|cos|tos|s3|oss|ks3|obs)://[^\s)\]>"']+`,
|
||||
`(?:local|minio|cos|tos|s3|oss|ks3|obs)://[^\s)\]>"']+|` +
|
||||
`summary/[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}`,
|
||||
)
|
||||
|
||||
// aliasShapeRE matches the alias syntax produced by EncodeText. It is used only
|
||||
|
||||
@@ -23,6 +23,25 @@ func TestRegistryAliasesLegacyPhysicalReferencesDuringRollout(t *testing.T) {
|
||||
require.Equal(t, "", r.DecodeText(encoded))
|
||||
}
|
||||
|
||||
func TestRegistryAliasesWikiSummarySlug(t *testing.T) {
|
||||
r := NewRegistry()
|
||||
slug := "summary/07a20bb1-a662-47cf-9929-06fb5d5b5b5e"
|
||||
// Inside a [[slug|display]] link the model must copy verbatim.
|
||||
encoded := r.EncodeText("see [[" + slug + "|Foo.md - Summary]]")
|
||||
require.Equal(t, "see [[res://0001|Foo.md - Summary]]", encoded)
|
||||
require.Equal(t, "see [["+slug+"|Foo.md - Summary]]", r.DecodeText(encoded))
|
||||
|
||||
// The same slug as a tool-call argument value resolves to one alias.
|
||||
require.Equal(t, "res://0001", r.EncodeText(slug))
|
||||
}
|
||||
|
||||
func TestRegistryLeavesEntitySlugUntouched(t *testing.T) {
|
||||
r := NewRegistry()
|
||||
// Entity slugs are low-entropy and semantically meaningful; not aliased.
|
||||
entity := "[[entity/weknora-error-log|WeKnora 试错记录]]"
|
||||
require.Equal(t, entity, r.EncodeText(entity))
|
||||
}
|
||||
|
||||
func TestRegistryEncodesMessageCopies(t *testing.T) {
|
||||
r := NewRegistry()
|
||||
ref := "resource://AbCdEfGhIjKlMnOpQrStUv"
|
||||
|
||||
@@ -39,6 +39,13 @@ type WikiPageService interface {
|
||||
// GetPageBySlug retrieves a wiki page by its slug within a knowledge base.
|
||||
GetPageBySlug(ctx context.Context, kbID string, slug string) (*types.WikiPage, error)
|
||||
|
||||
// RepairContentLinks rewrites dead [[slug]] references in content to their
|
||||
// most likely live target (rewrite-only — never strips). Used by the agent
|
||||
// write path to auto-correct LLM-mangled slugs (especially UUID-based
|
||||
// summary slugs) before persistence. Returns the possibly-updated content
|
||||
// and whether any rewrite happened. Best-effort: callers may ignore errors.
|
||||
RepairContentLinks(ctx context.Context, kbID, selfSlug, content string) (string, bool, error)
|
||||
|
||||
// GetPageByID retrieves a wiki page by its unique ID.
|
||||
GetPageByID(ctx context.Context, id string) (*types.WikiPage, error)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user