From 5d0326f987ddfd079eb2498919533fa85284d381 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Fri, 24 Jul 2026 15:06:49 +0800 Subject: [PATCH] fix(wiki): harden slug integrity across ingest and agent write paths Weak LLMs routinely mangle high-entropy wiki slugs (especially UUID-based summary/ slugs), producing dead cross-links and cross-item dedup hallucinations. This tightens integrity end to end: - Ingest dedup now groups each new item with only its own trigram candidates and validates merges against that per-item scope, rejecting cross-item mispairings; candidate top-K narrowed 20 -> 5. - Editor path aliases out-link slugs behind low-entropy ref-N tokens and translates them back after generation, removing the chance to mangle. - Agent write tool normalizes/validates slugs, blocks manual creation of system-owned summary pages, and auto-repairs dead [[slug]] links (rewrite-only, never strips) via bigram/display-text resolution. - llmresource registry aliases summary/ slugs the same way as storage handles so they round-trip safely through chat. Adds unit tests for slug validation, aliasing round-trip, dead-link repair, and dedup rejection rules. --- internal/agent/prompts_wiki.go | 22 +-- internal/agent/tools/wiki_write_page.go | 67 ++++++++ internal/agent/tools/wiki_write_page_test.go | 53 +++++++ internal/application/service/wiki_ingest.go | 144 +++++++++++------ .../application/service/wiki_ingest_batch.go | 25 ++- .../application/service/wiki_ingest_dedup.go | 52 ++++++- .../service/wiki_ingest_dedup_test.go | 55 +++++++ internal/application/service/wiki_page.go | 145 ++++++++++++++++++ .../application/service/wiki_page_test.go | 55 +++++++ .../application/service/wiki_slug_alias.go | 88 +++++++++++ .../service/wiki_slug_alias_test.go | 72 +++++++++ internal/llmresource/registry.go | 14 +- internal/llmresource/registry_test.go | 19 +++ internal/types/interfaces/wiki_page.go | 7 + 14 files changed, 750 insertions(+), 68 deletions(-) create mode 100644 internal/agent/tools/wiki_write_page_test.go create mode 100644 internal/application/service/wiki_slug_alias.go create mode 100644 internal/application/service/wiki_slug_alias_test.go diff --git a/internal/agent/prompts_wiki.go b/internal/agent/prompts_wiki.go index 3b5858cf..a57d6b79 100644 --- a/internal/agent/prompts_wiki.go +++ b/internal/agent/prompts_wiki.go @@ -453,21 +453,23 @@ const WikiLogEntryTemplate = `## [{{.Date}}] {{.Operation}} | {{.Title}} // WikiDeduplicationPrompt asks the LLM to identify duplicate entities/concepts // between newly extracted items and existing wiki pages. -const WikiDeduplicationPrompt = `You are a strict deduplication system. Given a list of newly extracted items and a list of existing wiki pages, determine which new items refer to the **exact same** real-world entity or concept as an existing page. +const WikiDeduplicationPrompt = `You are a strict deduplication system. You are given a list of newly extracted items. Each item carries its OWN short list of existing wiki pages that are surface-similar to it (its ). For each item, decide whether it refers to the **exact same** real-world entity or concept as ONE of its own candidates. - -{{.NewItems}} - - - -{{.ExistingPages}} - + +{{.Candidates}} + +### How to read the input +Each is a newly extracted entity/concept. The nested inside it are the ONLY existing pages you may merge that item into — they were pre-selected as similar to that specific item. A page listed under one item tells you NOTHING about any other item. + +### Hard constraints — a merge is only valid when ALL hold: +- The target slug is one of the candidate slugs listed **inside that same item**. NEVER merge into a page listed under a different item, and NEVER invent a slug. +- The types are compatible: entities merge with entities, concepts merge with concepts. **Never merge an entity into a concept or vice versa.** + ### Merge criteria — ALL must be true: -1. The new item and the existing page refer to the **same real-world thing** (same person, same organization, same specific concept). +1. The new item and the candidate page refer to the **same real-world thing** (same person, same organization, same specific concept). 2. The match is a **name variation**: abbreviation ↔ full name, translation, or minor spelling difference. -3. The types are compatible: entities merge with entities, concepts merge with concepts. **Never merge an entity into a concept or vice versa.** ### Examples of CORRECT merges: - "Acme Corp" → "Acme Corporation" (same company, abbreviation) diff --git a/internal/agent/tools/wiki_write_page.go b/internal/agent/tools/wiki_write_page.go index eeec6aac..d603f5c6 100644 --- a/internal/agent/tools/wiki_write_page.go +++ b/internal/agent/tools/wiki_write_page.go @@ -92,12 +92,45 @@ func (t *wikiWritePageTool) Execute(ctx context.Context, args json.RawMessage) ( return &types.ToolResult{Success: false, Error: "title, summary, content, and page_type are required for write action"}, nil } + // Validate + normalize the slug up front. The model routinely emits + // malformed slugs (stray characters, mangled UUIDs); persisting them + // verbatim creates unreachable pages and dead cross-links. + normalizedSlug, slugErr := normalizeAndValidateWikiSlug(params.Slug) + if slugErr != nil { + return &types.ToolResult{Success: false, Error: slugErr.Error()}, nil + } + params.Slug = normalizedSlug + // Try to get the existing page existingPage, err := t.wikiPageService.GetPageBySlug(ctx, kbID, params.Slug) if err != nil && !errors.Is(err, repository.ErrWikiPageNotFound) { return &types.ToolResult{Success: false, Error: "Failed to check existing page: " + err.Error()}, nil } + // Summary pages are system-owned: they are generated deterministically + // from a source document and keyed by its knowledge ID + // (summary/). Letting the agent CREATE one with an + // arbitrary/hand-typed slug is precisely how mangled-UUID ghost summary + // pages (e.g. summary/…b171c…) get into the KB. Allow updating an + // existing summary page, but never fabricating a new one. + if existingPage == nil && + (isSummaryNamespace(params.Slug) || strings.EqualFold(params.PageType, types.WikiPageTypeSummary)) { + return &types.ToolResult{ + Success: false, + Error: "summary pages are generated automatically from source documents and cannot be created manually. " + + "Use page_type 'synthesis'/'comparison'/'entity'/'concept' for authored pages, " + + "or target an existing summary page to update it.", + }, nil + } + + // Auto-repair dead [[slug]] references in the body before persisting. + // This rewrites LLM-mangled links (most importantly UUID-based summary + // slugs) back to their real target when a confident match exists, and + // leaves everything else untouched. Best-effort — never block the write. + if repaired, changed, rerr := t.wikiPageService.RepairContentLinks(ctx, kbID, params.Slug, params.Content); rerr == nil && changed { + params.Content = repaired + } + resolvedRefs := resolveSourceRefs(ctx, t.knowledgeService, params.SourceRefs) var action string @@ -164,3 +197,37 @@ func (t *wikiWritePageTool) Execute(ctx context.Context, args json.RawMessage) ( }, }, nil } + +// normalizeAndValidateWikiSlug lowercases + trims a model-supplied slug and +// rejects malformed ones. A valid slug contains only lowercase ASCII letters, +// digits, '-', '/', or CJK characters, has no leading/trailing/duplicate '/', +// and is non-empty. Keeping this strict stops the agent from persisting +// unreachable pages built from stray characters or garbled identifiers. +func normalizeAndValidateWikiSlug(raw string) (string, error) { + s := strings.ToLower(strings.TrimSpace(raw)) + s = strings.ReplaceAll(s, " ", "-") + if s == "" { + return "", errors.New("slug is required and must be non-empty") + } + if strings.Contains(s, "//") || strings.HasPrefix(s, "/") || strings.HasSuffix(s, "/") { + return "", fmt.Errorf("invalid slug %q: '/' separators are malformed", raw) + } + for _, r := range s { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '-', r == '/': + case r >= 0x4E00 && r <= 0x9FFF: // keep CJK characters + default: + return "", fmt.Errorf( + "invalid slug %q: character %q is not allowed (use lowercase letters, digits, '-', '/', or CJK)", + raw, string(r), + ) + } + } + return s, nil +} + +// isSummaryNamespace reports whether a slug lives in the system-owned summary +// namespace (summary/…). +func isSummaryNamespace(slug string) bool { + return strings.HasPrefix(slug, types.WikiPageTypeSummary+"/") +} diff --git a/internal/agent/tools/wiki_write_page_test.go b/internal/agent/tools/wiki_write_page_test.go new file mode 100644 index 00000000..f32b1b64 --- /dev/null +++ b/internal/agent/tools/wiki_write_page_test.go @@ -0,0 +1,53 @@ +package tools + +import "testing" + +func TestNormalizeAndValidateWikiSlug(t *testing.T) { + tests := []struct { + name string + in string + want string + wantErr bool + }{ + {name: "already valid", in: "entity/acme-corp", want: "entity/acme-corp"}, + {name: "lowercased and spaces", in: "Entity/Acme Corp", want: "entity/acme-corp"}, + {name: "trimmed", in: " concept/rag ", want: "concept/rag"}, + {name: "cjk kept", in: "entity/上海中心大厦", want: "entity/上海中心大厦"}, + {name: "uuid summary", in: "summary/07a20bb1-a662-47cf-9929-06fb5d5b5b5e", want: "summary/07a20bb1-a662-47cf-9929-06fb5d5b5b5e"}, + {name: "empty", in: " ", wantErr: true}, + {name: "leading slash", in: "/entity/x", wantErr: true}, + {name: "trailing slash", in: "entity/x/", wantErr: true}, + {name: "double slash", in: "entity//x", wantErr: true}, + {name: "invalid char", in: "entity/x!y", wantErr: true}, + {name: "invalid space-only becomes empty", in: " ", wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := normalizeAndValidateWikiSlug(tt.in) + if tt.wantErr { + if err == nil { + t.Fatalf("expected error for %q, got slug %q", tt.in, got) + } + return + } + if err != nil { + t.Fatalf("unexpected error for %q: %v", tt.in, err) + } + if got != tt.want { + t.Fatalf("normalizeAndValidateWikiSlug(%q) = %q, want %q", tt.in, got, tt.want) + } + }) + } +} + +func TestIsSummaryNamespace(t *testing.T) { + if !isSummaryNamespace("summary/abc") { + t.Fatal("summary/abc must be in the summary namespace") + } + if isSummaryNamespace("summary") { + t.Fatal("bare 'summary' (no slash) must not count as the summary namespace") + } + if isSummaryNamespace("entity/summary-of-x") { + t.Fatal("entity/summary-of-x must not count as the summary namespace") + } +} diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go index 6d95ff05..ba9c992b 100644 --- a/internal/application/service/wiki_ingest.go +++ b/internal/application/service/wiki_ingest.go @@ -2170,19 +2170,39 @@ func (s *wikiIngestService) publishDraftPages(ctx context.Context, kbID string, } } -// writeDedupItemXML renders a single entity/concept entry as a structured XML -// block for the deduplication prompt. Structured form (versus a single -// pipe-separated line) helps the LLM reliably tell name / aliases / type apart -// and reduces nonsensical merges like "居民身份证" → "工作居住证". -func writeDedupItemXML(buf *strings.Builder, slug, name, itemType string, aliases []string) { - fmt.Fprintf(buf, " \n", slug, itemType) - fmt.Fprintf(buf, " %s\n", xmlEscape(name)) - for _, alias := range aliases { +// writeDedupCandidateGroup renders one new item together with its own +// similarity-candidate existing pages, nested under a element. +// This per-item grouping is what constrains the dedup model to local +// decisions (see the grouping rationale in deduplicateExtractedBatch). The +// candidate pages keep their aliases so the model still has the acronym / +// translation signal it needs to accept a legitimate merge. +func writeDedupCandidateGroup( + buf *strings.Builder, item extractedItem, itemType string, candidates []*types.WikiPageLite, +) { + fmt.Fprintf(buf, " \n", item.Slug, itemType) + fmt.Fprintf(buf, " %s\n", xmlEscape(item.Name)) + for _, alias := range item.Aliases { if alias == "" { continue } fmt.Fprintf(buf, " %s\n", xmlEscape(alias)) } + buf.WriteString(" \n") + for _, p := range candidates { + if p == nil { + continue + } + fmt.Fprintf(buf, " \n", p.Slug, p.PageType) + fmt.Fprintf(buf, " %s\n", xmlEscape(p.Title)) + for _, alias := range []string(p.Aliases) { + if alias == "" { + continue + } + fmt.Fprintf(buf, " %s\n", xmlEscape(alias)) + } + buf.WriteString(" \n") + } + buf.WriteString(" \n") buf.WriteString(" \n") } @@ -2226,7 +2246,16 @@ func (s *wikiIngestService) deduplicateExtractedBatch( // Build the candidate set: for each new item, ask the repo for // the top-K trigram-similar pages and union the results. Dedup by // slug as we go so the prompt only carries each candidate once. + // + // itemCandidates additionally records, per new item, the slugs that + // surfaced for THAT item specifically. The prompt only ever sees the + // flattened union, so validMerge below uses this per-item scoping to + // reject a merge whose target was pulled in for a *different* item — + // the class of hallucination the union otherwise enables (e.g. weak + // models emitting entity/tencent-open → entity/hiring-agent, which + // share no trigram signal and were never candidates for each other). candidatePages := make(map[string]*types.WikiPageLite) + itemCandidates := make(map[string]map[string]bool) probe := func(item extractedItem) { queries := make([]string, 0, 1+len(item.Aliases)) if item.Name != "" { @@ -2237,6 +2266,11 @@ func (s *wikiIngestService) deduplicateExtractedBatch( queries = append(queries, alias) } } + own := itemCandidates[item.Slug] + if own == nil { + own = make(map[string]bool) + itemCandidates[item.Slug] = own + } for _, q := range queries { pages, err := s.wikiService.FindSimilarPages(ctx, kbID, q, []string{types.WikiPageTypeEntity, types.WikiPageTypeConcept}, @@ -2252,6 +2286,7 @@ func (s *wikiIngestService) deduplicateExtractedBatch( if _, ok := candidatePages[p.Slug]; !ok { candidatePages[p.Slug] = p } + own[p.Slug] = true } } } @@ -2270,25 +2305,59 @@ func (s *wikiIngestService) deduplicateExtractedBatch( logger.Infof(ctx, "wiki ingest: %d similar existing pages selected for %d new items", len(candidatePages), len(entities)+len(concepts)) - var existingBuf strings.Builder - for _, p := range candidatePages { - writeDedupItemXML(&existingBuf, p.Slug, p.Title, p.PageType, []string(p.Aliases)) + // Group each new item with ONLY the existing pages that surfaced for + // its own similarity probe. Presenting the model two flat lists (all + // new items × all candidates) invites cross-item mispairings — it has + // no way to tell which candidate is relevant to which item, so a weak + // model pairs unrelated slugs that merely coexist in the prompt. A + // per-item shortlist turns dedup into a local yes/no decision against + // a handful of genuinely-similar pages and makes cross-item pairings + // structurally unnatural to express. Items with no candidate are + // omitted entirely (they cannot merge and only add hallucination + // surface + tokens). + var candBuf strings.Builder + groups := 0 + renderGroup := func(item extractedItem, itemType string) { + cset := itemCandidates[item.Slug] + if len(cset) == 0 { + return + } + slugs := make([]string, 0, len(cset)) + for slug := range cset { + // Skip the item's own slug: an identically-slugged existing + // page is a re-ingest/update, not a merge target. + if slug == item.Slug { + continue + } + if _, ok := candidatePages[slug]; ok { + slugs = append(slugs, slug) + } + } + if len(slugs) == 0 { + return + } + sort.Strings(slugs) + pages := make([]*types.WikiPageLite, 0, len(slugs)) + for _, slug := range slugs { + pages = append(pages, candidatePages[slug]) + } + writeDedupCandidateGroup(&candBuf, item, itemType, pages) + groups++ } - if existingBuf.Len() == 0 { + for _, item := range entities { + renderGroup(item, "entity") + } + for _, item := range concepts { + renderGroup(item, "concept") + } + if groups == 0 { + // Every new item's candidate list is empty after scoping — + // nothing the model could safely merge. return entities, concepts } - var newBuf strings.Builder - for _, item := range entities { - writeDedupItemXML(&newBuf, item.Slug, item.Name, "entity", item.Aliases) - } - for _, item := range concepts { - writeDedupItemXML(&newBuf, item.Slug, item.Name, "concept", item.Aliases) - } - dedupeJSON, err := s.generateWithTemplate(ctx, chatModel, agent.WikiDeduplicationPrompt, map[string]string{ - "NewItems": newBuf.String(), - "ExistingPages": existingBuf.String(), + "Candidates": candBuf.String(), }) if err != nil { logger.Warnf(ctx, "wiki ingest: deduplication LLM call failed: %v", err) @@ -2309,36 +2378,9 @@ func (s *wikiIngestService) deduplicateExtractedBatch( return entities, concepts } - // Build the existing-slug set from the candidate map: anything not - // in candidates is rejected as an LLM hallucination, since by - // construction the model only ever saw those slugs as merge - // targets. Compare with the legacy "look up against allPages" - // path which had a wider acceptance window. - existingSlugs := make(map[string]bool, len(candidatePages)) - for slug := range candidatePages { - existingSlugs[slug] = true - } - validMerge := func(srcSlug, dstSlug string) bool { - if !existingSlugs[dstSlug] { - logger.Warnf(ctx, "wiki ingest: dedup rejected %s → %s (target slug does not exist in candidate set)", srcSlug, dstSlug) - return false - } - srcSlash := strings.Index(srcSlug, "/") - dstSlash := strings.Index(dstSlug, "/") - if srcSlash <= 0 || dstSlash <= 0 { - // A type-prefixed slug must look like "entity/foo" or - // "concept/bar". An LLM that emits an un-prefixed slug - // here is hallucinating; reject rather than fall through - // the prefix-equality check (which would treat both empty - // prefixes as a match). - logger.Warnf(ctx, "wiki ingest: dedup rejected %s → %s (missing type prefix)", srcSlug, dstSlug) - return false - } - srcPrefix := srcSlug[:srcSlash+1] - dstPrefix := dstSlug[:dstSlash+1] - if srcPrefix != dstPrefix { - logger.Warnf(ctx, "wiki ingest: dedup rejected %s → %s (type mismatch: %s vs %s)", srcSlug, dstSlug, srcPrefix, dstPrefix) + if reason := dedupMergeRejectReason(srcSlug, dstSlug, itemCandidates[srcSlug]); reason != "" { + logger.Warnf(ctx, "wiki ingest: dedup rejected %s → %s (%s)", srcSlug, dstSlug, reason) return false } return true diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go index 386d709b..e458b365 100644 --- a/internal/application/service/wiki_ingest_batch.go +++ b/internal/application/service/wiki_ingest_batch.go @@ -2003,9 +2003,27 @@ func (s *wikiIngestService) reduceSlugUpdates( if len(additions) > 0 || len(retracts) > 0 { titles := batchCtx.SlugTitleMany(ctx, []string(page.OutLinks)) + + // aliaser escapes high-entropy slugs behind short reference tokens + // (ref-1, ref-2, …) for the editor LLM, then translates them back to + // real slugs after generation (see dealiasContent below). + aliaser := newSlugAliaser() + + // Escape every out-link slug behind a reference token so the editor + // never has to retype a real slug — this is where UUID-based summary + // slugs (summary/) got mangled into 404-ing links. Both the + // listing AND the [[...]] refs already present in + // are aliased with the SAME token map, so the + // model sees one consistent, copy-safe identifier space; we translate + // the tokens back to real slugs after generation. + known := make(map[string]struct{}, len(page.OutLinks)) + for _, outSlug := range page.OutLinks { + known[outSlug] = struct{}{} + aliaser.token(outSlug) // pre-assign for stable ordering + } for _, outSlug := range page.OutLinks { if title := titles[outSlug]; title != "" { - fmt.Fprintf(&relatedSlugs, "- %s (%s)\n", outSlug, title) + fmt.Fprintf(&relatedSlugs, "- %s (%s)\n", aliaser.token(outSlug), title) } } @@ -2013,6 +2031,7 @@ func (s *wikiIngestService) reduceSlugUpdates( // [c003]. They are internal ingest metadata; keep the editor context // clean so a subsequent update cannot copy them into rewritten prose. existingContent := stripWikiInlineChunkCitations(page.Content) + existingContent = aliaser.aliasContent(existingContent, known) if !exists || existingContent == "" { existingContent = "(New page)" } @@ -2060,6 +2079,10 @@ func (s *wikiIngestService) reduceSlugUpdates( }) if err == nil && updatedContent != "" { + // Translate reference tokens (ref-N) the model copied from the + // aliased context back to their real slugs BEFORE the content is + // parsed/stored, so out_links reflect real pages again. + updatedContent = aliaser.dealiasContent(updatedContent) updatedSummary, updatedBody := splitSummaryLine(updatedContent) if updatedBody != "" { page.Content = updatedBody diff --git a/internal/application/service/wiki_ingest_dedup.go b/internal/application/service/wiki_ingest_dedup.go index be0dbac8..75cbf903 100644 --- a/internal/application/service/wiki_ingest_dedup.go +++ b/internal/application/service/wiki_ingest_dedup.go @@ -23,11 +23,15 @@ import ( // external calls, and it only ever *removes* candidates from the prompt — // the downstream validMerge check still guards the final write. const ( - // dedupCandidateTopK is how many existing pages we keep per new item - // even if none pass the score floor. Guarantees the LLM sees a small - // ordered shortlist rather than an empty list when nothing matches - // strongly. - dedupCandidateTopK = 20 + // dedupCandidateTopK bounds how many trigram-similar existing pages + // each new item's similarity probe returns (the LIMIT passed to + // FindSimilarPages, applied per query term = name + each alias). Kept + // deliberately small: the DB already orders by similarity desc behind a + // pg_trgm threshold, so a genuine same-entity target is essentially + // always the top hit. A tight K keeps each item's list in + // the dedup prompt short, which improves the model's precision and cuts + // tokens, at negligible recall cost. + dedupCandidateTopK = 5 // dedupCandidateScoreFloor is the Jaccard floor. Pairs at or above // this similarity are always included regardless of the top-K cap. @@ -170,6 +174,44 @@ func selectDedupCandidatePages( return out } +// dedupMergeRejectReason validates a single LLM-proposed merge (srcSlug → +// dstSlug) against deterministic, model-independent rules. It returns an +// empty string when the merge is allowed, or a short human-readable reason +// when it must be rejected. srcCandidates is the set of existing-page slugs +// that surfaced for srcSlug's OWN similarity probe (see itemCandidates in +// deduplicateExtractedBatch). +// +// The per-item scoping check is the key guard: the dedup prompt shows the +// model a flattened union of candidates across every new item, so nothing +// stops a weak model from pairing an item with a page that was only similar +// to a *different* item (observed: entity/tencent-open → entity/hiring-agent, +// which share no trigram signal). Requiring dstSlug to be one of srcSlug's +// own candidates rejects that entire class without depending on the model +// getting the semantic judgment right. +func dedupMergeRejectReason(srcSlug, dstSlug string, srcCandidates map[string]bool) string { + if !srcCandidates[dstSlug] { + // Covers both an outright hallucinated target (in no candidate + // set at all) and a real page that was only similar to another + // item. Either way the pair lacks a similarity signal for THIS + // item, so it is not a safe merge. + return "target is not a similarity candidate for this item" + } + srcSlash := strings.Index(srcSlug, "/") + dstSlash := strings.Index(dstSlug, "/") + if srcSlash <= 0 || dstSlash <= 0 { + // A type-prefixed slug must look like "entity/foo" or + // "concept/bar". An LLM that emits an un-prefixed slug here is + // hallucinating; reject rather than fall through the prefix- + // equality check (which would treat both empty prefixes as a + // match). + return "missing type prefix" + } + if srcSlug[:srcSlash+1] != dstSlug[:dstSlash+1] { + return "type mismatch: " + srcSlug[:srcSlash+1] + " vs " + dstSlug[:dstSlash+1] + } + return "" +} + // dedupPairScore is the max similarity between any surface form of a and // b (plus the slug-token similarity). Slug and name signals live in // different symbol spaces (ASCII pinyin vs raw surface form) so we take diff --git a/internal/application/service/wiki_ingest_dedup_test.go b/internal/application/service/wiki_ingest_dedup_test.go index cb9a61a8..42db5eff 100644 --- a/internal/application/service/wiki_ingest_dedup_test.go +++ b/internal/application/service/wiki_ingest_dedup_test.go @@ -7,6 +7,61 @@ import ( "github.com/Tencent/WeKnora/internal/types" ) +func TestDedupMergeRejectReason(t *testing.T) { + cases := []struct { + name string + src, dst string + candidates map[string]bool + wantAllowed bool + }{ + { + name: "allowed: dst is a candidate for this item, same type prefix", + src: "entity/acme-corp", + dst: "entity/acme-corporation", + candidates: map[string]bool{"entity/acme-corporation": true}, + wantAllowed: true, + }, + { + name: "rejected: dst similar to a DIFFERENT item (union hallucination)", + src: "entity/tencent-open", + dst: "entity/hiring-agent", + candidates: map[string]bool{"entity/tencent-ur": true}, // hiring-agent not here + wantAllowed: false, + }, + { + name: "rejected: dst is not a candidate at all (pure hallucination)", + src: "entity/hy3-preview", + dst: "entity/llm-cli-tool", + candidates: nil, + wantAllowed: false, + }, + { + name: "rejected: type mismatch even when dst is a candidate", + src: "entity/foo", + dst: "concept/foo", + candidates: map[string]bool{"concept/foo": true}, + wantAllowed: false, + }, + { + name: "rejected: missing type prefix", + src: "foo", + dst: "entity/foo", + candidates: map[string]bool{"entity/foo": true}, + wantAllowed: false, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + reason := dedupMergeRejectReason(tc.src, tc.dst, tc.candidates) + gotAllowed := reason == "" + if gotAllowed != tc.wantAllowed { + t.Fatalf("dedupMergeRejectReason(%q, %q) allowed=%v (reason=%q), want allowed=%v", + tc.src, tc.dst, gotAllowed, reason, tc.wantAllowed) + } + }) + } +} + // helper: build a minimal entity/concept WikiPage. func makePage(slug, title, typ string, aliases ...string) *types.WikiPage { return &types.WikiPage{ diff --git a/internal/application/service/wiki_page.go b/internal/application/service/wiki_page.go index 55889bbf..3657defd 100644 --- a/internal/application/service/wiki_page.go +++ b/internal/application/service/wiki_page.go @@ -918,6 +918,151 @@ func normalizeSlug(slug string) string { return slug } +// slugNamespace returns the prefix of a slug up to (but excluding) the first +// '/', e.g. "summary/abc" -> "summary". Slugs without a '/' map to "". +func slugNamespace(slug string) string { + if i := strings.IndexByte(slug, '/'); i >= 0 { + return slug[:i] + } + return "" +} + +// rewriteDeadWikiLinks walks every [[slug]] / [[slug|display]] occurrence in +// content and lets `resolve` decide, per link, whether to rewrite the slug. +// `resolve` receives the normalized slug and its display text and returns the +// replacement slug plus true to rewrite, or ("", false) to leave the link +// untouched. Display text is preserved verbatim. This is a pure helper — the +// resolution policy lives entirely in the callback. +func rewriteDeadWikiLinks(content string, resolve func(normSlug, display string) (string, bool)) (string, bool) { + changed := false + out := wikiLinkRegex.ReplaceAllStringFunc(content, func(match string) string { + inner := match[2 : len(match)-2] + rawSlug := inner + display := "" + if parts := strings.SplitN(inner, "|", 2); len(parts) == 2 { + rawSlug = parts[0] + display = strings.TrimSpace(parts[1]) + } + norm := normalizeSlug(rawSlug) + if norm == "" { + return match + } + newSlug, ok := resolve(norm, display) + if !ok || newSlug == "" || newSlug == norm { + return match + } + changed = true + if display != "" { + return "[[" + newSlug + "|" + display + "]]" + } + return "[[" + newSlug + "]]" + }) + return out, changed +} + +// RepairContentLinks rewrites [[slug]] / [[slug|display]] references in +// `content` whose target does not exist in the KB but is almost certainly a +// mangled form of a real page. The canonical case: an LLM re-typed a summary +// page's UUID-based slug and inserted/dropped a hex digit +// (summary/…06fb5d5b5b5e → summary/…06fb14d5b14b14e), producing a link that +// 404s and can never be recovered by exact lookup. +// +// Unlike stripDeadWikiLinks (the ingest cleanup pass), this method is +// REWRITE-ONLY: a dead link is corrected only when a confident live candidate +// exists; otherwise it is left completely untouched. It NEVER strips a link to +// plain text, so it is safe on any write path — including writes whose targets +// legitimately do not exist yet (those simply stay as-is until they do). +// +// The candidate pool for each dead link is scoped to live slugs that share the +// same namespace prefix (a dead `summary/` only resolves against live +// `summary/*` slugs). Scoping keeps the bigram-similarity lever safe: distinct +// high-entropy UUIDs in one namespace don't collide, while a one-character +// mangle stays comfortably above threshold against its true source. +// +// Returns the possibly-updated content and whether any rewrite happened. +// Errors are only returned for hard repo failures; callers may treat repair as +// best-effort and ignore them. +func (s *wikiPageService) RepairContentLinks( + ctx context.Context, kbID, selfSlug, content string, +) (string, bool, error) { + if strings.TrimSpace(content) == "" { + return content, false, nil + } + outLinks := s.parseOutLinks(content) + if len(outLinks) == 0 { + return content, false, nil + } + + existMap, err := s.repo.ExistsSlugs(ctx, kbID, outLinks) + if err != nil { + return content, false, err + } + deadPrefixes := make(map[string]struct{}) + for _, l := range outLinks { + if l == selfSlug || existMap[l] { + continue + } + deadPrefixes[slugNamespace(l)] = struct{}{} + } + if len(deadPrefixes) == 0 { + return content, false, nil + } + + allSlugs, err := s.repo.ListAllSlugs(ctx, kbID) + if err != nil { + return content, false, err + } + liveByPrefix := make(map[string]map[string]struct{}) + var candidateSlugs []string + for _, sl := range allSlugs { + ns := slugNamespace(sl) + if _, want := deadPrefixes[ns]; !want { + continue + } + set, ok := liveByPrefix[ns] + if !ok { + set = make(map[string]struct{}) + liveByPrefix[ns] = set + } + set[sl] = struct{}{} + candidateSlugs = append(candidateSlugs, sl) + } + if len(candidateSlugs) == 0 { + return content, false, nil + } + + // Build a title -> slug reverse lookup for candidate pages so the + // display-text lever (the safest, most precise one) can fire. Bounded to + // the relevant namespaces so this stays cheap even on large KBs. + titleToSlug := make(map[string]string) + if lites, lerr := s.repo.ListBySlugs(ctx, kbID, candidateSlugs); lerr == nil { + for _, lp := range lites { + if lp != nil && lp.Title != "" { + titleToSlug[lp.Title] = lp.Slug + } + } + } + + resolveCache := make(map[string]string) + newContent, changed := rewriteDeadWikiLinks(content, func(norm, display string) (string, bool) { + if norm == selfSlug || existMap[norm] { + return "", false + } + key := norm + "\x00" + display + if cached, ok := resolveCache[key]; ok { + return cached, cached != "" + } + resolved, ok := resolveDeadSlug(norm, display, liveByPrefix[slugNamespace(norm)], titleToSlug) + if !ok || resolved == norm { + resolveCache[key] = "" + return "", false + } + resolveCache[key] = resolved + return resolved, true + }) + return newContent, changed, nil +} + // updateInLinks adds the source slug to the in_links of target pages func (s *wikiPageService) updateInLinks(ctx context.Context, kbID string, sourceSlug string, targets types.StringArray) { for _, targetSlug := range targets { diff --git a/internal/application/service/wiki_page_test.go b/internal/application/service/wiki_page_test.go index 84c4b9ab..948dcaa8 100644 --- a/internal/application/service/wiki_page_test.go +++ b/internal/application/service/wiki_page_test.go @@ -8,6 +8,7 @@ import ( "github.com/Tencent/WeKnora/internal/application/repository" "github.com/Tencent/WeKnora/internal/types" + "github.com/google/uuid" "github.com/stretchr/testify/require" "gorm.io/driver/sqlite" "gorm.io/gorm" @@ -140,6 +141,60 @@ func TestParseOutLinks(t *testing.T) { } } +func TestRepairContentLinks(t *testing.T) { + db, err := gorm.Open(sqlite.Open(fmt.Sprintf("file:%s?mode=memory&cache=shared", t.Name())), &gorm.Config{}) + require.NoError(t, err) + require.NoError(t, db.AutoMigrate(&types.WikiFolder{}, &types.WikiPage{})) + + ctx := context.Background() + repo := repository.NewWikiPageRepository(db) + svc := NewWikiPageService(repo, nil, nil, nil, nil) + const kbID = "kb-repair" + now := time.Now() + + seed := func(slug, title, pageType string) { + require.NoError(t, repo.Create(ctx, &types.WikiPage{ + ID: uuid.New().String(), TenantID: 1, KnowledgeBaseID: kbID, + Slug: slug, Title: title, PageType: pageType, + Status: types.WikiPageStatusPublished, Version: 1, + CreatedAt: now, UpdatedAt: now, + })) + } + + // Real summary page (clean UUID slug) + a distractor summary + an entity. + realSummary := "summary/07a20bb1-a662-47cf-9929-06fb5d5b5b5e" + seed(realSummary, "Weknora 试错记录.md - Summary", types.WikiPageTypeSummary) + seed("summary/fcadcaab-e094-4037-b71c-9edfdcdba058", "Another Doc - Summary", types.WikiPageTypeSummary) + seed("entity/mongodb", "MongoDB", types.WikiPageTypeEntity) + + t.Run("mangled uuid summary link repaired via bigram", func(t *testing.T) { + // One hex digit inserted into the last group — 404s on exact lookup. + content := "See [[summary/07a20bb1-a662-47cf-9929-06fb14d5b14b14e|Weknora 试错记录.md - Summary]] for context." + got, changed, err := svc.RepairContentLinks(ctx, kbID, "synthesis/notes", content) + require.NoError(t, err) + require.True(t, changed, "expected the mangled summary link to be rewritten") + require.Contains(t, got, "[["+realSummary+"|Weknora 试错记录.md - Summary]]") + require.NotContains(t, got, "06fb14d5b14b14e") + }) + + t.Run("live links untouched", func(t *testing.T) { + content := "Fine: [[entity/mongodb]] and [[" + realSummary + "]]." + got, changed, err := svc.RepairContentLinks(ctx, kbID, "synthesis/notes", content) + require.NoError(t, err) + require.False(t, changed) + require.Equal(t, content, got) + }) + + t.Run("unresolvable dead link left as-is, never stripped", func(t *testing.T) { + // A future entity page that doesn't exist and isn't similar to anything. + content := "Pending: [[entity/some-brand-new-topic|New Topic]]." + got, changed, err := svc.RepairContentLinks(ctx, kbID, "synthesis/notes", content) + require.NoError(t, err) + require.False(t, changed) + require.Equal(t, content, got, "unresolvable links must be preserved verbatim, not stripped") + }) +} + func TestNormalizeSlug(t *testing.T) { tests := []struct { input string diff --git a/internal/application/service/wiki_slug_alias.go b/internal/application/service/wiki_slug_alias.go new file mode 100644 index 00000000..28d35223 --- /dev/null +++ b/internal/application/service/wiki_slug_alias.go @@ -0,0 +1,88 @@ +package service + +import ( + "fmt" + "strings" +) + +// slugAliaser maps real wiki slugs to short, low-entropy reference tokens +// (ref-1, ref-2, …) so the ingest editor LLM never has to reproduce a +// high-entropy slug verbatim — most importantly the UUID-based summary slugs +// (summary/) that models routinely mangle by inserting or +// dropping hex digits. The model copies the tiny token; we translate tokens +// back to real slugs on output. +// +// This is the generation-stage counterpart to wiki_write_page's slug +// validation: rather than repairing a mangled slug after the fact, it removes +// the opportunity to mangle at the source. It mirrors the chunk-alias +// (c000/c001) indirection already used by the chunk-citation pass in +// wiki_ingest_cite.go. +// +// A slugAliaser is single-use and NOT safe for concurrent use; construct one +// per LLM call. +type slugAliaser struct { + toToken map[string]string // real slug -> token + toSlug map[string]string // token -> real slug +} + +func newSlugAliaser() *slugAliaser { + return &slugAliaser{ + toToken: make(map[string]string), + toSlug: make(map[string]string), + } +} + +// token returns the stable token for a real slug, assigning a fresh one on +// first use. An empty slug returns "". Tokens have no '/' so they can never +// collide with a namespaced real slug (entity/…, summary/…). +func (a *slugAliaser) token(realSlug string) string { + realSlug = strings.TrimSpace(realSlug) + if realSlug == "" { + return "" + } + if t, ok := a.toToken[realSlug]; ok { + return t + } + t := fmt.Sprintf("ref-%d", len(a.toToken)+1) + a.toToken[realSlug] = t + a.toSlug[t] = realSlug + return t +} + +// empty reports whether no tokens have been assigned yet (nothing to alias). +func (a *slugAliaser) empty() bool { return len(a.toToken) == 0 } + +// aliasContent rewrites [[realSlug|disp]] / [[realSlug]] occurrences to their +// token form, but only for slugs present in `known`. Unknown links are left +// untouched. Assigns tokens on demand so a slug that appears only in the body +// (not in the listing) still gets a consistent token. +func (a *slugAliaser) aliasContent(content string, known map[string]struct{}) string { + if content == "" || len(known) == 0 { + return content + } + out, _ := rewriteDeadWikiLinks(content, func(norm, _ string) (string, bool) { + if _, ok := known[norm]; !ok { + return "", false + } + return a.token(norm), true + }) + return out +} + +// dealiasContent rewrites [[token|disp]] / [[token]] occurrences back to their +// real slug. Tokens with no mapping are left untouched — they fall through to +// the ordinary parse/dead-link-cleanup path, which is the correct behavior for +// anything the model invented that isn't a known reference. +func (a *slugAliaser) dealiasContent(content string) string { + if content == "" || a.empty() { + return content + } + out, _ := rewriteDeadWikiLinks(content, func(norm, _ string) (string, bool) { + real, ok := a.toSlug[norm] + if !ok { + return "", false + } + return real, true + }) + return out +} diff --git a/internal/application/service/wiki_slug_alias_test.go b/internal/application/service/wiki_slug_alias_test.go new file mode 100644 index 00000000..a3476d11 --- /dev/null +++ b/internal/application/service/wiki_slug_alias_test.go @@ -0,0 +1,72 @@ +package service + +import ( + "strings" + "testing" +) + +func TestSlugAliaser_RoundTripEscapesUUIDSlugs(t *testing.T) { + summarySlug := "summary/07a20bb1-a662-47cf-9929-06fb5d5b5b5e" + entitySlug := "entity/mongodb" + known := map[string]struct{}{summarySlug: {}, entitySlug: {}} + + a := newSlugAliaser() + // Pre-assign tokens the way the ingest listing loop does. + tSummary := a.token(summarySlug) + tEntity := a.token(entitySlug) + if tSummary == summarySlug || tEntity == entitySlug { + t.Fatalf("tokens must differ from real slugs: %q %q", tSummary, tEntity) + } + if strings.Contains(tSummary, "/") { + t.Fatalf("token %q must not contain '/'", tSummary) + } + + body := "See [[" + summarySlug + "|Weknora 试错记录.md - Summary]] and [[" + entitySlug + "]]." + aliased := a.aliasContent(body, known) + + // The high-entropy UUID must no longer be present in what the model sees. + if strings.Contains(aliased, "06fb5d5b5b5e") { + t.Fatalf("aliased content still exposes the UUID: %q", aliased) + } + if !strings.Contains(aliased, "[["+tSummary+"|Weknora 试错记录.md - Summary]]") { + t.Fatalf("summary link not aliased: %q", aliased) + } + if !strings.Contains(aliased, "[["+tEntity+"]]") { + t.Fatalf("entity link not aliased: %q", aliased) + } + + // Round-trip must restore the original body exactly. + if got := a.dealiasContent(aliased); got != body { + t.Fatalf("dealias round-trip mismatch:\n got %q\n want %q", got, body) + } +} + +func TestSlugAliaser_UnknownTokensAndSlugsUntouched(t *testing.T) { + a := newSlugAliaser() + a.token("summary/real-uuid") + + // A slug not in `known` must not be aliased. + known := map[string]struct{}{"summary/real-uuid": {}} + body := "keep [[entity/invented-by-model]] as-is" + if got := a.aliasContent(body, known); got != body { + t.Fatalf("unknown slug should be left untouched, got %q", got) + } + + // An unmapped token in output must be left untouched (falls through to the + // normal dead-link cleanup path). + out := "stray [[ref-999|X]] token" + if got := a.dealiasContent(out); got != out { + t.Fatalf("unmapped token should be left untouched, got %q", got) + } +} + +func TestSlugAliaser_EmptyIsNoop(t *testing.T) { + a := newSlugAliaser() + if !a.empty() { + t.Fatal("fresh aliaser must be empty") + } + body := "[[summary/x]] unchanged" + if got := a.dealiasContent(body); got != body { + t.Fatalf("empty aliaser dealias must be a no-op, got %q", got) + } +} diff --git a/internal/llmresource/registry.go b/internal/llmresource/registry.go index 142ea942..705553ac 100644 --- a/internal/llmresource/registry.go +++ b/internal/llmresource/registry.go @@ -17,10 +17,22 @@ import ( // resource:// handles, but old chunks and message history can still contain a // provider URL. Giving both forms the same request-local alias makes rollout // safe without a blocking full-table content rewrite. +// +// The final alternative aliases wiki summary-page slugs (summary/). They +// are not storage handles, but they share the exact failure mode this registry +// exists to prevent: a high-entropy identifier the model must reproduce +// verbatim (inside [[slug|display]] links and as wiki-tool slug arguments). +// Models routinely mangle the UUID by inserting or dropping hex digits, which +// yields dead cross-links. Aliasing the slug to a low-entropy res:// token +// removes the opportunity to mangle at the source; the token round-trips back +// to the real slug on stream output and in decoded tool-call arguments before +// anything is persisted. Entity slugs (entity/) are low-entropy +// and semantically meaningful, so they are deliberately left untouched. var storedRefRE = regexp.MustCompile( `resource://[0-9A-Za-z_-]{22}|` + `(?:storage://[0-9A-Za-z_-]+/)?` + - `(?:local|minio|cos|tos|s3|oss|ks3|obs)://[^\s)\]>"']+`, + `(?:local|minio|cos|tos|s3|oss|ks3|obs)://[^\s)\]>"']+|` + + `summary/[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}`, ) // aliasShapeRE matches the alias syntax produced by EncodeText. It is used only diff --git a/internal/llmresource/registry_test.go b/internal/llmresource/registry_test.go index 80a704d1..2483abb6 100644 --- a/internal/llmresource/registry_test.go +++ b/internal/llmresource/registry_test.go @@ -23,6 +23,25 @@ func TestRegistryAliasesLegacyPhysicalReferencesDuringRollout(t *testing.T) { require.Equal(t, "![image]("+ref+")", r.DecodeText(encoded)) } +func TestRegistryAliasesWikiSummarySlug(t *testing.T) { + r := NewRegistry() + slug := "summary/07a20bb1-a662-47cf-9929-06fb5d5b5b5e" + // Inside a [[slug|display]] link the model must copy verbatim. + encoded := r.EncodeText("see [[" + slug + "|Foo.md - Summary]]") + require.Equal(t, "see [[res://0001|Foo.md - Summary]]", encoded) + require.Equal(t, "see [["+slug+"|Foo.md - Summary]]", r.DecodeText(encoded)) + + // The same slug as a tool-call argument value resolves to one alias. + require.Equal(t, "res://0001", r.EncodeText(slug)) +} + +func TestRegistryLeavesEntitySlugUntouched(t *testing.T) { + r := NewRegistry() + // Entity slugs are low-entropy and semantically meaningful; not aliased. + entity := "[[entity/weknora-error-log|WeKnora 试错记录]]" + require.Equal(t, entity, r.EncodeText(entity)) +} + func TestRegistryEncodesMessageCopies(t *testing.T) { r := NewRegistry() ref := "resource://AbCdEfGhIjKlMnOpQrStUv" diff --git a/internal/types/interfaces/wiki_page.go b/internal/types/interfaces/wiki_page.go index 9885009e..e4eef89c 100644 --- a/internal/types/interfaces/wiki_page.go +++ b/internal/types/interfaces/wiki_page.go @@ -39,6 +39,13 @@ type WikiPageService interface { // GetPageBySlug retrieves a wiki page by its slug within a knowledge base. GetPageBySlug(ctx context.Context, kbID string, slug string) (*types.WikiPage, error) + // RepairContentLinks rewrites dead [[slug]] references in content to their + // most likely live target (rewrite-only — never strips). Used by the agent + // write path to auto-correct LLM-mangled slugs (especially UUID-based + // summary slugs) before persistence. Returns the possibly-updated content + // and whether any rewrite happened. Best-effort: callers may ignore errors. + RepairContentLinks(ctx context.Context, kbID, selfSlug, content string) (string, bool, error) + // GetPageByID retrieves a wiki page by its unique ID. GetPageByID(ctx context.Context, id string) (*types.WikiPage, error)