fix(wiki-ingest): reconcile summary links and log feed when reduce LLM fails
When a doc's entity/concept page generation hit a transient LLM error in
the reduce phase, the page never got written, but the doc's summary page
(generated in parallel with the slug list baked into its content) was
already persisted with [[entity/foo|name]] links pointing at the missing
page. The wiki log feed also surfaced the failed slugs as clickable
entries that 404'd. cleanDeadLinks only ran for retract batches, so the
debris persisted indefinitely.
reduceSlugUpdates now reports addition-path failures back to the batch
driver, which collects them and:
- rewrites this batch's summary pages, replacing dead [[slug|display]]
refs with plain display text (or a humanized slug tail);
- filters dead slugs out of docResults[].Pages before the wiki log
entries are flushed;
- broadens the cleanDeadLinks trigger to fire whenever any page was
touched or any slug failed, as a long-tail safety net.
Pure text replacement; no extra LLM calls. Failed slugs are picked up
naturally on the next ingest of the same document via the existing slug-
continuity rules, so the system self-heals without manual reingest.
This commit is contained in:
@@ -535,6 +535,96 @@ func previewStringSlice(items []string, limit int) string {
|
||||
return fmt.Sprintf("[%s]", strings.Join(out, ", "))
|
||||
}
|
||||
|
||||
// wikiLinkRE matches `[[slug]]` and `[[slug|display text]]` references
|
||||
// inside wiki page content. The slug capture group rejects whitespace and
|
||||
// the closing-bracket / pipe characters so we don't accidentally swallow
|
||||
// adjacent text. Display text (group 2) is optional.
|
||||
var wikiLinkRE = regexp.MustCompile(`\[\[([^\[\]\|\s]+)(?:\|([^\]]+))?\]\]`)
|
||||
|
||||
// sanitizeDeadSummaryLinks rewrites the summary pages produced by THIS
|
||||
// batch to remove `[[slug]]` / `[[slug|display]]` references that point
|
||||
// at slugs whose entity/concept page generation failed in reduce.
|
||||
//
|
||||
// Background: WikiSummaryPrompt instructs the LLM to embed wiki links
|
||||
// for every extracted slug it knows about, but slug extraction happens
|
||||
// during map (parallel with summary generation) and the actual page
|
||||
// creation happens later in reduce. When reduce's WikiPageModifyPrompt
|
||||
// fails on an entity/concept slug the page never gets written — and
|
||||
// the already-persisted summary is left holding a `[[entity/foo|name]]`
|
||||
// link that 404s. This pass replaces those dead links with their
|
||||
// display text so the summary degrades gracefully.
|
||||
//
|
||||
// Pure text replacement, no LLM call. Scoped to the doc-summary slugs
|
||||
// in this batch (`summary/<slugify(knowledgeID)>`), keeping the work
|
||||
// proportional to batch size.
|
||||
func (s *wikiIngestService) sanitizeDeadSummaryLinks(
|
||||
ctx context.Context,
|
||||
kbID string,
|
||||
docResults []*docIngestResult,
|
||||
failedSlugs map[string]struct{},
|
||||
) {
|
||||
if len(failedSlugs) == 0 || len(docResults) == 0 {
|
||||
return
|
||||
}
|
||||
for _, r := range docResults {
|
||||
if r == nil || r.KnowledgeID == "" {
|
||||
continue
|
||||
}
|
||||
summarySlug := "summary/" + slugify(r.KnowledgeID)
|
||||
page, err := s.wikiService.GetPageBySlug(ctx, kbID, summarySlug)
|
||||
if err != nil || page == nil {
|
||||
continue
|
||||
}
|
||||
newContent, changed := stripDeadWikiLinks(page.Content, failedSlugs)
|
||||
if !changed {
|
||||
continue
|
||||
}
|
||||
page.Content = newContent
|
||||
if err := s.wikiService.UpdateAutoLinkedContent(ctx, page); err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: failed to sanitize dead links in summary %s: %v", summarySlug, err)
|
||||
continue
|
||||
}
|
||||
logger.Infof(ctx, "wiki ingest: sanitized dead [[slug]] refs in summary %s", summarySlug)
|
||||
}
|
||||
}
|
||||
|
||||
// stripDeadWikiLinks replaces `[[slug]]` / `[[slug|display]]` references
|
||||
// with plain text whenever `slug` is in the dead set. The display text
|
||||
// (when present) is preserved verbatim; otherwise the slug's last path
|
||||
// segment is humanized into a fallback label so the surrounding sentence
|
||||
// still reads naturally.
|
||||
func stripDeadWikiLinks(content string, deadSlugs map[string]struct{}) (string, bool) {
|
||||
if len(deadSlugs) == 0 || content == "" {
|
||||
return content, false
|
||||
}
|
||||
changed := false
|
||||
out := wikiLinkRE.ReplaceAllStringFunc(content, func(match string) string {
|
||||
sub := wikiLinkRE.FindStringSubmatch(match)
|
||||
if len(sub) < 2 {
|
||||
return match
|
||||
}
|
||||
slug := sub[1]
|
||||
if _, dead := deadSlugs[slug]; !dead {
|
||||
return match
|
||||
}
|
||||
changed = true
|
||||
display := ""
|
||||
if len(sub) >= 3 {
|
||||
display = strings.TrimSpace(sub[2])
|
||||
}
|
||||
if display != "" {
|
||||
return display
|
||||
}
|
||||
// Derive a readable label from the slug's tail segment so we
|
||||
// don't leave a raw "entity/foo-bar" smear in the prose.
|
||||
parts := strings.Split(slug, "/")
|
||||
label := parts[len(parts)-1]
|
||||
label = strings.ReplaceAll(label, "-", " ")
|
||||
return label
|
||||
})
|
||||
return out, changed
|
||||
}
|
||||
|
||||
// cleanDeadLinks removes [[wiki-links]] that point to archived or deleted pages.
|
||||
// Scans all published pages, checks each out_link, and removes references to
|
||||
// pages that no longer exist or are archived. No LLM call — pure text cleanup.
|
||||
@@ -1041,7 +1131,7 @@ func (s *wikiIngestService) deduplicateExtractedBatch(
|
||||
// still alive), or generic "timeout"/"connection reset" wording.
|
||||
// 4xx (except 408/429) is a caller-side fault and fails fast.
|
||||
// - Backoff is exponential base 2s: 2s, 4s, 8s — roughly wikiLLMBackoffBase
|
||||
// * 2^(attempt-1). Honors ctx cancellation so the task can abort.
|
||||
// - 2^(attempt-1). Honors ctx cancellation so the task can abort.
|
||||
//
|
||||
// This exists because wiki ingest makes several independent LLM calls per
|
||||
// document (extraction, summary, dedup, citations, intro) and a single
|
||||
@@ -1142,7 +1232,7 @@ func isTransientLLMError(ctx context.Context, err error) bool {
|
||||
"connection reset",
|
||||
"connection refused",
|
||||
"broken pipe",
|
||||
"no such host", // DNS hiccup
|
||||
"no such host", // DNS hiccup
|
||||
"i/o timeout",
|
||||
"unexpected eof",
|
||||
"tls handshake",
|
||||
|
||||
@@ -372,12 +372,19 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
var allPagesAffected []string
|
||||
var ingestPagesAffected []string
|
||||
var retractPagesAffected []string
|
||||
// failedAdditionSlugs collects entity/concept slugs whose page
|
||||
// generation LLM call failed (so the page was never written). The
|
||||
// post-reduce cleanup step uses this set to (a) strip dead [[slug]]
|
||||
// references from the same batch's summary pages, and (b) prune the
|
||||
// slugs out of the wiki log feed so users don't see clickable entries
|
||||
// pointing at missing pages.
|
||||
failedAdditionSlugs := make(map[string]struct{})
|
||||
|
||||
for slug, updates := range slugUpdates {
|
||||
slug := slug
|
||||
updates := updates
|
||||
egReduce.Go(func() error {
|
||||
changed, affectedType, err := s.reduceSlugUpdates(reduceCtx, chatModel, payload.KnowledgeBaseID, slug, updates, payload.TenantID, batchCtx)
|
||||
changed, affectedType, additionFailed, err := s.reduceSlugUpdates(reduceCtx, chatModel, payload.KnowledgeBaseID, slug, updates, payload.TenantID, batchCtx)
|
||||
if err != nil {
|
||||
logger.Warnf(reduceCtx, "wiki ingest: reduce failed for slug %s: %v", slug, err)
|
||||
}
|
||||
@@ -391,11 +398,26 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
}
|
||||
reduceMu.Unlock()
|
||||
}
|
||||
if additionFailed {
|
||||
reduceMu.Lock()
|
||||
failedAdditionSlugs[slug] = struct{}{}
|
||||
reduceMu.Unlock()
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
_ = egReduce.Wait()
|
||||
|
||||
// Sanitize the doc summary pages produced by this batch BEFORE we
|
||||
// build log entries / rebuild the index. The summary LLM (run during
|
||||
// map) was free to inject [[entity/foo|name]] links to every slug it
|
||||
// saw extracted, but reduce may have failed to materialize some of
|
||||
// those slugs into actual pages. Rewrite those dead links to plain
|
||||
// text so the summary doesn't contain unresolvable references.
|
||||
if len(failedAdditionSlugs) > 0 && len(docResults) > 0 {
|
||||
s.sanitizeDeadSummaryLinks(ctx, payload.KnowledgeBaseID, docResults, failedAdditionSlugs)
|
||||
}
|
||||
|
||||
totalPagesAffected = len(allPagesAffected)
|
||||
|
||||
// Collect log entries for this batch and flush them in a single INSERT.
|
||||
@@ -428,7 +450,21 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
}
|
||||
}
|
||||
for _, r := range docResults {
|
||||
logEntries = append(logEntries, s.buildLogEntry(payload.TenantID, payload.KnowledgeBaseID, "ingest", r.KnowledgeID, r.DocTitle, r.Summary, r.Pages))
|
||||
// Drop any slugs whose page generation failed in reduce so the
|
||||
// log feed never offers a clickable entry that 404s. The summary
|
||||
// page itself (slug = summary/<knowledgeID>) is always created
|
||||
// unconditionally upstream, so it survives the filter.
|
||||
pages := r.Pages
|
||||
if len(failedAdditionSlugs) > 0 {
|
||||
pages = pages[:0:0]
|
||||
for _, ref := range r.Pages {
|
||||
if _, bad := failedAdditionSlugs[ref.Slug]; bad {
|
||||
continue
|
||||
}
|
||||
pages = append(pages, ref)
|
||||
}
|
||||
}
|
||||
logEntries = append(logEntries, s.buildLogEntry(payload.TenantID, payload.KnowledgeBaseID, "ingest", r.KnowledgeID, r.DocTitle, r.Summary, pages))
|
||||
}
|
||||
if len(logEntries) > 0 && s.logEntrySvc != nil {
|
||||
if err := s.logEntrySvc.AppendBatch(ctx, logEntries); err != nil {
|
||||
@@ -460,7 +496,13 @@ func (s *wikiIngestService) ProcessWikiIngest(ctx context.Context, t *asynq.Task
|
||||
}
|
||||
}
|
||||
|
||||
if len(retractPagesAffected) > 0 {
|
||||
// Clean dead [[slug]] references whenever ANY page was touched this
|
||||
// batch (not just retracts). Reduce-phase failures can leave stale
|
||||
// references in pages we just rewrote (e.g. summary pages cite
|
||||
// failed entity slugs); sanitizeDeadSummaryLinks above handles the
|
||||
// well-known summary case, and this pass is the safety net for the
|
||||
// long tail (cross-doc citations, prior batches' lingering refs).
|
||||
if len(retractPagesAffected) > 0 || len(failedAdditionSlugs) > 0 || len(allPagesAffected) > 0 {
|
||||
logger.Infof(ctx, "wiki ingest: cleaning dead links")
|
||||
s.cleanDeadLinks(ctx, payload.KnowledgeBaseID)
|
||||
}
|
||||
@@ -906,6 +948,15 @@ func (s *wikiIngestService) extractEntitiesAndConceptsNoUpsert(
|
||||
return result.Entities, result.Concepts, slugItems, nil
|
||||
}
|
||||
|
||||
// reduceSlugUpdates returns:
|
||||
// - changed: whether the wiki page was created or updated
|
||||
// - affectedType: "ingest" or "retract" — drives downstream bookkeeping
|
||||
// - additionFailed: true iff the slug had entity/concept additions queued
|
||||
// AND the WikiPageModifyPrompt LLM call failed, so no page exists/was
|
||||
// refreshed for it. Callers use this to sanitize dead [[slug]] links
|
||||
// elsewhere (e.g. in the doc's summary page) and to drop the slug from
|
||||
// the wiki log feed so users don't see a clickable entry that 404s.
|
||||
// - err: transport / repo error from the persisted upsert.
|
||||
func (s *wikiIngestService) reduceSlugUpdates(
|
||||
ctx context.Context,
|
||||
chatModel chat.Chat,
|
||||
@@ -914,7 +965,7 @@ func (s *wikiIngestService) reduceSlugUpdates(
|
||||
updates []SlugUpdate,
|
||||
tenantID uint64,
|
||||
batchCtx *WikiBatchContext,
|
||||
) (bool, string, error) {
|
||||
) (changed bool, affectedType string, additionFailed bool, err error) {
|
||||
// Final safety net for the ingest/delete race: between Map (which already
|
||||
// checks isKnowledgeGone) and Reduce there is a long LLM call where the
|
||||
// source document may be deleted. Drop any addition/summary updates whose
|
||||
@@ -923,10 +974,11 @@ func (s *wikiIngestService) reduceSlugUpdates(
|
||||
// want when the doc is gone.
|
||||
updates = s.filterLiveUpdates(ctx, kbID, updates)
|
||||
if len(updates) == 0 {
|
||||
return false, "", nil
|
||||
return false, "", false, nil
|
||||
}
|
||||
|
||||
page, err := s.wikiService.GetPageBySlug(ctx, kbID, slug)
|
||||
var page *types.WikiPage
|
||||
page, err = s.wikiService.GetPageBySlug(ctx, kbID, slug)
|
||||
exists := (err == nil && page != nil)
|
||||
|
||||
if !exists {
|
||||
@@ -938,7 +990,7 @@ func (s *wikiIngestService) reduceSlugUpdates(
|
||||
}
|
||||
}
|
||||
if !hasAdditions {
|
||||
return false, "", nil
|
||||
return false, "", false, nil
|
||||
}
|
||||
|
||||
page = &types.WikiPage{
|
||||
@@ -950,10 +1002,14 @@ func (s *wikiIngestService) reduceSlugUpdates(
|
||||
SourceRefs: types.StringArray{},
|
||||
Aliases: types.StringArray{},
|
||||
}
|
||||
// Reset err: GetPageBySlug returned "not found" which we just
|
||||
// handled by synthesizing the page. Don't leak that error to
|
||||
// the named return — subsequent assignments would mask it
|
||||
// anyway, but be explicit.
|
||||
err = nil
|
||||
}
|
||||
|
||||
changed := false
|
||||
affectedType := "ingest"
|
||||
affectedType = "ingest"
|
||||
|
||||
var summaryUpdate *SlugUpdate
|
||||
var retracts []SlugUpdate
|
||||
@@ -989,7 +1045,7 @@ func (s *wikiIngestService) reduceSlugUpdates(
|
||||
} else {
|
||||
_, err = s.wikiService.CreatePage(ctx, page)
|
||||
}
|
||||
return changed, affectedType, err
|
||||
return changed, affectedType, false, err
|
||||
}
|
||||
|
||||
var remainingSourcesContent strings.Builder
|
||||
@@ -1135,7 +1191,8 @@ func (s *wikiIngestService) reduceSlugUpdates(
|
||||
}
|
||||
pageAliases := strings.Join(page.Aliases, ", ")
|
||||
|
||||
updatedContent, err := s.generateWithTemplate(ctx, chatModel, agent.WikiPageModifyPrompt, map[string]string{
|
||||
var updatedContent string
|
||||
updatedContent, err = s.generateWithTemplate(ctx, chatModel, agent.WikiPageModifyPrompt, map[string]string{
|
||||
"HasAdditions": hasAdditionsStr,
|
||||
"HasRetractions": hasRetractionsStr,
|
||||
"PageSlug": slug,
|
||||
@@ -1163,6 +1220,19 @@ func (s *wikiIngestService) reduceSlugUpdates(
|
||||
changed = true
|
||||
} else if err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: update/retract failed for slug %s: %v", slug, err)
|
||||
// Flag addition failures so the batch can sanitize stale
|
||||
// [[slug]] references in the doc's summary page and prune
|
||||
// the slug from log entries — otherwise the wiki feed shows
|
||||
// a clickable entry whose target page doesn't exist.
|
||||
// Retract-only failures don't poison anything (they leave
|
||||
// the existing page unchanged), so don't flag those.
|
||||
if len(additions) > 0 {
|
||||
additionFailed = true
|
||||
}
|
||||
// Don't propagate the LLM error to the named return: it has
|
||||
// already been logged, and the eg.Go caller would otherwise
|
||||
// log it a second time as "reduce failed for slug".
|
||||
err = nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1177,10 +1247,10 @@ func (s *wikiIngestService) reduceSlugUpdates(
|
||||
} else {
|
||||
_, err = s.wikiService.CreatePage(ctx, page)
|
||||
}
|
||||
return true, affectedType, err
|
||||
return true, affectedType, additionFailed, err
|
||||
}
|
||||
|
||||
return false, "", nil
|
||||
return false, "", additionFailed, nil
|
||||
}
|
||||
|
||||
// mergeChunkRefs unions the chunk IDs currently on the page with the ones
|
||||
|
||||
Reference in New Issue
Block a user