fix(ingest): QA follow-up — preserve OCR text and close remaining filename leaks

Self-review of the prior commit surfaced one critical regression and
several smaller issues. This change addresses them.

Critical: the old stripImageMarkup stripped the WHOLE <image>...</image>
block, but searchutil.EnrichContentWithImageInfo wraps successful VLM OCR
output in exactly that shape — so a scanned PDF where OCR worked was
being rejected by hasSufficientTextContent and routed to the same
"failed" path as the OCR-empty case. The fix splits stripping into:
  * <image_original>...</image_original> blocks removed entirely
    (they only carry the redundant Markdown ref already stripped above)
  * <image>, <images>, <image_caption>, <image_ocr> tags removed at
    the tag level, INNER TEXT PRESERVED, so OCR / caption text reaches
    the threshold check and the LLM call.

A regression test pins this down with realistic enriched-image XML that
the previous code failed silently on.

Other fixes:
  * Threshold lowered from 50 to 10 runes — short legitimate documents
    (brief memos, single-line notes) must still pass.
  * Summary chunk content no longer prefixes "# Document\n<filename>";
    only the LLM-generated summary is indexed, so filenames cannot
    leak back into RAG retrieval context.
  * Wiki summary slug is now derived from the knowledge ID rather than
    the upload filename, eliminating filename-based slugs like
    "summary/mx5280-pdf" that downstream wiki-edit prompts can read.
  * Citation sourceRef likewise drops the "|<docTitle>" suffix.
  * Dead docTitle parameter removed from extractCandidateSlugs,
    extractEntitiesAndConceptsNoUpsert, classifyChunkCitations.
  * The redundant stripImageMarkup pass in the warning log is gone;
    realTextRuneCount caches the count once and uses
    utf8.RuneCountInString to avoid the rune-slice allocation.

Wiki-page display titles still flow through DocTitle (filename) into
later wiki-edit / index-rebuild prompts. That is a larger refactor of
the wiki pipeline and is intentionally left for a separate PR.
This commit is contained in:
Claude
2026-05-02 09:23:09 +00:00
committed by lyingbug
parent 15550d14d4
commit 6744e1cdd5
5 changed files with 123 additions and 47 deletions
+10 -4
View File
@@ -2308,10 +2308,11 @@ func (s *knowledgeService) getSummary(ctx context.Context,
// scanner model), and feeding that to the model would invite it to
// hallucinate a scanner manual instead of admitting the document had no
// extractable text.
if !hasSufficientTextContent(chunkContents) {
realTextLen := realTextRuneCount(chunkContents)
if realTextLen < minTextContentRunes {
logger.GetLogger(ctx).Warnf(
"getSummary: content for knowledge %s has insufficient text after stripping image markup (len=%d); skipping LLM call",
knowledge.ID, len([]rune(strings.TrimSpace(stripImageMarkup(chunkContents)))),
"getSummary: content for knowledge %s has insufficient text after stripping image markup (real_text_runes=%d, min=%d); skipping LLM call",
knowledge.ID, realTextLen, minTextContentRunes,
)
return "", errInsufficientSummaryContent
}
@@ -2536,12 +2537,17 @@ func (s *knowledgeService) ProcessSummaryGeneration(ctx context.Context, t *asyn
}
}
// Embed only the LLM-generated summary in the indexed chunk.
// We deliberately omit knowledge.FileName here: filenames are an
// unreliable signal (e.g. "MX5280.pdf" for a scanned legal letter)
// and surfacing them in retrieved RAG context can re-introduce the
// hallucination vector this branch is meant to close.
summaryChunk := &types.Chunk{
ID: uuid.New().String(),
TenantID: knowledge.TenantID,
KnowledgeID: knowledge.ID,
KnowledgeBaseID: knowledge.KnowledgeBaseID,
Content: fmt.Sprintf("# Document\n%s\n\n# Summary\n%s", knowledge.FileName, summary),
Content: fmt.Sprintf("# Summary\n%s", summary),
ChunkIndex: maxChunkIndex + 1,
IsEnabled: true,
CreatedAt: time.Now(),
+59 -25
View File
@@ -11,6 +11,7 @@ import (
"sync"
"text/template"
"time"
"unicode/utf8"
"github.com/Tencent/WeKnora/internal/agent"
"github.com/Tencent/WeKnora/internal/logger"
@@ -1217,43 +1218,76 @@ func appendUnique(arr types.StringArray, s string) types.StringArray {
// minTextContentRunes is the minimum number of non-whitespace, non-image-reference
// runes required for content to be considered substantive enough for LLM
// summarization or wiki extraction. Documents below this threshold (e.g. a
// scanned PDF where OCR yielded nothing) are routed to a deterministic empty-
// content fallback instead of being passed to the LLM, which would otherwise
// hallucinate based on metadata alone.
const minTextContentRunes = 50
// scanned PDF where OCR yielded nothing AND no caption either) are routed to
// a deterministic empty-content fallback instead of being passed to the LLM,
// which would otherwise hallucinate based on metadata alone.
//
// The threshold is intentionally low: legitimate short documents (brief
// memos, single-line notes) must still pass. The goal is only to catch
// the empty-image-only case.
const minTextContentRunes = 10
var (
// Markdown image references like ![alt](path) — pure visual placeholders
// with no extractable text, so the whole reference is removed.
mdImageRefRE = regexp.MustCompile(`!\[[^\]]*\]\([^)]*\)`)
// Strip the outer <images> wrapper before the inner <image> blocks so that
// nested structures like "<images><image>x</image></images>" don't leave a
// stray closing tag behind. RE2 has no backreferences, so we use two
// separate patterns instead of `<(image|images)\b...</\1>`.
htmlImagesBlockRE = regexp.MustCompile(`(?is)<images\b[^>]*>.*?</images>`)
htmlImageBlockRE = regexp.MustCompile(`(?is)<image\b[^>]*>.*?</image>`)
htmlImageTagRE = regexp.MustCompile(`(?i)<img\b[^>]*>`)
// <image_original>...</image_original> blocks wrap the verbatim Markdown
// image reference inside an enriched <image> block (see
// searchutil.EnrichContentWithImageInfo). The content is just a redundant
// copy of an already-stripped image link, so the whole block (tags +
// content) is removed.
imageOriginalBlockRE = regexp.MustCompile(`(?is)<image_original\b[^>]*>.*?</image_original>`)
// Self-closing or attribute-only HTML <img> tags.
htmlImgTagRE = regexp.MustCompile(`(?i)<img\b[^>]*/?>`)
// Wrapper-style <image>, <images>, <image_caption>, <image_ocr> tags
// (opening or closing). Matches ONLY the tag; the text content between
// open and close tags is preserved. This is critical: VLM-generated OCR
// and caption text live inside <image_ocr>...</image_ocr> and
// <image_caption>...</image_caption> blocks, and stripping the content
// would silently destroy the very text we want to keep.
imageWrapperTagRE = regexp.MustCompile(`(?i)</?image[a-z_]*\b[^>]*/?>`)
)
// stripImageMarkup removes markdown image references, raw <img> tags, and
// <image>/<images> XML blocks so that content consisting only of image
// placeholders (e.g. from PDFScannedParser pages without successful VLM OCR)
// is recognised as effectively empty.
// stripImageMarkup removes image-only placeholders (Markdown image refs,
// <img> tags, <image_original> redundancy blocks) and unwraps the
// <image>/<image_caption>/<image_ocr> XML wrappers produced by the search
// enrichment layer, leaving any OCR or caption text as plain inline text.
//
// This shape matters: when VLM OCR succeeds on a scanned PDF page, the
// extracted text reaches downstream code wrapped in <image_ocr> tags inside
// an <image> block. A naive "strip the whole <image>...</image> block"
// approach would discard the OCR text — the exact opposite of what we want.
func stripImageMarkup(s string) string {
s = htmlImagesBlockRE.ReplaceAllString(s, "")
s = htmlImageBlockRE.ReplaceAllString(s, "")
s = imageOriginalBlockRE.ReplaceAllString(s, "")
s = mdImageRefRE.ReplaceAllString(s, "")
s = htmlImageTagRE.ReplaceAllString(s, "")
s = htmlImgTagRE.ReplaceAllString(s, "")
s = imageWrapperTagRE.ReplaceAllString(s, "")
return s
}
// extractRealText returns the trimmed content with image markup stripped.
// Cached at the call site for use both in the threshold check and in any
// subsequent log message, avoiding redundant regex passes over large docs.
func extractRealText(content string) string {
return strings.TrimSpace(stripImageMarkup(content))
}
// hasSufficientTextContent reports whether the given content carries enough
// real text (after image markup is stripped) to warrant an LLM call. It is
// the primary defence against filename-driven hallucinations on scanned PDFs.
// real text (after image markup is stripped, with OCR/caption text retained)
// to warrant an LLM call. It is the primary defence against filename-driven
// hallucinations on scanned PDFs that have NO usable text at all.
func hasSufficientTextContent(content string) bool {
stripped := strings.TrimSpace(stripImageMarkup(content))
if stripped == "" {
return false
}
return len([]rune(stripped)) >= minTextContentRunes
return realTextRuneCount(content) >= minTextContentRunes
}
// realTextRuneCount returns the rune length of the content after image
// markup is stripped. Uses utf8.RuneCountInString to avoid allocating a
// rune slice for the count.
func realTextRuneCount(content string) int {
return utf8.RuneCountInString(extractRealText(content))
}
// cleanLLMJSON strips markdown code-fence wrappers and sanitizes control characters
@@ -500,7 +500,11 @@ func (s *wikiIngestService) mapOneDocument(
}
}
sourceRef := fmt.Sprintf("%s|%s", knowledgeID, docTitle)
// Citation source reference. We deliberately use only the knowledge ID
// (not docTitle, which is typically the upload filename) so the filename
// does not leak into citation strings that downstream LLM prompts may
// surface during wiki page editing.
sourceRef := knowledgeID
oldPageSlugs := s.getExistingPageSlugsForKnowledge(ctx, payload.KnowledgeBaseID, knowledgeID)
// Pass 0: lightweight candidate slug extraction (skeleton only).
@@ -513,11 +517,11 @@ func (s *wikiIngestService) mapOneDocument(
pass0Failed bool
)
logger.Infof(ctx, "wiki ingest: pass 0 — extracting candidate slugs for %s", knowledgeID)
extractedEntities, extractedConcepts, slugItems, err = s.extractCandidateSlugs(ctx, chatModel, content, docTitle, lang, oldPageSlugs, batchCtx)
extractedEntities, extractedConcepts, slugItems, err = s.extractCandidateSlugs(ctx, chatModel, content, lang, oldPageSlugs, batchCtx)
if err != nil {
logger.Warnf(ctx, "wiki ingest: pass 0 failed for %s (%v) — falling back to legacy extractor", knowledgeID, err)
pass0Failed = true
extractedEntities, extractedConcepts, slugItems, err = s.extractEntitiesAndConceptsNoUpsert(ctx, chatModel, content, docTitle, lang, oldPageSlugs, batchCtx)
extractedEntities, extractedConcepts, slugItems, err = s.extractEntitiesAndConceptsNoUpsert(ctx, chatModel, content, lang, oldPageSlugs, batchCtx)
if err != nil {
logger.Warnf(ctx, "wiki ingest: legacy fallback also failed for %s: %v", knowledgeID, err)
return nil, nil, err
@@ -529,7 +533,12 @@ func (s *wikiIngestService) mapOneDocument(
for slug := range slugItems {
summaryExtractedPages = append(summaryExtractedPages, slug)
}
summarySlug := fmt.Sprintf("summary/%s", slugify(docTitle))
// Wiki summary slug is derived from the knowledge ID rather than the
// docTitle (which is typically the upload filename). Filename-based slugs
// like "summary/mx5280-pdf" expose the filename in cross-link contexts
// that downstream LLM prompts read; a UUID-based slug is uglier but
// hallucination-safe.
summarySlug := fmt.Sprintf("summary/%s", slugify(knowledgeID))
var slugListing string
for _, slug := range summaryExtractedPages {
if item, ok := slugItems[slug]; ok {
@@ -574,7 +583,7 @@ func (s *wikiIngestService) mapOneDocument(
return
}
candidatesXML := renderCandidateSlugsXML(extractedEntities, extractedConcepts)
citations, newSlugs, batchCount = s.classifyChunkCitations(ctx, chatModel, candidatesXML, docTitle, chunks, lang)
citations, newSlugs, batchCount = s.classifyChunkCitations(ctx, chatModel, candidatesXML, chunks, lang)
}()
wg.Wait()
@@ -765,7 +774,7 @@ func (s *wikiIngestService) mapOneDocument(
func (s *wikiIngestService) extractEntitiesAndConceptsNoUpsert(
ctx context.Context,
chatModel chat.Chat,
content, docTitle, lang string,
content, lang string,
oldPageSlugs map[string]bool,
batchCtx *WikiBatchContext,
) ([]extractedItem, []extractedItem, map[string]extractedItem, error) {
@@ -71,7 +71,7 @@ type citationPipelineOutcome struct {
func (s *wikiIngestService) extractCandidateSlugs(
ctx context.Context,
chatModel chat.Chat,
content, docTitle, lang string,
content, lang string,
oldPageSlugs map[string]bool,
batchCtx *WikiBatchContext,
) ([]extractedItem, []extractedItem, map[string]extractedItem, error) {
@@ -276,7 +276,6 @@ func (s *wikiIngestService) classifyChunkCitations(
ctx context.Context,
chatModel chat.Chat,
candidatesXML string,
docTitle string,
chunks []*types.Chunk,
lang string,
) (map[string][]string, []newSlugFromCitation, int) {
@@ -122,7 +122,24 @@ func TestStripImageMarkup(t *testing.T) {
},
{"mixed text and image keeps text", "Intro paragraph.\n![fig](a.png)\nConclusion.", "Intro paragraph.\n\nConclusion."},
{"html img tag stripped", `Before <img src="x.png" alt="y"/> after`, "Before after"},
{"images xml block stripped", "Body <images><image>x</image></images> tail", "Body tail"},
{
// Regression guard: an earlier version stripped the WHOLE
// <image>...</image> block (including <image_ocr> content),
// silently destroying successful VLM OCR results. The fix must
// preserve the inner OCR / caption text.
"enriched <image> block keeps inner OCR + caption text",
`<image url="images/page_1.png">
<image_original>![p1](images/page_1.png)</image_original>
<image_caption>scanned letter on letterhead</image_caption>
<image_ocr>SEHR GEEHRTER HERR MUSTERMANN, ...</image_ocr>
</image>`,
"\n\nscanned letter on letterhead\nSEHR GEEHRTER HERR MUSTERMANN, ...\n",
},
{
"empty <image> block (OCR failed) reduces to whitespace",
`<image url="x"><image_original>![a](x)</image_original></image>`,
"",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@@ -147,20 +164,31 @@ func TestHasSufficientTextContent(t *testing.T) {
"![MX5280_page_1.png](images/MX5280_page_1.png)\n![MX5280_page_2.png](images/MX5280_page_2.png)",
false,
},
{"short text below threshold", "tiny", false},
{"too-short text below 10-rune threshold", "hi", false},
{
"short text mixed with images still below threshold",
"![a](x.png)\nshort\n![b](y.png)",
false,
},
{
"sufficient text passes",
"This is a substantial paragraph of real content describing a legal matter and the parties involved.",
"short legitimate note above threshold",
"Meeting at 3pm tomorrow.",
true,
},
{
"image-only with successful VLM OCR (the fix)",
`<image url="images/p1.png">
<image_original>![p1](images/p1.png)</image_original>
<image_caption>scanned letter</image_caption>
<image_ocr>Sehr geehrter Herr Mustermann, in der Sache 4711/2024 ...</image_ocr>
</image>`,
true,
},
{
"image-only with failed VLM OCR (still rejected)",
`<image url="images/p1.png">
<image_original>![p1](images/p1.png)</image_original>
</image>`,
false,
},
{
"sufficient text mixed with images still passes",
"![cover](cover.png)\nThis is a substantial paragraph of real content describing a legal matter and the parties involved.\n![sig](sig.png)",
"![cover](cover.png)\nDie Beklagte hat die Klage anerkannt.\n![sig](sig.png)",
true,
},
}