fix(ingest): QA follow-up — preserve OCR text and close remaining filename leaks
Self-review of the prior commit surfaced one critical regression and
several smaller issues. This change addresses them.
Critical: the old stripImageMarkup stripped the WHOLE <image>...</image>
block, but searchutil.EnrichContentWithImageInfo wraps successful VLM OCR
output in exactly that shape — so a scanned PDF where OCR worked was
being rejected by hasSufficientTextContent and routed to the same
"failed" path as the OCR-empty case. The fix splits stripping into:
* <image_original>...</image_original> blocks removed entirely
(they only carry the redundant Markdown ref already stripped above)
* <image>, <images>, <image_caption>, <image_ocr> tags removed at
the tag level, INNER TEXT PRESERVED, so OCR / caption text reaches
the threshold check and the LLM call.
A regression test pins this down with realistic enriched-image XML that
the previous code failed silently on.
Other fixes:
* Threshold lowered from 50 to 10 runes — short legitimate documents
(brief memos, single-line notes) must still pass.
* Summary chunk content no longer prefixes "# Document\n<filename>";
only the LLM-generated summary is indexed, so filenames cannot
leak back into RAG retrieval context.
* Wiki summary slug is now derived from the knowledge ID rather than
the upload filename, eliminating filename-based slugs like
"summary/mx5280-pdf" that downstream wiki-edit prompts can read.
* Citation sourceRef likewise drops the "|<docTitle>" suffix.
* Dead docTitle parameter removed from extractCandidateSlugs,
extractEntitiesAndConceptsNoUpsert, classifyChunkCitations.
* The redundant stripImageMarkup pass in the warning log is gone;
realTextRuneCount caches the count once and uses
utf8.RuneCountInString to avoid the rune-slice allocation.
Wiki-page display titles still flow through DocTitle (filename) into
later wiki-edit / index-rebuild prompts. That is a larger refactor of
the wiki pipeline and is intentionally left for a separate PR.
This commit is contained in:
@@ -2308,10 +2308,11 @@ func (s *knowledgeService) getSummary(ctx context.Context,
|
||||
// scanner model), and feeding that to the model would invite it to
|
||||
// hallucinate a scanner manual instead of admitting the document had no
|
||||
// extractable text.
|
||||
if !hasSufficientTextContent(chunkContents) {
|
||||
realTextLen := realTextRuneCount(chunkContents)
|
||||
if realTextLen < minTextContentRunes {
|
||||
logger.GetLogger(ctx).Warnf(
|
||||
"getSummary: content for knowledge %s has insufficient text after stripping image markup (len=%d); skipping LLM call",
|
||||
knowledge.ID, len([]rune(strings.TrimSpace(stripImageMarkup(chunkContents)))),
|
||||
"getSummary: content for knowledge %s has insufficient text after stripping image markup (real_text_runes=%d, min=%d); skipping LLM call",
|
||||
knowledge.ID, realTextLen, minTextContentRunes,
|
||||
)
|
||||
return "", errInsufficientSummaryContent
|
||||
}
|
||||
@@ -2536,12 +2537,17 @@ func (s *knowledgeService) ProcessSummaryGeneration(ctx context.Context, t *asyn
|
||||
}
|
||||
}
|
||||
|
||||
// Embed only the LLM-generated summary in the indexed chunk.
|
||||
// We deliberately omit knowledge.FileName here: filenames are an
|
||||
// unreliable signal (e.g. "MX5280.pdf" for a scanned legal letter)
|
||||
// and surfacing them in retrieved RAG context can re-introduce the
|
||||
// hallucination vector this branch is meant to close.
|
||||
summaryChunk := &types.Chunk{
|
||||
ID: uuid.New().String(),
|
||||
TenantID: knowledge.TenantID,
|
||||
KnowledgeID: knowledge.ID,
|
||||
KnowledgeBaseID: knowledge.KnowledgeBaseID,
|
||||
Content: fmt.Sprintf("# Document\n%s\n\n# Summary\n%s", knowledge.FileName, summary),
|
||||
Content: fmt.Sprintf("# Summary\n%s", summary),
|
||||
ChunkIndex: maxChunkIndex + 1,
|
||||
IsEnabled: true,
|
||||
CreatedAt: time.Now(),
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"sync"
|
||||
"text/template"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/agent"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
@@ -1217,43 +1218,76 @@ func appendUnique(arr types.StringArray, s string) types.StringArray {
|
||||
// minTextContentRunes is the minimum number of non-whitespace, non-image-reference
|
||||
// runes required for content to be considered substantive enough for LLM
|
||||
// summarization or wiki extraction. Documents below this threshold (e.g. a
|
||||
// scanned PDF where OCR yielded nothing) are routed to a deterministic empty-
|
||||
// content fallback instead of being passed to the LLM, which would otherwise
|
||||
// hallucinate based on metadata alone.
|
||||
const minTextContentRunes = 50
|
||||
// scanned PDF where OCR yielded nothing AND no caption either) are routed to
|
||||
// a deterministic empty-content fallback instead of being passed to the LLM,
|
||||
// which would otherwise hallucinate based on metadata alone.
|
||||
//
|
||||
// The threshold is intentionally low: legitimate short documents (brief
|
||||
// memos, single-line notes) must still pass. The goal is only to catch
|
||||
// the empty-image-only case.
|
||||
const minTextContentRunes = 10
|
||||
|
||||
var (
|
||||
// Markdown image references like  — pure visual placeholders
|
||||
// with no extractable text, so the whole reference is removed.
|
||||
mdImageRefRE = regexp.MustCompile(`!\[[^\]]*\]\([^)]*\)`)
|
||||
// Strip the outer <images> wrapper before the inner <image> blocks so that
|
||||
// nested structures like "<images><image>x</image></images>" don't leave a
|
||||
// stray closing tag behind. RE2 has no backreferences, so we use two
|
||||
// separate patterns instead of `<(image|images)\b...</\1>`.
|
||||
htmlImagesBlockRE = regexp.MustCompile(`(?is)<images\b[^>]*>.*?</images>`)
|
||||
htmlImageBlockRE = regexp.MustCompile(`(?is)<image\b[^>]*>.*?</image>`)
|
||||
htmlImageTagRE = regexp.MustCompile(`(?i)<img\b[^>]*>`)
|
||||
|
||||
// <image_original>...</image_original> blocks wrap the verbatim Markdown
|
||||
// image reference inside an enriched <image> block (see
|
||||
// searchutil.EnrichContentWithImageInfo). The content is just a redundant
|
||||
// copy of an already-stripped image link, so the whole block (tags +
|
||||
// content) is removed.
|
||||
imageOriginalBlockRE = regexp.MustCompile(`(?is)<image_original\b[^>]*>.*?</image_original>`)
|
||||
|
||||
// Self-closing or attribute-only HTML <img> tags.
|
||||
htmlImgTagRE = regexp.MustCompile(`(?i)<img\b[^>]*/?>`)
|
||||
|
||||
// Wrapper-style <image>, <images>, <image_caption>, <image_ocr> tags
|
||||
// (opening or closing). Matches ONLY the tag; the text content between
|
||||
// open and close tags is preserved. This is critical: VLM-generated OCR
|
||||
// and caption text live inside <image_ocr>...</image_ocr> and
|
||||
// <image_caption>...</image_caption> blocks, and stripping the content
|
||||
// would silently destroy the very text we want to keep.
|
||||
imageWrapperTagRE = regexp.MustCompile(`(?i)</?image[a-z_]*\b[^>]*/?>`)
|
||||
)
|
||||
|
||||
// stripImageMarkup removes markdown image references, raw <img> tags, and
|
||||
// <image>/<images> XML blocks so that content consisting only of image
|
||||
// placeholders (e.g. from PDFScannedParser pages without successful VLM OCR)
|
||||
// is recognised as effectively empty.
|
||||
// stripImageMarkup removes image-only placeholders (Markdown image refs,
|
||||
// <img> tags, <image_original> redundancy blocks) and unwraps the
|
||||
// <image>/<image_caption>/<image_ocr> XML wrappers produced by the search
|
||||
// enrichment layer, leaving any OCR or caption text as plain inline text.
|
||||
//
|
||||
// This shape matters: when VLM OCR succeeds on a scanned PDF page, the
|
||||
// extracted text reaches downstream code wrapped in <image_ocr> tags inside
|
||||
// an <image> block. A naive "strip the whole <image>...</image> block"
|
||||
// approach would discard the OCR text — the exact opposite of what we want.
|
||||
func stripImageMarkup(s string) string {
|
||||
s = htmlImagesBlockRE.ReplaceAllString(s, "")
|
||||
s = htmlImageBlockRE.ReplaceAllString(s, "")
|
||||
s = imageOriginalBlockRE.ReplaceAllString(s, "")
|
||||
s = mdImageRefRE.ReplaceAllString(s, "")
|
||||
s = htmlImageTagRE.ReplaceAllString(s, "")
|
||||
s = htmlImgTagRE.ReplaceAllString(s, "")
|
||||
s = imageWrapperTagRE.ReplaceAllString(s, "")
|
||||
return s
|
||||
}
|
||||
|
||||
// extractRealText returns the trimmed content with image markup stripped.
|
||||
// Cached at the call site for use both in the threshold check and in any
|
||||
// subsequent log message, avoiding redundant regex passes over large docs.
|
||||
func extractRealText(content string) string {
|
||||
return strings.TrimSpace(stripImageMarkup(content))
|
||||
}
|
||||
|
||||
// hasSufficientTextContent reports whether the given content carries enough
|
||||
// real text (after image markup is stripped) to warrant an LLM call. It is
|
||||
// the primary defence against filename-driven hallucinations on scanned PDFs.
|
||||
// real text (after image markup is stripped, with OCR/caption text retained)
|
||||
// to warrant an LLM call. It is the primary defence against filename-driven
|
||||
// hallucinations on scanned PDFs that have NO usable text at all.
|
||||
func hasSufficientTextContent(content string) bool {
|
||||
stripped := strings.TrimSpace(stripImageMarkup(content))
|
||||
if stripped == "" {
|
||||
return false
|
||||
}
|
||||
return len([]rune(stripped)) >= minTextContentRunes
|
||||
return realTextRuneCount(content) >= minTextContentRunes
|
||||
}
|
||||
|
||||
// realTextRuneCount returns the rune length of the content after image
|
||||
// markup is stripped. Uses utf8.RuneCountInString to avoid allocating a
|
||||
// rune slice for the count.
|
||||
func realTextRuneCount(content string) int {
|
||||
return utf8.RuneCountInString(extractRealText(content))
|
||||
}
|
||||
|
||||
// cleanLLMJSON strips markdown code-fence wrappers and sanitizes control characters
|
||||
|
||||
@@ -500,7 +500,11 @@ func (s *wikiIngestService) mapOneDocument(
|
||||
}
|
||||
}
|
||||
|
||||
sourceRef := fmt.Sprintf("%s|%s", knowledgeID, docTitle)
|
||||
// Citation source reference. We deliberately use only the knowledge ID
|
||||
// (not docTitle, which is typically the upload filename) so the filename
|
||||
// does not leak into citation strings that downstream LLM prompts may
|
||||
// surface during wiki page editing.
|
||||
sourceRef := knowledgeID
|
||||
oldPageSlugs := s.getExistingPageSlugsForKnowledge(ctx, payload.KnowledgeBaseID, knowledgeID)
|
||||
|
||||
// Pass 0: lightweight candidate slug extraction (skeleton only).
|
||||
@@ -513,11 +517,11 @@ func (s *wikiIngestService) mapOneDocument(
|
||||
pass0Failed bool
|
||||
)
|
||||
logger.Infof(ctx, "wiki ingest: pass 0 — extracting candidate slugs for %s", knowledgeID)
|
||||
extractedEntities, extractedConcepts, slugItems, err = s.extractCandidateSlugs(ctx, chatModel, content, docTitle, lang, oldPageSlugs, batchCtx)
|
||||
extractedEntities, extractedConcepts, slugItems, err = s.extractCandidateSlugs(ctx, chatModel, content, lang, oldPageSlugs, batchCtx)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: pass 0 failed for %s (%v) — falling back to legacy extractor", knowledgeID, err)
|
||||
pass0Failed = true
|
||||
extractedEntities, extractedConcepts, slugItems, err = s.extractEntitiesAndConceptsNoUpsert(ctx, chatModel, content, docTitle, lang, oldPageSlugs, batchCtx)
|
||||
extractedEntities, extractedConcepts, slugItems, err = s.extractEntitiesAndConceptsNoUpsert(ctx, chatModel, content, lang, oldPageSlugs, batchCtx)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "wiki ingest: legacy fallback also failed for %s: %v", knowledgeID, err)
|
||||
return nil, nil, err
|
||||
@@ -529,7 +533,12 @@ func (s *wikiIngestService) mapOneDocument(
|
||||
for slug := range slugItems {
|
||||
summaryExtractedPages = append(summaryExtractedPages, slug)
|
||||
}
|
||||
summarySlug := fmt.Sprintf("summary/%s", slugify(docTitle))
|
||||
// Wiki summary slug is derived from the knowledge ID rather than the
|
||||
// docTitle (which is typically the upload filename). Filename-based slugs
|
||||
// like "summary/mx5280-pdf" expose the filename in cross-link contexts
|
||||
// that downstream LLM prompts read; a UUID-based slug is uglier but
|
||||
// hallucination-safe.
|
||||
summarySlug := fmt.Sprintf("summary/%s", slugify(knowledgeID))
|
||||
var slugListing string
|
||||
for _, slug := range summaryExtractedPages {
|
||||
if item, ok := slugItems[slug]; ok {
|
||||
@@ -574,7 +583,7 @@ func (s *wikiIngestService) mapOneDocument(
|
||||
return
|
||||
}
|
||||
candidatesXML := renderCandidateSlugsXML(extractedEntities, extractedConcepts)
|
||||
citations, newSlugs, batchCount = s.classifyChunkCitations(ctx, chatModel, candidatesXML, docTitle, chunks, lang)
|
||||
citations, newSlugs, batchCount = s.classifyChunkCitations(ctx, chatModel, candidatesXML, chunks, lang)
|
||||
}()
|
||||
wg.Wait()
|
||||
|
||||
@@ -765,7 +774,7 @@ func (s *wikiIngestService) mapOneDocument(
|
||||
func (s *wikiIngestService) extractEntitiesAndConceptsNoUpsert(
|
||||
ctx context.Context,
|
||||
chatModel chat.Chat,
|
||||
content, docTitle, lang string,
|
||||
content, lang string,
|
||||
oldPageSlugs map[string]bool,
|
||||
batchCtx *WikiBatchContext,
|
||||
) ([]extractedItem, []extractedItem, map[string]extractedItem, error) {
|
||||
|
||||
@@ -71,7 +71,7 @@ type citationPipelineOutcome struct {
|
||||
func (s *wikiIngestService) extractCandidateSlugs(
|
||||
ctx context.Context,
|
||||
chatModel chat.Chat,
|
||||
content, docTitle, lang string,
|
||||
content, lang string,
|
||||
oldPageSlugs map[string]bool,
|
||||
batchCtx *WikiBatchContext,
|
||||
) ([]extractedItem, []extractedItem, map[string]extractedItem, error) {
|
||||
@@ -276,7 +276,6 @@ func (s *wikiIngestService) classifyChunkCitations(
|
||||
ctx context.Context,
|
||||
chatModel chat.Chat,
|
||||
candidatesXML string,
|
||||
docTitle string,
|
||||
chunks []*types.Chunk,
|
||||
lang string,
|
||||
) (map[string][]string, []newSlugFromCitation, int) {
|
||||
|
||||
@@ -122,7 +122,24 @@ func TestStripImageMarkup(t *testing.T) {
|
||||
},
|
||||
{"mixed text and image keeps text", "Intro paragraph.\n\nConclusion.", "Intro paragraph.\n\nConclusion."},
|
||||
{"html img tag stripped", `Before <img src="x.png" alt="y"/> after`, "Before after"},
|
||||
{"images xml block stripped", "Body <images><image>x</image></images> tail", "Body tail"},
|
||||
{
|
||||
// Regression guard: an earlier version stripped the WHOLE
|
||||
// <image>...</image> block (including <image_ocr> content),
|
||||
// silently destroying successful VLM OCR results. The fix must
|
||||
// preserve the inner OCR / caption text.
|
||||
"enriched <image> block keeps inner OCR + caption text",
|
||||
`<image url="images/page_1.png">
|
||||
<image_original></image_original>
|
||||
<image_caption>scanned letter on letterhead</image_caption>
|
||||
<image_ocr>SEHR GEEHRTER HERR MUSTERMANN, ...</image_ocr>
|
||||
</image>`,
|
||||
"\n\nscanned letter on letterhead\nSEHR GEEHRTER HERR MUSTERMANN, ...\n",
|
||||
},
|
||||
{
|
||||
"empty <image> block (OCR failed) reduces to whitespace",
|
||||
`<image url="x"><image_original></image_original></image>`,
|
||||
"",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -147,20 +164,31 @@ func TestHasSufficientTextContent(t *testing.T) {
|
||||
"\n",
|
||||
false,
|
||||
},
|
||||
{"short text below threshold", "tiny", false},
|
||||
{"too-short text below 10-rune threshold", "hi", false},
|
||||
{
|
||||
"short text mixed with images still below threshold",
|
||||
"\nshort\n",
|
||||
false,
|
||||
},
|
||||
{
|
||||
"sufficient text passes",
|
||||
"This is a substantial paragraph of real content describing a legal matter and the parties involved.",
|
||||
"short legitimate note above threshold",
|
||||
"Meeting at 3pm tomorrow.",
|
||||
true,
|
||||
},
|
||||
{
|
||||
"image-only with successful VLM OCR (the fix)",
|
||||
`<image url="images/p1.png">
|
||||
<image_original></image_original>
|
||||
<image_caption>scanned letter</image_caption>
|
||||
<image_ocr>Sehr geehrter Herr Mustermann, in der Sache 4711/2024 ...</image_ocr>
|
||||
</image>`,
|
||||
true,
|
||||
},
|
||||
{
|
||||
"image-only with failed VLM OCR (still rejected)",
|
||||
`<image url="images/p1.png">
|
||||
<image_original></image_original>
|
||||
</image>`,
|
||||
false,
|
||||
},
|
||||
{
|
||||
"sufficient text mixed with images still passes",
|
||||
"\nThis is a substantial paragraph of real content describing a legal matter and the parties involved.\n",
|
||||
"\nDie Beklagte hat die Klage anerkannt.\n",
|
||||
true,
|
||||
},
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user