diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index a4deab65..d7ee6bf8 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -2308,10 +2308,11 @@ func (s *knowledgeService) getSummary(ctx context.Context, // scanner model), and feeding that to the model would invite it to // hallucinate a scanner manual instead of admitting the document had no // extractable text. - if !hasSufficientTextContent(chunkContents) { + realTextLen := realTextRuneCount(chunkContents) + if realTextLen < minTextContentRunes { logger.GetLogger(ctx).Warnf( - "getSummary: content for knowledge %s has insufficient text after stripping image markup (len=%d); skipping LLM call", - knowledge.ID, len([]rune(strings.TrimSpace(stripImageMarkup(chunkContents)))), + "getSummary: content for knowledge %s has insufficient text after stripping image markup (real_text_runes=%d, min=%d); skipping LLM call", + knowledge.ID, realTextLen, minTextContentRunes, ) return "", errInsufficientSummaryContent } @@ -2536,12 +2537,17 @@ func (s *knowledgeService) ProcessSummaryGeneration(ctx context.Context, t *asyn } } + // Embed only the LLM-generated summary in the indexed chunk. + // We deliberately omit knowledge.FileName here: filenames are an + // unreliable signal (e.g. "MX5280.pdf" for a scanned legal letter) + // and surfacing them in retrieved RAG context can re-introduce the + // hallucination vector this branch is meant to close. summaryChunk := &types.Chunk{ ID: uuid.New().String(), TenantID: knowledge.TenantID, KnowledgeID: knowledge.ID, KnowledgeBaseID: knowledge.KnowledgeBaseID, - Content: fmt.Sprintf("# Document\n%s\n\n# Summary\n%s", knowledge.FileName, summary), + Content: fmt.Sprintf("# Summary\n%s", summary), ChunkIndex: maxChunkIndex + 1, IsEnabled: true, CreatedAt: time.Now(), diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go index 0f9560d9..517df3c8 100644 --- a/internal/application/service/wiki_ingest.go +++ b/internal/application/service/wiki_ingest.go @@ -11,6 +11,7 @@ import ( "sync" "text/template" "time" + "unicode/utf8" "github.com/Tencent/WeKnora/internal/agent" "github.com/Tencent/WeKnora/internal/logger" @@ -1217,43 +1218,76 @@ func appendUnique(arr types.StringArray, s string) types.StringArray { // minTextContentRunes is the minimum number of non-whitespace, non-image-reference // runes required for content to be considered substantive enough for LLM // summarization or wiki extraction. Documents below this threshold (e.g. a -// scanned PDF where OCR yielded nothing) are routed to a deterministic empty- -// content fallback instead of being passed to the LLM, which would otherwise -// hallucinate based on metadata alone. -const minTextContentRunes = 50 +// scanned PDF where OCR yielded nothing AND no caption either) are routed to +// a deterministic empty-content fallback instead of being passed to the LLM, +// which would otherwise hallucinate based on metadata alone. +// +// The threshold is intentionally low: legitimate short documents (brief +// memos, single-line notes) must still pass. The goal is only to catch +// the empty-image-only case. +const minTextContentRunes = 10 var ( + // Markdown image references like ![alt](path) — pure visual placeholders + // with no extractable text, so the whole reference is removed. mdImageRefRE = regexp.MustCompile(`!\[[^\]]*\]\([^)]*\)`) - // Strip the outer wrapper before the inner blocks so that - // nested structures like "x" don't leave a - // stray closing tag behind. RE2 has no backreferences, so we use two - // separate patterns instead of `<(image|images)\b...`. - htmlImagesBlockRE = regexp.MustCompile(`(?is)]*>.*?`) - htmlImageBlockRE = regexp.MustCompile(`(?is)]*>.*?`) - htmlImageTagRE = regexp.MustCompile(`(?i)]*>`) + + // ... blocks wrap the verbatim Markdown + // image reference inside an enriched block (see + // searchutil.EnrichContentWithImageInfo). The content is just a redundant + // copy of an already-stripped image link, so the whole block (tags + + // content) is removed. + imageOriginalBlockRE = regexp.MustCompile(`(?is)]*>.*?`) + + // Self-closing or attribute-only HTML tags. + htmlImgTagRE = regexp.MustCompile(`(?i)]*/?>`) + + // Wrapper-style , , , tags + // (opening or closing). Matches ONLY the tag; the text content between + // open and close tags is preserved. This is critical: VLM-generated OCR + // and caption text live inside ... and + // ... blocks, and stripping the content + // would silently destroy the very text we want to keep. + imageWrapperTagRE = regexp.MustCompile(`(?i)]*/?>`) ) -// stripImageMarkup removes markdown image references, raw tags, and -// / XML blocks so that content consisting only of image -// placeholders (e.g. from PDFScannedParser pages without successful VLM OCR) -// is recognised as effectively empty. +// stripImageMarkup removes image-only placeholders (Markdown image refs, +// tags, redundancy blocks) and unwraps the +// // XML wrappers produced by the search +// enrichment layer, leaving any OCR or caption text as plain inline text. +// +// This shape matters: when VLM OCR succeeds on a scanned PDF page, the +// extracted text reaches downstream code wrapped in tags inside +// an block. A naive "strip the whole ... block" +// approach would discard the OCR text — the exact opposite of what we want. func stripImageMarkup(s string) string { - s = htmlImagesBlockRE.ReplaceAllString(s, "") - s = htmlImageBlockRE.ReplaceAllString(s, "") + s = imageOriginalBlockRE.ReplaceAllString(s, "") s = mdImageRefRE.ReplaceAllString(s, "") - s = htmlImageTagRE.ReplaceAllString(s, "") + s = htmlImgTagRE.ReplaceAllString(s, "") + s = imageWrapperTagRE.ReplaceAllString(s, "") return s } +// extractRealText returns the trimmed content with image markup stripped. +// Cached at the call site for use both in the threshold check and in any +// subsequent log message, avoiding redundant regex passes over large docs. +func extractRealText(content string) string { + return strings.TrimSpace(stripImageMarkup(content)) +} + // hasSufficientTextContent reports whether the given content carries enough -// real text (after image markup is stripped) to warrant an LLM call. It is -// the primary defence against filename-driven hallucinations on scanned PDFs. +// real text (after image markup is stripped, with OCR/caption text retained) +// to warrant an LLM call. It is the primary defence against filename-driven +// hallucinations on scanned PDFs that have NO usable text at all. func hasSufficientTextContent(content string) bool { - stripped := strings.TrimSpace(stripImageMarkup(content)) - if stripped == "" { - return false - } - return len([]rune(stripped)) >= minTextContentRunes + return realTextRuneCount(content) >= minTextContentRunes +} + +// realTextRuneCount returns the rune length of the content after image +// markup is stripped. Uses utf8.RuneCountInString to avoid allocating a +// rune slice for the count. +func realTextRuneCount(content string) int { + return utf8.RuneCountInString(extractRealText(content)) } // cleanLLMJSON strips markdown code-fence wrappers and sanitizes control characters diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go index c06a7e28..87e37b4e 100644 --- a/internal/application/service/wiki_ingest_batch.go +++ b/internal/application/service/wiki_ingest_batch.go @@ -500,7 +500,11 @@ func (s *wikiIngestService) mapOneDocument( } } - sourceRef := fmt.Sprintf("%s|%s", knowledgeID, docTitle) + // Citation source reference. We deliberately use only the knowledge ID + // (not docTitle, which is typically the upload filename) so the filename + // does not leak into citation strings that downstream LLM prompts may + // surface during wiki page editing. + sourceRef := knowledgeID oldPageSlugs := s.getExistingPageSlugsForKnowledge(ctx, payload.KnowledgeBaseID, knowledgeID) // Pass 0: lightweight candidate slug extraction (skeleton only). @@ -513,11 +517,11 @@ func (s *wikiIngestService) mapOneDocument( pass0Failed bool ) logger.Infof(ctx, "wiki ingest: pass 0 — extracting candidate slugs for %s", knowledgeID) - extractedEntities, extractedConcepts, slugItems, err = s.extractCandidateSlugs(ctx, chatModel, content, docTitle, lang, oldPageSlugs, batchCtx) + extractedEntities, extractedConcepts, slugItems, err = s.extractCandidateSlugs(ctx, chatModel, content, lang, oldPageSlugs, batchCtx) if err != nil { logger.Warnf(ctx, "wiki ingest: pass 0 failed for %s (%v) — falling back to legacy extractor", knowledgeID, err) pass0Failed = true - extractedEntities, extractedConcepts, slugItems, err = s.extractEntitiesAndConceptsNoUpsert(ctx, chatModel, content, docTitle, lang, oldPageSlugs, batchCtx) + extractedEntities, extractedConcepts, slugItems, err = s.extractEntitiesAndConceptsNoUpsert(ctx, chatModel, content, lang, oldPageSlugs, batchCtx) if err != nil { logger.Warnf(ctx, "wiki ingest: legacy fallback also failed for %s: %v", knowledgeID, err) return nil, nil, err @@ -529,7 +533,12 @@ func (s *wikiIngestService) mapOneDocument( for slug := range slugItems { summaryExtractedPages = append(summaryExtractedPages, slug) } - summarySlug := fmt.Sprintf("summary/%s", slugify(docTitle)) + // Wiki summary slug is derived from the knowledge ID rather than the + // docTitle (which is typically the upload filename). Filename-based slugs + // like "summary/mx5280-pdf" expose the filename in cross-link contexts + // that downstream LLM prompts read; a UUID-based slug is uglier but + // hallucination-safe. + summarySlug := fmt.Sprintf("summary/%s", slugify(knowledgeID)) var slugListing string for _, slug := range summaryExtractedPages { if item, ok := slugItems[slug]; ok { @@ -574,7 +583,7 @@ func (s *wikiIngestService) mapOneDocument( return } candidatesXML := renderCandidateSlugsXML(extractedEntities, extractedConcepts) - citations, newSlugs, batchCount = s.classifyChunkCitations(ctx, chatModel, candidatesXML, docTitle, chunks, lang) + citations, newSlugs, batchCount = s.classifyChunkCitations(ctx, chatModel, candidatesXML, chunks, lang) }() wg.Wait() @@ -765,7 +774,7 @@ func (s *wikiIngestService) mapOneDocument( func (s *wikiIngestService) extractEntitiesAndConceptsNoUpsert( ctx context.Context, chatModel chat.Chat, - content, docTitle, lang string, + content, lang string, oldPageSlugs map[string]bool, batchCtx *WikiBatchContext, ) ([]extractedItem, []extractedItem, map[string]extractedItem, error) { diff --git a/internal/application/service/wiki_ingest_cite.go b/internal/application/service/wiki_ingest_cite.go index e399f515..c97fa1b7 100644 --- a/internal/application/service/wiki_ingest_cite.go +++ b/internal/application/service/wiki_ingest_cite.go @@ -71,7 +71,7 @@ type citationPipelineOutcome struct { func (s *wikiIngestService) extractCandidateSlugs( ctx context.Context, chatModel chat.Chat, - content, docTitle, lang string, + content, lang string, oldPageSlugs map[string]bool, batchCtx *WikiBatchContext, ) ([]extractedItem, []extractedItem, map[string]extractedItem, error) { @@ -276,7 +276,6 @@ func (s *wikiIngestService) classifyChunkCitations( ctx context.Context, chatModel chat.Chat, candidatesXML string, - docTitle string, chunks []*types.Chunk, lang string, ) (map[string][]string, []newSlugFromCitation, int) { diff --git a/internal/application/service/wiki_ingest_test.go b/internal/application/service/wiki_ingest_test.go index 0f44d1bf..86ddb732 100644 --- a/internal/application/service/wiki_ingest_test.go +++ b/internal/application/service/wiki_ingest_test.go @@ -122,7 +122,24 @@ func TestStripImageMarkup(t *testing.T) { }, {"mixed text and image keeps text", "Intro paragraph.\n![fig](a.png)\nConclusion.", "Intro paragraph.\n\nConclusion."}, {"html img tag stripped", `Before y after`, "Before after"}, - {"images xml block stripped", "Body x tail", "Body tail"}, + { + // Regression guard: an earlier version stripped the WHOLE + // ... block (including content), + // silently destroying successful VLM OCR results. The fix must + // preserve the inner OCR / caption text. + "enriched block keeps inner OCR + caption text", + ` +![p1](images/page_1.png) +scanned letter on letterhead +SEHR GEEHRTER HERR MUSTERMANN, ... +`, + "\n\nscanned letter on letterhead\nSEHR GEEHRTER HERR MUSTERMANN, ...\n", + }, + { + "empty block (OCR failed) reduces to whitespace", + `![a](x)`, + "", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -147,20 +164,31 @@ func TestHasSufficientTextContent(t *testing.T) { "![MX5280_page_1.png](images/MX5280_page_1.png)\n![MX5280_page_2.png](images/MX5280_page_2.png)", false, }, - {"short text below threshold", "tiny", false}, + {"too-short text below 10-rune threshold", "hi", false}, { - "short text mixed with images still below threshold", - "![a](x.png)\nshort\n![b](y.png)", - false, - }, - { - "sufficient text passes", - "This is a substantial paragraph of real content describing a legal matter and the parties involved.", + "short legitimate note above threshold", + "Meeting at 3pm tomorrow.", true, }, + { + "image-only with successful VLM OCR (the fix)", + ` +![p1](images/p1.png) +scanned letter +Sehr geehrter Herr Mustermann, in der Sache 4711/2024 ... +`, + true, + }, + { + "image-only with failed VLM OCR (still rejected)", + ` +![p1](images/p1.png) +`, + false, + }, { "sufficient text mixed with images still passes", - "![cover](cover.png)\nThis is a substantial paragraph of real content describing a legal matter and the parties involved.\n![sig](sig.png)", + "![cover](cover.png)\nDie Beklagte hat die Klage anerkannt.\n![sig](sig.png)", true, }, }