From 15550d14d484e9a466be750a4c95a18d4ab7aa41 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 2 May 2026 09:06:32 +0000 Subject: [PATCH] fix(ingest): stop hallucinating summaries from filenames on empty PDFs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scanned PDFs without a successful OCR pass (e.g. when no VLM is configured on the KB, or when the VLM returned a "no text content" reply that the sanitizer discarded) reach the summary and wiki-ingest pipelines with nothing but markdown image references for content. The downstream LLM prompts used to receive the filename, file type, and title alongside that empty content, and the generate_summary template explicitly forbade the model from refusing — so the model would invent a topic from the filename alone, producing e.g. a "Canon scanner manual" summary for an MX5280.pdf that is actually a scanned legal letter. Two changes shut this down: 1. Drop filename / title / file-type from every LLM prompt input. The wiki summary, knowledge extract, candidate slug, and chunk citation prompts now see only the document content. The knowledge summary path no longer prepends the "Document Type: X / File Name: Y" intro. 2. Add a content-sufficiency guard in front of the LLM call. A new helper strips markdown image references and HTML image tags and refuses to invoke the model when fewer than 50 real-text runes remain. getSummary returns a sentinel error in that case and the caller marks the knowledge as SummaryStatusFailed instead of surfacing the first chunk's bare image reference. Wiki ingest skips LLM extraction with a clear log message. The generate_summary.yaml prompt is also updated: the "NEVER output refusal phrases" clause is removed and replaced with an explicit empty-content rule that returns a deterministic placeholder line. Real OCR (Tesseract / PaddleOCR) and filename-pattern sanitisation are intentionally left for separate PRs. --- config/prompt_templates/generate_summary.yaml | 25 ++++--- internal/agent/prompts_wiki.go | 19 ++++-- internal/application/service/knowledge.go | 53 ++++++++++----- internal/application/service/wiki_ingest.go | 43 ++++++++++++ .../application/service/wiki_ingest_batch.go | 16 +++-- .../application/service/wiki_ingest_cite.go | 2 - .../application/service/wiki_ingest_test.go | 67 +++++++++++++++++++ 7 files changed, 183 insertions(+), 42 deletions(-) diff --git a/config/prompt_templates/generate_summary.yaml b/config/prompt_templates/generate_summary.yaml index c3fad973..d6267433 100644 --- a/config/prompt_templates/generate_summary.yaml +++ b/config/prompt_templates/generate_summary.yaml @@ -5,22 +5,21 @@ templates: description: "Generate a concise document summary" default: true content: | - You are a precise document summarization expert. Your task is to extract and summarize the core content of the document provided by the user. + You are a precise document summarization expert. Your task is to summarize the core content of the document provided by the user, basing the summary STRICTLY on the text the user supplies — never on a filename, title, file extension, or any other external clue. ## Steps - 1. Identify the document type from the metadata (e.g., technical doc, meeting notes, research paper, code, etc.) - 2. Extract 3-5 key points or main topics covered in the document - 3. Write a coherent summary incorporating these key points + 1. Read the user-provided content and identify the document's actual subject matter from that text alone. + 2. Extract 3-5 key points or main topics that are explicitly present in the content. + 3. Write a coherent summary incorporating these key points. ## Core Requirements - Summary length: 100-500 words, adjusted based on content complexity - Short/simple documents: 100-200 words - Long/complex documents: 300-500 words - - Generate the summary entirely based on the provided content, without adding any information not present in the document - - Ensure the summary captures key information points and main conclusions - - If the content contains "[...content omitted...]" markers, it is a sampled excerpt from a longer document — cover ALL topics that appear across the provided sections, not just the beginning - - Even for complex or specialized content, you must attempt to extract core points for summarization - - Output the summary directly, without any preamble, prefix, or explanation + - Generate the summary entirely based on the provided content, without adding any information not present in the document. + - Ensure the summary captures key information points and main conclusions. + - If the content contains "[...content omitted...]" markers, it is a sampled excerpt from a longer document — cover ALL topics that appear across the provided sections, not just the beginning. + - Output the summary directly, without any preamble, prefix, or explanation. ## Format and Style - Use an objective, neutral third-person narrative tone @@ -29,10 +28,10 @@ templates: - For technical documents: preserve key terms, metrics, and specific details - For meeting notes/reports: highlight decisions, action items, and conclusions - ## Important Notes - - NEVER output refusal phrases such as "unable to generate", "unable to summarize", or "insufficient content" - - Do not copy or reference any content from examples; ensure the summary is entirely based on the user's document - - Make every effort to extract key points and summarize for any text, regardless of length or complexity + ## Empty or Insufficient Content + - If the user-provided content is empty, contains only image/figure references with no extracted text, or otherwise carries no substantive textual information, you MUST output exactly the single line: "No textual content was extractable from this document." and nothing else. + - Do NOT fabricate a topic, do NOT guess from any other clue, and do NOT copy content from examples or unrelated sources. + - It is correct and expected to refuse to summarise when the content is absent. This is preferred over inventing a plausible-sounding but unsupported summary. ## Language - Use {{language}} for all outputs diff --git a/internal/agent/prompts_wiki.go b/internal/agent/prompts_wiki.go index 2a6ba999..cb28e400 100644 --- a/internal/agent/prompts_wiki.go +++ b/internal/agent/prompts_wiki.go @@ -5,12 +5,16 @@ package agent // knowledge from raw documents and build/update wiki pages. // WikiSummaryPrompt generates a summary page for a newly ingested document. +// +// Filename and title are intentionally NOT passed to the LLM: documents +// uploaded to WeKnora often carry filenames that say nothing about the +// content (e.g. scanned PDFs named after the scanner model "MX5280.pdf"), +// and feeding such filenames to the model invites hallucinated summaries +// when the actual extracted content is thin. The model must rely solely on +// the document content provided below. const WikiSummaryPrompt = `You are a wiki editor. Given the following document content, create a structured wiki summary page in Markdown format. -{{.Title}} -{{.FileName}} -{{.FileType}} {{.Content}} @@ -30,6 +34,7 @@ const WikiSummaryPrompt = `You are a wiki editor. Given the following document c 7. At the end, include a "## Key Takeaways" section with bullet points. 8. Write in {{.Language}}. 9. Keep the summary concise but thorough (500-1500 words depending on document length). +10. **Empty content rule**: If the block above is empty, contains only image references with no extracted text, or otherwise carries no substantive information, output exactly: "SUMMARY: No textual content was extractable from this document." followed by a brief note explaining that the document could not be summarised. Do NOT invent a topic, do NOT guess from any other clue. Output the SUMMARY line first, then the Markdown content. Do not include any other preamble.` @@ -40,7 +45,6 @@ Output the SUMMARY line first, then the Markdown content. Do not include any oth const WikiKnowledgeExtractPrompt = `You are a knowledge extraction system. Analyze the following document and extract all significant entities AND key concepts. -{{.Title}} {{.Content}} @@ -54,6 +58,8 @@ const WikiKnowledgeExtractPrompt = `You are a knowledge extraction system. Analy Return a JSON object with two arrays: "entities" and "concepts". **IMPORTANT: Write ALL names, descriptions, and details in {{.Language}}**. +If the block above is empty, contains only image references with no extracted text, or otherwise carries no substantive information, return {"entities": [], "concepts": []}. Do NOT invent entities or concepts from any other source. + ### Slug Continuity Rules If previous slugs are provided above, you MUST follow these rules: - If an entity or concept from the previous extraction still exists in the current document, **reuse its exact slug** from the previous list. Do NOT generate a new slug for the same thing. @@ -121,7 +127,6 @@ Output ONLY valid JSON. Example: const WikiCandidateSlugPrompt = `You are a knowledge extraction system. Analyze the following document and list all significant entities AND key concepts as a lightweight candidate set. Another pass will later attach concrete supporting chunks to each item, so you do NOT need to write exhaustive per-item facts here. -{{.Title}} {{.Content}} @@ -135,6 +140,8 @@ const WikiCandidateSlugPrompt = `You are a knowledge extraction system. Analyze Return a JSON object with two arrays: "entities" and "concepts". **IMPORTANT: Write ALL names, descriptions, and details in {{.Language}}**. +If the block above is empty, contains only image references with no extracted text, or otherwise carries no substantive information, return {"entities": [], "concepts": []}. Do NOT invent entities or concepts from any other source. + ### Extraction Scope (Granularity: {{.Granularity}}) {{.GranularityGuidance}} @@ -202,8 +209,6 @@ Output ONLY valid JSON. Example: // their verbatim form (the chunk text) instead of asking the LLM to paraphrase. const WikiChunkCitationPrompt = `You are a precise citation system. Your job is to scan a batch of document chunks and decide, for each candidate entity/concept below, which chunks substantively discuss it. -{{.DocTitle}} - {{.CandidateSlugs}} diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index ea5250ef..a4deab65 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -2242,6 +2242,13 @@ func (s *knowledgeService) processChunks(ctx context.Context, // defaultMaxInputChars is the default maximum characters used as input for summary generation. const defaultMaxInputChars = 1024 * 24 +// errInsufficientSummaryContent signals that getSummary refused to call the +// LLM because the document had no usable text after image markup was stripped +// (typical for scanned PDFs where VLM OCR yielded nothing). Callers should +// mark the knowledge's summary as failed instead of falling back to the first +// chunk's raw content (which would just be a bare image reference). +var errInsufficientSummaryContent = errors.New("insufficient text content for summary generation") + // getSummary generates a summary for knowledge content using an AI model func (s *knowledgeService) getSummary(ctx context.Context, summaryModel chat.Chat, knowledge *types.Knowledge, chunks []*types.Chunk, @@ -2295,22 +2302,23 @@ func (s *knowledgeService) getSummary(ctx context.Context, logger.GetLogger(ctx).Infof("getSummary: content length=%d chars (max=%d) for knowledge %s", len([]rune(chunkContents)), maxInputChars, knowledge.ID) - // Prepare content with metadata for summary generation - contentWithMetadata := chunkContents - - // Add knowledge metadata if available - if knowledge != nil { - metadataIntro := fmt.Sprintf("Document Type: %s\nFile Name: %s\n", knowledge.FileType, knowledge.FileName) - - // Add additional metadata if available - if knowledge.Type != "" { - metadataIntro += fmt.Sprintf("Knowledge Type: %s\n", knowledge.Type) - } - - // Prepend metadata to content - contentWithMetadata = metadataIntro + "\nContent:\n" + contentWithMetadata + // Bail out before the LLM call when there is not enough actual text to + // summarise. We deliberately do not pass filename/file-type metadata to the + // LLM: scanned PDFs frequently carry filenames like "MX5280.pdf" (the + // scanner model), and feeding that to the model would invite it to + // hallucinate a scanner manual instead of admitting the document had no + // extractable text. + if !hasSufficientTextContent(chunkContents) { + logger.GetLogger(ctx).Warnf( + "getSummary: content for knowledge %s has insufficient text after stripping image markup (len=%d); skipping LLM call", + knowledge.ID, len([]rune(strings.TrimSpace(stripImageMarkup(chunkContents)))), + ) + return "", errInsufficientSummaryContent } + // Pass the raw chunk text to the LLM with no filename / file-type framing. + contentWithMetadata := chunkContents + // Determine max output tokens from config maxTokens := 2048 if s.config.Conversation.Summary != nil && s.config.Conversation.Summary.MaxCompletionTokens > 0 { @@ -2482,11 +2490,24 @@ func (s *knowledgeService) ProcessSummaryGeneration(ctx context.Context, t *asyn summary, err := s.getSummary(ctx, chatModel, knowledge, textChunks) if err != nil { logger.Errorf(ctx, "Failed to generate summary for knowledge %s: %v", payload.KnowledgeID, err) - // Use first chunk content as fallback + // For the insufficient-content case (scanned PDF without OCR, etc.) + // we deliberately do NOT fall back to the first chunk's raw content, + // since that chunk is typically just a bare markdown image reference + // and surfacing it in the description is misleading. + if errors.Is(err, errInsufficientSummaryContent) { + knowledge.Description = "" + knowledge.SummaryStatus = types.SummaryStatusFailed + knowledge.UpdatedAt = time.Now() + if updateErr := s.repo.UpdateKnowledge(ctx, knowledge); updateErr != nil { + logger.Errorf(ctx, "Failed to mark summary as failed: %v", updateErr) + return fmt.Errorf("failed to update knowledge: %w", updateErr) + } + return nil + } + // For other errors (LLM API issues etc.), fall back to the first chunk. if len(textChunks) > 0 { summary = textChunks[0].Content if len(summary) > 500 { - // Use rune-based truncation to avoid cutting UTF-8 multi-byte characters runes := []rune(summary) if len(runes) > 500 { summary = string(runes[:500]) diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go index 736b8820..0f9560d9 100644 --- a/internal/application/service/wiki_ingest.go +++ b/internal/application/service/wiki_ingest.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "regexp" "sort" "strings" "sync" @@ -1213,6 +1214,48 @@ func appendUnique(arr types.StringArray, s string) types.StringArray { return append(arr, s) } +// minTextContentRunes is the minimum number of non-whitespace, non-image-reference +// runes required for content to be considered substantive enough for LLM +// summarization or wiki extraction. Documents below this threshold (e.g. a +// scanned PDF where OCR yielded nothing) are routed to a deterministic empty- +// content fallback instead of being passed to the LLM, which would otherwise +// hallucinate based on metadata alone. +const minTextContentRunes = 50 + +var ( + mdImageRefRE = regexp.MustCompile(`!\[[^\]]*\]\([^)]*\)`) + // Strip the outer wrapper before the inner blocks so that + // nested structures like "x" don't leave a + // stray closing tag behind. RE2 has no backreferences, so we use two + // separate patterns instead of `<(image|images)\b...`. + htmlImagesBlockRE = regexp.MustCompile(`(?is)]*>.*?`) + htmlImageBlockRE = regexp.MustCompile(`(?is)]*>.*?`) + htmlImageTagRE = regexp.MustCompile(`(?i)]*>`) +) + +// stripImageMarkup removes markdown image references, raw tags, and +// / XML blocks so that content consisting only of image +// placeholders (e.g. from PDFScannedParser pages without successful VLM OCR) +// is recognised as effectively empty. +func stripImageMarkup(s string) string { + s = htmlImagesBlockRE.ReplaceAllString(s, "") + s = htmlImageBlockRE.ReplaceAllString(s, "") + s = mdImageRefRE.ReplaceAllString(s, "") + s = htmlImageTagRE.ReplaceAllString(s, "") + return s +} + +// hasSufficientTextContent reports whether the given content carries enough +// real text (after image markup is stripped) to warrant an LLM call. It is +// the primary defence against filename-driven hallucinations on scanned PDFs. +func hasSufficientTextContent(content string) bool { + stripped := strings.TrimSpace(stripImageMarkup(content)) + if stripped == "" { + return false + } + return len([]rune(stripped)) >= minTextContentRunes +} + // cleanLLMJSON strips markdown code-fence wrappers and sanitizes control characters // from LLM-generated JSON output so it can be safely unmarshalled. func cleanLLMJSON(s string) string { diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go index dee902c2..c06a7e28 100644 --- a/internal/application/service/wiki_ingest_batch.go +++ b/internal/application/service/wiki_ingest_batch.go @@ -473,6 +473,18 @@ func (s *wikiIngestService) mapOneDocument( } logger.Infof(ctx, "wiki ingest: doc %s chunks=%d content_len(raw=%d,truncated=%d)", knowledgeID, len(chunks), rawRuneCount, len([]rune(content))) + // Refuse to run LLM-based extraction when the document carries no real + // text — e.g. a scanned PDF whose pages were converted to images but where + // VLM OCR produced nothing usable. Without this guard the LLM would have + // only image markup left and would happily fabricate entities/concepts. + if !hasSufficientTextContent(content) { + logger.Warnf(ctx, + "wiki ingest: doc %s has insufficient text content after stripping image markup (raw_len=%d), skipping LLM extraction", + knowledgeID, rawRuneCount, + ) + return nil, nil, nil + } + docTitle := knowledgeID if kn, err := s.knowledgeSvc.GetKnowledgeByIDOnly(ctx, knowledgeID); err == nil && kn != nil && kn.Title != "" { docTitle = kn.Title @@ -547,9 +559,6 @@ func (s *wikiIngestService) mapOneDocument( go func() { defer wg.Done() summaryContent, summaryErr = s.generateWithTemplate(ctx, chatModel, agent.WikiSummaryPrompt, map[string]string{ - "Title": docTitle, - "FileName": docTitle, - "FileType": "document", "Content": content, "Language": lang, "ExtractedSlugs": slugListing, @@ -780,7 +789,6 @@ func (s *wikiIngestService) extractEntitiesAndConceptsNoUpsert( } extractionJSON, err := s.generateWithTemplate(ctx, chatModel, agent.WikiKnowledgeExtractPrompt, map[string]string{ - "Title": docTitle, "Content": content, "Language": lang, "PreviousSlugs": prevSlugsText, diff --git a/internal/application/service/wiki_ingest_cite.go b/internal/application/service/wiki_ingest_cite.go index a23bc4a1..e399f515 100644 --- a/internal/application/service/wiki_ingest_cite.go +++ b/internal/application/service/wiki_ingest_cite.go @@ -92,7 +92,6 @@ func (s *wikiIngestService) extractCandidateSlugs( granularity := batchCtx.ExtractionGranularity.Normalize() raw, err := s.generateWithTemplate(ctx, chatModel, agent.WikiCandidateSlugPrompt, map[string]string{ - "Title": docTitle, "Content": content, "Language": lang, "PreviousSlugs": prevSlugsText, @@ -301,7 +300,6 @@ func (s *wikiIngestService) classifyChunkCitations( eg.Go(func() error { chunksXML := renderChunksXML(batch) raw, err := s.generateWithTemplate(ectx, chatModel, agent.WikiChunkCitationPrompt, map[string]string{ - "DocTitle": docTitle, "CandidateSlugs": candidatesXML, "ChunksXML": chunksXML, "Language": lang, diff --git a/internal/application/service/wiki_ingest_test.go b/internal/application/service/wiki_ingest_test.go index 6de9e1c3..0f44d1bf 100644 --- a/internal/application/service/wiki_ingest_test.go +++ b/internal/application/service/wiki_ingest_test.go @@ -106,3 +106,70 @@ func TestReconstructContentEmpty(t *testing.T) { t.Errorf("Empty chunks should produce empty content, got %q", content) } } + +func TestStripImageMarkup(t *testing.T) { + tests := []struct { + name string + input string + want string + }{ + {"plain text untouched", "Hello world.", "Hello world."}, + {"single markdown image removed", "![alt](images/page_1.png)", ""}, + { + "scanned-pdf style page references all stripped", + "![MX5280_page_1.png](images/MX5280_page_1.png)\n\n![MX5280_page_2.png](images/MX5280_page_2.png)", + "\n\n", + }, + {"mixed text and image keeps text", "Intro paragraph.\n![fig](a.png)\nConclusion.", "Intro paragraph.\n\nConclusion."}, + {"html img tag stripped", `Before y after`, "Before after"}, + {"images xml block stripped", "Body x tail", "Body tail"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := stripImageMarkup(tt.input) + if got != tt.want { + t.Errorf("stripImageMarkup(%q) = %q, want %q", tt.input, got, tt.want) + } + }) + } +} + +func TestHasSufficientTextContent(t *testing.T) { + tests := []struct { + name string + input string + want bool + }{ + {"empty string", "", false}, + {"only whitespace", " \n\n\t ", false}, + { + "only image references (scanned PDF without OCR)", + "![MX5280_page_1.png](images/MX5280_page_1.png)\n![MX5280_page_2.png](images/MX5280_page_2.png)", + false, + }, + {"short text below threshold", "tiny", false}, + { + "short text mixed with images still below threshold", + "![a](x.png)\nshort\n![b](y.png)", + false, + }, + { + "sufficient text passes", + "This is a substantial paragraph of real content describing a legal matter and the parties involved.", + true, + }, + { + "sufficient text mixed with images still passes", + "![cover](cover.png)\nThis is a substantial paragraph of real content describing a legal matter and the parties involved.\n![sig](sig.png)", + true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := hasSufficientTextContent(tt.input) + if got != tt.want { + t.Errorf("hasSufficientTextContent(%q) = %v, want %v", tt.input, got, tt.want) + } + }) + } +}