diff --git a/config/prompt_templates/generate_summary.yaml b/config/prompt_templates/generate_summary.yaml index c3fad973..d6267433 100644 --- a/config/prompt_templates/generate_summary.yaml +++ b/config/prompt_templates/generate_summary.yaml @@ -5,22 +5,21 @@ templates: description: "Generate a concise document summary" default: true content: | - You are a precise document summarization expert. Your task is to extract and summarize the core content of the document provided by the user. + You are a precise document summarization expert. Your task is to summarize the core content of the document provided by the user, basing the summary STRICTLY on the text the user supplies — never on a filename, title, file extension, or any other external clue. ## Steps - 1. Identify the document type from the metadata (e.g., technical doc, meeting notes, research paper, code, etc.) - 2. Extract 3-5 key points or main topics covered in the document - 3. Write a coherent summary incorporating these key points + 1. Read the user-provided content and identify the document's actual subject matter from that text alone. + 2. Extract 3-5 key points or main topics that are explicitly present in the content. + 3. Write a coherent summary incorporating these key points. ## Core Requirements - Summary length: 100-500 words, adjusted based on content complexity - Short/simple documents: 100-200 words - Long/complex documents: 300-500 words - - Generate the summary entirely based on the provided content, without adding any information not present in the document - - Ensure the summary captures key information points and main conclusions - - If the content contains "[...content omitted...]" markers, it is a sampled excerpt from a longer document — cover ALL topics that appear across the provided sections, not just the beginning - - Even for complex or specialized content, you must attempt to extract core points for summarization - - Output the summary directly, without any preamble, prefix, or explanation + - Generate the summary entirely based on the provided content, without adding any information not present in the document. + - Ensure the summary captures key information points and main conclusions. + - If the content contains "[...content omitted...]" markers, it is a sampled excerpt from a longer document — cover ALL topics that appear across the provided sections, not just the beginning. + - Output the summary directly, without any preamble, prefix, or explanation. ## Format and Style - Use an objective, neutral third-person narrative tone @@ -29,10 +28,10 @@ templates: - For technical documents: preserve key terms, metrics, and specific details - For meeting notes/reports: highlight decisions, action items, and conclusions - ## Important Notes - - NEVER output refusal phrases such as "unable to generate", "unable to summarize", or "insufficient content" - - Do not copy or reference any content from examples; ensure the summary is entirely based on the user's document - - Make every effort to extract key points and summarize for any text, regardless of length or complexity + ## Empty or Insufficient Content + - If the user-provided content is empty, contains only image/figure references with no extracted text, or otherwise carries no substantive textual information, you MUST output exactly the single line: "No textual content was extractable from this document." and nothing else. + - Do NOT fabricate a topic, do NOT guess from any other clue, and do NOT copy content from examples or unrelated sources. + - It is correct and expected to refuse to summarise when the content is absent. This is preferred over inventing a plausible-sounding but unsupported summary. ## Language - Use {{language}} for all outputs diff --git a/internal/agent/prompts_wiki.go b/internal/agent/prompts_wiki.go index 2a6ba999..cb28e400 100644 --- a/internal/agent/prompts_wiki.go +++ b/internal/agent/prompts_wiki.go @@ -5,12 +5,16 @@ package agent // knowledge from raw documents and build/update wiki pages. // WikiSummaryPrompt generates a summary page for a newly ingested document. +// +// Filename and title are intentionally NOT passed to the LLM: documents +// uploaded to WeKnora often carry filenames that say nothing about the +// content (e.g. scanned PDFs named after the scanner model "MX5280.pdf"), +// and feeding such filenames to the model invites hallucinated summaries +// when the actual extracted content is thin. The model must rely solely on +// the document content provided below. const WikiSummaryPrompt = `You are a wiki editor. Given the following document content, create a structured wiki summary page in Markdown format. -{{.Title}} -{{.FileName}} -{{.FileType}} {{.Content}} @@ -30,6 +34,7 @@ const WikiSummaryPrompt = `You are a wiki editor. Given the following document c 7. At the end, include a "## Key Takeaways" section with bullet points. 8. Write in {{.Language}}. 9. Keep the summary concise but thorough (500-1500 words depending on document length). +10. **Empty content rule**: If the block above is empty, contains only image references with no extracted text, or otherwise carries no substantive information, output exactly: "SUMMARY: No textual content was extractable from this document." followed by a brief note explaining that the document could not be summarised. Do NOT invent a topic, do NOT guess from any other clue. Output the SUMMARY line first, then the Markdown content. Do not include any other preamble.` @@ -40,7 +45,6 @@ Output the SUMMARY line first, then the Markdown content. Do not include any oth const WikiKnowledgeExtractPrompt = `You are a knowledge extraction system. Analyze the following document and extract all significant entities AND key concepts. -{{.Title}} {{.Content}} @@ -54,6 +58,8 @@ const WikiKnowledgeExtractPrompt = `You are a knowledge extraction system. Analy Return a JSON object with two arrays: "entities" and "concepts". **IMPORTANT: Write ALL names, descriptions, and details in {{.Language}}**. +If the block above is empty, contains only image references with no extracted text, or otherwise carries no substantive information, return {"entities": [], "concepts": []}. Do NOT invent entities or concepts from any other source. + ### Slug Continuity Rules If previous slugs are provided above, you MUST follow these rules: - If an entity or concept from the previous extraction still exists in the current document, **reuse its exact slug** from the previous list. Do NOT generate a new slug for the same thing. @@ -121,7 +127,6 @@ Output ONLY valid JSON. Example: const WikiCandidateSlugPrompt = `You are a knowledge extraction system. Analyze the following document and list all significant entities AND key concepts as a lightweight candidate set. Another pass will later attach concrete supporting chunks to each item, so you do NOT need to write exhaustive per-item facts here. -{{.Title}} {{.Content}} @@ -135,6 +140,8 @@ const WikiCandidateSlugPrompt = `You are a knowledge extraction system. Analyze Return a JSON object with two arrays: "entities" and "concepts". **IMPORTANT: Write ALL names, descriptions, and details in {{.Language}}**. +If the block above is empty, contains only image references with no extracted text, or otherwise carries no substantive information, return {"entities": [], "concepts": []}. Do NOT invent entities or concepts from any other source. + ### Extraction Scope (Granularity: {{.Granularity}}) {{.GranularityGuidance}} @@ -202,8 +209,6 @@ Output ONLY valid JSON. Example: // their verbatim form (the chunk text) instead of asking the LLM to paraphrase. const WikiChunkCitationPrompt = `You are a precise citation system. Your job is to scan a batch of document chunks and decide, for each candidate entity/concept below, which chunks substantively discuss it. -{{.DocTitle}} - {{.CandidateSlugs}} diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index ea5250ef..a4deab65 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -2242,6 +2242,13 @@ func (s *knowledgeService) processChunks(ctx context.Context, // defaultMaxInputChars is the default maximum characters used as input for summary generation. const defaultMaxInputChars = 1024 * 24 +// errInsufficientSummaryContent signals that getSummary refused to call the +// LLM because the document had no usable text after image markup was stripped +// (typical for scanned PDFs where VLM OCR yielded nothing). Callers should +// mark the knowledge's summary as failed instead of falling back to the first +// chunk's raw content (which would just be a bare image reference). +var errInsufficientSummaryContent = errors.New("insufficient text content for summary generation") + // getSummary generates a summary for knowledge content using an AI model func (s *knowledgeService) getSummary(ctx context.Context, summaryModel chat.Chat, knowledge *types.Knowledge, chunks []*types.Chunk, @@ -2295,22 +2302,23 @@ func (s *knowledgeService) getSummary(ctx context.Context, logger.GetLogger(ctx).Infof("getSummary: content length=%d chars (max=%d) for knowledge %s", len([]rune(chunkContents)), maxInputChars, knowledge.ID) - // Prepare content with metadata for summary generation - contentWithMetadata := chunkContents - - // Add knowledge metadata if available - if knowledge != nil { - metadataIntro := fmt.Sprintf("Document Type: %s\nFile Name: %s\n", knowledge.FileType, knowledge.FileName) - - // Add additional metadata if available - if knowledge.Type != "" { - metadataIntro += fmt.Sprintf("Knowledge Type: %s\n", knowledge.Type) - } - - // Prepend metadata to content - contentWithMetadata = metadataIntro + "\nContent:\n" + contentWithMetadata + // Bail out before the LLM call when there is not enough actual text to + // summarise. We deliberately do not pass filename/file-type metadata to the + // LLM: scanned PDFs frequently carry filenames like "MX5280.pdf" (the + // scanner model), and feeding that to the model would invite it to + // hallucinate a scanner manual instead of admitting the document had no + // extractable text. + if !hasSufficientTextContent(chunkContents) { + logger.GetLogger(ctx).Warnf( + "getSummary: content for knowledge %s has insufficient text after stripping image markup (len=%d); skipping LLM call", + knowledge.ID, len([]rune(strings.TrimSpace(stripImageMarkup(chunkContents)))), + ) + return "", errInsufficientSummaryContent } + // Pass the raw chunk text to the LLM with no filename / file-type framing. + contentWithMetadata := chunkContents + // Determine max output tokens from config maxTokens := 2048 if s.config.Conversation.Summary != nil && s.config.Conversation.Summary.MaxCompletionTokens > 0 { @@ -2482,11 +2490,24 @@ func (s *knowledgeService) ProcessSummaryGeneration(ctx context.Context, t *asyn summary, err := s.getSummary(ctx, chatModel, knowledge, textChunks) if err != nil { logger.Errorf(ctx, "Failed to generate summary for knowledge %s: %v", payload.KnowledgeID, err) - // Use first chunk content as fallback + // For the insufficient-content case (scanned PDF without OCR, etc.) + // we deliberately do NOT fall back to the first chunk's raw content, + // since that chunk is typically just a bare markdown image reference + // and surfacing it in the description is misleading. + if errors.Is(err, errInsufficientSummaryContent) { + knowledge.Description = "" + knowledge.SummaryStatus = types.SummaryStatusFailed + knowledge.UpdatedAt = time.Now() + if updateErr := s.repo.UpdateKnowledge(ctx, knowledge); updateErr != nil { + logger.Errorf(ctx, "Failed to mark summary as failed: %v", updateErr) + return fmt.Errorf("failed to update knowledge: %w", updateErr) + } + return nil + } + // For other errors (LLM API issues etc.), fall back to the first chunk. if len(textChunks) > 0 { summary = textChunks[0].Content if len(summary) > 500 { - // Use rune-based truncation to avoid cutting UTF-8 multi-byte characters runes := []rune(summary) if len(runes) > 500 { summary = string(runes[:500]) diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.go index 736b8820..0f9560d9 100644 --- a/internal/application/service/wiki_ingest.go +++ b/internal/application/service/wiki_ingest.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "regexp" "sort" "strings" "sync" @@ -1213,6 +1214,48 @@ func appendUnique(arr types.StringArray, s string) types.StringArray { return append(arr, s) } +// minTextContentRunes is the minimum number of non-whitespace, non-image-reference +// runes required for content to be considered substantive enough for LLM +// summarization or wiki extraction. Documents below this threshold (e.g. a +// scanned PDF where OCR yielded nothing) are routed to a deterministic empty- +// content fallback instead of being passed to the LLM, which would otherwise +// hallucinate based on metadata alone. +const minTextContentRunes = 50 + +var ( + mdImageRefRE = regexp.MustCompile(`!\[[^\]]*\]\([^)]*\)`) + // Strip the outer wrapper before the inner blocks so that + // nested structures like "x" don't leave a + // stray closing tag behind. RE2 has no backreferences, so we use two + // separate patterns instead of `<(image|images)\b...`. + htmlImagesBlockRE = regexp.MustCompile(`(?is)]*>.*?`) + htmlImageBlockRE = regexp.MustCompile(`(?is)]*>.*?`) + htmlImageTagRE = regexp.MustCompile(`(?i)]*>`) +) + +// stripImageMarkup removes markdown image references, raw tags, and +// / XML blocks so that content consisting only of image +// placeholders (e.g. from PDFScannedParser pages without successful VLM OCR) +// is recognised as effectively empty. +func stripImageMarkup(s string) string { + s = htmlImagesBlockRE.ReplaceAllString(s, "") + s = htmlImageBlockRE.ReplaceAllString(s, "") + s = mdImageRefRE.ReplaceAllString(s, "") + s = htmlImageTagRE.ReplaceAllString(s, "") + return s +} + +// hasSufficientTextContent reports whether the given content carries enough +// real text (after image markup is stripped) to warrant an LLM call. It is +// the primary defence against filename-driven hallucinations on scanned PDFs. +func hasSufficientTextContent(content string) bool { + stripped := strings.TrimSpace(stripImageMarkup(content)) + if stripped == "" { + return false + } + return len([]rune(stripped)) >= minTextContentRunes +} + // cleanLLMJSON strips markdown code-fence wrappers and sanitizes control characters // from LLM-generated JSON output so it can be safely unmarshalled. func cleanLLMJSON(s string) string { diff --git a/internal/application/service/wiki_ingest_batch.go b/internal/application/service/wiki_ingest_batch.go index dee902c2..c06a7e28 100644 --- a/internal/application/service/wiki_ingest_batch.go +++ b/internal/application/service/wiki_ingest_batch.go @@ -473,6 +473,18 @@ func (s *wikiIngestService) mapOneDocument( } logger.Infof(ctx, "wiki ingest: doc %s chunks=%d content_len(raw=%d,truncated=%d)", knowledgeID, len(chunks), rawRuneCount, len([]rune(content))) + // Refuse to run LLM-based extraction when the document carries no real + // text — e.g. a scanned PDF whose pages were converted to images but where + // VLM OCR produced nothing usable. Without this guard the LLM would have + // only image markup left and would happily fabricate entities/concepts. + if !hasSufficientTextContent(content) { + logger.Warnf(ctx, + "wiki ingest: doc %s has insufficient text content after stripping image markup (raw_len=%d), skipping LLM extraction", + knowledgeID, rawRuneCount, + ) + return nil, nil, nil + } + docTitle := knowledgeID if kn, err := s.knowledgeSvc.GetKnowledgeByIDOnly(ctx, knowledgeID); err == nil && kn != nil && kn.Title != "" { docTitle = kn.Title @@ -547,9 +559,6 @@ func (s *wikiIngestService) mapOneDocument( go func() { defer wg.Done() summaryContent, summaryErr = s.generateWithTemplate(ctx, chatModel, agent.WikiSummaryPrompt, map[string]string{ - "Title": docTitle, - "FileName": docTitle, - "FileType": "document", "Content": content, "Language": lang, "ExtractedSlugs": slugListing, @@ -780,7 +789,6 @@ func (s *wikiIngestService) extractEntitiesAndConceptsNoUpsert( } extractionJSON, err := s.generateWithTemplate(ctx, chatModel, agent.WikiKnowledgeExtractPrompt, map[string]string{ - "Title": docTitle, "Content": content, "Language": lang, "PreviousSlugs": prevSlugsText, diff --git a/internal/application/service/wiki_ingest_cite.go b/internal/application/service/wiki_ingest_cite.go index a23bc4a1..e399f515 100644 --- a/internal/application/service/wiki_ingest_cite.go +++ b/internal/application/service/wiki_ingest_cite.go @@ -92,7 +92,6 @@ func (s *wikiIngestService) extractCandidateSlugs( granularity := batchCtx.ExtractionGranularity.Normalize() raw, err := s.generateWithTemplate(ctx, chatModel, agent.WikiCandidateSlugPrompt, map[string]string{ - "Title": docTitle, "Content": content, "Language": lang, "PreviousSlugs": prevSlugsText, @@ -301,7 +300,6 @@ func (s *wikiIngestService) classifyChunkCitations( eg.Go(func() error { chunksXML := renderChunksXML(batch) raw, err := s.generateWithTemplate(ectx, chatModel, agent.WikiChunkCitationPrompt, map[string]string{ - "DocTitle": docTitle, "CandidateSlugs": candidatesXML, "ChunksXML": chunksXML, "Language": lang, diff --git a/internal/application/service/wiki_ingest_test.go b/internal/application/service/wiki_ingest_test.go index 6de9e1c3..0f44d1bf 100644 --- a/internal/application/service/wiki_ingest_test.go +++ b/internal/application/service/wiki_ingest_test.go @@ -106,3 +106,70 @@ func TestReconstructContentEmpty(t *testing.T) { t.Errorf("Empty chunks should produce empty content, got %q", content) } } + +func TestStripImageMarkup(t *testing.T) { + tests := []struct { + name string + input string + want string + }{ + {"plain text untouched", "Hello world.", "Hello world."}, + {"single markdown image removed", "![alt](images/page_1.png)", ""}, + { + "scanned-pdf style page references all stripped", + "![MX5280_page_1.png](images/MX5280_page_1.png)\n\n![MX5280_page_2.png](images/MX5280_page_2.png)", + "\n\n", + }, + {"mixed text and image keeps text", "Intro paragraph.\n![fig](a.png)\nConclusion.", "Intro paragraph.\n\nConclusion."}, + {"html img tag stripped", `Before y after`, "Before after"}, + {"images xml block stripped", "Body x tail", "Body tail"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := stripImageMarkup(tt.input) + if got != tt.want { + t.Errorf("stripImageMarkup(%q) = %q, want %q", tt.input, got, tt.want) + } + }) + } +} + +func TestHasSufficientTextContent(t *testing.T) { + tests := []struct { + name string + input string + want bool + }{ + {"empty string", "", false}, + {"only whitespace", " \n\n\t ", false}, + { + "only image references (scanned PDF without OCR)", + "![MX5280_page_1.png](images/MX5280_page_1.png)\n![MX5280_page_2.png](images/MX5280_page_2.png)", + false, + }, + {"short text below threshold", "tiny", false}, + { + "short text mixed with images still below threshold", + "![a](x.png)\nshort\n![b](y.png)", + false, + }, + { + "sufficient text passes", + "This is a substantial paragraph of real content describing a legal matter and the parties involved.", + true, + }, + { + "sufficient text mixed with images still passes", + "![cover](cover.png)\nThis is a substantial paragraph of real content describing a legal matter and the parties involved.\n![sig](sig.png)", + true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := hasSufficientTextContent(tt.input) + if got != tt.want { + t.Errorf("hasSufficientTextContent(%q) = %v, want %v", tt.input, got, tt.want) + } + }) + } +}