diff --git a/config/prompt_templates/agent_system_prompt.yaml b/config/prompt_templates/agent_system_prompt.yaml index 51418a75..eb56a8c8 100644 --- a/config/prompt_templates/agent_system_prompt.yaml +++ b/config/prompt_templates/agent_system_prompt.yaml @@ -161,7 +161,7 @@ templates: - WRONG (all grouped at end of answer): * **Structured:** Clear hierarchy and logic. - * **Rich Media (Markdown with Images):** When retrieved chunks contain images (indicated by the "images" field with URLs), you MUST include them in your response using standard Markdown image syntax: ![description](image_url). Place images at contextually appropriate positions within the answer to create a well-formatted, visually rich response. Images help users better understand the content, especially for diagrams, charts, screenshots, or visual explanations. + * **Rich Media (Markdown with Images):** When retrieved chunks contain Markdown images or an "images" field with URLs, reuse relevant images in your response using standard Markdown image syntax: ![description](image_url). Preserve each image URL exactly and place images at contextually appropriate positions within the answer. Images help users better understand the content, especially for diagrams, charts, screenshots, or visual explanations. ### System Status Web Search: {{web_search_status}} @@ -263,7 +263,7 @@ templates: - **External MCP Tools (if any are exposed in your tool list):** Use them when the user's question requires real-time data, external system lookups, or actions that do NOT live inside the wiki (e.g. querying a ticketing system, calling an internal API, fetching live metrics). Treat their responses as additional evidence, not as a replacement for wiki content. - **Skills (if an `### Available Skills` section appears later in this prompt):** Before answering, scan the listed skills. If the user's intent matches a skill's triggers, call `read_skill(skill_name="...")` to load its full instructions, then follow them. Skills are especially useful for specialized output formatting or domain-specific procedures. - When the user @MCP or @Skill, a short `` block also appears before the question — follow it with highest priority; wiki content remains the primary source of truth for domain facts, and MCP/skills are complementary. - 6. **Synthesize:** Once you have gathered sufficient information from reading multiple interconnected pages (and optionally source documents, MCP results, or skill guidance), synthesize your answer and deliver it by writing it as your reply, then stop (no further tool calls in that final message). If you encountered images (e.g. ``) in the source documents that are relevant to the user's query, be sure to include them in your answer using Markdown format (`![alt](url)`). + 6. **Synthesize:** Once you have gathered sufficient information from reading multiple interconnected pages (and optionally source documents, MCP results, or skill guidance), synthesize your answer and deliver it by writing it as your reply, then stop (no further tool calls in that final message). If you encountered relevant Markdown images in the source documents, reuse them in your answer and preserve their URLs exactly. 7. **Flag Issues (If Necessary):** If you discover that a wiki page contains factual errors, mixed entities (e.g., two different products combined into one page), or outdated information, OR if the user points out such errors, use the `wiki_flag_issue` tool to submit a maintenance report for that page before you write your final answer. diff --git a/frontend/src/utils/chatMarkdownRenderer.test.ts b/frontend/src/utils/chatMarkdownRenderer.test.ts index 2356f514..3754e90d 100644 --- a/frontend/src/utils/chatMarkdownRenderer.test.ts +++ b/frontend/src/utils/chatMarkdownRenderer.test.ts @@ -6,6 +6,7 @@ import { closeDanglingStreamingEmphasis, createChatMarkdownRenderer, markStandaloneStrongParagraphs, + normalizeLegacyImageContextMarkup, preprocessMathDelimiters, renderChatMarkdown, repairFlankingEmphasis, @@ -45,6 +46,79 @@ test('replaceIncompleteImageWithPlaceholder hides an unfinished streaming image' ) }) +test('normalizeLegacyImageContextMarkup converts copied image XML to Markdown', () => { + const input = [ + 'before', + '', + '目标说话人提取的流程图 [测试]', + '目标说话人提取', + '', + 'after', + ].join('\n') + + const output = normalizeLegacyImageContextMarkup(input) + assert.ok( + output.includes('![目标说话人提取的流程图 \\[测试\\]](resource://AbCdEfGhIjKlMnOpQrStUv)'), + ) + assert.doesNotMatch(output, / { + const input = [ + '', + '', + '![原图](resource://AbCdEfGhIjKlMnOpQrStUv)', + 'description', + '', + '', + ].join('\n') + + assert.equal( + normalizeLegacyImageContextMarkup(input).trim(), + '![原图](resource://AbCdEfGhIjKlMnOpQrStUv)', + ) +}) + +test('normalizeLegacyImageContextMarkup hides an unfinished XML block while streaming', () => { + const prefix = '测试阶段主要流程\n\n' + for (const partial of [ + '', + '\n流程图', + ]) { + const output = normalizeLegacyImageContextMarkup(prefix + partial, true) + assert.equal(output, prefix + '') + assert.doesNotMatch(output, /resource:\/\/|image_caption| { + const code = '```xml\n\n\n```' + assert.equal(normalizeLegacyImageContextMarkup(code, true), code) + assert.equal(normalizeLegacyImageContextMarkup(code, false), code) + assert.equal(normalizeLegacyImageContextMarkup('ordinary { + const renderer = createChatMarkdownRenderer({ + imageRenderer: ({ href, text }) => `${text}`, + isValidImageUrl: (href) => href.startsWith('resource://'), + }) + const html = renderChatMarkdown( + '流程图', + { + renderer, + escapeMarkdown: (text) => text, + sanitizeHtml: (value) => value, + streaming: false, + }, + ) + + assert.match(html, /流程图/) + assert.doesNotMatch(html, /image_caption|<image/) +}) + test('stripIncompleteCitationTag hides only an unfinished streaming citation tail', () => { const prefix = 'Source ' const complete = '' diff --git a/frontend/src/utils/chatMarkdownRenderer.ts b/frontend/src/utils/chatMarkdownRenderer.ts index eadb1a17..60f9427d 100644 --- a/frontend/src/utils/chatMarkdownRenderer.ts +++ b/frontend/src/utils/chatMarkdownRenderer.ts @@ -78,6 +78,84 @@ export function replaceIncompleteImageWithPlaceholder(content: string): string { return content } +const LEGACY_IMAGE_BLOCK_RE = /]*)>([\s\S]*?)<\/images?>/gi +const LEGACY_IMAGES_WRAPPER_RE = /<\/?images\b[^>]*>/gi +const LEGACY_IMAGE_ORIGINAL_RE = /([\s\S]*?)<\/image_original>/i +const LEGACY_IMAGE_CAPTION_RE = /([\s\S]*?)<\/image_caption>/i +const LEGACY_IMAGE_OCR_RE = /([\s\S]*?)<\/image_ocr>/i +const COMPLETE_MARKDOWN_CODE_RE = /(```[\s\S]*?```|~~~[\s\S]*?~~~|`[^`\n]*`)/g + +function legacyImageField(body: string, pattern: RegExp): string { + const value = body.match(pattern)?.[1] || '' + return value.replace(/<[^>]*>/g, '').replace(/\s+/g, ' ').trim() +} + +function escapeMarkdownImageAlt(value: string): string { + return value.replace(/\\/g, '\\\\').replace(/\[/g, '\\[').replace(/\]/g, '\\]') +} + +function legacyImageBlockToMarkdown(attributes: string, body: string): string { + const original = body.match(LEGACY_IMAGE_ORIGINAL_RE)?.[1]?.trim() + if (original) return original + + const urlMatch = attributes.match(/\burl\s*=\s*(["'])(.*?)\1/i) + const url = urlMatch?.[2]?.trim() || '' + const caption = legacyImageField(body, LEGACY_IMAGE_CAPTION_RE) + const ocr = legacyImageField(body, LEGACY_IMAGE_OCR_RE) + if (!url) return caption || ocr + + const alt = escapeMarkdownImageAlt(caption || ocr || 'image') + return `![${alt}](${url})` +} + +function normalizeLegacyImageSegment(content: string): string { + return content + .replace( + LEGACY_IMAGE_BLOCK_RE, + (_match, attributes: string, body: string) => legacyImageBlockToMarkdown(attributes, body), + ) + .replace(LEGACY_IMAGES_WRAPPER_RE, '') +} + +/** + * Normalize the legacy image-context protocol if a model copies it into an + * answer. New chat context is Markdown-native, but persisted conversations and + * older deployments can still contain `` blocks. + * + * Complete blocks become ordinary Markdown images and therefore pass through + * the same URL validation/sanitization as model-authored images. While a block + * is still streaming, its unfinished tail is replaced by the existing image + * skeleton so internal tags, captions, and OCR never flash as prose. + */ +export function normalizeLegacyImageContextMarkup(content: string, streaming = false): string { + if (!content) return content + const hasLegacyImageTag = /<\/?images?\b/i.test(content) + const partialTagPattern = /(^|\n)[ \t]*\n]*)?)?)?)?)?$/i + const hasPartialStreamingTag = streaming && partialTagPattern.test(content) + if (!hasLegacyImageTag && !hasPartialStreamingTag) return content + + const parts = content.split(COMPLETE_MARKDOWN_CODE_RE) + for (let i = 0; i < parts.length; i += 2) { + parts[i] = normalizeLegacyImageSegment(parts[i]) + } + if (streaming) { + // The split always leaves normal Markdown in the final even-indexed part, + // so code examples containing literal tags stay untouched. + const tailIndex = parts.length - 1 + let tail = parts[tailIndex] + const openingMatches = Array.from(tail.matchAll(/)/gi)) + const lastOpening = openingMatches[openingMatches.length - 1] + if (lastOpening?.index !== undefined) { + tail = tail.slice(0, lastOpening.index) + STREAMING_IMAGE_PLACEHOLDER + } else { + // Also hide the few frames before the opening tag name itself is complete. + tail = tail.replace(partialTagPattern, `$1${STREAMING_IMAGE_PLACEHOLDER}`) + } + parts[tailIndex] = tail + } + return parts.join('') +} + /** * Hide a trailing Markdown horizontal-rule candidate while content is streaming. * @@ -353,7 +431,11 @@ export function renderChatMarkdown(rawMarkdown: unknown, options: RenderChatMark const streamingSafeText = options.streaming ? stripTrailingStreamingListMarker(stripTrailingStreamingHorizontalRule(rawText)) : rawText - const citationSafeText = stripIncompleteCitationTag(streamingSafeText) + const imageContextSafeText = normalizeLegacyImageContextMarkup( + streamingSafeText, + Boolean(options.streaming), + ) + const citationSafeText = stripIncompleteCitationTag(imageContextSafeText) const { text: tagSafe, tags } = preserveCitationTags(citationSafeText) const imageSafe = replaceIncompleteImageWithPlaceholder(tagSafe) const mathSafe = preprocessMathDelimiters(imageSafe) diff --git a/internal/agent/tools/image_markdown_output_test.go b/internal/agent/tools/image_markdown_output_test.go new file mode 100644 index 00000000..8e431161 --- /dev/null +++ b/internal/agent/tools/image_markdown_output_test.go @@ -0,0 +1,46 @@ +package tools + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/Tencent/WeKnora/internal/types" +) + +func TestLLMToolOutputsUseMarkdownImages(t *testing.T) { + imageInfo, err := json.Marshal([]types.ImageInfo{ + { + URL: "resource://AbCdEfGhIjKlMnOpQrStUv", + Caption: "目标说话人提取流程图", + OCRText: "输入\n输出", + }, + }) + if err != nil { + t.Fatal(err) + } + + chunk := &types.Chunk{ + ID: "chunk-1", + ChunkIndex: 0, + ChunkType: types.ChunkTypeText, + Content: "测试阶段主要流程", + ImageInfo: string(imageInfo), + } + + listOutput := (&ListKnowledgeChunksTool{}).buildOutput("knowledge-1", "测试文档", 1, 1, []*types.Chunk{chunk}) + wikiOutput := enrichChunkContent(chunk) + for name, output := range map[string]string{ + "list_knowledge_chunks": listOutput, + "wiki_read_source_doc": wikiOutput, + } { + t.Run(name, func(t *testing.T) { + if !strings.Contains(output, "![目标说话人提取流程图](resource://AbCdEfGhIjKlMnOpQrStUv)") { + t.Fatalf("expected Markdown image in tool output:\n%s", output) + } + if strings.Contains(output, "") || strings.Contains(output, "") { + t.Fatalf("tool output leaked legacy image XML:\n%s", output) + } + }) + } +} diff --git a/internal/agent/tools/knowledge_search.go b/internal/agent/tools/knowledge_search.go index 5f9d1620..08afd875 100644 --- a/internal/agent/tools/knowledge_search.go +++ b/internal/agent/tools/knowledge_search.go @@ -1247,14 +1247,10 @@ func (t *KnowledgeSearchTool) formatOutput( var imageInfos []types.ImageInfo if err := json.Unmarshal([]byte(result.ImageInfo), &imageInfos); err == nil && len(imageInfos) > 0 { for _, img := range imageInfos { - ob.WriteString(fmt.Sprintf("\n", xmlEscape(img.URL))) - if img.Caption != "" { - ob.WriteString(fmt.Sprintf("%s\n", xmlEscape(img.Caption))) + if imageMarkdown := searchutil.BuildImageInfoMarkdownWithURL(img.URL, &img); imageMarkdown != "" { + ob.WriteString(imageMarkdown) + ob.WriteString("\n") } - if img.OCRText != "" { - ob.WriteString(fmt.Sprintf("%s\n", xmlEscape(img.OCRText))) - } - ob.WriteString("\n") } } } diff --git a/internal/agent/tools/list_knowledge_chunks.go b/internal/agent/tools/list_knowledge_chunks.go index 6f6bd55a..060a7167 100644 --- a/internal/agent/tools/list_knowledge_chunks.go +++ b/internal/agent/tools/list_knowledge_chunks.go @@ -412,7 +412,7 @@ func (t *ListKnowledgeChunksTool) buildOutput( for _, c := range chunks { if c.ChunkType == types.ChunkTypeFAQ { writeFAQEntryXML(&b, c) - writeChunkImagesXML(&b, c) + writeChunkImagesMarkdown(&b, c) continue } @@ -424,7 +424,7 @@ func (t *ListKnowledgeChunksTool) buildOutput( c.ID, c.ChunkIndex, c.ChunkType) } fmt.Fprintf(&b, "%s\n", summarizeContent(c.Content)) - writeChunkImagesXML(&b, c) + writeChunkImagesMarkdown(&b, c) b.WriteString("\n") } @@ -436,7 +436,7 @@ func (t *ListKnowledgeChunksTool) buildOutput( return b.String() } -func writeChunkImagesXML(b *strings.Builder, c *types.Chunk) { +func writeChunkImagesMarkdown(b *strings.Builder, c *types.Chunk) { if c == nil || c.ImageInfo == "" { return } @@ -445,18 +445,10 @@ func writeChunkImagesXML(b *strings.Builder, c *types.Chunk) { return } for _, img := range imageInfos { - if img.URL != "" { - fmt.Fprintf(b, "\n", img.URL) - } else { - b.WriteString("\n") + if imageMarkdown := searchutil.BuildImageInfoMarkdownWithURL(img.URL, &img); imageMarkdown != "" { + b.WriteString(imageMarkdown) + b.WriteString("\n") } - if img.Caption != "" { - fmt.Fprintf(b, "%s\n", img.Caption) - } - if img.OCRText != "" { - fmt.Fprintf(b, "%s\n", img.OCRText) - } - b.WriteString("\n") } } diff --git a/internal/agent/tools/wiki_read_source_doc.go b/internal/agent/tools/wiki_read_source_doc.go index 009e54fd..564bb359 100644 --- a/internal/agent/tools/wiki_read_source_doc.go +++ b/internal/agent/tools/wiki_read_source_doc.go @@ -96,15 +96,9 @@ func enrichChunkContent(c *types.Chunk) string { if err := json.Unmarshal([]byte(c.ImageInfo), &imgInfos); err == nil && len(imgInfos) > 0 { var imgBuilder strings.Builder for _, img := range imgInfos { - if img.URL != "" { - imgBuilder.WriteString(fmt.Sprintf("\n\n", img.URL)) - if img.Caption != "" { - imgBuilder.WriteString(fmt.Sprintf("%s\n", img.Caption)) - } - if img.OCRText != "" { - imgBuilder.WriteString(fmt.Sprintf("%s\n", img.OCRText)) - } - imgBuilder.WriteString("") + if imageMarkdown := searchutil.BuildImageInfoMarkdownWithURL(img.URL, &img); imageMarkdown != "" { + imgBuilder.WriteString("\n") + imgBuilder.WriteString(imageMarkdown) } } content += imgBuilder.String() @@ -163,7 +157,7 @@ func (t *wikiReadSourceDocTool) Execute(ctx context.Context, args json.RawMessag pageSize := 100 page := 1 if hasRange { - page = (params.StartChunkIndex - 1) / pageSize + 1 + page = (params.StartChunkIndex-1)/pageSize + 1 } var chunksOutput strings.Builder @@ -293,7 +287,7 @@ func (t *wikiReadSourceDocTool) Execute(ctx context.Context, args json.RawMessag } sb.WriteString(fmt.Sprintf("%d\n\n", totalChunks)) - + if matchCount > 0 { sb.WriteString(fmt.Sprintf("\n", matchCount)) sb.WriteString(chunksOutput.String()) diff --git a/internal/searchutil/imageinfo.go b/internal/searchutil/imageinfo.go index 5499781c..62b5e21c 100644 --- a/internal/searchutil/imageinfo.go +++ b/internal/searchutil/imageinfo.go @@ -260,10 +260,15 @@ func EnrichContentWithImageInfo(content string, imageInfoJSON string) string { return content } -// EnrichContentWithImageInfoForChat is like EnrichContentWithImageInfo but only -// wraps Markdown images that have a matching image_info entry. This avoids -// turning every parent thumbnail into an block when only a few pages -// were retrieved, and skips appending orphan image_info extras. +// EnrichContentWithImageInfoForChat enriches matching Markdown images with +// caption / OCR text while keeping the image itself as Markdown. Chat context is +// deliberately answer-ready: if a model copies a relevant image from its +// context, the copied content should still render instead of leaking the +// internal XML protocol into the answer. +// +// Only images with a matching image_info entry are enriched. This avoids adding +// every parent thumbnail when only a few pages were retrieved, and skips orphan +// image_info extras. func EnrichContentWithImageInfoForChat(content string, imageInfoJSON string) string { var imageInfos []types.ImageInfo if err := json.Unmarshal([]byte(imageInfoJSON), &imageInfos); err != nil { @@ -283,29 +288,77 @@ func EnrichContentWithImageInfoForChat(content string, imageInfoJSON string) str } } - matches := MarkdownImageRegex.FindAllStringSubmatch(content, -1) - for _, match := range matches { + content = MarkdownImageRegex.ReplaceAllStringFunc(content, func(markdownImage string) string { + match := MarkdownImageRegex.FindStringSubmatch(markdownImage) if len(match) < 3 { - continue + return markdownImage } imgURL := match[2] imgInfo, found := imageInfoMap[imgURL] if !found || imgInfo == nil { - continue + return markdownImage } - inner := BuildImageInfoXML(imgInfo) - if inner == "" { - continue + metadata := buildImageInfoMarkdownMetadata(imgInfo) + if metadata == "" { + return markdownImage } - var b strings.Builder - b.WriteString(fmt.Sprintf("\n", imgURL)) - b.WriteString(inner) - b.WriteString("") - content = strings.Replace(content, match[0], b.String(), 1) - } + return markdownImage + "\n\n" + metadata + }) return content } +// buildImageInfoMarkdownMetadata keeps image-derived text explicit for the LLM +// without introducing a second, user-visible markup protocol. Blockquotes keep +// multiline OCR attached to its image and remain harmless if copied verbatim. +func buildImageInfoMarkdownMetadata(img *types.ImageInfo) string { + if img == nil { + return "" + } + + var lines []string + if caption := strings.TrimSpace(img.Caption); caption != "" { + lines = append(lines, "**Image caption:** "+caption) + } + if ocr := strings.TrimSpace(img.OCRText); ocr != "" { + lines = append(lines, "**Image text (OCR):** "+ocr) + } + if len(lines) == 0 { + return "" + } + + return "> " + strings.ReplaceAll(strings.Join(lines, "\n\n"), "\n", "\n> ") +} + +// BuildImageInfoMarkdownWithURL formats one image as answer-ready Markdown for +// LLM-facing chat/tool context. The URL is intentionally preserved verbatim; +// resource and provider URLs are opaque handles resolved by the frontend. +func BuildImageInfoMarkdownWithURL(url string, img *types.ImageInfo) string { + if img == nil { + return "" + } + url = strings.TrimSpace(url) + metadata := buildImageInfoMarkdownMetadata(img) + if url == "" { + return metadata + } + + alt := strings.Join(strings.Fields(img.Caption), " ") + if alt == "" { + alt = "image" + } + alt = strings.NewReplacer( + `\`, `\\`, + `[`, `\[`, + `]`, `\]`, + ).Replace(alt) + + image := fmt.Sprintf("![%s](%s)", alt, url) + if metadata == "" { + return image + } + return image + "\n\n" + metadata +} + // BuildImageInfoXML returns XML-tagged caption / ocr for one image. func BuildImageInfoXML(img *types.ImageInfo) string { var b strings.Builder diff --git a/internal/searchutil/imageinfo_match_test.go b/internal/searchutil/imageinfo_match_test.go index 75fae7be..63bcc973 100644 --- a/internal/searchutil/imageinfo_match_test.go +++ b/internal/searchutil/imageinfo_match_test.go @@ -72,17 +72,80 @@ func TestEnrichContentWithImageInfoForChat_SkipsUnmatched(t *testing.T) { content := "![p1](u1)\n\n![p2](u2)" raw, _ := json.Marshal([]types.ImageInfo{{URL: "u2", OCRText: "two"}}) got := EnrichContentWithImageInfoForChat(content, string(raw)) - if strings.Count(got, "two") { + if !strings.Contains(got, "![p2](u2)") { + t.Fatalf("matched markdown should remain renderable: %s", got) + } + if !strings.Contains(got, "> **Image text (OCR):** two") { t.Fatalf("matched image should be enriched: %s", got) } - if strings.Contains(got, "") { - t.Fatalf("chat enrich should not duplicate markdown in image_original: %s", got) + if strings.Count(got, "![") != 2 { + t.Fatalf("chat enrich should not duplicate markdown images: %s", got) + } +} + +func TestEnrichContentWithImageInfoForChat_UsesMarkdownForMultilineMetadata(t *testing.T) { + content := "before\n\n![flow](resource://AbCdEfGhIjKlMnOpQrStUv)\n\nafter" + raw, _ := json.Marshal([]types.ImageInfo{ + { + URL: "resource://AbCdEfGhIjKlMnOpQrStUv", + Caption: "目标说话人提取流程图", + OCRText: "输入\n目标说话人提取\n输出", + }, + }) + + got := EnrichContentWithImageInfoForChat(content, string(raw)) + for _, want := range []string{ + "![flow](resource://AbCdEfGhIjKlMnOpQrStUv)", + "> **Image caption:** 目标说话人提取流程图", + "> **Image text (OCR):** 输入", + "> 目标说话人提取", + "> 输出", + } { + if !strings.Contains(got, want) { + t.Fatalf("expected %q in enriched Markdown:\n%s", want, got) + } + } + if strings.Contains(got, " **Image caption:** same caption") != 2 { + t.Fatalf("expected each image to be enriched exactly once: %s", got) + } +} + +func TestBuildImageInfoMarkdownWithURL(t *testing.T) { + got := BuildImageInfoMarkdownWithURL( + "resource://AbCdEfGhIjKlMnOpQrStUv", + &types.ImageInfo{Caption: "流程图 [测试]", OCRText: "输入\n输出"}, + ) + for _, want := range []string{ + `![流程图 \[测试\]](resource://AbCdEfGhIjKlMnOpQrStUv)`, + "> **Image caption:** 流程图 [测试]", + "> **Image text (OCR):** 输入", + "> 输出", + } { + if !strings.Contains(got, want) { + t.Fatalf("expected %q in image Markdown:\n%s", want, got) + } + } + if strings.Contains(got, "