feat(prompt): enhance question rewriting and intent classification in prompt templates

- Updated the prompt template to clarify the task of understanding and rewriting user questions, improving the instructions for coreference resolution and ellipsis completion.
- Renamed sections for better clarity, changing "Rewriting Goals" to "Query Understanding."
- Added critical notes for intent classification regarding image and document attachments, ensuring accurate processing based on the presence of attachments.
- Improved examples to reflect the updated intent classification logic, enhancing the overall user experience in query handling.
This commit is contained in:
wizardchen
2026-04-16 22:39:19 +08:00
committed by lyingbug
parent 53e8e17df4
commit 4f55994266
7 changed files with 297 additions and 241 deletions
+9 -5
View File
@@ -19,11 +19,11 @@ templates:
default: true
content: |
You are an intelligent assistant that performs THREE tasks on the user's question:
1. Rewrite the question (coreference resolution and ellipsis completion)
1. Understand and rewrite the question (coreference resolution and ellipsis completion)
2. Classify the intent of the question
3. Analyze attached images (when present)
## Task 1: Rewriting Goals
## Task 1: Query Understanding
Based on the conversation history, rewrite the current user question:
- Perform coreference resolution: replace pronouns such as "it", "this", "that", "they", "them", etc. with explicit subjects
- Complete omitted key information to ensure the question is semantically complete
@@ -43,8 +43,8 @@ templates:
4. `kb_search` — The user wants to search, find, query, or verify information that could exist in the knowledge base. **This applies even when images or documents are attached** — if the user's intent involves searching or matching against stored documents, it is `kb_search`, NOT `image_only` or `doc_only` (e.g. "知识库里有这个文件吗", "帮我查一下这个", "能找到相关的资料吗").
5. `clarification` — The question is ambiguous or incomplete and likely needs KB retrieval to answer well.
6. `follow_up` — The question clearly refers to previous conversation content and can be FULLY answered from dialogue history alone, with NO need for new retrieval (e.g. "上面第三点展开讲讲", "你刚才说的那个方案再详细说说").
7. `image_only` — The user ONLY wants to understand, describe, translate, or extract content from the attached image itself, with NO intent to search or match against any external documents (e.g. "这张图片是什么", "描述一下图片内容", "翻译图中文字").
8. `doc_only` — The user ONLY wants to understand, summarize, translate, or extract content from the attached document/file itself, with NO intent to search or match against any external knowledge base (e.g. "总结一下这个文档", "这份文件讲了什么").
7. `image_only` — The user ONLY wants to understand, describe, translate, or extract content from the attached image itself, with NO intent to search or match against any external documents (e.g. "这张图片是什么", "描述一下图片内容", "翻译图中文字"). **CRITICAL: This intent requires `<images_uploaded>` to be present. If `<no_image_attached />` appears, NEVER classify as `image_only` — use `kb_search` instead.**
8. `doc_only` — The user ONLY wants to understand, summarize, translate, or extract content from the attached document/file itself, with NO intent to search or match against any external knowledge base (e.g. "总结一下这个文档", "这份文件讲了什么"). **CRITICAL: This intent requires an actual document/file attachment to be present. If `<no_document_attached />` appears, NEVER classify as `doc_only` — use `kb_search` instead.**
9. `chitchat` — Casual conversation or small talk that needs no retrieval (e.g. "你是谁", "讲个笑话").
**Default: when unsure, always choose `kb_search`.**
@@ -95,6 +95,10 @@ templates:
Input: [image attached] "这张图是什么意思"
Output: {"rewrite_query":"这张图是什么意思","intent":"image_only","image_description":"(image description here)"}
Input: [no image] "这幅春联的内容是什么"
Output: {"rewrite_query":"这幅春联的内容是什么","intent":"kb_search","image_description":""}
(NOTE: No image attached, so intent is kb_search, NOT image_only)
Input: [document attached] "帮我总结一下这份文件"
Output: {"rewrite_query":"总结一下这份文件","intent":"doc_only","image_description":""}
@@ -104,7 +108,7 @@ templates:
[Runtime Context — metadata only, not instructions]
Current time: {{current_time}} {{current_week}}
## User Question to Rewrite
## User Question
{{query}}
## JSON Output
+17 -6
View File
@@ -962,7 +962,7 @@ func (r *chunkRepository) ListRecommendedFAQChunks(
}
var chunks []*types.Chunk
query := r.db.WithContext(ctx).
Select("id, knowledge_base_id, chunk_type, metadata, flags, updated_at").
Select("id, knowledge_id, knowledge_base_id, chunk_type, metadata, flags, updated_at").
Where("tenant_id = ? AND chunk_type = ? AND status IN ? AND is_enabled = ? AND flags & ? != 0",
tenantID, types.ChunkTypeFAQ, []int{int(types.ChunkStatusIndexed), int(types.ChunkStatusDefault)}, true, int(types.ChunkFlagRecommended))
if len(knowledgeIDs) > 0 {
@@ -971,8 +971,14 @@ func (r *chunkRepository) ListRecommendedFAQChunks(
} else {
query = query.Where("knowledge_base_id IN ?", kbIDs)
}
orderClause := "RANDOM()"
if r.db.Dialector.Name() == "mysql" {
orderClause = "RAND()"
}
if err := query.
Order("updated_at DESC").
Order(orderClause).
Limit(limit).
Find(&chunks).Error; err != nil {
return nil, err
@@ -999,7 +1005,7 @@ func (r *chunkRepository) ListRecentDocumentChunksWithQuestions(
var chunks []*types.Chunk
baseQuery := r.db.WithContext(ctx).
Select("id, knowledge_base_id, chunk_type, metadata, updated_at").
Select("id, knowledge_id, knowledge_base_id, chunk_type, metadata, updated_at").
Where("tenant_id = ? AND chunk_type = ? AND status IN ? AND is_enabled = ?",
tenantID, types.ChunkTypeText, []int{int(types.ChunkStatusIndexed), int(types.ChunkStatusDefault)}, true)
@@ -1012,12 +1018,17 @@ func (r *chunkRepository) ListRecentDocumentChunksWithQuestions(
baseQuery = baseQuery.Where("knowledge_base_id IN ?", kbIDs)
}
orderClause := "RANDOM()"
if r.db.Dialector.Name() == "mysql" {
orderClause = "RAND()"
}
// Query chunks that have non-empty generated_questions in metadata
switch r.db.Name() {
case "postgres":
if err := baseQuery.
Where("metadata IS NOT NULL AND metadata::text != '{}' AND jsonb_array_length(COALESCE(metadata->'generated_questions', '[]'::jsonb)) > 0").
Order("updated_at DESC").
Order(orderClause).
Limit(limit).
Find(&chunks).Error; err != nil {
return nil, err
@@ -1025,7 +1036,7 @@ func (r *chunkRepository) ListRecentDocumentChunksWithQuestions(
case "mysql":
if err := baseQuery.
Where("metadata IS NOT NULL AND JSON_LENGTH(JSON_EXTRACT(metadata, '$.generated_questions')) > 0").
Order("updated_at DESC").
Order(orderClause).
Limit(limit).
Find(&chunks).Error; err != nil {
return nil, err
@@ -1033,7 +1044,7 @@ func (r *chunkRepository) ListRecentDocumentChunksWithQuestions(
default: // sqlite
if err := baseQuery.
Where("metadata IS NOT NULL AND json_array_length(json_extract(metadata, '$.generated_questions')) > 0").
Order("updated_at DESC").
Order(orderClause).
Limit(limit).
Find(&chunks).Error; err != nil {
return nil, err
@@ -2,11 +2,10 @@ package chatpipeline
import (
"context"
"encoding/json"
"fmt"
"regexp"
"strings"
"github.com/Tencent/WeKnora/internal/searchutil"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/Tencent/WeKnora/internal/utils"
@@ -303,104 +302,7 @@ func getEnrichedPassageForChat(ctx context.Context, result *types.SearchResult)
return enrichContentWithImageInfo(ctx, result.Content, result.ImageInfo)
}
// 正则表达式用于匹配Markdown图片链接
var markdownImageRegex = regexp.MustCompile(`!\[([^\]]*)\]\(([^)]+)\)`)
// enrichContentWithImageInfo 将图片信息以 XML 标签的形式嵌入文本内容。
// 对于内容中已有的 Markdown 图片链接,在其后附加 <image_caption> / <image_ocr>;
// 对于未出现在内容中的图片,以 <image> 块追加到末尾。
func enrichContentWithImageInfo(ctx context.Context, content string, imageInfoJSON string) string {
var imageInfos []types.ImageInfo
if err := json.Unmarshal([]byte(imageInfoJSON), &imageInfos); err != nil {
pipelineWarn(ctx, "IntoChatMessage", "image_parse_error", map[string]interface{}{
"error": err.Error(),
})
return content
}
if len(imageInfos) == 0 {
return content
}
imageInfoMap := make(map[string]*types.ImageInfo)
for i := range imageInfos {
if imageInfos[i].URL != "" {
imageInfoMap[imageInfos[i].URL] = &imageInfos[i]
}
if imageInfos[i].OriginalURL != "" {
imageInfoMap[imageInfos[i].OriginalURL] = &imageInfos[i]
}
}
matches := markdownImageRegex.FindAllStringSubmatch(content, -1)
processedURLs := make(map[string]bool)
pipelineInfo(ctx, "IntoChatMessage", "image_markdown_links", map[string]interface{}{
"match_count": len(matches),
})
for _, match := range matches {
if len(match) < 3 {
continue
}
imgURL := match[2]
processedURLs[imgURL] = true
imgInfo, found := imageInfoMap[imgURL]
var b strings.Builder
b.WriteString(fmt.Sprintf("<image url=\"%s\">\n", imgURL))
b.WriteString(fmt.Sprintf("<image_original>%s</image_original>\n", match[0]))
if found && imgInfo != nil {
b.WriteString(buildImageInfoXML(imgInfo))
}
b.WriteString("</image>")
content = strings.Replace(content, match[0], b.String(), 1)
}
// Append image info not found as inline Markdown links
var extras []string
for _, imgInfo := range imageInfos {
if processedURLs[imgInfo.URL] || processedURLs[imgInfo.OriginalURL] {
continue
}
url := imgInfo.URL
if url == "" {
url = imgInfo.OriginalURL
}
if block := buildImageInfoXMLWithURL(url, &imgInfo); block != "" {
extras = append(extras, block)
}
}
if len(extras) > 0 {
if content != "" {
content += "\n"
}
content += strings.Join(extras, "\n")
}
pipelineInfo(ctx, "IntoChatMessage", "image_enrich_summary", map[string]interface{}{
"markdown_images": len(matches),
"additional_imgs": len(extras),
})
return content
}
// buildImageInfoXML returns XML-tagged caption / ocr for one image.
func buildImageInfoXML(img *types.ImageInfo) string {
var b strings.Builder
if img.Caption != "" {
b.WriteString(fmt.Sprintf("<image_caption>%s</image_caption>\n", img.Caption))
}
if img.OCRText != "" {
b.WriteString(fmt.Sprintf("<image_ocr>%s</image_ocr>\n", img.OCRText))
}
return b.String()
}
// buildImageInfoXMLWithURL wraps image info in an <image> element carrying the URL.
func buildImageInfoXMLWithURL(url string, img *types.ImageInfo) string {
inner := buildImageInfoXML(img)
if inner == "" {
return ""
}
return fmt.Sprintf("<image url=\"%s\">\n%s</image>", url, inner)
// enrichContentWithImageInfo delegates to the shared searchutil implementation.
func enrichContentWithImageInfo(_ context.Context, content string, imageInfoJSON string) string {
return searchutil.EnrichContentWithImageInfo(content, imageInfoJSON)
}
@@ -321,9 +321,13 @@ func (p *PluginQueryUnderstand) buildPrompts(chatManage *types.ChatManage, histo
queryContent := chatManage.Query
if len(chatManage.Images) > 0 {
queryContent += fmt.Sprintf("\n\n<images_uploaded count=\"%d\" />", len(chatManage.Images))
} else {
queryContent += "\n\n<no_image_attached />"
}
if len(chatManage.Attachments) > 0 {
queryContent += chatManage.Attachments.BuildPrompt()
} else {
queryContent += "\n<no_document_attached />"
}
vals := types.PlaceholderValues{
+44 -33
View File
@@ -3,7 +3,7 @@ package service
import (
"context"
"errors"
"sort"
"math/rand"
"strings"
"time"
@@ -503,22 +503,19 @@ func (s *customAgentService) GetSuggestedQuestions(
return s.truncateQuestions(result, limit), nil
}
// 3. Collect all candidate chunks from both FAQ and Document KBs,
// then sort by updated_at uniformly (not FAQ-first).
type candidate struct {
question types.SuggestedQuestion
updatedAt time.Time
}
var candidates []candidate
// 3. Collect candidate chunks from both FAQ and Document KBs,
// grouped by knowledge_id for diversity.
// knowledgeID -> list of questions
buckets := make(map[string][]types.SuggestedQuestion)
// Determine query scope
queryKBIDs := effectiveKBIDs
queryKnowledgeIDs := knowledgeIDs
// Fetch more than needed from each source, we'll merge-sort and truncate
fetchLimit := remaining * 2
if fetchLimit < 10 {
fetchLimit = 10
// Fetch a large pool so DB-level random sampling covers multiple documents.
fetchLimit := remaining * 5
if fetchLimit < 20 {
fetchLimit = 20
}
// Collect FAQ recommended chunks
@@ -537,13 +534,10 @@ func (s *customAgentService) GetSuggestedQuestions(
continue
}
seen[meta.StandardQuestion] = true
candidates = append(candidates, candidate{
question: types.SuggestedQuestion{
Question: meta.StandardQuestion,
Source: "faq",
KnowledgeBaseID: chunk.KnowledgeBaseID,
},
updatedAt: chunk.UpdatedAt,
buckets[chunk.KnowledgeID] = append(buckets[chunk.KnowledgeID], types.SuggestedQuestion{
Question: meta.StandardQuestion,
Source: "faq",
KnowledgeBaseID: chunk.KnowledgeBaseID,
})
}
}
@@ -565,28 +559,45 @@ func (s *customAgentService) GetSuggestedQuestions(
continue
}
seen[q] = true
candidates = append(candidates, candidate{
question: types.SuggestedQuestion{
Question: q,
Source: "document",
KnowledgeBaseID: chunk.KnowledgeBaseID,
},
updatedAt: chunk.UpdatedAt,
buckets[chunk.KnowledgeID] = append(buckets[chunk.KnowledgeID], types.SuggestedQuestion{
Question: q,
Source: "document",
KnowledgeBaseID: chunk.KnowledgeBaseID,
})
}
}
// 4. Sort all candidates by updated_at descending (newest first)
sort.Slice(candidates, func(i, j int) bool {
return candidates[i].updatedAt.After(candidates[j].updatedAt)
// 4. Shuffle within each bucket, then round-robin across buckets
// to ensure diversity across different documents.
bucketKeys := make([]string, 0, len(buckets))
for k, qs := range buckets {
bucketKeys = append(bucketKeys, k)
rand.Shuffle(len(qs), func(i, j int) { qs[i], qs[j] = qs[j], qs[i] })
buckets[k] = qs
}
rand.Shuffle(len(bucketKeys), func(i, j int) {
bucketKeys[i], bucketKeys[j] = bucketKeys[j], bucketKeys[i]
})
// 5. Pick top N
for _, c := range candidates {
if len(result) >= limit {
// Round-robin pick one question from each document in turn.
offsets := make(map[string]int, len(bucketKeys))
for len(result) < limit {
picked := false
for _, key := range bucketKeys {
if len(result) >= limit {
break
}
qs := buckets[key]
idx := offsets[key]
if idx < len(qs) {
result = append(result, qs[idx])
offsets[key] = idx + 1
picked = true
}
}
if !picked {
break
}
result = append(result, c.question)
}
return s.truncateQuestions(result, limit), nil
+35 -95
View File
@@ -13,7 +13,6 @@ import (
"net/url"
"os"
"path"
"regexp"
"runtime"
"slices"
"sort"
@@ -30,6 +29,7 @@ import (
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/models/embedding"
"github.com/Tencent/WeKnora/internal/searchutil"
"github.com/Tencent/WeKnora/internal/tracing"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
@@ -1943,7 +1943,7 @@ func (s *knowledgeService) processChunks(ctx context.Context,
}
// defaultMaxInputChars is the default maximum characters used as input for summary generation.
const defaultMaxInputChars = 16384
const defaultMaxInputChars = 1024 * 24
// getSummary generates a summary for knowledge content using an AI model
func (s *knowledgeService) getSummary(ctx context.Context,
@@ -1960,52 +1960,36 @@ func (s *knowledgeService) getSummary(ctx context.Context,
maxInputChars = s.config.Conversation.Summary.MaxInputChars
}
// concat chunk contents
chunkContents := ""
allImageInfos := make([]*types.ImageInfo, 0)
// then, sort chunks by StartAt
// Sort chunks by StartAt for proper concatenation
sortedChunks := make([]*types.Chunk, len(chunks))
copy(sortedChunks, chunks)
sort.Slice(sortedChunks, func(i, j int) bool {
return sortedChunks[i].StartAt < sortedChunks[j].StartAt
})
// concat ALL chunk contents (no early truncation) and collect image infos
// Concatenate original chunk contents by StartAt offset to reconstruct the
// document, then enrich with image info in a second pass. Enrichment must
// happen AFTER concatenation because StartAt is based on original document
// offsets — enriched (longer) content would break the positioning.
chunkContents := ""
for _, chunk := range sortedChunks {
// Ensure we don't slice beyond the current content length
runes := []rune(chunkContents)
if chunk.StartAt <= len(runes) {
chunkContents = string(runes[:chunk.StartAt]) + chunk.Content
} else {
// If StartAt is beyond current content, just append
chunkContents = chunkContents + chunk.Content
}
if chunk.ImageInfo != "" {
var images []*types.ImageInfo
if err := json.Unmarshal([]byte(chunk.ImageInfo), &images); err == nil {
allImageInfos = append(allImageInfos, images...)
}
}
}
// remove markdown image syntax
re := regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`)
chunkContents = re.ReplaceAllString(chunkContents, "")
// collect all image infos
if len(allImageInfos) > 0 {
// add image infos to chunk contents
var imageAnnotations string
for _, img := range allImageInfos {
if img.Caption != "" {
imageAnnotations += fmt.Sprintf("\n[Image Description: %s]", img.Caption)
}
if img.OCRText != "" {
imageAnnotations += fmt.Sprintf("\n[Image OCR Text: %s]", img.OCRText)
}
}
// concat chunk contents and image annotations
chunkContents = chunkContents + imageAnnotations
// Collect image_info from image_ocr/image_caption children and enrich
chunkIDs := make([]string, len(sortedChunks))
for i, c := range sortedChunks {
chunkIDs[i] = c.ID
}
imageInfoMap := searchutil.CollectImageInfoByChunkIDs(ctx, s.chunkRepo, knowledge.TenantID, chunkIDs)
mergedImageInfo := searchutil.MergeImageInfoJSON(imageInfoMap)
if mergedImageInfo != "" {
chunkContents = searchutil.EnrichContentCaptionOnly(chunkContents, mergedImageInfo)
}
// Apply length limit: sample long content to fit within maxInputChars
@@ -2108,63 +2092,6 @@ func sampleLongContent(content string, maxChars int) string {
return head + omitMarker + middle + omitMarker + tail
}
// enqueueQuestionGenerationTask enqueues an async task for question generation
func (s *knowledgeService) enqueueQuestionGenerationTask(ctx context.Context,
kbID, knowledgeID string, questionCount int,
) {
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
lang, _ := types.LanguageFromContext(ctx)
payload := types.QuestionGenerationPayload{
TenantID: tenantID,
KnowledgeBaseID: kbID,
KnowledgeID: knowledgeID,
QuestionCount: questionCount,
Language: lang,
}
payloadBytes, err := json.Marshal(payload)
if err != nil {
logger.Errorf(ctx, "Failed to marshal question generation payload: %v", err)
return
}
task := asynq.NewTask(types.TypeQuestionGeneration, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3))
info, err := s.task.Enqueue(task)
if err != nil {
logger.Errorf(ctx, "Failed to enqueue question generation task: %v", err)
return
}
logger.Infof(ctx, "Enqueued question generation task: %s for knowledge: %s", info.ID, knowledgeID)
}
// enqueueSummaryGenerationTask enqueues an async task for summary generation
func (s *knowledgeService) enqueueSummaryGenerationTask(ctx context.Context,
kbID, knowledgeID string,
) {
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
lang, _ := types.LanguageFromContext(ctx)
payload := types.SummaryGenerationPayload{
TenantID: tenantID,
KnowledgeBaseID: kbID,
KnowledgeID: knowledgeID,
Language: lang,
}
payloadBytes, err := json.Marshal(payload)
if err != nil {
logger.Errorf(ctx, "Failed to marshal summary generation payload: %v", err)
return
}
task := asynq.NewTask(types.TypeSummaryGeneration, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3))
info, err := s.task.Enqueue(task)
if err != nil {
logger.Errorf(ctx, "Failed to enqueue summary generation task: %v", err)
return
}
logger.Infof(ctx, "Enqueued summary generation task: %s for knowledge: %s", info.ID, knowledgeID)
}
// ProcessSummaryGeneration handles async summary generation task
func (s *knowledgeService) ProcessSummaryGeneration(ctx context.Context, t *asynq.Task) error {
var payload types.SummaryGenerationPayload
@@ -2452,27 +2379,40 @@ func (s *knowledgeService) ProcessQuestionGeneration(ctx context.Context, t *asy
questionCount = 10
}
// Collect image info for all text chunks so question generation can
// see caption / OCR text instead of bare image links.
textChunkIDs := make([]string, len(textChunks))
for i, c := range textChunks {
textChunkIDs[i] = c.ID
}
imageInfoMap := searchutil.CollectImageInfoByChunkIDs(ctx, s.chunkRepo, payload.TenantID, textChunkIDs)
enrichContent := func(chunk *types.Chunk) string {
if info, ok := imageInfoMap[chunk.ID]; ok && info != "" {
return searchutil.EnrichContentWithImageInfo(chunk.Content, info)
}
return chunk.Content
}
// Generate questions for each chunk with context
var indexInfoList []*types.IndexInfo
for i, chunk := range textChunks {
// Build context from adjacent chunks
var prevContent, nextContent string
if i > 0 {
prevContent = textChunks[i-1].Content
// Limit context size
prevContent = enrichContent(textChunks[i-1])
if len(prevContent) > 500 {
prevContent = prevContent[len(prevContent)-500:]
}
}
if i < len(textChunks)-1 {
nextContent = textChunks[i+1].Content
// Limit context size
nextContent = enrichContent(textChunks[i+1])
if len(nextContent) > 500 {
nextContent = nextContent[:500]
}
}
questions, err := s.generateQuestionsWithContext(ctx, chatModel, chunk.Content, prevContent, nextContent, knowledge.Title, questionCount)
questions, err := s.generateQuestionsWithContext(ctx, chatModel, enrichContent(chunk), prevContent, nextContent, knowledge.Title, questionCount)
if err != nil {
logger.Warnf(ctx, "Failed to generate questions for chunk %s: %v", chunk.ID, err)
continue
+184
View File
@@ -3,11 +3,17 @@ package searchutil
import (
"context"
"encoding/json"
"fmt"
"regexp"
"strings"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
// MarkdownImageRegex matches Markdown image links: ![alt](url)
var MarkdownImageRegex = regexp.MustCompile(`!\[([^\]]*)\]\(([^)]+)\)`)
// CollectImageInfoByChunkIDs collects merged image_info JSON for each given
// chunk ID by querying child chunks (image_ocr / image_caption). It supports
// two-level resolution:
@@ -154,3 +160,181 @@ func EnrichSearchResultsImageInfo(
}
}
}
// MergeImageInfoJSON combines per-chunk image_info JSON strings (from
// CollectImageInfoByChunkIDs) into a single JSON array, deduplicating by URL.
func MergeImageInfoJSON(perChunk map[string]string) string {
if len(perChunk) == 0 {
return ""
}
seen := make(map[string]bool)
var all []types.ImageInfo
for _, raw := range perChunk {
var infos []types.ImageInfo
if err := json.Unmarshal([]byte(raw), &infos); err != nil {
continue
}
for _, info := range infos {
key := info.URL
if key == "" {
key = info.OriginalURL
}
if key != "" && !seen[key] {
seen[key] = true
all = append(all, info)
}
}
}
if len(all) == 0 {
return ""
}
data, err := json.Marshal(all)
if err != nil {
return ""
}
return string(data)
}
// EnrichContentWithImageInfo embeds image info as XML tags into text content.
// Inline Markdown image links get wrapped in <image> with <image_caption> / <image_ocr>;
// images not found in content are appended as <image> blocks.
func EnrichContentWithImageInfo(content string, imageInfoJSON string) string {
var imageInfos []types.ImageInfo
if err := json.Unmarshal([]byte(imageInfoJSON), &imageInfos); err != nil {
return content
}
if len(imageInfos) == 0 {
return content
}
imageInfoMap := make(map[string]*types.ImageInfo)
for i := range imageInfos {
if imageInfos[i].URL != "" {
imageInfoMap[imageInfos[i].URL] = &imageInfos[i]
}
if imageInfos[i].OriginalURL != "" {
imageInfoMap[imageInfos[i].OriginalURL] = &imageInfos[i]
}
}
matches := MarkdownImageRegex.FindAllStringSubmatch(content, -1)
processedURLs := make(map[string]bool)
for _, match := range matches {
if len(match) < 3 {
continue
}
imgURL := match[2]
processedURLs[imgURL] = true
imgInfo, found := imageInfoMap[imgURL]
var b strings.Builder
b.WriteString(fmt.Sprintf("<image url=\"%s\">\n", imgURL))
b.WriteString(fmt.Sprintf("<image_original>%s</image_original>\n", match[0]))
if found && imgInfo != nil {
b.WriteString(BuildImageInfoXML(imgInfo))
}
b.WriteString("</image>")
content = strings.Replace(content, match[0], b.String(), 1)
}
var extras []string
for _, imgInfo := range imageInfos {
if processedURLs[imgInfo.URL] || processedURLs[imgInfo.OriginalURL] {
continue
}
url := imgInfo.URL
if url == "" {
url = imgInfo.OriginalURL
}
if block := BuildImageInfoXMLWithURL(url, &imgInfo); block != "" {
extras = append(extras, block)
}
}
if len(extras) > 0 {
if content != "" {
content += "\n"
}
content += strings.Join(extras, "\n")
}
return content
}
// BuildImageInfoXML returns XML-tagged caption / ocr for one image.
func BuildImageInfoXML(img *types.ImageInfo) string {
var b strings.Builder
if img.Caption != "" {
b.WriteString(fmt.Sprintf("<image_caption>%s</image_caption>\n", img.Caption))
}
if img.OCRText != "" {
b.WriteString(fmt.Sprintf("<image_ocr>%s</image_ocr>\n", img.OCRText))
}
return b.String()
}
// BuildImageInfoXMLWithURL wraps image info in an <image> element carrying the URL.
func BuildImageInfoXMLWithURL(url string, img *types.ImageInfo) string {
inner := BuildImageInfoXML(img)
if inner == "" {
return ""
}
return fmt.Sprintf("<image url=\"%s\">\n%s</image>", url, inner)
}
// EnrichContentCaptionOnly is like EnrichContentWithImageInfo but only
// includes image captions (no OCR text). Original content (including Markdown
// image links) is preserved. Useful for summary generation where OCR would
// add too much noise.
func EnrichContentCaptionOnly(content string, imageInfoJSON string) string {
var imageInfos []types.ImageInfo
if err := json.Unmarshal([]byte(imageInfoJSON), &imageInfos); err != nil {
return content
}
if len(imageInfos) == 0 {
return content
}
imageInfoMap := make(map[string]*types.ImageInfo)
for i := range imageInfos {
if imageInfos[i].URL != "" {
imageInfoMap[imageInfos[i].URL] = &imageInfos[i]
}
if imageInfos[i].OriginalURL != "" {
imageInfoMap[imageInfos[i].OriginalURL] = &imageInfos[i]
}
}
matches := MarkdownImageRegex.FindAllStringSubmatch(content, -1)
processedURLs := make(map[string]bool)
for _, match := range matches {
if len(match) < 3 {
continue
}
imgURL := match[2]
processedURLs[imgURL] = true
imgInfo, found := imageInfoMap[imgURL]
if found && imgInfo != nil && imgInfo.Caption != "" {
replacement := match[0] + "\n" + fmt.Sprintf("<image_caption>%s</image_caption>", imgInfo.Caption)
content = strings.Replace(content, match[0], replacement, 1)
}
}
var extras []string
for _, imgInfo := range imageInfos {
if processedURLs[imgInfo.URL] || processedURLs[imgInfo.OriginalURL] {
continue
}
if imgInfo.Caption != "" {
extras = append(extras, fmt.Sprintf("<image_caption>%s</image_caption>", imgInfo.Caption))
}
}
if len(extras) > 0 {
if content != "" {
content += "\n"
}
content += strings.Join(extras, "\n")
}
return content
}