diff --git a/config/prompt_templates/rewrite.yaml b/config/prompt_templates/rewrite.yaml index 94b9950e..e026f347 100644 --- a/config/prompt_templates/rewrite.yaml +++ b/config/prompt_templates/rewrite.yaml @@ -19,11 +19,11 @@ templates: default: true content: | You are an intelligent assistant that performs THREE tasks on the user's question: - 1. Rewrite the question (coreference resolution and ellipsis completion) + 1. Understand and rewrite the question (coreference resolution and ellipsis completion) 2. Classify the intent of the question 3. Analyze attached images (when present) - ## Task 1: Rewriting Goals + ## Task 1: Query Understanding Based on the conversation history, rewrite the current user question: - Perform coreference resolution: replace pronouns such as "it", "this", "that", "they", "them", etc. with explicit subjects - Complete omitted key information to ensure the question is semantically complete @@ -43,8 +43,8 @@ templates: 4. `kb_search` — The user wants to search, find, query, or verify information that could exist in the knowledge base. **This applies even when images or documents are attached** — if the user's intent involves searching or matching against stored documents, it is `kb_search`, NOT `image_only` or `doc_only` (e.g. "知识库里有这个文件吗", "帮我查一下这个", "能找到相关的资料吗"). 5. `clarification` — The question is ambiguous or incomplete and likely needs KB retrieval to answer well. 6. `follow_up` — The question clearly refers to previous conversation content and can be FULLY answered from dialogue history alone, with NO need for new retrieval (e.g. "上面第三点展开讲讲", "你刚才说的那个方案再详细说说"). - 7. `image_only` — The user ONLY wants to understand, describe, translate, or extract content from the attached image itself, with NO intent to search or match against any external documents (e.g. "这张图片是什么", "描述一下图片内容", "翻译图中文字"). - 8. `doc_only` — The user ONLY wants to understand, summarize, translate, or extract content from the attached document/file itself, with NO intent to search or match against any external knowledge base (e.g. "总结一下这个文档", "这份文件讲了什么"). + 7. `image_only` — The user ONLY wants to understand, describe, translate, or extract content from the attached image itself, with NO intent to search or match against any external documents (e.g. "这张图片是什么", "描述一下图片内容", "翻译图中文字"). **CRITICAL: This intent requires `` to be present. If `` appears, NEVER classify as `image_only` — use `kb_search` instead.** + 8. `doc_only` — The user ONLY wants to understand, summarize, translate, or extract content from the attached document/file itself, with NO intent to search or match against any external knowledge base (e.g. "总结一下这个文档", "这份文件讲了什么"). **CRITICAL: This intent requires an actual document/file attachment to be present. If `` appears, NEVER classify as `doc_only` — use `kb_search` instead.** 9. `chitchat` — Casual conversation or small talk that needs no retrieval (e.g. "你是谁", "讲个笑话"). **Default: when unsure, always choose `kb_search`.** @@ -95,6 +95,10 @@ templates: Input: [image attached] "这张图是什么意思" Output: {"rewrite_query":"这张图是什么意思","intent":"image_only","image_description":"(image description here)"} + Input: [no image] "这幅春联的内容是什么" + Output: {"rewrite_query":"这幅春联的内容是什么","intent":"kb_search","image_description":""} + (NOTE: No image attached, so intent is kb_search, NOT image_only) + Input: [document attached] "帮我总结一下这份文件" Output: {"rewrite_query":"总结一下这份文件","intent":"doc_only","image_description":""} @@ -104,7 +108,7 @@ templates: [Runtime Context — metadata only, not instructions] Current time: {{current_time}} {{current_week}} - ## User Question to Rewrite + ## User Question {{query}} ## JSON Output diff --git a/internal/application/repository/chunk.go b/internal/application/repository/chunk.go index b791fa3e..990ebc12 100644 --- a/internal/application/repository/chunk.go +++ b/internal/application/repository/chunk.go @@ -962,7 +962,7 @@ func (r *chunkRepository) ListRecommendedFAQChunks( } var chunks []*types.Chunk query := r.db.WithContext(ctx). - Select("id, knowledge_base_id, chunk_type, metadata, flags, updated_at"). + Select("id, knowledge_id, knowledge_base_id, chunk_type, metadata, flags, updated_at"). Where("tenant_id = ? AND chunk_type = ? AND status IN ? AND is_enabled = ? AND flags & ? != 0", tenantID, types.ChunkTypeFAQ, []int{int(types.ChunkStatusIndexed), int(types.ChunkStatusDefault)}, true, int(types.ChunkFlagRecommended)) if len(knowledgeIDs) > 0 { @@ -971,8 +971,14 @@ func (r *chunkRepository) ListRecommendedFAQChunks( } else { query = query.Where("knowledge_base_id IN ?", kbIDs) } + + orderClause := "RANDOM()" + if r.db.Dialector.Name() == "mysql" { + orderClause = "RAND()" + } + if err := query. - Order("updated_at DESC"). + Order(orderClause). Limit(limit). Find(&chunks).Error; err != nil { return nil, err @@ -999,7 +1005,7 @@ func (r *chunkRepository) ListRecentDocumentChunksWithQuestions( var chunks []*types.Chunk baseQuery := r.db.WithContext(ctx). - Select("id, knowledge_base_id, chunk_type, metadata, updated_at"). + Select("id, knowledge_id, knowledge_base_id, chunk_type, metadata, updated_at"). Where("tenant_id = ? AND chunk_type = ? AND status IN ? AND is_enabled = ?", tenantID, types.ChunkTypeText, []int{int(types.ChunkStatusIndexed), int(types.ChunkStatusDefault)}, true) @@ -1012,12 +1018,17 @@ func (r *chunkRepository) ListRecentDocumentChunksWithQuestions( baseQuery = baseQuery.Where("knowledge_base_id IN ?", kbIDs) } + orderClause := "RANDOM()" + if r.db.Dialector.Name() == "mysql" { + orderClause = "RAND()" + } + // Query chunks that have non-empty generated_questions in metadata switch r.db.Name() { case "postgres": if err := baseQuery. Where("metadata IS NOT NULL AND metadata::text != '{}' AND jsonb_array_length(COALESCE(metadata->'generated_questions', '[]'::jsonb)) > 0"). - Order("updated_at DESC"). + Order(orderClause). Limit(limit). Find(&chunks).Error; err != nil { return nil, err @@ -1025,7 +1036,7 @@ func (r *chunkRepository) ListRecentDocumentChunksWithQuestions( case "mysql": if err := baseQuery. Where("metadata IS NOT NULL AND JSON_LENGTH(JSON_EXTRACT(metadata, '$.generated_questions')) > 0"). - Order("updated_at DESC"). + Order(orderClause). Limit(limit). Find(&chunks).Error; err != nil { return nil, err @@ -1033,7 +1044,7 @@ func (r *chunkRepository) ListRecentDocumentChunksWithQuestions( default: // sqlite if err := baseQuery. Where("metadata IS NOT NULL AND json_array_length(json_extract(metadata, '$.generated_questions')) > 0"). - Order("updated_at DESC"). + Order(orderClause). Limit(limit). Find(&chunks).Error; err != nil { return nil, err diff --git a/internal/application/service/chat_pipeline/into_chat_message.go b/internal/application/service/chat_pipeline/into_chat_message.go index 9139a59c..9baec373 100644 --- a/internal/application/service/chat_pipeline/into_chat_message.go +++ b/internal/application/service/chat_pipeline/into_chat_message.go @@ -2,11 +2,10 @@ package chatpipeline import ( "context" - "encoding/json" "fmt" - "regexp" "strings" + "github.com/Tencent/WeKnora/internal/searchutil" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" "github.com/Tencent/WeKnora/internal/utils" @@ -303,104 +302,7 @@ func getEnrichedPassageForChat(ctx context.Context, result *types.SearchResult) return enrichContentWithImageInfo(ctx, result.Content, result.ImageInfo) } -// 正则表达式用于匹配Markdown图片链接 -var markdownImageRegex = regexp.MustCompile(`!\[([^\]]*)\]\(([^)]+)\)`) - -// enrichContentWithImageInfo 将图片信息以 XML 标签的形式嵌入文本内容。 -// 对于内容中已有的 Markdown 图片链接,在其后附加 / ; -// 对于未出现在内容中的图片,以 块追加到末尾。 -func enrichContentWithImageInfo(ctx context.Context, content string, imageInfoJSON string) string { - var imageInfos []types.ImageInfo - if err := json.Unmarshal([]byte(imageInfoJSON), &imageInfos); err != nil { - pipelineWarn(ctx, "IntoChatMessage", "image_parse_error", map[string]interface{}{ - "error": err.Error(), - }) - return content - } - if len(imageInfos) == 0 { - return content - } - - imageInfoMap := make(map[string]*types.ImageInfo) - for i := range imageInfos { - if imageInfos[i].URL != "" { - imageInfoMap[imageInfos[i].URL] = &imageInfos[i] - } - if imageInfos[i].OriginalURL != "" { - imageInfoMap[imageInfos[i].OriginalURL] = &imageInfos[i] - } - } - - matches := markdownImageRegex.FindAllStringSubmatch(content, -1) - processedURLs := make(map[string]bool) - - pipelineInfo(ctx, "IntoChatMessage", "image_markdown_links", map[string]interface{}{ - "match_count": len(matches), - }) - - for _, match := range matches { - if len(match) < 3 { - continue - } - imgURL := match[2] - processedURLs[imgURL] = true - - imgInfo, found := imageInfoMap[imgURL] - var b strings.Builder - b.WriteString(fmt.Sprintf("\n", imgURL)) - b.WriteString(fmt.Sprintf("%s\n", match[0])) - if found && imgInfo != nil { - b.WriteString(buildImageInfoXML(imgInfo)) - } - b.WriteString("") - content = strings.Replace(content, match[0], b.String(), 1) - } - - // Append image info not found as inline Markdown links - var extras []string - for _, imgInfo := range imageInfos { - if processedURLs[imgInfo.URL] || processedURLs[imgInfo.OriginalURL] { - continue - } - url := imgInfo.URL - if url == "" { - url = imgInfo.OriginalURL - } - if block := buildImageInfoXMLWithURL(url, &imgInfo); block != "" { - extras = append(extras, block) - } - } - if len(extras) > 0 { - if content != "" { - content += "\n" - } - content += strings.Join(extras, "\n") - } - - pipelineInfo(ctx, "IntoChatMessage", "image_enrich_summary", map[string]interface{}{ - "markdown_images": len(matches), - "additional_imgs": len(extras), - }) - return content -} - -// buildImageInfoXML returns XML-tagged caption / ocr for one image. -func buildImageInfoXML(img *types.ImageInfo) string { - var b strings.Builder - if img.Caption != "" { - b.WriteString(fmt.Sprintf("%s\n", img.Caption)) - } - if img.OCRText != "" { - b.WriteString(fmt.Sprintf("%s\n", img.OCRText)) - } - return b.String() -} - -// buildImageInfoXMLWithURL wraps image info in an element carrying the URL. -func buildImageInfoXMLWithURL(url string, img *types.ImageInfo) string { - inner := buildImageInfoXML(img) - if inner == "" { - return "" - } - return fmt.Sprintf("\n%s", url, inner) +// enrichContentWithImageInfo delegates to the shared searchutil implementation. +func enrichContentWithImageInfo(_ context.Context, content string, imageInfoJSON string) string { + return searchutil.EnrichContentWithImageInfo(content, imageInfoJSON) } diff --git a/internal/application/service/chat_pipeline/query_understand.go b/internal/application/service/chat_pipeline/query_understand.go index 795b379c..805fa49b 100644 --- a/internal/application/service/chat_pipeline/query_understand.go +++ b/internal/application/service/chat_pipeline/query_understand.go @@ -321,9 +321,13 @@ func (p *PluginQueryUnderstand) buildPrompts(chatManage *types.ChatManage, histo queryContent := chatManage.Query if len(chatManage.Images) > 0 { queryContent += fmt.Sprintf("\n\n", len(chatManage.Images)) + } else { + queryContent += "\n\n" } if len(chatManage.Attachments) > 0 { queryContent += chatManage.Attachments.BuildPrompt() + } else { + queryContent += "\n" } vals := types.PlaceholderValues{ diff --git a/internal/application/service/custom_agent.go b/internal/application/service/custom_agent.go index b302d495..bcf74138 100644 --- a/internal/application/service/custom_agent.go +++ b/internal/application/service/custom_agent.go @@ -3,7 +3,7 @@ package service import ( "context" "errors" - "sort" + "math/rand" "strings" "time" @@ -503,22 +503,19 @@ func (s *customAgentService) GetSuggestedQuestions( return s.truncateQuestions(result, limit), nil } - // 3. Collect all candidate chunks from both FAQ and Document KBs, - // then sort by updated_at uniformly (not FAQ-first). - type candidate struct { - question types.SuggestedQuestion - updatedAt time.Time - } - var candidates []candidate + // 3. Collect candidate chunks from both FAQ and Document KBs, + // grouped by knowledge_id for diversity. + // knowledgeID -> list of questions + buckets := make(map[string][]types.SuggestedQuestion) // Determine query scope queryKBIDs := effectiveKBIDs queryKnowledgeIDs := knowledgeIDs - // Fetch more than needed from each source, we'll merge-sort and truncate - fetchLimit := remaining * 2 - if fetchLimit < 10 { - fetchLimit = 10 + // Fetch a large pool so DB-level random sampling covers multiple documents. + fetchLimit := remaining * 5 + if fetchLimit < 20 { + fetchLimit = 20 } // Collect FAQ recommended chunks @@ -537,13 +534,10 @@ func (s *customAgentService) GetSuggestedQuestions( continue } seen[meta.StandardQuestion] = true - candidates = append(candidates, candidate{ - question: types.SuggestedQuestion{ - Question: meta.StandardQuestion, - Source: "faq", - KnowledgeBaseID: chunk.KnowledgeBaseID, - }, - updatedAt: chunk.UpdatedAt, + buckets[chunk.KnowledgeID] = append(buckets[chunk.KnowledgeID], types.SuggestedQuestion{ + Question: meta.StandardQuestion, + Source: "faq", + KnowledgeBaseID: chunk.KnowledgeBaseID, }) } } @@ -565,28 +559,45 @@ func (s *customAgentService) GetSuggestedQuestions( continue } seen[q] = true - candidates = append(candidates, candidate{ - question: types.SuggestedQuestion{ - Question: q, - Source: "document", - KnowledgeBaseID: chunk.KnowledgeBaseID, - }, - updatedAt: chunk.UpdatedAt, + buckets[chunk.KnowledgeID] = append(buckets[chunk.KnowledgeID], types.SuggestedQuestion{ + Question: q, + Source: "document", + KnowledgeBaseID: chunk.KnowledgeBaseID, }) } } - // 4. Sort all candidates by updated_at descending (newest first) - sort.Slice(candidates, func(i, j int) bool { - return candidates[i].updatedAt.After(candidates[j].updatedAt) + // 4. Shuffle within each bucket, then round-robin across buckets + // to ensure diversity across different documents. + bucketKeys := make([]string, 0, len(buckets)) + for k, qs := range buckets { + bucketKeys = append(bucketKeys, k) + rand.Shuffle(len(qs), func(i, j int) { qs[i], qs[j] = qs[j], qs[i] }) + buckets[k] = qs + } + rand.Shuffle(len(bucketKeys), func(i, j int) { + bucketKeys[i], bucketKeys[j] = bucketKeys[j], bucketKeys[i] }) - // 5. Pick top N - for _, c := range candidates { - if len(result) >= limit { + // Round-robin pick one question from each document in turn. + offsets := make(map[string]int, len(bucketKeys)) + for len(result) < limit { + picked := false + for _, key := range bucketKeys { + if len(result) >= limit { + break + } + qs := buckets[key] + idx := offsets[key] + if idx < len(qs) { + result = append(result, qs[idx]) + offsets[key] = idx + 1 + picked = true + } + } + if !picked { break } - result = append(result, c.question) } return s.truncateQuestions(result, limit), nil diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index 703e443b..33cb7d3c 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -13,7 +13,6 @@ import ( "net/url" "os" "path" - "regexp" "runtime" "slices" "sort" @@ -30,6 +29,7 @@ import ( "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/models/chat" "github.com/Tencent/WeKnora/internal/models/embedding" + "github.com/Tencent/WeKnora/internal/searchutil" "github.com/Tencent/WeKnora/internal/tracing" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" @@ -1943,7 +1943,7 @@ func (s *knowledgeService) processChunks(ctx context.Context, } // defaultMaxInputChars is the default maximum characters used as input for summary generation. -const defaultMaxInputChars = 16384 +const defaultMaxInputChars = 1024 * 24 // getSummary generates a summary for knowledge content using an AI model func (s *knowledgeService) getSummary(ctx context.Context, @@ -1960,52 +1960,36 @@ func (s *knowledgeService) getSummary(ctx context.Context, maxInputChars = s.config.Conversation.Summary.MaxInputChars } - // concat chunk contents - chunkContents := "" - allImageInfos := make([]*types.ImageInfo, 0) - - // then, sort chunks by StartAt + // Sort chunks by StartAt for proper concatenation sortedChunks := make([]*types.Chunk, len(chunks)) copy(sortedChunks, chunks) sort.Slice(sortedChunks, func(i, j int) bool { return sortedChunks[i].StartAt < sortedChunks[j].StartAt }) - // concat ALL chunk contents (no early truncation) and collect image infos + // Concatenate original chunk contents by StartAt offset to reconstruct the + // document, then enrich with image info in a second pass. Enrichment must + // happen AFTER concatenation because StartAt is based on original document + // offsets — enriched (longer) content would break the positioning. + chunkContents := "" for _, chunk := range sortedChunks { - // Ensure we don't slice beyond the current content length runes := []rune(chunkContents) if chunk.StartAt <= len(runes) { chunkContents = string(runes[:chunk.StartAt]) + chunk.Content } else { - // If StartAt is beyond current content, just append chunkContents = chunkContents + chunk.Content } - if chunk.ImageInfo != "" { - var images []*types.ImageInfo - if err := json.Unmarshal([]byte(chunk.ImageInfo), &images); err == nil { - allImageInfos = append(allImageInfos, images...) - } - } } - // remove markdown image syntax - re := regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`) - chunkContents = re.ReplaceAllString(chunkContents, "") - // collect all image infos - if len(allImageInfos) > 0 { - // add image infos to chunk contents - var imageAnnotations string - for _, img := range allImageInfos { - if img.Caption != "" { - imageAnnotations += fmt.Sprintf("\n[Image Description: %s]", img.Caption) - } - if img.OCRText != "" { - imageAnnotations += fmt.Sprintf("\n[Image OCR Text: %s]", img.OCRText) - } - } - // concat chunk contents and image annotations - chunkContents = chunkContents + imageAnnotations + // Collect image_info from image_ocr/image_caption children and enrich + chunkIDs := make([]string, len(sortedChunks)) + for i, c := range sortedChunks { + chunkIDs[i] = c.ID + } + imageInfoMap := searchutil.CollectImageInfoByChunkIDs(ctx, s.chunkRepo, knowledge.TenantID, chunkIDs) + mergedImageInfo := searchutil.MergeImageInfoJSON(imageInfoMap) + if mergedImageInfo != "" { + chunkContents = searchutil.EnrichContentCaptionOnly(chunkContents, mergedImageInfo) } // Apply length limit: sample long content to fit within maxInputChars @@ -2108,63 +2092,6 @@ func sampleLongContent(content string, maxChars int) string { return head + omitMarker + middle + omitMarker + tail } -// enqueueQuestionGenerationTask enqueues an async task for question generation -func (s *knowledgeService) enqueueQuestionGenerationTask(ctx context.Context, - kbID, knowledgeID string, questionCount int, -) { - tenantID := ctx.Value(types.TenantIDContextKey).(uint64) - lang, _ := types.LanguageFromContext(ctx) - payload := types.QuestionGenerationPayload{ - TenantID: tenantID, - KnowledgeBaseID: kbID, - KnowledgeID: knowledgeID, - QuestionCount: questionCount, - Language: lang, - } - - payloadBytes, err := json.Marshal(payload) - if err != nil { - logger.Errorf(ctx, "Failed to marshal question generation payload: %v", err) - return - } - - task := asynq.NewTask(types.TypeQuestionGeneration, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3)) - info, err := s.task.Enqueue(task) - if err != nil { - logger.Errorf(ctx, "Failed to enqueue question generation task: %v", err) - return - } - logger.Infof(ctx, "Enqueued question generation task: %s for knowledge: %s", info.ID, knowledgeID) -} - -// enqueueSummaryGenerationTask enqueues an async task for summary generation -func (s *knowledgeService) enqueueSummaryGenerationTask(ctx context.Context, - kbID, knowledgeID string, -) { - tenantID := ctx.Value(types.TenantIDContextKey).(uint64) - lang, _ := types.LanguageFromContext(ctx) - payload := types.SummaryGenerationPayload{ - TenantID: tenantID, - KnowledgeBaseID: kbID, - KnowledgeID: knowledgeID, - Language: lang, - } - - payloadBytes, err := json.Marshal(payload) - if err != nil { - logger.Errorf(ctx, "Failed to marshal summary generation payload: %v", err) - return - } - - task := asynq.NewTask(types.TypeSummaryGeneration, payloadBytes, asynq.Queue("low"), asynq.MaxRetry(3)) - info, err := s.task.Enqueue(task) - if err != nil { - logger.Errorf(ctx, "Failed to enqueue summary generation task: %v", err) - return - } - logger.Infof(ctx, "Enqueued summary generation task: %s for knowledge: %s", info.ID, knowledgeID) -} - // ProcessSummaryGeneration handles async summary generation task func (s *knowledgeService) ProcessSummaryGeneration(ctx context.Context, t *asynq.Task) error { var payload types.SummaryGenerationPayload @@ -2452,27 +2379,40 @@ func (s *knowledgeService) ProcessQuestionGeneration(ctx context.Context, t *asy questionCount = 10 } + // Collect image info for all text chunks so question generation can + // see caption / OCR text instead of bare image links. + textChunkIDs := make([]string, len(textChunks)) + for i, c := range textChunks { + textChunkIDs[i] = c.ID + } + imageInfoMap := searchutil.CollectImageInfoByChunkIDs(ctx, s.chunkRepo, payload.TenantID, textChunkIDs) + + enrichContent := func(chunk *types.Chunk) string { + if info, ok := imageInfoMap[chunk.ID]; ok && info != "" { + return searchutil.EnrichContentWithImageInfo(chunk.Content, info) + } + return chunk.Content + } + // Generate questions for each chunk with context var indexInfoList []*types.IndexInfo for i, chunk := range textChunks { // Build context from adjacent chunks var prevContent, nextContent string if i > 0 { - prevContent = textChunks[i-1].Content - // Limit context size + prevContent = enrichContent(textChunks[i-1]) if len(prevContent) > 500 { prevContent = prevContent[len(prevContent)-500:] } } if i < len(textChunks)-1 { - nextContent = textChunks[i+1].Content - // Limit context size + nextContent = enrichContent(textChunks[i+1]) if len(nextContent) > 500 { nextContent = nextContent[:500] } } - questions, err := s.generateQuestionsWithContext(ctx, chatModel, chunk.Content, prevContent, nextContent, knowledge.Title, questionCount) + questions, err := s.generateQuestionsWithContext(ctx, chatModel, enrichContent(chunk), prevContent, nextContent, knowledge.Title, questionCount) if err != nil { logger.Warnf(ctx, "Failed to generate questions for chunk %s: %v", chunk.ID, err) continue diff --git a/internal/searchutil/imageinfo.go b/internal/searchutil/imageinfo.go index 18295ec2..b526ff91 100644 --- a/internal/searchutil/imageinfo.go +++ b/internal/searchutil/imageinfo.go @@ -3,11 +3,17 @@ package searchutil import ( "context" "encoding/json" + "fmt" + "regexp" + "strings" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" ) +// MarkdownImageRegex matches Markdown image links: ![alt](url) +var MarkdownImageRegex = regexp.MustCompile(`!\[([^\]]*)\]\(([^)]+)\)`) + // CollectImageInfoByChunkIDs collects merged image_info JSON for each given // chunk ID by querying child chunks (image_ocr / image_caption). It supports // two-level resolution: @@ -154,3 +160,181 @@ func EnrichSearchResultsImageInfo( } } } + +// MergeImageInfoJSON combines per-chunk image_info JSON strings (from +// CollectImageInfoByChunkIDs) into a single JSON array, deduplicating by URL. +func MergeImageInfoJSON(perChunk map[string]string) string { + if len(perChunk) == 0 { + return "" + } + seen := make(map[string]bool) + var all []types.ImageInfo + for _, raw := range perChunk { + var infos []types.ImageInfo + if err := json.Unmarshal([]byte(raw), &infos); err != nil { + continue + } + for _, info := range infos { + key := info.URL + if key == "" { + key = info.OriginalURL + } + if key != "" && !seen[key] { + seen[key] = true + all = append(all, info) + } + } + } + if len(all) == 0 { + return "" + } + data, err := json.Marshal(all) + if err != nil { + return "" + } + return string(data) +} + +// EnrichContentWithImageInfo embeds image info as XML tags into text content. +// Inline Markdown image links get wrapped in with / ; +// images not found in content are appended as blocks. +func EnrichContentWithImageInfo(content string, imageInfoJSON string) string { + var imageInfos []types.ImageInfo + if err := json.Unmarshal([]byte(imageInfoJSON), &imageInfos); err != nil { + return content + } + if len(imageInfos) == 0 { + return content + } + + imageInfoMap := make(map[string]*types.ImageInfo) + for i := range imageInfos { + if imageInfos[i].URL != "" { + imageInfoMap[imageInfos[i].URL] = &imageInfos[i] + } + if imageInfos[i].OriginalURL != "" { + imageInfoMap[imageInfos[i].OriginalURL] = &imageInfos[i] + } + } + + matches := MarkdownImageRegex.FindAllStringSubmatch(content, -1) + processedURLs := make(map[string]bool) + + for _, match := range matches { + if len(match) < 3 { + continue + } + imgURL := match[2] + processedURLs[imgURL] = true + + imgInfo, found := imageInfoMap[imgURL] + var b strings.Builder + b.WriteString(fmt.Sprintf("\n", imgURL)) + b.WriteString(fmt.Sprintf("%s\n", match[0])) + if found && imgInfo != nil { + b.WriteString(BuildImageInfoXML(imgInfo)) + } + b.WriteString("") + content = strings.Replace(content, match[0], b.String(), 1) + } + + var extras []string + for _, imgInfo := range imageInfos { + if processedURLs[imgInfo.URL] || processedURLs[imgInfo.OriginalURL] { + continue + } + url := imgInfo.URL + if url == "" { + url = imgInfo.OriginalURL + } + if block := BuildImageInfoXMLWithURL(url, &imgInfo); block != "" { + extras = append(extras, block) + } + } + if len(extras) > 0 { + if content != "" { + content += "\n" + } + content += strings.Join(extras, "\n") + } + return content +} + +// BuildImageInfoXML returns XML-tagged caption / ocr for one image. +func BuildImageInfoXML(img *types.ImageInfo) string { + var b strings.Builder + if img.Caption != "" { + b.WriteString(fmt.Sprintf("%s\n", img.Caption)) + } + if img.OCRText != "" { + b.WriteString(fmt.Sprintf("%s\n", img.OCRText)) + } + return b.String() +} + +// BuildImageInfoXMLWithURL wraps image info in an element carrying the URL. +func BuildImageInfoXMLWithURL(url string, img *types.ImageInfo) string { + inner := BuildImageInfoXML(img) + if inner == "" { + return "" + } + return fmt.Sprintf("\n%s", url, inner) +} + +// EnrichContentCaptionOnly is like EnrichContentWithImageInfo but only +// includes image captions (no OCR text). Original content (including Markdown +// image links) is preserved. Useful for summary generation where OCR would +// add too much noise. +func EnrichContentCaptionOnly(content string, imageInfoJSON string) string { + var imageInfos []types.ImageInfo + if err := json.Unmarshal([]byte(imageInfoJSON), &imageInfos); err != nil { + return content + } + if len(imageInfos) == 0 { + return content + } + + imageInfoMap := make(map[string]*types.ImageInfo) + for i := range imageInfos { + if imageInfos[i].URL != "" { + imageInfoMap[imageInfos[i].URL] = &imageInfos[i] + } + if imageInfos[i].OriginalURL != "" { + imageInfoMap[imageInfos[i].OriginalURL] = &imageInfos[i] + } + } + + matches := MarkdownImageRegex.FindAllStringSubmatch(content, -1) + processedURLs := make(map[string]bool) + + for _, match := range matches { + if len(match) < 3 { + continue + } + imgURL := match[2] + processedURLs[imgURL] = true + + imgInfo, found := imageInfoMap[imgURL] + if found && imgInfo != nil && imgInfo.Caption != "" { + replacement := match[0] + "\n" + fmt.Sprintf("%s", imgInfo.Caption) + content = strings.Replace(content, match[0], replacement, 1) + } + } + + var extras []string + for _, imgInfo := range imageInfos { + if processedURLs[imgInfo.URL] || processedURLs[imgInfo.OriginalURL] { + continue + } + if imgInfo.Caption != "" { + extras = append(extras, fmt.Sprintf("%s", imgInfo.Caption)) + } + } + if len(extras) > 0 { + if content != "" { + content += "\n" + } + content += strings.Join(extras, "\n") + } + return content +}