2906c6656e
Scope image info to matched text children during parent resolution and add range/URL filtering utilities with unit tests.
149 lines
4.1 KiB
Go
149 lines
4.1 KiB
Go
package searchutil
|
|
|
|
import (
|
|
"encoding/json"
|
|
"strings"
|
|
"unicode/utf8"
|
|
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
)
|
|
|
|
// SliceContentByDocumentRange extracts the substring of content whose
|
|
// document-level rune offsets fall within [rangeStart, rangeEnd).
|
|
// contentStartAt is the document offset where content begins.
|
|
func SliceContentByDocumentRange(content string, contentStartAt, rangeStart, rangeEnd int) string {
|
|
if content == "" {
|
|
return content
|
|
}
|
|
relStart := rangeStart - contentStartAt
|
|
relEnd := rangeEnd - contentStartAt
|
|
runes := []rune(content)
|
|
if relStart < 0 {
|
|
relStart = 0
|
|
}
|
|
if relEnd > len(runes) {
|
|
relEnd = len(runes)
|
|
}
|
|
if relStart >= relEnd {
|
|
return ""
|
|
}
|
|
return string(runes[relStart:relEnd])
|
|
}
|
|
|
|
// ImageURLsInContent returns the set of image URLs referenced as Markdown
|
|
// images in content.
|
|
func ImageURLsInContent(content string) map[string]bool {
|
|
urls := make(map[string]bool)
|
|
for _, match := range MarkdownImageRegex.FindAllStringSubmatch(content, -1) {
|
|
if len(match) < 3 || match[2] == "" {
|
|
continue
|
|
}
|
|
urls[match[2]] = true
|
|
}
|
|
return urls
|
|
}
|
|
|
|
// FilterImageInfoByContentURLs keeps only image_info entries whose URL or
|
|
// OriginalURL appears as a Markdown image in content. Returns empty string
|
|
// when nothing matches or imageInfoJSON is invalid.
|
|
func FilterImageInfoByContentURLs(content string, imageInfoJSON string) string {
|
|
if imageInfoJSON == "" {
|
|
return ""
|
|
}
|
|
var infos []types.ImageInfo
|
|
if err := json.Unmarshal([]byte(imageInfoJSON), &infos); err != nil || len(infos) == 0 {
|
|
return ""
|
|
}
|
|
urls := ImageURLsInContent(content)
|
|
if len(urls) == 0 {
|
|
return ""
|
|
}
|
|
filtered := make([]types.ImageInfo, 0, len(infos))
|
|
for _, info := range infos {
|
|
if urls[info.URL] || urls[info.OriginalURL] {
|
|
filtered = append(filtered, info)
|
|
}
|
|
}
|
|
return marshalImageInfos(filtered)
|
|
}
|
|
|
|
// FilterImageInfoByMatchRange keeps image_info entries whose Markdown image
|
|
// reference falls within the document rune range [matchStart, matchEnd).
|
|
// Used when parent content is expanded for context but multimodal enrichment
|
|
// should only cover the retrieved child window.
|
|
func FilterImageInfoByMatchRange(
|
|
parentContent string,
|
|
parentStartAt, matchStart, matchEnd int,
|
|
imageInfoJSON string,
|
|
) string {
|
|
if imageInfoJSON == "" {
|
|
return ""
|
|
}
|
|
var infos []types.ImageInfo
|
|
if err := json.Unmarshal([]byte(imageInfoJSON), &infos); err != nil || len(infos) == 0 {
|
|
return ""
|
|
}
|
|
window := SliceContentByDocumentRange(parentContent, parentStartAt, matchStart, matchEnd)
|
|
urls := ImageURLsInContent(window)
|
|
if len(urls) == 0 {
|
|
return ""
|
|
}
|
|
filtered := make([]types.ImageInfo, 0, len(infos))
|
|
for _, info := range infos {
|
|
if urls[info.URL] || urls[info.OriginalURL] {
|
|
filtered = append(filtered, info)
|
|
}
|
|
}
|
|
return marshalImageInfos(filtered)
|
|
}
|
|
|
|
func marshalImageInfos(infos []types.ImageInfo) string {
|
|
if len(infos) == 0 {
|
|
return ""
|
|
}
|
|
data, err := json.Marshal(infos)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return string(data)
|
|
}
|
|
|
|
// PruneMarkdownImagesOutsideRange removes Markdown image lines whose
|
|
// document-level rune offsets fall outside [matchStart, matchEnd). Non-image
|
|
// text is preserved so parent-child expansion still provides full textual
|
|
// context while dropping irrelevant page thumbnails.
|
|
func PruneMarkdownImagesOutsideRange(
|
|
content string,
|
|
contentStartAt, matchStart, matchEnd int,
|
|
) string {
|
|
locs := MarkdownImageRegex.FindAllStringIndex(content, -1)
|
|
if len(locs) == 0 {
|
|
return content
|
|
}
|
|
var b strings.Builder
|
|
last := 0
|
|
for _, loc := range locs {
|
|
if loc[0] < last || loc[1] > len(content) {
|
|
continue
|
|
}
|
|
docStart := contentStartAt + utf8.RuneCountInString(content[:loc[0]])
|
|
docEnd := contentStartAt + utf8.RuneCountInString(content[:loc[1]])
|
|
inRange := docStart < matchEnd && docEnd > matchStart
|
|
if inRange {
|
|
b.WriteString(content[last:loc[1]])
|
|
} else {
|
|
b.WriteString(content[last:loc[0]])
|
|
}
|
|
last = loc[1]
|
|
}
|
|
b.WriteString(content[last:])
|
|
return collapseBlankLines(b.String())
|
|
}
|
|
|
|
func collapseBlankLines(s string) string {
|
|
for strings.Contains(s, "\n\n\n") {
|
|
s = strings.ReplaceAll(s, "\n\n\n", "\n\n")
|
|
}
|
|
return strings.TrimSpace(s)
|
|
}
|