Files
Enterprise-Knowledge-Base/internal/searchutil/imageinfo_match.go
T
wizardchen 2906c6656e feat(backend): improve parent-child chunk merge and imageinfo matching
Scope image info to matched text children during parent resolution and
add range/URL filtering utilities with unit tests.
2026-06-16 01:09:07 +08:00

149 lines
4.1 KiB
Go

package searchutil
import (
"encoding/json"
"strings"
"unicode/utf8"
"github.com/Tencent/WeKnora/internal/types"
)
// SliceContentByDocumentRange extracts the substring of content whose
// document-level rune offsets fall within [rangeStart, rangeEnd).
// contentStartAt is the document offset where content begins.
func SliceContentByDocumentRange(content string, contentStartAt, rangeStart, rangeEnd int) string {
if content == "" {
return content
}
relStart := rangeStart - contentStartAt
relEnd := rangeEnd - contentStartAt
runes := []rune(content)
if relStart < 0 {
relStart = 0
}
if relEnd > len(runes) {
relEnd = len(runes)
}
if relStart >= relEnd {
return ""
}
return string(runes[relStart:relEnd])
}
// ImageURLsInContent returns the set of image URLs referenced as Markdown
// images in content.
func ImageURLsInContent(content string) map[string]bool {
urls := make(map[string]bool)
for _, match := range MarkdownImageRegex.FindAllStringSubmatch(content, -1) {
if len(match) < 3 || match[2] == "" {
continue
}
urls[match[2]] = true
}
return urls
}
// FilterImageInfoByContentURLs keeps only image_info entries whose URL or
// OriginalURL appears as a Markdown image in content. Returns empty string
// when nothing matches or imageInfoJSON is invalid.
func FilterImageInfoByContentURLs(content string, imageInfoJSON string) string {
if imageInfoJSON == "" {
return ""
}
var infos []types.ImageInfo
if err := json.Unmarshal([]byte(imageInfoJSON), &infos); err != nil || len(infos) == 0 {
return ""
}
urls := ImageURLsInContent(content)
if len(urls) == 0 {
return ""
}
filtered := make([]types.ImageInfo, 0, len(infos))
for _, info := range infos {
if urls[info.URL] || urls[info.OriginalURL] {
filtered = append(filtered, info)
}
}
return marshalImageInfos(filtered)
}
// FilterImageInfoByMatchRange keeps image_info entries whose Markdown image
// reference falls within the document rune range [matchStart, matchEnd).
// Used when parent content is expanded for context but multimodal enrichment
// should only cover the retrieved child window.
func FilterImageInfoByMatchRange(
parentContent string,
parentStartAt, matchStart, matchEnd int,
imageInfoJSON string,
) string {
if imageInfoJSON == "" {
return ""
}
var infos []types.ImageInfo
if err := json.Unmarshal([]byte(imageInfoJSON), &infos); err != nil || len(infos) == 0 {
return ""
}
window := SliceContentByDocumentRange(parentContent, parentStartAt, matchStart, matchEnd)
urls := ImageURLsInContent(window)
if len(urls) == 0 {
return ""
}
filtered := make([]types.ImageInfo, 0, len(infos))
for _, info := range infos {
if urls[info.URL] || urls[info.OriginalURL] {
filtered = append(filtered, info)
}
}
return marshalImageInfos(filtered)
}
func marshalImageInfos(infos []types.ImageInfo) string {
if len(infos) == 0 {
return ""
}
data, err := json.Marshal(infos)
if err != nil {
return ""
}
return string(data)
}
// PruneMarkdownImagesOutsideRange removes Markdown image lines whose
// document-level rune offsets fall outside [matchStart, matchEnd). Non-image
// text is preserved so parent-child expansion still provides full textual
// context while dropping irrelevant page thumbnails.
func PruneMarkdownImagesOutsideRange(
content string,
contentStartAt, matchStart, matchEnd int,
) string {
locs := MarkdownImageRegex.FindAllStringIndex(content, -1)
if len(locs) == 0 {
return content
}
var b strings.Builder
last := 0
for _, loc := range locs {
if loc[0] < last || loc[1] > len(content) {
continue
}
docStart := contentStartAt + utf8.RuneCountInString(content[:loc[0]])
docEnd := contentStartAt + utf8.RuneCountInString(content[:loc[1]])
inRange := docStart < matchEnd && docEnd > matchStart
if inRange {
b.WriteString(content[last:loc[1]])
} else {
b.WriteString(content[last:loc[0]])
}
last = loc[1]
}
b.WriteString(content[last:])
return collapseBlankLines(b.String())
}
func collapseBlankLines(s string) string {
for strings.Contains(s, "\n\n\n") {
s = strings.ReplaceAll(s, "\n\n\n", "\n\n")
}
return strings.TrimSpace(s)
}