feat(logger): implement LLM debug logging functionality
- Added a new logger for LLM calls, enabling detailed logging of model interactions, including request and response data. - Introduced configuration options for enabling and specifying the log directory for LLM debug logs. - Implemented cleanup for old log files to manage disk space effectively. - Wrapped existing chat implementations to log calls when debug logging is enabled, enhancing traceability for model interactions.
This commit is contained in:
@@ -29,14 +29,23 @@ func (r *chunkRepository) CreateChunks(ctx context.Context, chunks []*types.Chun
|
||||
for _, chunk := range chunks {
|
||||
chunk.Content = common.CleanInvalidUTF8(chunk.Content)
|
||||
}
|
||||
// Pre-assign SeqIDs for SQLite compatibility (autoIncrement on non-PK columns
|
||||
// doesn't work in SQLite). This is a no-op if all chunks already have SeqIDs.
|
||||
if err := types.AssignChunkSeqIDs(r.db.WithContext(ctx), chunks); err != nil {
|
||||
return fmt.Errorf("failed to assign chunk seq_ids: %w", err)
|
||||
|
||||
db := r.db.WithContext(ctx)
|
||||
|
||||
// SQLite doesn't support autoIncrement on non-PK columns,
|
||||
// so we must pre-assign SeqIDs manually (safe: single connection).
|
||||
// PostgreSQL / MySQL use DB sequences — skip to avoid duplicate key
|
||||
// races under concurrent inserts.
|
||||
if db.Dialector.Name() == "sqlite" {
|
||||
if err := types.AssignChunkSeqIDs(db, chunks); err != nil {
|
||||
return fmt.Errorf("failed to assign chunk seq_ids: %w", err)
|
||||
}
|
||||
}
|
||||
// Use Select("*") to ensure all fields including zero values (IsEnabled=false, Flags=0)
|
||||
// are inserted, bypassing GORM's default value behavior for zero values
|
||||
return r.db.WithContext(ctx).Select("*").CreateInBatches(chunks, 100).Error
|
||||
|
||||
// Select("*") ensures zero-value fields (IsEnabled=false, Flags=0) are
|
||||
// explicitly inserted, bypassing GORM's default value behavior.
|
||||
// SeqID=0 is skipped by GORM automatically (autoIncrement tag).
|
||||
return db.Select("*").CreateInBatches(chunks, 100).Error
|
||||
}
|
||||
|
||||
// GetChunkByID retrieves a chunk by its ID and tenant ID
|
||||
|
||||
@@ -133,37 +133,32 @@ func (p *PluginIntoChatMessage) OnEvent(ctx context.Context,
|
||||
|
||||
// Build contexts string based on FAQ priority strategy
|
||||
if chatManage.FAQPriorityEnabled && len(faqResults) > 0 {
|
||||
// Build structured context with FAQ prioritization
|
||||
contextsBuilder.WriteString("### Source 1: FAQ Knowledge Base\n")
|
||||
contextsBuilder.WriteString("[High Confidence - Prioritize these results]\n")
|
||||
contextsBuilder.WriteString("<source type=\"faq\" priority=\"high\">\n")
|
||||
for i, result := range faqResults {
|
||||
passage := getEnrichedPassageForChat(ctx, result)
|
||||
if hasHighConfidenceFAQ && i == 0 {
|
||||
contextsBuilder.WriteString(fmt.Sprintf("[FAQ-%d] Exact Match: %s\n", i+1, passage))
|
||||
contextsBuilder.WriteString(fmt.Sprintf("<context id=\"FAQ-%d\" match=\"exact\">%s</context>\n", i+1, passage))
|
||||
} else {
|
||||
contextsBuilder.WriteString(fmt.Sprintf("[FAQ-%d] %s\n", i+1, passage))
|
||||
contextsBuilder.WriteString(fmt.Sprintf("<context id=\"FAQ-%d\">%s</context>\n", i+1, passage))
|
||||
}
|
||||
}
|
||||
contextsBuilder.WriteString("</source>\n")
|
||||
|
||||
if len(docResults) > 0 {
|
||||
contextsBuilder.WriteString("\n### Source 2: Reference Documents\n")
|
||||
contextsBuilder.WriteString("[Supplementary - Use only when FAQ cannot answer the question]\n")
|
||||
contextsBuilder.WriteString("<source type=\"document\" priority=\"supplementary\">\n")
|
||||
for i, result := range docResults {
|
||||
passage := getEnrichedPassageForChat(ctx, result)
|
||||
contextsBuilder.WriteString(fmt.Sprintf("[DOC-%d] %s\n", i+1, passage))
|
||||
contextsBuilder.WriteString(fmt.Sprintf("<context id=\"DOC-%d\">%s</context>\n", i+1, passage))
|
||||
}
|
||||
contextsBuilder.WriteString("</source>")
|
||||
}
|
||||
} else {
|
||||
// Original behavior: simple numbered list
|
||||
passages := make([]string, len(chatManage.MergeResult))
|
||||
for i, result := range chatManage.MergeResult {
|
||||
passages[i] = getEnrichedPassageForChat(ctx, result)
|
||||
}
|
||||
for i, passage := range passages {
|
||||
passage := getEnrichedPassageForChat(ctx, result)
|
||||
if i > 0 {
|
||||
contextsBuilder.WriteString("\n\n")
|
||||
contextsBuilder.WriteString("\n")
|
||||
}
|
||||
contextsBuilder.WriteString(fmt.Sprintf("[%d] %s", i+1, passage))
|
||||
contextsBuilder.WriteString(fmt.Sprintf("<context id=\"%d\">%s</context>", i+1, passage))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -279,15 +274,16 @@ func buildDocumentHeader(results []*types.SearchResult) string {
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString("### Referenced Documents\n")
|
||||
for i, d := range docs {
|
||||
// if d.description != "" {
|
||||
// b.WriteString(fmt.Sprintf("%d. %s — %s\n", i+1, d.title, d.description))
|
||||
// } else {
|
||||
// b.WriteString(fmt.Sprintf("%d. %s\n", i+1, d.title))
|
||||
// }
|
||||
b.WriteString(fmt.Sprintf("%d. %s\n", i+1, d.title))
|
||||
b.WriteString("<documents>\n")
|
||||
for _, d := range docs {
|
||||
b.WriteString("<document>\n")
|
||||
b.WriteString(fmt.Sprintf("<title>%s</title>\n", d.title))
|
||||
if d.description != "" {
|
||||
b.WriteString(fmt.Sprintf("<description>%s</description>\n", d.description))
|
||||
}
|
||||
b.WriteString("</document>\n")
|
||||
}
|
||||
b.WriteString("</documents>")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
@@ -310,105 +306,101 @@ func getEnrichedPassageForChat(ctx context.Context, result *types.SearchResult)
|
||||
// 正则表达式用于匹配Markdown图片链接
|
||||
var markdownImageRegex = regexp.MustCompile(`!\[([^\]]*)\]\(([^)]+)\)`)
|
||||
|
||||
// enrichContentWithImageInfo 将图片信息与文本内容合并
|
||||
// enrichContentWithImageInfo 将图片信息以 XML 标签的形式嵌入文本内容。
|
||||
// 对于内容中已有的 Markdown 图片链接,在其后附加 <image_caption> / <image_ocr>;
|
||||
// 对于未出现在内容中的图片,以 <image> 块追加到末尾。
|
||||
func enrichContentWithImageInfo(ctx context.Context, content string, imageInfoJSON string) string {
|
||||
// 解析ImageInfo
|
||||
var imageInfos []types.ImageInfo
|
||||
err := json.Unmarshal([]byte(imageInfoJSON), &imageInfos)
|
||||
if err != nil {
|
||||
if err := json.Unmarshal([]byte(imageInfoJSON), &imageInfos); err != nil {
|
||||
pipelineWarn(ctx, "IntoChatMessage", "image_parse_error", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return content
|
||||
}
|
||||
|
||||
if len(imageInfos) == 0 {
|
||||
return content
|
||||
}
|
||||
|
||||
// 创建图片URL到信息的映射
|
||||
imageInfoMap := make(map[string]*types.ImageInfo)
|
||||
for i := range imageInfos {
|
||||
if imageInfos[i].URL != "" {
|
||||
imageInfoMap[imageInfos[i].URL] = &imageInfos[i]
|
||||
}
|
||||
// 同时检查原始URL
|
||||
if imageInfos[i].OriginalURL != "" {
|
||||
imageInfoMap[imageInfos[i].OriginalURL] = &imageInfos[i]
|
||||
}
|
||||
}
|
||||
|
||||
// 查找内容中的所有Markdown图片链接
|
||||
matches := markdownImageRegex.FindAllStringSubmatch(content, -1)
|
||||
|
||||
// 用于存储已处理的图片URL
|
||||
processedURLs := make(map[string]bool)
|
||||
|
||||
pipelineInfo(ctx, "IntoChatMessage", "image_markdown_links", map[string]interface{}{
|
||||
"match_count": len(matches),
|
||||
})
|
||||
|
||||
// 替换每个图片链接,添加描述和OCR文本
|
||||
for _, match := range matches {
|
||||
if len(match) < 3 {
|
||||
continue
|
||||
}
|
||||
|
||||
// 提取图片URL,忽略alt文本
|
||||
imgURL := match[2]
|
||||
|
||||
// 标记该URL已处理
|
||||
processedURLs[imgURL] = true
|
||||
|
||||
// 查找匹配的图片信息
|
||||
imgInfo, found := imageInfoMap[imgURL]
|
||||
|
||||
// 如果找到匹配的图片信息,添加描述和OCR文本
|
||||
var b strings.Builder
|
||||
b.WriteString(fmt.Sprintf("<image url=\"%s\">\n", imgURL))
|
||||
b.WriteString(fmt.Sprintf("<image_original>%s</image_original>\n", match[0]))
|
||||
if found && imgInfo != nil {
|
||||
replacement := match[0] + "\n"
|
||||
if imgInfo.Caption != "" {
|
||||
replacement += fmt.Sprintf("Image Caption: %s\n", imgInfo.Caption)
|
||||
}
|
||||
if imgInfo.OCRText != "" {
|
||||
replacement += fmt.Sprintf("Image Text: %s\n", imgInfo.OCRText)
|
||||
}
|
||||
content = strings.Replace(content, match[0], replacement, 1)
|
||||
b.WriteString(buildImageInfoXML(imgInfo))
|
||||
}
|
||||
b.WriteString("</image>")
|
||||
content = strings.Replace(content, match[0], b.String(), 1)
|
||||
}
|
||||
|
||||
// 处理未在内容中找到但存在于ImageInfo中的图片
|
||||
var additionalImageTexts []string
|
||||
// Append image info not found as inline Markdown links
|
||||
var extras []string
|
||||
for _, imgInfo := range imageInfos {
|
||||
// 如果图片URL已经处理过,跳过
|
||||
if processedURLs[imgInfo.URL] || processedURLs[imgInfo.OriginalURL] {
|
||||
continue
|
||||
}
|
||||
|
||||
var imgTexts []string
|
||||
if imgInfo.Caption != "" {
|
||||
imgTexts = append(imgTexts, fmt.Sprintf("Image %s caption: %s", imgInfo.URL, imgInfo.Caption))
|
||||
url := imgInfo.URL
|
||||
if url == "" {
|
||||
url = imgInfo.OriginalURL
|
||||
}
|
||||
if imgInfo.OCRText != "" {
|
||||
imgTexts = append(imgTexts, fmt.Sprintf("Image %s text: %s", imgInfo.URL, imgInfo.OCRText))
|
||||
}
|
||||
|
||||
if len(imgTexts) > 0 {
|
||||
additionalImageTexts = append(additionalImageTexts, imgTexts...)
|
||||
if block := buildImageInfoXMLWithURL(url, &imgInfo); block != "" {
|
||||
extras = append(extras, block)
|
||||
}
|
||||
}
|
||||
|
||||
// 如果有额外的图片信息,添加到内容末尾
|
||||
if len(additionalImageTexts) > 0 {
|
||||
if len(extras) > 0 {
|
||||
if content != "" {
|
||||
content += "\n\n"
|
||||
content += "\n"
|
||||
}
|
||||
content += "Additional Image Info:\n" + strings.Join(additionalImageTexts, "\n")
|
||||
content += strings.Join(extras, "\n")
|
||||
}
|
||||
|
||||
pipelineInfo(ctx, "IntoChatMessage", "image_enrich_summary", map[string]interface{}{
|
||||
"markdown_images": len(matches),
|
||||
"additional_imgs": len(additionalImageTexts),
|
||||
"additional_imgs": len(extras),
|
||||
})
|
||||
|
||||
return content
|
||||
}
|
||||
|
||||
// buildImageInfoXML returns XML-tagged caption / ocr for one image.
|
||||
func buildImageInfoXML(img *types.ImageInfo) string {
|
||||
var b strings.Builder
|
||||
if img.Caption != "" {
|
||||
b.WriteString(fmt.Sprintf("<image_caption>%s</image_caption>\n", img.Caption))
|
||||
}
|
||||
if img.OCRText != "" {
|
||||
b.WriteString(fmt.Sprintf("<image_ocr>%s</image_ocr>\n", img.OCRText))
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// buildImageInfoXMLWithURL wraps image info in an <image> element carrying the URL.
|
||||
func buildImageInfoXMLWithURL(url string, img *types.ImageInfo) string {
|
||||
inner := buildImageInfoXML(img)
|
||||
if inner == "" {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("<image url=\"%s\">\n%s</image>", url, inner)
|
||||
}
|
||||
|
||||
@@ -260,136 +260,207 @@ func (p *PluginMerge) resolveParentChunks(
|
||||
parentMap[c.ID] = c
|
||||
}
|
||||
|
||||
// Check if any results are image chunks; only then do we need
|
||||
// grandparent resolution and the extra DB round-trip.
|
||||
hasImageResults := false
|
||||
for _, r := range results {
|
||||
if r.ChunkType == string(types.ChunkTypeImageOCR) || r.ChunkType == string(types.ChunkTypeImageCaption) {
|
||||
hasImageResults = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
var grandparentIDs []string
|
||||
if hasImageResults {
|
||||
// Fetch grandparent chunks for the image → text → parent_text chain.
|
||||
for _, pc := range parentChunks {
|
||||
if pc.ParentChunkID != "" && pc.ChunkType == types.ChunkTypeText {
|
||||
if _, already := parentMap[pc.ParentChunkID]; !already {
|
||||
grandparentIDs = append(grandparentIDs, pc.ParentChunkID)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(grandparentIDs) > 0 {
|
||||
gpChunks, err := p.chunkRepo.ListChunksByID(ctx, tenantID, grandparentIDs)
|
||||
if err != nil {
|
||||
pipelineWarn(ctx, "Merge", "grandparent_fetch_failed", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
} else {
|
||||
for _, c := range gpChunks {
|
||||
parentMap[c.ID] = c
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Collect merged ImageInfo for each parent by fetching ALL sibling
|
||||
// child chunks. Individual child chunks only carry ImageInfo for images
|
||||
// within their own range, but the parent content spans all children.
|
||||
parentImageInfoMap := p.collectParentImageInfo(ctx, tenantID, ids)
|
||||
imageInfoIDs := ids
|
||||
if len(grandparentIDs) > 0 {
|
||||
imageInfoIDs = append(append([]string(nil), ids...), grandparentIDs...)
|
||||
}
|
||||
parentImageInfoMap := p.collectParentImageInfo(ctx, tenantID, imageInfoIDs)
|
||||
|
||||
// Replace child content with parent content.
|
||||
// Only replace for text chunks whose parent is a parent_text chunk
|
||||
// (i.e., the parent-child chunking strategy). Summary chunks also carry
|
||||
// a ParentChunkID that points to their source chunk, but that is a
|
||||
// different semantic — replacing summary content with its source would
|
||||
// degrade quality.
|
||||
for _, r := range results {
|
||||
if r.ParentChunkID == "" {
|
||||
continue
|
||||
}
|
||||
// Skip non-text chunks (e.g., summary, image_caption) — their
|
||||
// ParentChunkID has a different meaning than the parent-child
|
||||
// chunking strategy.
|
||||
if r.ChunkType != string(types.ChunkTypeText) {
|
||||
continue
|
||||
}
|
||||
parent, ok := parentMap[r.ParentChunkID]
|
||||
if !ok || parent.Content == "" {
|
||||
continue
|
||||
}
|
||||
// Only replace if the parent is actually a parent_text chunk from
|
||||
// the parent-child chunking strategy.
|
||||
if parent.ChunkType != types.ChunkTypeParentText {
|
||||
continue
|
||||
}
|
||||
pipelineInfo(ctx, "Merge", "parent_resolve", map[string]interface{}{
|
||||
"child_id": r.ID,
|
||||
"parent_id": r.ParentChunkID,
|
||||
"child_len": runeLen(r.Content),
|
||||
"parent_len": runeLen(parent.Content),
|
||||
})
|
||||
r.Content = parent.Content
|
||||
r.StartAt = parent.StartAt
|
||||
r.EndAt = parent.EndAt
|
||||
if mergedImageInfo, ok := parentImageInfoMap[r.ParentChunkID]; ok && mergedImageInfo != "" {
|
||||
r.ImageInfo = mergedImageInfo
|
||||
}
|
||||
// Track the original child as a sub-chunk
|
||||
if !containsID(r.SubChunkID, r.ID) {
|
||||
r.SubChunkID = append(r.SubChunkID, r.ID)
|
||||
|
||||
switch r.ChunkType {
|
||||
case string(types.ChunkTypeText):
|
||||
// text → parent_text resolution (parent-child chunking strategy).
|
||||
// Summary chunks also carry a ParentChunkID that points to their
|
||||
// source chunk, but that is a different semantic — replacing
|
||||
// summary content with its source would degrade quality.
|
||||
parent, ok := parentMap[r.ParentChunkID]
|
||||
if !ok || parent.Content == "" || parent.ChunkType != types.ChunkTypeParentText {
|
||||
continue
|
||||
}
|
||||
pipelineInfo(ctx, "Merge", "parent_resolve", map[string]interface{}{
|
||||
"child_id": r.ID,
|
||||
"parent_id": r.ParentChunkID,
|
||||
"child_len": runeLen(r.Content),
|
||||
"parent_len": runeLen(parent.Content),
|
||||
})
|
||||
r.Content = parent.Content
|
||||
r.StartAt = parent.StartAt
|
||||
r.EndAt = parent.EndAt
|
||||
if mergedImageInfo, ok := parentImageInfoMap[r.ParentChunkID]; ok && mergedImageInfo != "" {
|
||||
r.ImageInfo = mergedImageInfo
|
||||
}
|
||||
if !containsID(r.SubChunkID, r.ID) {
|
||||
r.SubChunkID = append(r.SubChunkID, r.ID)
|
||||
}
|
||||
|
||||
case string(types.ChunkTypeImageOCR), string(types.ChunkTypeImageCaption):
|
||||
// image_ocr/image_caption → text parent → optional parent_text grandparent.
|
||||
// Replace content with parent text for surrounding context.
|
||||
parent, ok := parentMap[r.ParentChunkID]
|
||||
if !ok || parent.Content == "" {
|
||||
continue
|
||||
}
|
||||
resolvedParent := parent
|
||||
// If parent text uses parent-child chunking, resolve one more level
|
||||
if parent.ChunkType == types.ChunkTypeText && parent.ParentChunkID != "" {
|
||||
if gp, gpOK := parentMap[parent.ParentChunkID]; gpOK && gp.ChunkType == types.ChunkTypeParentText && gp.Content != "" {
|
||||
resolvedParent = gp
|
||||
}
|
||||
}
|
||||
pipelineInfo(ctx, "Merge", "image_parent_resolve", map[string]interface{}{
|
||||
"child_id": r.ID,
|
||||
"child_type": r.ChunkType,
|
||||
"resolved_id": resolvedParent.ID,
|
||||
"child_len": runeLen(r.Content),
|
||||
"parent_len": runeLen(resolvedParent.Content),
|
||||
})
|
||||
r.Content = resolvedParent.Content
|
||||
r.StartAt = resolvedParent.StartAt
|
||||
r.EndAt = resolvedParent.EndAt
|
||||
if mergedInfo, ok := parentImageInfoMap[resolvedParent.ID]; ok && mergedInfo != "" {
|
||||
r.ImageInfo = mergedInfo
|
||||
}
|
||||
if !containsID(r.SubChunkID, r.ID) {
|
||||
r.SubChunkID = append(r.SubChunkID, r.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
// collectParentImageInfo batch-fetches all child chunks for the given parents
|
||||
// and merges their ImageInfo into a single JSON string per parent. This ensures
|
||||
// that when child content is replaced with parent content, the complete set of
|
||||
// image descriptions across all sibling chunks is preserved.
|
||||
// collectParentImageInfo batch-fetches image info for the given parent_text
|
||||
// chunk IDs using a two-level query:
|
||||
//
|
||||
// Level 1: parent_text → text children
|
||||
// Level 2: text children → image_ocr / image_caption grandchildren (carry image_info)
|
||||
func (p *PluginMerge) collectParentImageInfo(
|
||||
ctx context.Context,
|
||||
tenantID uint64,
|
||||
parentIDs []string,
|
||||
) map[string]string {
|
||||
result := make(map[string]string, len(parentIDs))
|
||||
|
||||
allChildren, err := p.chunkRepo.ListChunksByParentIDs(ctx, tenantID, parentIDs)
|
||||
// Level 1: get direct children of parent_text chunks
|
||||
children, err := p.chunkRepo.ListChunksByParentIDs(ctx, tenantID, parentIDs)
|
||||
if err != nil {
|
||||
pipelineWarn(ctx, "Merge", "parent_imageinfo_fetch_failed", map[string]interface{}{
|
||||
"parent_cnt": len(parentIDs),
|
||||
"error": err.Error(),
|
||||
})
|
||||
return result
|
||||
return nil
|
||||
}
|
||||
|
||||
// Group children by parent chunk ID, collecting unique ImageInfo entries
|
||||
type parentAgg struct {
|
||||
imageInfos []types.ImageInfo
|
||||
uniqueURLs map[string]bool
|
||||
siblingCnt int
|
||||
type agg struct {
|
||||
infos []types.ImageInfo
|
||||
seenURLs map[string]bool
|
||||
}
|
||||
aggMap := make(map[string]*parentAgg, len(parentIDs))
|
||||
|
||||
for _, child := range allChildren {
|
||||
agg, ok := aggMap[child.ParentChunkID]
|
||||
if !ok {
|
||||
agg = &parentAgg{uniqueURLs: make(map[string]bool)}
|
||||
aggMap[child.ParentChunkID] = agg
|
||||
}
|
||||
agg.siblingCnt++
|
||||
|
||||
if child.ImageInfo == "" {
|
||||
continue
|
||||
aggMap := make(map[string]*agg)
|
||||
addInfo := func(targetID string, chunk *types.Chunk) {
|
||||
if chunk.ImageInfo == "" {
|
||||
return
|
||||
}
|
||||
var infos []types.ImageInfo
|
||||
if err := json.Unmarshal([]byte(child.ImageInfo), &infos); err != nil {
|
||||
pipelineWarn(ctx, "Merge", "parent_imageinfo_parse", map[string]interface{}{
|
||||
"chunk_id": child.ID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
continue
|
||||
if err := json.Unmarshal([]byte(chunk.ImageInfo), &infos); err != nil || len(infos) == 0 {
|
||||
return
|
||||
}
|
||||
a, ok := aggMap[targetID]
|
||||
if !ok {
|
||||
a = &agg{seenURLs: make(map[string]bool)}
|
||||
aggMap[targetID] = a
|
||||
}
|
||||
for _, info := range infos {
|
||||
key := info.URL
|
||||
if key == "" {
|
||||
key = info.OriginalURL
|
||||
}
|
||||
if key != "" && !agg.uniqueURLs[key] {
|
||||
agg.uniqueURLs[key] = true
|
||||
agg.imageInfos = append(agg.imageInfos, info)
|
||||
if key != "" && !a.seenURLs[key] {
|
||||
a.seenURLs[key] = true
|
||||
a.infos = append(a.infos, info)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for parentID, agg := range aggMap {
|
||||
if len(agg.imageInfos) == 0 {
|
||||
continue
|
||||
var textChildIDs []string
|
||||
textToParent := make(map[string]string, len(children))
|
||||
for _, child := range children {
|
||||
switch child.ChunkType {
|
||||
case types.ChunkTypeImageOCR, types.ChunkTypeImageCaption:
|
||||
addInfo(child.ParentChunkID, child)
|
||||
case types.ChunkTypeText:
|
||||
textChildIDs = append(textChildIDs, child.ID)
|
||||
textToParent[child.ID] = child.ParentChunkID
|
||||
}
|
||||
merged, err := json.Marshal(agg.imageInfos)
|
||||
if err != nil {
|
||||
pipelineWarn(ctx, "Merge", "parent_imageinfo_marshal", map[string]interface{}{
|
||||
"parent_id": parentID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
continue
|
||||
}
|
||||
result[parentID] = string(merged)
|
||||
|
||||
pipelineInfo(ctx, "Merge", "parent_imageinfo_collected", map[string]interface{}{
|
||||
"parent_id": parentID,
|
||||
"sibling_cnt": agg.siblingCnt,
|
||||
"image_cnt": len(agg.imageInfos),
|
||||
})
|
||||
}
|
||||
|
||||
// Level 2: text children → image grandchildren
|
||||
if len(textChildIDs) > 0 {
|
||||
grandChildren, err := p.chunkRepo.ListChunksByParentIDs(ctx, tenantID, textChildIDs)
|
||||
if err != nil {
|
||||
pipelineWarn(ctx, "Merge", "parent_imageinfo_l2_fetch_failed", map[string]interface{}{
|
||||
"text_cnt": len(textChildIDs),
|
||||
"error": err.Error(),
|
||||
})
|
||||
} else {
|
||||
for _, gc := range grandChildren {
|
||||
if gc.ChunkType != types.ChunkTypeImageOCR && gc.ChunkType != types.ChunkTypeImageCaption {
|
||||
continue
|
||||
}
|
||||
if parentTextID, ok := textToParent[gc.ParentChunkID]; ok {
|
||||
addInfo(parentTextID, gc)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result := make(map[string]string, len(aggMap))
|
||||
for id, a := range aggMap {
|
||||
if len(a.infos) == 0 {
|
||||
continue
|
||||
}
|
||||
data, err := json.Marshal(a.infos)
|
||||
if err == nil {
|
||||
result[id] = string(data)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
@@ -611,6 +611,8 @@ func (p *PluginSearch) tryDirectChunkLoading(ctx context.Context, tenantID uint6
|
||||
results = append(results, res)
|
||||
}
|
||||
|
||||
searchutil.EnrichSearchResultsImageInfo(ctx, p.chunkService.GetRepository(), tenantID, results)
|
||||
|
||||
return results, skippedIDs
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"sync"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/searchutil"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
)
|
||||
@@ -161,10 +162,13 @@ func (p *PluginSearchEntity) OnEvent(ctx context.Context,
|
||||
for _, knowledge := range knowledges {
|
||||
knowledgeMap[knowledge.ID] = knowledge
|
||||
}
|
||||
var entityResults []*types.SearchResult
|
||||
for _, chunk := range chunks {
|
||||
searchResult := chunk2SearchResult(chunk, knowledgeMap[chunk.KnowledgeID])
|
||||
chatManage.SearchResult = append(chatManage.SearchResult, searchResult)
|
||||
entityResults = append(entityResults, searchResult)
|
||||
}
|
||||
searchutil.EnrichSearchResultsImageInfo(ctx, p.chunkRepo, types.MustTenantIDFromContext(ctx), entityResults)
|
||||
chatManage.SearchResult = append(chatManage.SearchResult, entityResults...)
|
||||
// remove duplicate results
|
||||
chatManage.SearchResult = removeDuplicateResults(chatManage.SearchResult)
|
||||
if len(chatManage.SearchResult) == 0 {
|
||||
|
||||
@@ -172,13 +172,11 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) e
|
||||
}
|
||||
}
|
||||
|
||||
if payload.EnableCaption {
|
||||
caption, capErr := vlmModel.Predict(ctx, [][]byte{imgBytes}, vlmCaptionPrompt)
|
||||
if capErr != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Caption failed for %s: %v", payload.ImageURL, capErr)
|
||||
} else if caption != "" {
|
||||
imageInfo.Caption = caption
|
||||
}
|
||||
caption, capErr := vlmModel.Predict(ctx, [][]byte{imgBytes}, vlmCaptionPrompt)
|
||||
if capErr != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Caption failed for %s: %v", payload.ImageURL, capErr)
|
||||
} else if caption != "" {
|
||||
imageInfo.Caption = caption
|
||||
}
|
||||
|
||||
// Build child chunks for OCR and caption results
|
||||
@@ -220,8 +218,6 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) e
|
||||
}
|
||||
|
||||
if len(newChunks) == 0 {
|
||||
// Even if OCR/caption both failed, mark knowledge as completed
|
||||
s.finalizeImageKnowledge(ctx, payload, "")
|
||||
s.checkAndFinalizeAllImages(ctx, payload)
|
||||
return nil
|
||||
}
|
||||
@@ -238,13 +234,6 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) e
|
||||
// Index chunks so they can be retrieved
|
||||
s.indexChunks(ctx, payload, newChunks)
|
||||
|
||||
// Update the parent text chunk's ImageInfo (mirrors old docreader behaviour)
|
||||
s.updateParentChunkImageInfo(ctx, payload, imageInfo)
|
||||
|
||||
// For standalone image files, use caption as the knowledge description
|
||||
// and mark the knowledge as completed (it was kept in "processing" until now).
|
||||
s.finalizeImageKnowledge(ctx, payload, imageInfo.Caption)
|
||||
|
||||
// Enqueue question generation for the caption/OCR content if KB has it enabled.
|
||||
// During initial processChunks, question generation is skipped for image-type
|
||||
// knowledge because the text chunk is just a markdown reference. Now that we
|
||||
@@ -256,36 +245,6 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) e
|
||||
return nil
|
||||
}
|
||||
|
||||
// finalizeImageKnowledge updates the knowledge after multimodal processing:
|
||||
// - For standalone image files: sets Description from caption and marks ParseStatus as completed
|
||||
// (processChunks kept it in "processing" to wait for multimodal results).
|
||||
// - For images extracted from PDFs: no-op (description comes from summary generation).
|
||||
func (s *ImageMultimodalService) finalizeImageKnowledge(ctx context.Context, payload types.ImageMultimodalPayload, caption string) {
|
||||
knowledge, err := s.knowledgeRepo.GetKnowledgeByIDOnly(ctx, payload.KnowledgeID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Failed to get knowledge %s: %v", payload.KnowledgeID, err)
|
||||
return
|
||||
}
|
||||
if knowledge == nil {
|
||||
return
|
||||
}
|
||||
if !IsImageType(knowledge.FileType) {
|
||||
return
|
||||
}
|
||||
|
||||
if caption != "" {
|
||||
knowledge.Description = caption
|
||||
}
|
||||
knowledge.ParseStatus = types.ParseStatusCompleted
|
||||
knowledge.UpdatedAt = time.Now()
|
||||
if err := s.knowledgeRepo.UpdateKnowledge(ctx, knowledge); err != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Failed to finalize knowledge: %v", err)
|
||||
} else {
|
||||
logger.Infof(ctx, "[ImageMultimodal] Finalized image knowledge %s (status=completed, description=%d chars)",
|
||||
payload.KnowledgeID, len(knowledge.Description))
|
||||
}
|
||||
}
|
||||
|
||||
// indexChunks indexes the newly created multimodal chunks into the retrieval engine
|
||||
// so they can participate in semantic search.
|
||||
func (s *ImageMultimodalService) indexChunks(ctx context.Context, payload types.ImageMultimodalPayload, chunks []*types.Chunk) {
|
||||
@@ -348,47 +307,6 @@ func (s *ImageMultimodalService) indexChunks(ctx context.Context, payload types.
|
||||
logger.Infof(ctx, "[ImageMultimodal] Indexed %d multimodal chunks for image %s", len(chunks), payload.ImageURL)
|
||||
}
|
||||
|
||||
// updateParentChunkImageInfo updates the parent text chunk's ImageInfo field,
|
||||
// replicating the behaviour of the old docreader flow where the parent chunk
|
||||
// carried the full image metadata (URL, OCR, caption).
|
||||
func (s *ImageMultimodalService) updateParentChunkImageInfo(ctx context.Context, payload types.ImageMultimodalPayload, imageInfo types.ImageInfo) {
|
||||
if payload.ChunkID == "" {
|
||||
return
|
||||
}
|
||||
|
||||
chunk, err := s.chunkService.GetChunkByIDOnly(ctx, payload.ChunkID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Failed to get parent chunk %s: %v", payload.ChunkID, err)
|
||||
return
|
||||
}
|
||||
|
||||
var existingInfos []types.ImageInfo
|
||||
if chunk.ImageInfo != "" {
|
||||
_ = json.Unmarshal([]byte(chunk.ImageInfo), &existingInfos)
|
||||
}
|
||||
|
||||
found := false
|
||||
for i, info := range existingInfos {
|
||||
if info.URL == imageInfo.URL {
|
||||
existingInfos[i] = imageInfo
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
existingInfos = append(existingInfos, imageInfo)
|
||||
}
|
||||
|
||||
imageInfoJSON, _ := json.Marshal(existingInfos)
|
||||
chunk.ImageInfo = string(imageInfoJSON)
|
||||
chunk.UpdatedAt = time.Now()
|
||||
if err := s.chunkService.UpdateChunk(ctx, chunk); err != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Failed to update parent chunk %s ImageInfo: %v", chunk.ID, err)
|
||||
} else {
|
||||
logger.Infof(ctx, "[ImageMultimodal] Updated parent chunk %s ImageInfo for image %s", chunk.ID, payload.ImageURL)
|
||||
}
|
||||
}
|
||||
|
||||
// resolveVLM creates a vlm.VLM instance for the given knowledge base,
|
||||
// supporting both new-style (ModelID) and legacy (inline BaseURL) configs.
|
||||
func (s *ImageMultimodalService) resolveVLM(ctx context.Context, kbID string) (vlm.VLM, error) {
|
||||
@@ -464,7 +382,6 @@ func (s *ImageMultimodalService) checkAndFinalizeAllImages(ctx context.Context,
|
||||
logger.Infof(ctx, "[ImageMultimodal] All images processed for knowledge %s. Finalizing...", payload.KnowledgeID)
|
||||
s.redisClient.Del(ctx, redisKey)
|
||||
|
||||
// Enqueue the post process task to handle all downstream tasks (summary, question generation, etc)
|
||||
s.enqueueKnowledgePostProcessTask(ctx, payload)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"slices"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/searchutil"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
@@ -59,12 +60,32 @@ func (s *knowledgeBaseService) processSearchResults(ctx context.Context,
|
||||
for _, chunk := range additionalChunks {
|
||||
chunkMap[chunk.ID] = chunk
|
||||
}
|
||||
// Second round: only needed when image chunks are among primary
|
||||
// results (image → text resolved above, now text → parent_text).
|
||||
// For normal text-only results this is a no-op.
|
||||
if s.hasImageChunks(allChunks) {
|
||||
parentIDs := s.collectParentChunkIDs(additionalChunks, index)
|
||||
if len(parentIDs) > 0 {
|
||||
logger.Infof(ctx, "Fetching %d second-level parent chunks", len(parentIDs))
|
||||
parentChunks, err := s.listChunksByIDWithShared(ctx, tenantID, parentIDs)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "Failed to fetch second-level parent chunks: %v", err)
|
||||
} else {
|
||||
for _, chunk := range parentChunks {
|
||||
chunkMap[chunk.ID] = chunk
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build final search results
|
||||
searchResults := s.assembleSearchResults(ctx, chunks, chunkMap, knowledgeMap, index, skipEnrichment)
|
||||
|
||||
searchutil.EnrichSearchResultsImageInfo(ctx, s.chunkRepo, tenantID, searchResults)
|
||||
|
||||
logger.Infof(ctx, "Search results processed, total: %d", len(searchResults))
|
||||
return searchResults, nil
|
||||
}
|
||||
@@ -151,6 +172,36 @@ func (s *knowledgeBaseService) collectEnrichmentChunkIDs(
|
||||
return additionalIDs
|
||||
}
|
||||
|
||||
// collectParentChunkIDs returns unprocessed parent IDs from the given chunks.
|
||||
// Unlike collectEnrichmentChunkIDs, this only resolves parent links without
|
||||
// expanding nearby or related chunks, making it suitable for second-round
|
||||
// parent chain resolution (e.g., text → parent_text after image → text).
|
||||
func (s *knowledgeBaseService) collectParentChunkIDs(
|
||||
chunks []*types.Chunk,
|
||||
idx *chunkIndex,
|
||||
) []string {
|
||||
var ids []string
|
||||
for _, chunk := range chunks {
|
||||
if chunk.ParentChunkID != "" && !idx.processedIDs[chunk.ParentChunkID] {
|
||||
ids = append(ids, chunk.ParentChunkID)
|
||||
idx.processedIDs[chunk.ParentChunkID] = true
|
||||
idx.scores[chunk.ParentChunkID] = idx.scores[chunk.ID]
|
||||
idx.matchTypes[chunk.ParentChunkID] = types.MatchTypeParentChunk
|
||||
}
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
// hasImageChunks returns true if any chunk is an image_ocr or image_caption type.
|
||||
func (s *knowledgeBaseService) hasImageChunks(chunks []*types.Chunk) bool {
|
||||
for _, c := range chunks {
|
||||
if c.ChunkType == types.ChunkTypeImageOCR || c.ChunkType == types.ChunkTypeImageCaption {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// assembleSearchResults builds the final []*types.SearchResult from chunk data and knowledge data.
|
||||
// Primary results (from input chunks) are added first in order, then enrichment results.
|
||||
func (s *knowledgeBaseService) assembleSearchResults(
|
||||
@@ -174,7 +225,7 @@ func (s *knowledgeBaseService) assembleSearchResults(
|
||||
logger.Debugf(ctx, "Chunk not found in chunkMap: %s", inputChunk.ChunkID)
|
||||
continue
|
||||
}
|
||||
if !s.isValidTextChunk(chunk) {
|
||||
if !s.isSearchableChunk(chunk) {
|
||||
invalidChunkCnt++
|
||||
if len(invalidChunkSamples) < maxInvalidChunkLog {
|
||||
invalidChunkSamples = append(invalidChunkSamples, chunk.ID+":"+chunk.ChunkType)
|
||||
@@ -197,7 +248,7 @@ func (s *knowledgeBaseService) assembleSearchResults(
|
||||
}
|
||||
if invalidChunkCnt > 0 {
|
||||
logger.Debugf(ctx,
|
||||
"Skip non-text chunks in search results: total=%d sampled=%d samples=%v",
|
||||
"Skip non-searchable chunks in search results: total=%d sampled=%d samples=%v",
|
||||
invalidChunkCnt, len(invalidChunkSamples), invalidChunkSamples,
|
||||
)
|
||||
}
|
||||
@@ -205,7 +256,7 @@ func (s *knowledgeBaseService) assembleSearchResults(
|
||||
// Second pass: Add enrichment chunks (parent, nearby, relation)
|
||||
if !skipEnrichment {
|
||||
for chunkID, chunk := range chunkMap {
|
||||
if addedChunkIDs[chunkID] || !s.isValidTextChunk(chunk) {
|
||||
if addedChunkIDs[chunkID] || !s.isSearchableChunk(chunk) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -280,11 +331,12 @@ func (s *knowledgeBaseService) buildSearchResult(chunk *types.Chunk,
|
||||
}
|
||||
}
|
||||
|
||||
// isValidTextChunk checks if a chunk is a valid text chunk.
|
||||
func (s *knowledgeBaseService) isValidTextChunk(chunk *types.Chunk) bool {
|
||||
// isSearchableChunk checks if a chunk type should be included in search results.
|
||||
func (s *knowledgeBaseService) isSearchableChunk(chunk *types.Chunk) bool {
|
||||
return slices.Contains([]types.ChunkType{
|
||||
types.ChunkTypeText, types.ChunkTypeSummary,
|
||||
types.ChunkTypeTableColumn, types.ChunkTypeTableSummary,
|
||||
types.ChunkTypeFAQ,
|
||||
types.ChunkTypeImageOCR, types.ChunkTypeImageCaption,
|
||||
}, chunk.ChunkType)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user