feat(faq): enhance FAQ handling and localization support
- Updated the agent system prompt to clarify mandatory deep read requirements for FAQ entries, ensuring accurate retrieval and response generation. - Added new localization strings for FAQ-related labels in English and Chinese, improving user experience and consistency across the application. - Refactored the document and grep results components to support distinct FAQ identification, enhancing clarity in the output. - Introduced new tests for FAQ metadata handling, ensuring robust functionality and correctness in the FAQ processing logic.
This commit is contained in:
@@ -73,7 +73,7 @@ templates:
|
||||
|
||||
### Critical Constraints (ABSOLUTE RULES)
|
||||
1. **Evidence-Based Facts:** For factual claims about documents or domain knowledge, rely on KB/Web retrieval rather than internal knowledge. However, you MAY answer directly when the user's question is about image content you can see, conversational context, or general interaction.
|
||||
2. **Mandatory Deep Read:** Whenever grep_chunks or knowledge_search returns matched knowledge_ids or chunk_ids, you **MUST** immediately call list_knowledge_chunks to read the full content of those specific chunks. Do not rely on search snippets alone.
|
||||
2. **Mandatory Deep Read:** Whenever grep_chunks or knowledge_search returns matches, you **MUST** read full content before answering. For **FAQ** hits use `list_knowledge_chunks(faq_id=...)` or `get_document_info(faq_ids=[...])` with the **faq_id** from the result — do **not** use the parent knowledge_id of the FAQ container. For **documents**, use `list_knowledge_chunks(knowledge_id=...)` or `get_document_info(knowledge_ids=...)`. Do not rely on search snippets alone.
|
||||
3. **Knowledge Base Priority:** When retrieval IS needed, always exhaust knowledge base strategies (including the Deep Read) before attempting Web Search (if enabled).
|
||||
4. **Always Re-Retrieve for Each New Question:** You MUST perform fresh knowledge base retrieval for EVERY new user question that requires factual or domain-specific information, even if a similar or identical question was asked earlier in the conversation. NEVER rely on previously retrieved knowledge base content from the conversation history — the knowledge base may have been updated, switched, or had content removed since the last retrieval. Treat each new question as if you have no prior knowledge from previous retrievals.
|
||||
5. **User-Friendly Communication:** In ALL outputs visible to users (including your thinking/reasoning process), you MUST:
|
||||
|
||||
@@ -2701,6 +2701,9 @@ export default {
|
||||
referenceIconAlt: 'Reference materials icon',
|
||||
chunkIdLabel: 'Chunk ID:',
|
||||
documentIdLabel: 'Document ID:',
|
||||
faqIdLabel: 'FAQ ID:',
|
||||
faqContainerIdLabel: 'Container ID:',
|
||||
faqAnswersLabel: 'Answers:',
|
||||
chunkOrdinal: 'Chunk {index}',
|
||||
previewContent: 'Preview content',
|
||||
noPlanSteps: 'No detailed steps provided',
|
||||
@@ -4570,6 +4573,7 @@ export default {
|
||||
getDocument: 'Get document: {title}',
|
||||
document: 'Document',
|
||||
listChunks: 'View {title}',
|
||||
listFaqEntry: 'View FAQ: {question}',
|
||||
deepThinking: 'Deep Thinking',
|
||||
},
|
||||
plan: {
|
||||
@@ -4588,6 +4592,7 @@ export default {
|
||||
chunkHits: '{count} chunks',
|
||||
keywordHits: '{count} hits',
|
||||
titleMatch: 'title',
|
||||
faqEntry: 'FAQ entry',
|
||||
},
|
||||
toolStatus: {
|
||||
calling: 'Calling {name}...',
|
||||
|
||||
@@ -806,6 +806,9 @@ export default {
|
||||
referenceIconAlt: "参考内容图标",
|
||||
chunkIdLabel: "片段ID:",
|
||||
documentIdLabel: "文档ID:",
|
||||
faqIdLabel: "FAQ ID:",
|
||||
faqContainerIdLabel: "所属文档ID:",
|
||||
faqAnswersLabel: "答案:",
|
||||
chunkOrdinal: "片段 {index}",
|
||||
previewContent: "预览内容",
|
||||
noPlanSteps: "未提供具体步骤",
|
||||
@@ -4564,6 +4567,7 @@ export default {
|
||||
getDocument: "获取文档:{title}",
|
||||
document: "文档",
|
||||
listChunks: "查看 {title}",
|
||||
listFaqEntry: "查看 FAQ:{question}",
|
||||
deepThinking: "深度思考",
|
||||
},
|
||||
plan: {
|
||||
@@ -4582,6 +4586,7 @@ export default {
|
||||
chunkHits: "{count} 片段",
|
||||
keywordHits: "{count} 次",
|
||||
titleMatch: "标题匹配",
|
||||
faqEntry: "FAQ 条目",
|
||||
},
|
||||
toolStatus: {
|
||||
calling: "正在调用 {name}...",
|
||||
|
||||
@@ -105,8 +105,14 @@ export interface KnowledgeBaseListData {
|
||||
|
||||
// Document info data
|
||||
export interface DocumentInfoDocument {
|
||||
knowledge_id: string;
|
||||
knowledge_id?: string;
|
||||
faq_id?: string;
|
||||
chunk_id?: string;
|
||||
title: string;
|
||||
faq_question?: string;
|
||||
faq_answers?: string[];
|
||||
faq_similar_questions?: string[];
|
||||
is_faq?: boolean;
|
||||
description?: string;
|
||||
type?: string;
|
||||
source?: string;
|
||||
@@ -204,13 +210,11 @@ export interface WebFetchResultsData {
|
||||
count?: number;
|
||||
}
|
||||
|
||||
// Grep knowledge aggregation item
|
||||
// Grep knowledge aggregation item (legacy, grouped by knowledge_id)
|
||||
export interface GrepKnowledgeResult {
|
||||
knowledge_id: string;
|
||||
knowledge_base_id: string;
|
||||
knowledge_title: string;
|
||||
// Standard question of the first matched FAQ entry, used as the row label
|
||||
// since FAQ entries share the owning document's title.
|
||||
faq_question?: string;
|
||||
title_match?: boolean;
|
||||
chunk_hit_count: number;
|
||||
@@ -220,11 +224,28 @@ export interface GrepKnowledgeResult {
|
||||
distinct_patterns: number;
|
||||
}
|
||||
|
||||
// Per-chunk grep hit (preferred for UI — one row per FAQ entry or chunk)
|
||||
export interface GrepChunkResult {
|
||||
chunk_id: string;
|
||||
faq_id?: string;
|
||||
knowledge_id: string;
|
||||
knowledge_base_id: string;
|
||||
knowledge_title: string;
|
||||
chunk_type?: string;
|
||||
index?: number;
|
||||
chunk_index?: number;
|
||||
faq_question?: string;
|
||||
title_match?: boolean;
|
||||
match_snippet?: string;
|
||||
score?: number;
|
||||
}
|
||||
|
||||
// Grep results data
|
||||
export interface GrepResultsData {
|
||||
display_type: 'grep_results';
|
||||
query?: string;
|
||||
patterns: string[];
|
||||
chunk_results?: GrepChunkResult[];
|
||||
knowledge_results: GrepKnowledgeResult[];
|
||||
result_count: number;
|
||||
total_matches: number;
|
||||
|
||||
@@ -2080,6 +2080,9 @@ const getToolSummary = (event: any): string => {
|
||||
return t('agentStream.toolSummary.getDocument', { title: toolData.title });
|
||||
}
|
||||
} else if (toolName === 'list_knowledge_chunks') {
|
||||
if (toolData?.faq_question) {
|
||||
return t('agentStream.toolSummary.listFaqEntry', { question: toolData.faq_question });
|
||||
}
|
||||
if (toolData?.fetched_chunks !== undefined) {
|
||||
const title = toolData?.knowledge_title || toolData?.knowledge_id || t('agentStream.toolSummary.document');
|
||||
return t('agentStream.toolSummary.listChunks', { title, fetched: toolData.fetched_chunks, total: toolData.total_chunks ?? '?' });
|
||||
@@ -2238,7 +2241,8 @@ const getGrepResultsSummary = (toolData: any): string => {
|
||||
if (!toolData) return '';
|
||||
|
||||
const totalChunks = toolData.total_matches || 0;
|
||||
const docCount = toolData.result_count || 0;
|
||||
const rowCount =
|
||||
toolData.chunk_results?.length ?? toolData.result_count ?? toolData.knowledge_results?.length ?? 0;
|
||||
|
||||
if (totalChunks === 0) {
|
||||
return t('agentStream.search.noResults');
|
||||
@@ -2246,7 +2250,7 @@ const getGrepResultsSummary = (toolData: any): string => {
|
||||
|
||||
return t('agentStream.search.grepSummary', {
|
||||
chunks: `<strong>${totalChunks}</strong>`,
|
||||
docs: `<strong>${docCount}</strong>`,
|
||||
docs: `<strong>${rowCount}</strong>`,
|
||||
});
|
||||
};
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
<div v-if="documents.length" class="documents-list">
|
||||
<div
|
||||
v-for="(doc, index) in documents"
|
||||
:key="doc.knowledge_id || index"
|
||||
:key="doc.faq_id || doc.knowledge_id || index"
|
||||
class="result-card document-card"
|
||||
>
|
||||
<div class="result-header document-header">
|
||||
@@ -20,10 +20,23 @@
|
||||
</div>
|
||||
<div class="result-content expanded">
|
||||
<div class="info-section">
|
||||
<div class="info-field">
|
||||
<span class="field-label">{{ $t('chat.documentIdLabel') }}</span>
|
||||
<div class="info-field" v-if="doc.is_faq && doc.faq_id">
|
||||
<span class="field-label">{{ $t('chat.faqIdLabel') }}</span>
|
||||
<span class="field-value"><code>{{ doc.faq_id }}</code></span>
|
||||
</div>
|
||||
<div class="info-field" v-if="doc.knowledge_id">
|
||||
<span class="field-label">{{ doc.is_faq ? $t('chat.faqContainerIdLabel') : $t('chat.documentIdLabel') }}</span>
|
||||
<span class="field-value"><code>{{ doc.knowledge_id }}</code></span>
|
||||
</div>
|
||||
<div
|
||||
v-if="doc.is_faq && doc.faq_answers?.length"
|
||||
class="info-field info-field--block"
|
||||
>
|
||||
<span class="field-label">{{ $t('chat.faqAnswersLabel') }}</span>
|
||||
<ul class="faq-answers-list">
|
||||
<li v-for="(ans, aIdx) in doc.faq_answers" :key="aIdx">{{ ans }}</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="info-field" v-if="doc.description">
|
||||
<span class="field-label">{{ $t('chat.documentDescriptionLabel') }}</span>
|
||||
<span class="field-value">{{ doc.description }}</span>
|
||||
@@ -205,6 +218,21 @@ const formatMetadataValue = (value: unknown) => {
|
||||
}
|
||||
}
|
||||
|
||||
.faq-answers-list {
|
||||
list-style: none;
|
||||
margin: 4px 0 0;
|
||||
padding: 0;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 4px;
|
||||
|
||||
li {
|
||||
font-size: 12px;
|
||||
line-height: 1.5;
|
||||
color: var(--td-text-color-primary);
|
||||
}
|
||||
}
|
||||
|
||||
.info-field {
|
||||
display: flex;
|
||||
gap: 10px;
|
||||
@@ -212,6 +240,11 @@ const formatMetadataValue = (value: unknown) => {
|
||||
font-size: 12px;
|
||||
line-height: 1.5;
|
||||
|
||||
&--block {
|
||||
flex-direction: column;
|
||||
gap: 4px;
|
||||
}
|
||||
|
||||
.field-label {
|
||||
color: var(--td-text-color-secondary);
|
||||
min-width: 90px;
|
||||
|
||||
@@ -1,16 +1,17 @@
|
||||
<template>
|
||||
<div class="grep-results">
|
||||
<div v-if="results.length" class="results-list">
|
||||
<div v-if="rows.length" class="results-list">
|
||||
<ResultRow
|
||||
v-for="(result, index) in results"
|
||||
:key="result.knowledge_id"
|
||||
v-for="(result, index) in rows"
|
||||
:key="result.key"
|
||||
:index="index + 1"
|
||||
:title="result.faq_question || result.knowledge_title || $t('knowledge.untitledDocument')"
|
||||
:meta="formatMeta(result)"
|
||||
:popup-key="result.knowledge_id || index"
|
||||
:show-popup="!!cleanedSnippet(result)"
|
||||
:content="result.match_snippet"
|
||||
:knowledge-id="result.knowledge_id"
|
||||
:title="result.title"
|
||||
:meta="result.meta"
|
||||
:popup-key="result.key"
|
||||
:show-popup="!!result.snippet"
|
||||
:content="result.snippet"
|
||||
:chunk-id="result.chunkId"
|
||||
:knowledge-id="result.knowledgeId"
|
||||
:highlight="searchPattern"
|
||||
:regex="true"
|
||||
/>
|
||||
@@ -27,7 +28,7 @@ import { computed } from 'vue';
|
||||
import { useI18n } from 'vue-i18n';
|
||||
import { cleanSnippet } from './contentClean';
|
||||
import ResultRow from './ResultRow.vue';
|
||||
import type { GrepKnowledgeResult, GrepResultsData } from '@/types/tool-results';
|
||||
import type { GrepChunkResult, GrepKnowledgeResult, GrepResultsData } from '@/types/tool-results';
|
||||
|
||||
const props = defineProps<{
|
||||
data: GrepResultsData;
|
||||
@@ -35,10 +36,18 @@ const props = defineProps<{
|
||||
|
||||
const { t } = useI18n();
|
||||
|
||||
const results = computed(() => props.data.knowledge_results ?? []);
|
||||
const searchPattern = computed(() => props.data.query ?? props.data.patterns?.[0] ?? '');
|
||||
|
||||
const formatMeta = (result: GrepKnowledgeResult): string => {
|
||||
type GrepRow = {
|
||||
key: string;
|
||||
title: string;
|
||||
meta: string;
|
||||
snippet: string;
|
||||
chunkId?: string;
|
||||
knowledgeId?: string;
|
||||
};
|
||||
|
||||
const formatKnowledgeMeta = (result: GrepKnowledgeResult): string => {
|
||||
const parts: string[] = [];
|
||||
const chunks = result.chunk_hit_count ?? 0;
|
||||
if (chunks > 0) {
|
||||
@@ -54,8 +63,46 @@ const formatMeta = (result: GrepKnowledgeResult): string => {
|
||||
return parts.join(' · ');
|
||||
};
|
||||
|
||||
const cleanedSnippet = (result: GrepKnowledgeResult): string =>
|
||||
cleanSnippet(result.match_snippet ?? '');
|
||||
const rowFromChunk = (result: GrepChunkResult): GrepRow => {
|
||||
const isFAQ = !!result.faq_id || result.chunk_type === 'faq';
|
||||
const title = result.faq_question || result.knowledge_title || t('knowledge.untitledDocument');
|
||||
const meta = isFAQ
|
||||
? t('agentStream.grepResults.faqEntry')
|
||||
: formatKnowledgeMeta({
|
||||
knowledge_id: result.knowledge_id,
|
||||
knowledge_base_id: result.knowledge_base_id,
|
||||
knowledge_title: result.knowledge_title,
|
||||
chunk_hit_count: 1,
|
||||
total_pattern_hits: 1,
|
||||
distinct_patterns: 1,
|
||||
pattern_counts: {},
|
||||
title_match: !!result.title_match,
|
||||
});
|
||||
return {
|
||||
key: result.faq_id || result.chunk_id || String(result.index ?? '') || result.knowledge_id,
|
||||
title,
|
||||
meta,
|
||||
snippet: cleanSnippet(result.match_snippet ?? ''),
|
||||
chunkId: result.faq_id || result.chunk_id,
|
||||
knowledgeId: result.knowledge_id,
|
||||
};
|
||||
};
|
||||
|
||||
const rowFromKnowledge = (result: GrepKnowledgeResult): GrepRow => ({
|
||||
key: result.knowledge_id,
|
||||
title: result.faq_question || result.knowledge_title || t('knowledge.untitledDocument'),
|
||||
meta: formatKnowledgeMeta(result),
|
||||
snippet: cleanSnippet(result.match_snippet ?? ''),
|
||||
knowledgeId: result.knowledge_id,
|
||||
});
|
||||
|
||||
const rows = computed((): GrepRow[] => {
|
||||
const chunkRows = props.data.chunk_results;
|
||||
if (chunkRows?.length) {
|
||||
return chunkRows.map(rowFromChunk);
|
||||
}
|
||||
return (props.data.knowledge_results ?? []).map(rowFromKnowledge);
|
||||
});
|
||||
</script>
|
||||
|
||||
<style lang="less" scoped>
|
||||
|
||||
@@ -0,0 +1,287 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// faqMaxSimilarQuestionsDisplay caps similar_question rows in tool output.
|
||||
const faqMaxSimilarQuestionsDisplay = 5
|
||||
|
||||
func truncateSimilarQuestionsForDisplay(questions []string) (display []string, omitted int) {
|
||||
if len(questions) == 0 {
|
||||
return nil, 0
|
||||
}
|
||||
if len(questions) <= faqMaxSimilarQuestionsDisplay {
|
||||
return questions, 0
|
||||
}
|
||||
return questions[:faqMaxSimilarQuestionsDisplay], len(questions) - faqMaxSimilarQuestionsDisplay
|
||||
}
|
||||
|
||||
func writeSimilarQuestionsXML(b *strings.Builder, questions []string) {
|
||||
display, omitted := truncateSimilarQuestionsForDisplay(questions)
|
||||
for _, sq := range display {
|
||||
fmt.Fprintf(b, "<similar_question>%s</similar_question>\n", xmlEscape(sq))
|
||||
}
|
||||
if omitted > 0 {
|
||||
fmt.Fprintf(b, "<similar_questions_omitted count=\"%d\" />\n", omitted)
|
||||
}
|
||||
}
|
||||
|
||||
func appendSimilarQuestionsToChunkData(chunkData map[string]interface{}, questions []string) {
|
||||
display, omitted := truncateSimilarQuestionsForDisplay(questions)
|
||||
if len(display) == 0 {
|
||||
return
|
||||
}
|
||||
chunkData["faq_similar_questions"] = display
|
||||
if omitted > 0 {
|
||||
chunkData["faq_similar_questions_omitted"] = omitted
|
||||
}
|
||||
}
|
||||
|
||||
// writeFAQFieldsXML emits question / similar_question / answer children (no wrapper).
|
||||
func writeFAQFieldsXML(b *strings.Builder, meta *types.FAQChunkMetadata) {
|
||||
if meta == nil {
|
||||
return
|
||||
}
|
||||
if meta.StandardQuestion != "" {
|
||||
fmt.Fprintf(b, "<question>%s</question>\n", xmlEscape(meta.StandardQuestion))
|
||||
}
|
||||
writeSimilarQuestionsXML(b, meta.SimilarQuestions)
|
||||
for _, ans := range meta.Answers {
|
||||
if strings.TrimSpace(ans) == "" {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(b, "<answer>%s</answer>\n", xmlEscape(ans))
|
||||
}
|
||||
}
|
||||
|
||||
func faqFieldsEmpty(meta *types.FAQChunkMetadata) bool {
|
||||
if meta == nil {
|
||||
return true
|
||||
}
|
||||
return meta.StandardQuestion == "" && len(meta.SimilarQuestions) == 0 && len(meta.Answers) == 0
|
||||
}
|
||||
|
||||
// writeFAQMetadataXML emits a nested <faq> block (used inside knowledge_search <chunk>).
|
||||
func writeFAQMetadataXML(b *strings.Builder, meta *types.FAQChunkMetadata) {
|
||||
if faqFieldsEmpty(meta) {
|
||||
return
|
||||
}
|
||||
b.WriteString("<faq>\n")
|
||||
writeFAQFieldsXML(b, meta)
|
||||
b.WriteString("</faq>\n")
|
||||
}
|
||||
|
||||
// writeFAQEntryXML emits a top-level FAQ entry for list_knowledge_chunks (not wrapped in <chunk>).
|
||||
func writeFAQEntryXML(b *strings.Builder, c *types.Chunk) {
|
||||
if c == nil || c.ChunkType != types.ChunkTypeFAQ {
|
||||
return
|
||||
}
|
||||
meta, err := c.FAQMetadata()
|
||||
if err != nil {
|
||||
meta = nil
|
||||
}
|
||||
|
||||
q := faqStandardQuestion(c)
|
||||
questionAttr := ""
|
||||
if q != "" {
|
||||
questionAttr = fmt.Sprintf(" question=\"%s\"", xmlEscape(q))
|
||||
}
|
||||
fmt.Fprintf(b, "<faq faq_id=\"%s\" index=\"%d\"%s>\n",
|
||||
xmlEscape(c.ID),
|
||||
c.ChunkIndex,
|
||||
questionAttr,
|
||||
)
|
||||
|
||||
if !faqFieldsEmpty(meta) {
|
||||
writeFAQFieldsXML(b, meta)
|
||||
} else if q != "" {
|
||||
fmt.Fprintf(b, "<question>%s</question>\n", xmlEscape(q))
|
||||
}
|
||||
|
||||
b.WriteString("</faq>\n")
|
||||
}
|
||||
|
||||
// normalizeFAQChunkDataMap uses faq_id / index instead of chunk_id / chunk_index in JSON payloads.
|
||||
func normalizeFAQChunkDataMap(chunkData map[string]interface{}, c *types.Chunk) {
|
||||
if c == nil || c.ChunkType != types.ChunkTypeFAQ || chunkData == nil {
|
||||
return
|
||||
}
|
||||
chunkData["faq_id"] = c.ID
|
||||
chunkData["index"] = c.ChunkIndex
|
||||
delete(chunkData, "chunk_id")
|
||||
delete(chunkData, "chunk_index")
|
||||
}
|
||||
|
||||
// appendFAQChunkData adds FAQ metadata fields to structured tool result maps.
|
||||
func appendFAQChunkData(chunkData map[string]interface{}, c *types.Chunk) {
|
||||
if c == nil || c.ChunkType != types.ChunkTypeFAQ {
|
||||
return
|
||||
}
|
||||
meta, err := c.FAQMetadata()
|
||||
if err != nil || meta == nil {
|
||||
return
|
||||
}
|
||||
if q := strings.TrimSpace(meta.StandardQuestion); q != "" {
|
||||
chunkData["faq_question"] = q
|
||||
}
|
||||
appendSimilarQuestionsToChunkData(chunkData, meta.SimilarQuestions)
|
||||
if len(meta.Answers) > 0 {
|
||||
chunkData["faq_answers"] = meta.Answers
|
||||
}
|
||||
}
|
||||
|
||||
// Bounds for retrieval-tool match snippets (grep_chunks, knowledge_search).
|
||||
const (
|
||||
snippetContextRunes = 200
|
||||
snippetMaxMatchRunes = 200
|
||||
snippetMaxTotalRunes = 800
|
||||
snippetMaxAnswerRunes = 600
|
||||
)
|
||||
|
||||
// faqMatchSnippetFromQueries builds "Q: … | A: …" for knowledge_search hits.
|
||||
func faqMatchSnippetFromQueries(meta *types.FAQChunkMetadata, queries []string) string {
|
||||
if meta == nil {
|
||||
return ""
|
||||
}
|
||||
question := faqMatchedQuestionFromQueries(meta, queries)
|
||||
if question == "" {
|
||||
question = strings.TrimSpace(meta.StandardQuestion)
|
||||
}
|
||||
return formatFAQMatchSnippet(question, meta.Answers)
|
||||
}
|
||||
|
||||
// faqMatchSnippet builds "Q: … | A: …" for grep_chunks regex hits.
|
||||
func faqMatchSnippet(chunk *types.Chunk, compiled []*regexp.Regexp) string {
|
||||
if chunk == nil {
|
||||
return ""
|
||||
}
|
||||
meta, err := chunk.FAQMetadata()
|
||||
if err != nil || meta == nil {
|
||||
return ""
|
||||
}
|
||||
question := faqMatchedQuestionFromRegex(meta, compiled)
|
||||
if question == "" {
|
||||
question = strings.TrimSpace(meta.StandardQuestion)
|
||||
}
|
||||
if question == "" {
|
||||
return ""
|
||||
}
|
||||
return formatFAQMatchSnippet(question, meta.Answers)
|
||||
}
|
||||
|
||||
func formatFAQMatchSnippet(question string, answers []string) string {
|
||||
question = strings.TrimSpace(question)
|
||||
if question == "" {
|
||||
return ""
|
||||
}
|
||||
var b strings.Builder
|
||||
b.WriteString("Q: ")
|
||||
b.WriteString(question)
|
||||
if answer := faqAnswersForSnippet(answers); answer != "" {
|
||||
b.WriteString(" | A: ")
|
||||
b.WriteString(answer)
|
||||
}
|
||||
snippet := strings.TrimSpace(b.String())
|
||||
if len([]rune(snippet)) > snippetMaxTotalRunes {
|
||||
snippet = truncateRunes(snippet, snippetMaxTotalRunes)
|
||||
}
|
||||
return snippet
|
||||
}
|
||||
|
||||
func faqMatchedQuestionFromRegex(meta *types.FAQChunkMetadata, compiled []*regexp.Regexp) string {
|
||||
if meta == nil {
|
||||
return ""
|
||||
}
|
||||
for _, sq := range meta.SimilarQuestions {
|
||||
if regexMatchesAny(sq, compiled) {
|
||||
return sq
|
||||
}
|
||||
}
|
||||
if regexMatchesAny(meta.StandardQuestion, compiled) {
|
||||
return meta.StandardQuestion
|
||||
}
|
||||
return meta.StandardQuestion
|
||||
}
|
||||
|
||||
func faqMatchedQuestionFromQueries(meta *types.FAQChunkMetadata, queries []string) string {
|
||||
if meta == nil {
|
||||
return ""
|
||||
}
|
||||
tokens := searchQueryTokens(queries)
|
||||
for _, sq := range meta.SimilarQuestions {
|
||||
if textMatchesSearchQueries(sq, queries, tokens) {
|
||||
return sq
|
||||
}
|
||||
}
|
||||
if textMatchesSearchQueries(meta.StandardQuestion, queries, tokens) {
|
||||
return meta.StandardQuestion
|
||||
}
|
||||
return meta.StandardQuestion
|
||||
}
|
||||
|
||||
func faqAnswersForSnippet(answers []string) string {
|
||||
if len(answers) == 0 {
|
||||
return ""
|
||||
}
|
||||
parts := make([]string, 0, len(answers))
|
||||
for _, a := range answers {
|
||||
a = strings.TrimSpace(a)
|
||||
if a != "" {
|
||||
parts = append(parts, a)
|
||||
}
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return ""
|
||||
}
|
||||
return truncateRunes(strings.Join(parts, " | "), snippetMaxAnswerRunes)
|
||||
}
|
||||
|
||||
func textMatchesSearchQueries(text string, queries []string, tokens []string) bool {
|
||||
if text == "" {
|
||||
return false
|
||||
}
|
||||
lowered := strings.ToLower(text)
|
||||
for _, q := range queries {
|
||||
q = strings.ToLower(strings.TrimSpace(q))
|
||||
if q != "" && strings.Contains(lowered, q) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, tok := range tokens {
|
||||
if strings.Contains(lowered, tok) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func searchQueryTokens(queries []string) []string {
|
||||
tokens := make([]string, 0, 8)
|
||||
seen := make(map[string]struct{})
|
||||
for _, q := range queries {
|
||||
for _, tok := range strings.FieldsFunc(q, func(r rune) bool {
|
||||
switch r {
|
||||
case ' ', '\t', '\n', '\r', ',', '.', ';', ':', '?', '!',
|
||||
'(', ')', '[', ']', '{', '}', '"', '\'':
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}) {
|
||||
tok = strings.ToLower(strings.TrimSpace(tok))
|
||||
if len([]rune(tok)) < 2 {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[tok]; ok {
|
||||
continue
|
||||
}
|
||||
seen[tok] = struct{}{}
|
||||
tokens = append(tokens, tok)
|
||||
}
|
||||
}
|
||||
return tokens
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
func TestWriteFAQMetadataXML_IncludesAnswers(t *testing.T) {
|
||||
var b strings.Builder
|
||||
meta := &types.FAQChunkMetadata{
|
||||
StandardQuestion: "如何创建知识库?",
|
||||
SimilarQuestions: []string{"怎么创建知识库?"},
|
||||
Answers: []string{"在控制台点击新建知识库。"},
|
||||
}
|
||||
writeFAQMetadataXML(&b, meta)
|
||||
out := b.String()
|
||||
|
||||
for _, want := range []string{
|
||||
"<faq>", "<question>如何创建知识库?</question>",
|
||||
"<similar_question>怎么创建知识库?</similar_question>",
|
||||
"<answer>在控制台点击新建知识库。</answer>", "</faq>",
|
||||
} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Fatalf("missing %q in:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteSimilarQuestionsXML_TruncatesWhenTooMany(t *testing.T) {
|
||||
var b strings.Builder
|
||||
questions := make([]string, 8)
|
||||
for i := range questions {
|
||||
questions[i] = fmt.Sprintf("相似问%d", i+1)
|
||||
}
|
||||
writeSimilarQuestionsXML(&b, questions)
|
||||
out := b.String()
|
||||
|
||||
if strings.Count(out, "<similar_question>") != faqMaxSimilarQuestionsDisplay {
|
||||
t.Fatalf("want %d similar_question tags, got:\n%s", faqMaxSimilarQuestionsDisplay, out)
|
||||
}
|
||||
if !strings.Contains(out, `<similar_questions_omitted count="3"`) {
|
||||
t.Fatalf("want omitted marker, got:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteFAQEntryXML_UsesFaqNotChunk(t *testing.T) {
|
||||
chunk := &types.Chunk{
|
||||
ID: "faq-chunk-1",
|
||||
ChunkType: types.ChunkTypeFAQ,
|
||||
ChunkIndex: 0,
|
||||
KnowledgeID: "kb-doc-1",
|
||||
Content: "Q: test\nSimilar Questions:\n- alt",
|
||||
}
|
||||
meta := &types.FAQChunkMetadata{
|
||||
StandardQuestion: "如何创建知识库?",
|
||||
Answers: []string{"点新建即可。"},
|
||||
}
|
||||
if err := chunk.SetFAQMetadata(meta); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
writeFAQEntryXML(&b, chunk)
|
||||
out := b.String()
|
||||
|
||||
if strings.Contains(out, "<chunk") {
|
||||
t.Fatalf("FAQ list output must not use <chunk>, got:\n%s", out)
|
||||
}
|
||||
for _, want := range []string{
|
||||
`<faq faq_id="faq-chunk-1"`,
|
||||
"<question>如何创建知识库?</question>",
|
||||
"<answer>点新建即可。</answer>",
|
||||
"</faq>",
|
||||
} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Fatalf("missing %q in:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
@@ -40,13 +41,18 @@ Do not use when:
|
||||
|
||||
- Concurrent query for multiple documents provides better performance
|
||||
- Returns complete document metadata, not just title
|
||||
- Can check document processing status (parse_status)`,
|
||||
- Can check document processing status (parse_status)
|
||||
|
||||
## IDs
|
||||
- knowledge_ids: regular documents knowledges
|
||||
- faq_ids: individual FAQ entries. Returns the standard question and answers, not the container title.`,
|
||||
schema: utils.GenerateSchema[GetDocumentInfoInput](),
|
||||
}
|
||||
|
||||
// GetDocumentInfoInput defines the input parameters for get document info tool
|
||||
type GetDocumentInfoInput struct {
|
||||
KnowledgeIDs []string `json:"knowledge_ids" jsonschema:"Array of document/knowledge IDs, obtained from knowledge_id field in search results, supports concurrent batch queries"`
|
||||
KnowledgeIDs []string `json:"knowledge_ids" jsonschema:"Document/knowledge IDs for regular documents or FAQ containers"`
|
||||
FAQIDs []string `json:"faq_ids" jsonschema:"FAQ entry IDs (= chunk_id from grep_chunks). Use instead of knowledge_ids for a single FAQ Q&A."`
|
||||
}
|
||||
|
||||
// GetDocumentInfoTool retrieves detailed information about a document/knowledge
|
||||
@@ -82,18 +88,19 @@ func (t *GetDocumentInfoTool) Execute(ctx context.Context, args json.RawMessage)
|
||||
}, err
|
||||
}
|
||||
|
||||
// Extract knowledge_ids array
|
||||
knowledgeIDs := input.KnowledgeIDs
|
||||
if len(knowledgeIDs) == 0 {
|
||||
faqIDs := input.FAQIDs
|
||||
if len(knowledgeIDs) == 0 && len(faqIDs) == 0 {
|
||||
return &types.ToolResult{
|
||||
Success: false,
|
||||
Error: "knowledge_ids is required and must be a non-empty array",
|
||||
}, fmt.Errorf("knowledge_ids is required")
|
||||
Error: "knowledge_ids or faq_ids is required (non-empty array)",
|
||||
}, fmt.Errorf("missing ids")
|
||||
}
|
||||
|
||||
// Concurrently get info for each knowledge ID
|
||||
type docInfo struct {
|
||||
knowledge *types.Knowledge
|
||||
chunk *types.Chunk
|
||||
faqMeta *types.FAQChunkMetadata
|
||||
chunkCount int
|
||||
err error
|
||||
}
|
||||
@@ -102,7 +109,37 @@ func (t *GetDocumentInfoTool) Execute(ctx context.Context, args json.RawMessage)
|
||||
var mu sync.Mutex
|
||||
results := make(map[string]*docInfo)
|
||||
|
||||
// Concurrently get info for each knowledge ID
|
||||
for _, faqID := range faqIDs {
|
||||
faqID = strings.TrimSpace(faqID)
|
||||
if faqID == "" {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(id string) {
|
||||
defer wg.Done()
|
||||
chunk, err := t.chunkService.GetChunkByIDOnly(ctx, id)
|
||||
if err != nil || chunk == nil {
|
||||
mu.Lock()
|
||||
results["faq:"+id] = &docInfo{err: fmt.Errorf("faq entry not found: %v", err)}
|
||||
mu.Unlock()
|
||||
return
|
||||
}
|
||||
if !t.searchTargets.ContainsKB(chunk.KnowledgeBaseID) {
|
||||
mu.Lock()
|
||||
results["faq:"+id] = &docInfo{err: fmt.Errorf("knowledge base %s is not accessible", chunk.KnowledgeBaseID)}
|
||||
mu.Unlock()
|
||||
return
|
||||
}
|
||||
var meta *types.FAQChunkMetadata
|
||||
if chunk.ChunkType == types.ChunkTypeFAQ {
|
||||
meta, _ = chunk.FAQMetadata()
|
||||
}
|
||||
mu.Lock()
|
||||
results["faq:"+id] = &docInfo{chunk: chunk, faqMeta: meta, chunkCount: 1}
|
||||
mu.Unlock()
|
||||
}(faqID)
|
||||
}
|
||||
|
||||
for _, knowledgeID := range knowledgeIDs {
|
||||
wg.Add(1)
|
||||
go func(id string) {
|
||||
@@ -158,18 +195,38 @@ func (t *GetDocumentInfoTool) Execute(ctx context.Context, args json.RawMessage)
|
||||
|
||||
wg.Wait()
|
||||
|
||||
// Collect successful results and errors
|
||||
requested := len(knowledgeIDs) + len(faqIDs)
|
||||
successDocs := make([]*docInfo, 0)
|
||||
var errors []string
|
||||
|
||||
for _, knowledgeID := range knowledgeIDs {
|
||||
result := results[knowledgeID]
|
||||
if result == nil {
|
||||
errors = append(errors, fmt.Sprintf("%s: not found", knowledgeID))
|
||||
continue
|
||||
}
|
||||
if result.err != nil {
|
||||
errors = append(errors, fmt.Sprintf("%s: %v", knowledgeID, result.err))
|
||||
} else if result.knowledge != nil {
|
||||
successDocs = append(successDocs, result)
|
||||
}
|
||||
}
|
||||
for _, faqID := range faqIDs {
|
||||
faqID = strings.TrimSpace(faqID)
|
||||
if faqID == "" {
|
||||
continue
|
||||
}
|
||||
result := results["faq:"+faqID]
|
||||
if result == nil {
|
||||
errors = append(errors, fmt.Sprintf("faq:%s: not found", faqID))
|
||||
continue
|
||||
}
|
||||
if result.err != nil {
|
||||
errors = append(errors, fmt.Sprintf("faq:%s: %v", faqID, result.err))
|
||||
} else if result.chunk != nil {
|
||||
successDocs = append(successDocs, result)
|
||||
}
|
||||
}
|
||||
|
||||
if len(successDocs) == 0 {
|
||||
return &types.ToolResult{
|
||||
@@ -178,9 +235,8 @@ func (t *GetDocumentInfoTool) Execute(ctx context.Context, args json.RawMessage)
|
||||
}, fmt.Errorf("all document retrievals failed")
|
||||
}
|
||||
|
||||
// Format output
|
||||
output := "=== Document Info ===\n\n"
|
||||
output += fmt.Sprintf("Successfully retrieved %d / %d documents\n\n", len(successDocs), len(knowledgeIDs))
|
||||
output += fmt.Sprintf("Successfully retrieved %d / %d entries\n\n", len(successDocs), requested)
|
||||
|
||||
if len(errors) > 0 {
|
||||
output += "=== Partial Failures ===\n"
|
||||
@@ -192,9 +248,15 @@ func (t *GetDocumentInfoTool) Execute(ctx context.Context, args json.RawMessage)
|
||||
|
||||
formattedDocs := make([]map[string]interface{}, 0, len(successDocs))
|
||||
for i, doc := range successDocs {
|
||||
k := doc.knowledge
|
||||
output += fmt.Sprintf("[Entry #%d]\n", i+1)
|
||||
|
||||
output += fmt.Sprintf("[Document #%d]\n", i+1)
|
||||
if doc.chunk != nil {
|
||||
formatted := formatFAQEntryInfo(&output, doc.chunk, doc.faqMeta)
|
||||
formattedDocs = append(formattedDocs, formatted)
|
||||
continue
|
||||
}
|
||||
|
||||
k := doc.knowledge
|
||||
output += fmt.Sprintf(" ID: %s\n", k.ID)
|
||||
output += fmt.Sprintf(" Title: %s\n", k.Title)
|
||||
|
||||
@@ -236,13 +298,15 @@ func (t *GetDocumentInfoTool) Execute(ctx context.Context, args json.RawMessage)
|
||||
"parse_status": k.ParseStatus,
|
||||
"chunk_count": doc.chunkCount,
|
||||
"metadata": k.GetMetadata(),
|
||||
"is_faq": false,
|
||||
})
|
||||
}
|
||||
|
||||
// Extract first document title for summary
|
||||
var firstTitle string
|
||||
if len(successDocs) > 0 && successDocs[0].knowledge != nil {
|
||||
firstTitle = successDocs[0].knowledge.Title
|
||||
if len(formattedDocs) > 0 {
|
||||
if t, ok := formattedDocs[0]["title"].(string); ok {
|
||||
firstTitle = t
|
||||
}
|
||||
}
|
||||
|
||||
return &types.ToolResult{
|
||||
@@ -251,14 +315,64 @@ func (t *GetDocumentInfoTool) Execute(ctx context.Context, args json.RawMessage)
|
||||
Data: map[string]interface{}{
|
||||
"documents": formattedDocs,
|
||||
"total_docs": len(successDocs),
|
||||
"requested": len(knowledgeIDs),
|
||||
"requested": requested,
|
||||
"errors": errors,
|
||||
"display_type": "document_info",
|
||||
"title": firstTitle, // For frontend summary display
|
||||
"title": firstTitle,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func formatFAQEntryInfo(output *string, chunk *types.Chunk, meta *types.FAQChunkMetadata) map[string]interface{} {
|
||||
title := faqStandardQuestion(chunk)
|
||||
if title == "" && meta != nil {
|
||||
title = strings.TrimSpace(meta.StandardQuestion)
|
||||
}
|
||||
if title == "" {
|
||||
title = "FAQ Entry"
|
||||
}
|
||||
|
||||
*output += fmt.Sprintf(" FAQ ID: %s\n", chunk.ID)
|
||||
*output += fmt.Sprintf(" Question: %s\n", title)
|
||||
if chunk.KnowledgeID != "" {
|
||||
*output += fmt.Sprintf(" Container ID: %s\n", chunk.KnowledgeID)
|
||||
}
|
||||
if meta != nil && len(meta.Answers) > 0 {
|
||||
*output += " Answers:\n"
|
||||
for _, ans := range meta.Answers {
|
||||
*output += fmt.Sprintf(" - %s\n", ans)
|
||||
}
|
||||
}
|
||||
if meta != nil && len(meta.SimilarQuestions) > 0 {
|
||||
display, omitted := truncateSimilarQuestionsForDisplay(meta.SimilarQuestions)
|
||||
*output += " Similar Questions:\n"
|
||||
for _, sq := range display {
|
||||
*output += fmt.Sprintf(" - %s\n", sq)
|
||||
}
|
||||
if omitted > 0 {
|
||||
*output += fmt.Sprintf(" ... and %d more omitted\n", omitted)
|
||||
}
|
||||
}
|
||||
*output += "\n"
|
||||
|
||||
entry := map[string]interface{}{
|
||||
"faq_id": chunk.ID,
|
||||
"knowledge_id": chunk.KnowledgeID,
|
||||
"title": title,
|
||||
"faq_question": title,
|
||||
"type": "faq",
|
||||
"is_faq": true,
|
||||
"chunk_count": 1,
|
||||
}
|
||||
if meta != nil {
|
||||
if len(meta.Answers) > 0 {
|
||||
entry["faq_answers"] = meta.Answers
|
||||
}
|
||||
appendSimilarQuestionsToChunkData(entry, meta.SimilarQuestions)
|
||||
}
|
||||
return entry
|
||||
}
|
||||
|
||||
func formatSource(knowledgeType, source string) string {
|
||||
switch knowledgeType {
|
||||
case "file":
|
||||
|
||||
@@ -27,7 +27,11 @@ Examples:
|
||||
- Word boundary / anchor: "\\brag\\b" or "^chapter\\s+\\d+"
|
||||
- Plain text: "engine" (matches literal substring anywhere in chunk content)
|
||||
IMPORTANT — JSON escaping: every backslash in a regex MUST be written as \\ inside the JSON tool arguments (e.g. to search for literal "C++" write "C\\+\\+", NOT "C\+\+"; for "\d+" write "\\d+"). Plain "\+" / "\d" etc. are invalid JSON escapes and will fail to parse.
|
||||
Use this to locate candidate chunks by exact identifiers, error codes, product names, or recurring terms. Pair with list_knowledge_chunks afterwards to read the full context around any promising chunk_id.`,
|
||||
Use this to locate candidate chunks by exact identifiers, error codes, product names, or recurring terms.
|
||||
|
||||
## Deep read after grep:
|
||||
- **FAQ hit** (chunk type faq): call list_knowledge_chunks with **faq_id** from the grep result (NOT the parent knowledge_id).
|
||||
- **Document hit**: call list_knowledge_chunks with **knowledge_id**, or get_document_info with **knowledge_ids**.`,
|
||||
schema: json.RawMessage(`{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -186,7 +190,10 @@ func (t *GrepChunksTool) Execute(ctx context.Context, args json.RawMessage) (*ty
|
||||
finalResults = finalResults[:limit]
|
||||
}
|
||||
|
||||
// Aggregation by knowledge is still useful for the frontend summary view.
|
||||
// Per-chunk rows for the UI (FAQ entries are one chunk each — do not fold
|
||||
// them into the parent knowledge container). Knowledge-level aggregation is
|
||||
// kept for backward compatibility.
|
||||
chunkResults := buildGrepChunkResults(finalResults, compiled)
|
||||
aggregatedResults := t.aggregateByKnowledge(finalResults, queries, compiled)
|
||||
if len(aggregatedResults) > 20 {
|
||||
aggregatedResults = aggregatedResults[:20]
|
||||
@@ -201,8 +208,9 @@ func (t *GrepChunksTool) Execute(ctx context.Context, args json.RawMessage) (*ty
|
||||
"query": query,
|
||||
"queries": queries, // legacy alias for older frontends
|
||||
"patterns": queries, // legacy alias for older frontends
|
||||
"chunk_results": chunkResults,
|
||||
"knowledge_results": aggregatedResults,
|
||||
"result_count": len(aggregatedResults),
|
||||
"result_count": len(chunkResults),
|
||||
"total_matches": len(finalResults),
|
||||
"knowledge_base_ids": kbIDs,
|
||||
"limit": limit,
|
||||
@@ -373,60 +381,59 @@ func (t *GrepChunksTool) formatOutput(
|
||||
|
||||
for _, r := range results {
|
||||
counts := countRegexHits(r.Content, compiled, queries)
|
||||
snippet := extractSnippetRegex(r.Content, compiled)
|
||||
snippet := extractChunkMatchSnippet(&r.Chunk, compiled)
|
||||
|
||||
// FAQ entries share the owning knowledge's title, so expose the
|
||||
// standard question as a per-chunk identifier when available.
|
||||
faqAttr := ""
|
||||
extraAttr := ""
|
||||
if q := faqStandardQuestion(&r.Chunk); q != "" {
|
||||
faqAttr = fmt.Sprintf(" faq_question=\"%s\"", xmlEscape(q))
|
||||
extraAttr = fmt.Sprintf(" faq_question=\"%s\"", xmlEscape(q))
|
||||
}
|
||||
isFAQ := r.ChunkType == types.ChunkTypeFAQ
|
||||
|
||||
t.mu.Lock()
|
||||
seen := t.seenChunks[r.ID]
|
||||
t.seenChunks[r.ID] = true
|
||||
t.mu.Unlock()
|
||||
|
||||
if seen {
|
||||
b.WriteString(fmt.Sprintf(
|
||||
"<chunk chunk_id=\"%s\" knowledge_id=\"%s\" knowledge_title=\"%s\"%s chunk_index=\"%d\" score=\"%.3f\" already_seen=\"true\">\n",
|
||||
xmlEscape(r.ID),
|
||||
xmlEscape(r.KnowledgeID),
|
||||
xmlEscape(r.KnowledgeTitle),
|
||||
faqAttr,
|
||||
r.ChunkIndex,
|
||||
r.MatchScore,
|
||||
))
|
||||
for _, q := range queries {
|
||||
if c := counts[q]; c > 0 {
|
||||
b.WriteString(fmt.Sprintf("<query_hit query=\"%s\" count=\"%d\" />\n",
|
||||
xmlEscape(q), c))
|
||||
}
|
||||
if isFAQ {
|
||||
if seen {
|
||||
fmt.Fprintf(&b,
|
||||
"<faq faq_id=\"%s\" knowledge_title=\"%s\"%s index=\"%d\" score=\"%.3f\" already_seen=\"true\">\n",
|
||||
xmlEscape(r.ID), xmlEscape(r.KnowledgeTitle),
|
||||
extraAttr, r.ChunkIndex, r.MatchScore)
|
||||
} else {
|
||||
fmt.Fprintf(&b,
|
||||
"<faq faq_id=\"%s\" knowledge_title=\"%s\"%s index=\"%d\" score=\"%.3f\">\n",
|
||||
xmlEscape(r.ID), xmlEscape(r.KnowledgeTitle),
|
||||
extraAttr, r.ChunkIndex, r.MatchScore)
|
||||
}
|
||||
b.WriteString("<note>(snippet omitted, already returned in a previous grep_chunks call this session)</note>\n")
|
||||
b.WriteString("</chunk>\n")
|
||||
continue
|
||||
} else if seen {
|
||||
fmt.Fprintf(&b,
|
||||
"<chunk chunk_id=\"%s\" knowledge_id=\"%s\" knowledge_title=\"%s\"%s chunk_index=\"%d\" score=\"%.3f\" already_seen=\"true\">\n",
|
||||
xmlEscape(r.ID), xmlEscape(r.KnowledgeID), xmlEscape(r.KnowledgeTitle),
|
||||
extraAttr, r.ChunkIndex, r.MatchScore)
|
||||
} else {
|
||||
fmt.Fprintf(&b,
|
||||
"<chunk chunk_id=\"%s\" knowledge_id=\"%s\" knowledge_title=\"%s\"%s chunk_index=\"%d\" score=\"%.3f\">\n",
|
||||
xmlEscape(r.ID), xmlEscape(r.KnowledgeID), xmlEscape(r.KnowledgeTitle),
|
||||
extraAttr, r.ChunkIndex, r.MatchScore)
|
||||
}
|
||||
|
||||
b.WriteString(fmt.Sprintf(
|
||||
"<chunk chunk_id=\"%s\" knowledge_id=\"%s\" knowledge_title=\"%s\"%s chunk_index=\"%d\" score=\"%.3f\">\n",
|
||||
xmlEscape(r.ID),
|
||||
xmlEscape(r.KnowledgeID),
|
||||
xmlEscape(r.KnowledgeTitle),
|
||||
faqAttr,
|
||||
r.ChunkIndex,
|
||||
r.MatchScore,
|
||||
))
|
||||
for _, q := range queries {
|
||||
if c := counts[q]; c > 0 {
|
||||
b.WriteString(fmt.Sprintf("<query_hit query=\"%s\" count=\"%d\" />\n",
|
||||
xmlEscape(q), c))
|
||||
}
|
||||
}
|
||||
if snippet != "" {
|
||||
if seen {
|
||||
b.WriteString("<note>(snippet omitted, already returned in a previous grep_chunks call this session)</note>\n")
|
||||
} else if snippet != "" {
|
||||
b.WriteString(fmt.Sprintf("<match_snippet>%s</match_snippet>\n", xmlEscape(snippet)))
|
||||
}
|
||||
b.WriteString("</chunk>\n")
|
||||
if isFAQ {
|
||||
b.WriteString("</faq>\n")
|
||||
} else {
|
||||
b.WriteString("</chunk>\n")
|
||||
}
|
||||
}
|
||||
|
||||
b.WriteString("</grep_results>")
|
||||
@@ -503,7 +510,7 @@ func (t *GrepChunksTool) aggregateByKnowledge(
|
||||
}
|
||||
}
|
||||
if entry.MatchSnippet == "" {
|
||||
if snippet := extractSnippetRegex(chunk.Content, compiled); snippet != "" {
|
||||
if snippet := extractChunkMatchSnippet(&chunk.Chunk, compiled); snippet != "" {
|
||||
entry.MatchSnippet = snippet
|
||||
}
|
||||
}
|
||||
@@ -551,6 +558,49 @@ func (t *GrepChunksTool) aggregateByKnowledge(
|
||||
return resultSlice
|
||||
}
|
||||
|
||||
type grepChunkResult struct {
|
||||
ChunkID string `json:"chunk_id,omitempty"`
|
||||
FAQID string `json:"faq_id,omitempty"`
|
||||
KnowledgeID string `json:"knowledge_id"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||||
KnowledgeTitle string `json:"knowledge_title"`
|
||||
ChunkType string `json:"chunk_type"`
|
||||
Index int `json:"index,omitempty"`
|
||||
ChunkIndex int `json:"chunk_index,omitempty"`
|
||||
FAQQuestion string `json:"faq_question,omitempty"`
|
||||
TitleMatch bool `json:"title_match,omitempty"`
|
||||
MatchSnippet string `json:"match_snippet,omitempty"`
|
||||
Score float64 `json:"score"`
|
||||
}
|
||||
|
||||
func buildGrepChunkResults(results []chunkWithTitle, compiled []*regexp.Regexp) []grepChunkResult {
|
||||
if len(results) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]grepChunkResult, 0, len(results))
|
||||
for _, r := range results {
|
||||
item := grepChunkResult{
|
||||
KnowledgeID: r.KnowledgeID,
|
||||
KnowledgeBaseID: r.KnowledgeBaseID,
|
||||
KnowledgeTitle: r.KnowledgeTitle,
|
||||
ChunkType: string(r.ChunkType),
|
||||
TitleMatch: r.TitleMatch,
|
||||
MatchSnippet: extractChunkMatchSnippet(&r.Chunk, compiled),
|
||||
Score: r.MatchScore,
|
||||
}
|
||||
if r.ChunkType == types.ChunkTypeFAQ {
|
||||
item.FAQID = r.ID
|
||||
item.Index = r.ChunkIndex
|
||||
item.FAQQuestion = faqStandardQuestion(&r.Chunk)
|
||||
} else {
|
||||
item.ChunkID = r.ID
|
||||
item.ChunkIndex = r.ChunkIndex
|
||||
}
|
||||
out = append(out, item)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// regexMatchesAny reports whether text matches at least one of the compiled
|
||||
// patterns. Used to flag title hits without counting occurrences.
|
||||
func regexMatchesAny(text string, compiled []*regexp.Regexp) bool {
|
||||
@@ -582,6 +632,22 @@ func countRegexHits(content string, compiled []*regexp.Regexp, patterns []string
|
||||
return counts
|
||||
}
|
||||
|
||||
// extractChunkMatchSnippet returns a preview for tool output. FAQ chunks only
|
||||
// surface the matched question plus answers from metadata (answers are not
|
||||
// stored in chunk content for question_only index mode). Other chunk types
|
||||
// use regex context around the first body match.
|
||||
func extractChunkMatchSnippet(chunk *types.Chunk, compiled []*regexp.Regexp) string {
|
||||
if chunk != nil && chunk.ChunkType == types.ChunkTypeFAQ {
|
||||
if s := faqMatchSnippet(chunk, compiled); s != "" {
|
||||
return s
|
||||
}
|
||||
}
|
||||
if chunk == nil {
|
||||
return ""
|
||||
}
|
||||
return extractSnippetRegex(chunk.Content, compiled)
|
||||
}
|
||||
|
||||
// extractSnippetRegex returns a short context snippet around the earliest
|
||||
// regex match across any of the provided compiled patterns. Result is
|
||||
// compressed to a single line and bounded in length on both sides of the
|
||||
@@ -614,18 +680,17 @@ func extractSnippetRegex(content string, compiled []*regexp.Regexp) string {
|
||||
before := content[:earliest]
|
||||
after := content[earliestEnd:]
|
||||
|
||||
const contextRunes = 60
|
||||
beforeRunes := []rune(before)
|
||||
if len(beforeRunes) > contextRunes {
|
||||
beforeRunes = beforeRunes[len(beforeRunes)-contextRunes:]
|
||||
if len(beforeRunes) > snippetContextRunes {
|
||||
beforeRunes = beforeRunes[len(beforeRunes)-snippetContextRunes:]
|
||||
}
|
||||
afterRunes := []rune(after)
|
||||
if len(afterRunes) > contextRunes {
|
||||
afterRunes = afterRunes[:contextRunes]
|
||||
if len(afterRunes) > snippetContextRunes {
|
||||
afterRunes = afterRunes[:snippetContextRunes]
|
||||
}
|
||||
matchRunes := []rune(matchStr)
|
||||
if len(matchRunes) > 120 {
|
||||
matchRunes = append(matchRunes[:120], []rune("...")...)
|
||||
if len(matchRunes) > snippetMaxMatchRunes {
|
||||
matchRunes = append(matchRunes[:snippetMaxMatchRunes], []rune("...")...)
|
||||
}
|
||||
|
||||
snippet := string(beforeRunes) + string(matchRunes) + string(afterRunes)
|
||||
@@ -633,7 +698,11 @@ func extractSnippetRegex(content string, compiled []*regexp.Regexp) string {
|
||||
for strings.Contains(snippet, " ") {
|
||||
snippet = strings.ReplaceAll(snippet, " ", " ")
|
||||
}
|
||||
return "... " + strings.TrimSpace(snippet) + " ..."
|
||||
snippet = strings.TrimSpace(snippet)
|
||||
if len([]rune(snippet)) > snippetMaxTotalRunes {
|
||||
snippet = string([]rune(snippet)[:snippetMaxTotalRunes]) + "..."
|
||||
}
|
||||
return "... " + snippet + " ..."
|
||||
}
|
||||
|
||||
// xmlEscape replaces characters that would break simple XML attribute /
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
func faqChunkForSnippetTest(t *testing.T) *types.Chunk {
|
||||
t.Helper()
|
||||
chunk := &types.Chunk{
|
||||
ChunkType: types.ChunkTypeFAQ,
|
||||
Content: `Q: 离职原因是否可以自定义
|
||||
Similar Questions:
|
||||
- 离职原因自定义
|
||||
- 设置离职原因`,
|
||||
}
|
||||
meta := &types.FAQChunkMetadata{
|
||||
StandardQuestion: "离职原因是否可以自定义",
|
||||
SimilarQuestions: []string{"离职原因自定义", "设置离职原因"},
|
||||
Answers: []string{"可以,在系统设置中自定义离职原因。"},
|
||||
}
|
||||
if err := chunk.SetFAQMetadata(meta); err != nil {
|
||||
t.Fatalf("SetFAQMetadata: %v", err)
|
||||
}
|
||||
return chunk
|
||||
}
|
||||
|
||||
func TestFaqMatchSnippet_StandardQuestionAndMetadataAnswer(t *testing.T) {
|
||||
chunk := faqChunkForSnippetTest(t)
|
||||
re := regexp.MustCompile(`(?i)是否可以自定义`)
|
||||
snippet := faqMatchSnippet(chunk, []*regexp.Regexp{re})
|
||||
|
||||
if !strings.Contains(snippet, "Q: 离职原因是否可以自定义") {
|
||||
t.Fatalf("want standard question in snippet, got: %s", snippet)
|
||||
}
|
||||
if !strings.Contains(snippet, "A: 可以,在系统设置中") {
|
||||
t.Fatalf("want metadata answer in snippet, got: %s", snippet)
|
||||
}
|
||||
for _, unwanted := range []string{"Similar Questions", "离职原因自定义", "设置离职原因"} {
|
||||
if strings.Contains(snippet, unwanted) {
|
||||
t.Errorf("snippet should not include similar-question noise %q: %s", unwanted, snippet)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFaqMatchSnippetFromQueries_StandardQuestionAndMetadataAnswer(t *testing.T) {
|
||||
meta := &types.FAQChunkMetadata{
|
||||
StandardQuestion: "离职原因是否可以自定义",
|
||||
SimilarQuestions: []string{"离职原因自定义", "设置离职原因"},
|
||||
Answers: []string{"可以,在系统设置中自定义离职原因。"},
|
||||
}
|
||||
snippet := faqMatchSnippetFromQueries(meta, []string{"是否可以自定义"})
|
||||
|
||||
if !strings.Contains(snippet, "Q: 离职原因是否可以自定义") {
|
||||
t.Fatalf("want standard question, got: %s", snippet)
|
||||
}
|
||||
if !strings.Contains(snippet, "A: 可以,在系统设置中") {
|
||||
t.Fatalf("want metadata answer, got: %s", snippet)
|
||||
}
|
||||
if strings.Contains(snippet, "Similar Questions") {
|
||||
t.Fatalf("should not include similar-question list: %s", snippet)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFaqMatchSnippetFromQueries_SimilarQuestionHitShowsMatchedVariant(t *testing.T) {
|
||||
meta := &types.FAQChunkMetadata{
|
||||
StandardQuestion: "离职原因是否可以自定义",
|
||||
SimilarQuestions: []string{"离职原因自定义", "设置离职原因"},
|
||||
Answers: []string{"可以自定义。"},
|
||||
}
|
||||
snippet := faqMatchSnippetFromQueries(meta, []string{"离职原因自定义"})
|
||||
|
||||
if !strings.Contains(snippet, "Q: 离职原因自定义") {
|
||||
t.Fatalf("want matched similar question, got: %s", snippet)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFaqMatchSnippet_SimilarQuestionHitShowsMatchedVariant(t *testing.T) {
|
||||
chunk := faqChunkForSnippetTest(t)
|
||||
re := regexp.MustCompile(`(?i)离职原因自定义`)
|
||||
snippet := faqMatchSnippet(chunk, []*regexp.Regexp{re})
|
||||
|
||||
if !strings.Contains(snippet, "Q: 离职原因自定义") {
|
||||
t.Fatalf("want matched similar question, got: %s", snippet)
|
||||
}
|
||||
if strings.Contains(snippet, "是否可以自定义") {
|
||||
t.Fatalf("should not show standard question when similar question matched: %s", snippet)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractChunkMatchSnippet_NonFAQUsesBodyContext(t *testing.T) {
|
||||
content := strings.Repeat("前置。", 50) + "TARGET" + strings.Repeat("后续。", 50)
|
||||
chunk := &types.Chunk{ChunkType: types.ChunkTypeText, Content: content}
|
||||
re := regexp.MustCompile(`TARGET`)
|
||||
snippet := extractChunkMatchSnippet(chunk, []*regexp.Regexp{re})
|
||||
|
||||
if !strings.Contains(snippet, "TARGET") {
|
||||
t.Fatalf("missing match: %s", snippet)
|
||||
}
|
||||
if !strings.Contains(snippet, "前置") || !strings.Contains(snippet, "后续") {
|
||||
t.Fatalf("expected expanded body context: %s", snippet)
|
||||
}
|
||||
}
|
||||
@@ -1164,36 +1164,74 @@ func (t *KnowledgeSearchTool) formatOutput(
|
||||
t.seenChunks[result.ID] = true
|
||||
t.seenMu.Unlock()
|
||||
|
||||
isFAQ := faqMeta != nil
|
||||
if seen {
|
||||
// Compact rendering for chunks we already returned in a previous
|
||||
// knowledge_search call during this session. The model has the
|
||||
// content in context already, so re-emitting it only burns tokens.
|
||||
ob.WriteString(fmt.Sprintf(
|
||||
"<chunk rank=\"%d\" chunk_id=\"%s\" chunk_index=\"%d\" knowledge_id=\"%s\" knowledge_base_id=\"%s\" knowledge_title=\"%s\" score=\"%.3f\" source_query=\"%s\" already_seen=\"true\">\n",
|
||||
i+1,
|
||||
xmlEscape(result.ID),
|
||||
result.ChunkIndex,
|
||||
xmlEscape(result.KnowledgeID),
|
||||
xmlEscape(result.KnowledgeBaseID),
|
||||
xmlEscape(result.KnowledgeTitle),
|
||||
result.Score,
|
||||
xmlEscape(result.SourceQuery),
|
||||
))
|
||||
if isFAQ {
|
||||
ob.WriteString(fmt.Sprintf(
|
||||
"<faq rank=\"%d\" faq_id=\"%s\" index=\"%d\" knowledge_base_id=\"%s\" knowledge_title=\"%s\" score=\"%.3f\" source_query=\"%s\" already_seen=\"true\">\n",
|
||||
i+1,
|
||||
xmlEscape(result.ID),
|
||||
result.ChunkIndex,
|
||||
xmlEscape(result.KnowledgeBaseID),
|
||||
xmlEscape(result.KnowledgeTitle),
|
||||
result.Score,
|
||||
xmlEscape(result.SourceQuery),
|
||||
))
|
||||
} else {
|
||||
ob.WriteString(fmt.Sprintf(
|
||||
"<chunk rank=\"%d\" chunk_id=\"%s\" chunk_index=\"%d\" knowledge_id=\"%s\" knowledge_base_id=\"%s\" knowledge_title=\"%s\" score=\"%.3f\" source_query=\"%s\" already_seen=\"true\">\n",
|
||||
i+1,
|
||||
xmlEscape(result.ID),
|
||||
result.ChunkIndex,
|
||||
xmlEscape(result.KnowledgeID),
|
||||
xmlEscape(result.KnowledgeBaseID),
|
||||
xmlEscape(result.KnowledgeTitle),
|
||||
result.Score,
|
||||
xmlEscape(result.SourceQuery),
|
||||
))
|
||||
}
|
||||
ob.WriteString("<note>(content omitted, already returned in a previous knowledge_search call this session)</note>\n")
|
||||
ob.WriteString("</chunk>\n")
|
||||
if isFAQ {
|
||||
ob.WriteString("</faq>\n")
|
||||
} else {
|
||||
ob.WriteString("</chunk>\n")
|
||||
}
|
||||
} else {
|
||||
ob.WriteString(fmt.Sprintf(
|
||||
"<chunk rank=\"%d\" chunk_id=\"%s\" chunk_index=\"%d\" knowledge_id=\"%s\" knowledge_base_id=\"%s\" knowledge_title=\"%s\" score=\"%.3f\" source_query=\"%s\">\n",
|
||||
i+1,
|
||||
xmlEscape(result.ID),
|
||||
result.ChunkIndex,
|
||||
xmlEscape(result.KnowledgeID),
|
||||
xmlEscape(result.KnowledgeBaseID),
|
||||
xmlEscape(result.KnowledgeTitle),
|
||||
result.Score,
|
||||
xmlEscape(result.SourceQuery),
|
||||
))
|
||||
if snippet := extractSnippetForQueries(result.Content, queries); snippet != "" {
|
||||
if isFAQ {
|
||||
ob.WriteString(fmt.Sprintf(
|
||||
"<faq rank=\"%d\" faq_id=\"%s\" index=\"%d\" knowledge_base_id=\"%s\" knowledge_title=\"%s\" score=\"%.3f\" source_query=\"%s\">\n",
|
||||
i+1,
|
||||
xmlEscape(result.ID),
|
||||
result.ChunkIndex,
|
||||
xmlEscape(result.KnowledgeBaseID),
|
||||
xmlEscape(result.KnowledgeTitle),
|
||||
result.Score,
|
||||
xmlEscape(result.SourceQuery),
|
||||
))
|
||||
} else {
|
||||
ob.WriteString(fmt.Sprintf(
|
||||
"<chunk rank=\"%d\" chunk_id=\"%s\" chunk_index=\"%d\" knowledge_id=\"%s\" knowledge_base_id=\"%s\" knowledge_title=\"%s\" score=\"%.3f\" source_query=\"%s\">\n",
|
||||
i+1,
|
||||
xmlEscape(result.ID),
|
||||
result.ChunkIndex,
|
||||
xmlEscape(result.KnowledgeID),
|
||||
xmlEscape(result.KnowledgeBaseID),
|
||||
xmlEscape(result.KnowledgeTitle),
|
||||
result.Score,
|
||||
xmlEscape(result.SourceQuery),
|
||||
))
|
||||
}
|
||||
snippet := ""
|
||||
if faqMeta != nil {
|
||||
snippet = faqMatchSnippetFromQueries(faqMeta, queries)
|
||||
}
|
||||
if snippet == "" {
|
||||
snippet = extractSnippetForQueries(result.Content, queries)
|
||||
}
|
||||
if snippet != "" {
|
||||
ob.WriteString(fmt.Sprintf("<match_snippet>%s</match_snippet>\n", xmlEscape(snippet)))
|
||||
}
|
||||
ob.WriteString(fmt.Sprintf("<content>%s</content>\n", result.Content))
|
||||
@@ -1214,30 +1252,16 @@ func (t *KnowledgeSearchTool) formatOutput(
|
||||
}
|
||||
}
|
||||
|
||||
if faqMeta != nil {
|
||||
ob.WriteString("<faq>\n")
|
||||
if faqMeta.StandardQuestion != "" {
|
||||
ob.WriteString(fmt.Sprintf("<question>%s</question>\n", xmlEscape(faqMeta.StandardQuestion)))
|
||||
}
|
||||
if len(faqMeta.SimilarQuestions) > 0 {
|
||||
for _, sq := range faqMeta.SimilarQuestions {
|
||||
ob.WriteString(fmt.Sprintf("<similar_question>%s</similar_question>\n", xmlEscape(sq)))
|
||||
}
|
||||
}
|
||||
if len(faqMeta.Answers) > 0 {
|
||||
for _, ans := range faqMeta.Answers {
|
||||
ob.WriteString(fmt.Sprintf("<answer>%s</answer>\n", xmlEscape(ans)))
|
||||
}
|
||||
}
|
||||
if isFAQ {
|
||||
writeFAQFieldsXML(&ob, faqMeta)
|
||||
ob.WriteString("</faq>\n")
|
||||
} else {
|
||||
ob.WriteString("</chunk>\n")
|
||||
}
|
||||
|
||||
ob.WriteString("</chunk>\n")
|
||||
}
|
||||
|
||||
formattedResults = append(formattedResults, map[string]interface{}{
|
||||
"result_index": i + 1,
|
||||
"chunk_id": result.ID,
|
||||
"content": result.Content,
|
||||
"knowledge_id": result.KnowledgeID,
|
||||
"knowledge_title": result.KnowledgeTitle,
|
||||
@@ -1275,15 +1299,18 @@ func (t *KnowledgeSearchTool) formatOutput(
|
||||
}
|
||||
|
||||
if faqMeta != nil {
|
||||
last["faq_id"] = result.ID
|
||||
last["index"] = result.ChunkIndex
|
||||
if faqMeta.StandardQuestion != "" {
|
||||
last["faq_standard_question"] = faqMeta.StandardQuestion
|
||||
}
|
||||
if len(faqMeta.SimilarQuestions) > 0 {
|
||||
last["faq_similar_questions"] = faqMeta.SimilarQuestions
|
||||
}
|
||||
appendSimilarQuestionsToChunkData(last, faqMeta.SimilarQuestions)
|
||||
if len(faqMeta.Answers) > 0 {
|
||||
last["faq_answers"] = faqMeta.Answers
|
||||
}
|
||||
} else {
|
||||
last["chunk_id"] = result.ID
|
||||
last["chunk_index"] = result.ChunkIndex
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1509,31 +1536,7 @@ func extractSnippetForQueries(content string, queries []string) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
tokens := make([]string, 0, 8)
|
||||
seen := make(map[string]struct{})
|
||||
for _, q := range queries {
|
||||
for _, tok := range strings.FieldsFunc(q, func(r rune) bool {
|
||||
// Split on whitespace and common punctuation; keep CJK as whole tokens.
|
||||
switch r {
|
||||
case ' ', '\t', '\n', '\r', ',', '.', ';', ':', '?', '!',
|
||||
'(', ')', '[', ']', '{', '}', '"', '\'':
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}) {
|
||||
tok = strings.ToLower(strings.TrimSpace(tok))
|
||||
// Skip trivially-short stopwords. Three rune floor covers most
|
||||
// English function words without chopping CJK bigrams.
|
||||
if len([]rune(tok)) < 2 {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[tok]; ok {
|
||||
continue
|
||||
}
|
||||
seen[tok] = struct{}{}
|
||||
tokens = append(tokens, tok)
|
||||
}
|
||||
}
|
||||
tokens := searchQueryTokens(queries)
|
||||
|
||||
lowered := strings.ToLower(content)
|
||||
earliest := -1
|
||||
@@ -1550,11 +1553,10 @@ func extractSnippetForQueries(content string, queries []string) string {
|
||||
}
|
||||
}
|
||||
|
||||
const contextRunes = 60
|
||||
if earliest < 0 {
|
||||
runes := []rune(content)
|
||||
if len(runes) > contextRunes*2 {
|
||||
return strings.TrimSpace(string(runes[:contextRunes*2])) + " ..."
|
||||
if len(runes) > snippetContextRunes*2 {
|
||||
return strings.TrimSpace(string(runes[:snippetContextRunes*2])) + " ..."
|
||||
}
|
||||
return content
|
||||
}
|
||||
@@ -1564,12 +1566,12 @@ func extractSnippetForQueries(content string, queries []string) string {
|
||||
after := content[earliestEnd:]
|
||||
|
||||
beforeRunes := []rune(before)
|
||||
if len(beforeRunes) > contextRunes {
|
||||
beforeRunes = beforeRunes[len(beforeRunes)-contextRunes:]
|
||||
if len(beforeRunes) > snippetContextRunes {
|
||||
beforeRunes = beforeRunes[len(beforeRunes)-snippetContextRunes:]
|
||||
}
|
||||
afterRunes := []rune(after)
|
||||
if len(afterRunes) > contextRunes {
|
||||
afterRunes = afterRunes[:contextRunes]
|
||||
if len(afterRunes) > snippetContextRunes {
|
||||
afterRunes = afterRunes[:snippetContextRunes]
|
||||
}
|
||||
|
||||
snippet := string(beforeRunes) + matchStr + string(afterRunes)
|
||||
|
||||
@@ -13,60 +13,57 @@ import (
|
||||
|
||||
var listKnowledgeChunksTool = BaseTool{
|
||||
name: ToolListKnowledgeChunks,
|
||||
description: `Retrieve full chunk content for a document by knowledge_id.
|
||||
description: `Retrieve full chunk content for a document or a single FAQ entry.
|
||||
|
||||
## Use After grep_chunks or knowledge_search:
|
||||
1. grep_chunks(["keyword", "variant"]) → get knowledge_id
|
||||
2. list_knowledge_chunks(knowledge_id) → read full content
|
||||
- **FAQ hit** (type faq): list_knowledge_chunks(faq_id="<chunk_id from search>") — reads that one FAQ entry with answers from metadata.
|
||||
- **Document hit**: list_knowledge_chunks(knowledge_id="<document id>") — pages through all chunks.
|
||||
|
||||
## When to Use:
|
||||
- Need full content of chunks from a known document
|
||||
- Want to see context around specific chunks
|
||||
- Check how many chunks a document has
|
||||
|
||||
## Parameters:
|
||||
- knowledge_id (required): Document ID
|
||||
- limit (optional): Chunks per page (default 20, max 100)
|
||||
- offset (optional): 0-based position in the chunk list (NOT chunk_index).
|
||||
Must be < total. Use offset=0 to read from the first chunk, offset=N to
|
||||
skip the first N chunks. If offset >= total the tool returns an explicit
|
||||
out-of-range error with the valid total so you can retry.
|
||||
## Parameters (provide exactly one id target):
|
||||
- faq_id (optional): FAQ entry ID from grep_chunks / knowledge_search.
|
||||
- chunk_id (optional): Single non-FAQ chunk ID (do not use for FAQ — use faq_id).
|
||||
- knowledge_id (optional): Document/knowledge ID to page through all chunks.
|
||||
- limit / offset: Only for knowledge_id paging (default limit 20, max 100).
|
||||
|
||||
## Output:
|
||||
Full chunk content with chunk_id, chunk_index, and content text.
|
||||
The response attributes include:
|
||||
- total: total number of chunks in the document
|
||||
- fetched: how many chunks this page returned
|
||||
If fetched=0 with total>0, your offset is past the end — retry with a
|
||||
smaller offset (e.g. offset = max(0, total - limit)).`,
|
||||
Full chunk content. FAQ entries include <faq> with <answer> from metadata.`,
|
||||
schema: json.RawMessage(`{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"faq_id": {
|
||||
"type": "string",
|
||||
"description": "FAQ entry ID (same as chunk_id). Use for FAQ hits instead of knowledge_id."
|
||||
},
|
||||
"chunk_id": {
|
||||
"type": "string",
|
||||
"description": "Single chunk ID (alias of faq_id)"
|
||||
},
|
||||
"knowledge_id": {
|
||||
"type": "string",
|
||||
"description": "Document ID to retrieve chunks from"
|
||||
"description": "Document/knowledge ID to list all chunks"
|
||||
},
|
||||
"limit": {
|
||||
"type": "integer",
|
||||
"description": "Chunks per page (default 20, max 100)",
|
||||
"description": "Chunks per page when using knowledge_id (default 20, max 100)",
|
||||
"default": 20,
|
||||
"minimum": 1,
|
||||
"maximum": 100
|
||||
},
|
||||
"offset": {
|
||||
"type": "integer",
|
||||
"description": "Start position (default 0)",
|
||||
"description": "Start position when using knowledge_id (default 0)",
|
||||
"default": 0,
|
||||
"minimum": 0
|
||||
}
|
||||
},
|
||||
"required": ["knowledge_id"]
|
||||
}
|
||||
}`),
|
||||
}
|
||||
|
||||
// ListKnowledgeChunksInput defines the input parameters for list knowledge chunks tool
|
||||
type ListKnowledgeChunksInput struct {
|
||||
KnowledgeID string `json:"knowledge_id"`
|
||||
KnowledgeID string `json:"knowledge_id,omitempty"`
|
||||
FAQID string `json:"faq_id,omitempty"`
|
||||
ChunkID string `json:"chunk_id,omitempty"`
|
||||
Limit int `json:"limit"`
|
||||
Offset int `json:"offset"`
|
||||
}
|
||||
@@ -104,15 +101,21 @@ func (t *ListKnowledgeChunksTool) Execute(ctx context.Context, args json.RawMess
|
||||
}, err
|
||||
}
|
||||
|
||||
knowledgeID := input.KnowledgeID
|
||||
ok := knowledgeID != ""
|
||||
if !ok || strings.TrimSpace(knowledgeID) == "" {
|
||||
chunkID := strings.TrimSpace(input.FAQID)
|
||||
if chunkID == "" {
|
||||
chunkID = strings.TrimSpace(input.ChunkID)
|
||||
}
|
||||
if chunkID != "" {
|
||||
return t.executeByChunkID(ctx, chunkID)
|
||||
}
|
||||
|
||||
knowledgeID := strings.TrimSpace(input.KnowledgeID)
|
||||
if knowledgeID == "" {
|
||||
return &types.ToolResult{
|
||||
Success: false,
|
||||
Error: "knowledge_id is required",
|
||||
}, fmt.Errorf("knowledge_id is required")
|
||||
Error: "one of faq_id, chunk_id, or knowledge_id is required",
|
||||
}, fmt.Errorf("missing id parameter")
|
||||
}
|
||||
knowledgeID = strings.TrimSpace(knowledgeID)
|
||||
|
||||
// Get knowledge info without tenant filter to support shared KB
|
||||
knowledge, err := t.knowledgeService.GetKnowledgeByIDOnly(ctx, knowledgeID)
|
||||
@@ -230,10 +233,8 @@ func (t *ListKnowledgeChunksTool) Execute(ctx context.Context, args json.RawMess
|
||||
"parent_chunk_id": c.ParentChunkID,
|
||||
}
|
||||
|
||||
// FAQ 条目共用所属 Knowledge 的标题,补上标准问作为单条身份标识
|
||||
if q := faqStandardQuestion(c); q != "" {
|
||||
chunkData["faq_question"] = q
|
||||
}
|
||||
appendFAQChunkData(chunkData, c)
|
||||
normalizeFAQChunkDataMap(chunkData, c)
|
||||
|
||||
// 添加图片信息
|
||||
if c.ImageInfo != "" {
|
||||
@@ -279,6 +280,72 @@ func (t *ListKnowledgeChunksTool) Execute(ctx context.Context, args json.RawMess
|
||||
}, nil
|
||||
}
|
||||
|
||||
// executeByChunkID loads one chunk by faq_id / chunk_id (FAQ entry or any chunk).
|
||||
func (t *ListKnowledgeChunksTool) executeByChunkID(ctx context.Context, chunkID string) (*types.ToolResult, error) {
|
||||
chunk, err := t.chunkService.GetChunkByIDOnly(ctx, chunkID)
|
||||
if err != nil || chunk == nil {
|
||||
return &types.ToolResult{
|
||||
Success: false,
|
||||
Error: fmt.Sprintf("chunk not found: %v", err),
|
||||
}, err
|
||||
}
|
||||
if !t.searchTargets.ContainsKB(chunk.KnowledgeBaseID) {
|
||||
return &types.ToolResult{
|
||||
Success: false,
|
||||
Error: fmt.Sprintf("knowledge base %s is not accessible", chunk.KnowledgeBaseID),
|
||||
}, fmt.Errorf("knowledge base not in search targets")
|
||||
}
|
||||
|
||||
chunks := []*types.Chunk{chunk}
|
||||
if chunk.ImageInfo == "" {
|
||||
effectiveTenantID := t.searchTargets.GetTenantIDForKB(chunk.KnowledgeBaseID)
|
||||
if effectiveTenantID > 0 {
|
||||
infoMap := searchutil.CollectImageInfoByChunkIDs(ctx, t.chunkService.GetRepository(), effectiveTenantID, []string{chunk.ID})
|
||||
if merged, ok := infoMap[chunk.ID]; ok {
|
||||
chunk.ImageInfo = merged
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
knowledgeTitle := t.lookupKnowledgeTitle(ctx, chunk.KnowledgeID)
|
||||
output := t.buildOutput(chunk.KnowledgeID, knowledgeTitle, 1, 1, chunks)
|
||||
|
||||
formattedChunks := []map[string]interface{}{
|
||||
{
|
||||
"seq": 1,
|
||||
"chunk_id": chunk.ID,
|
||||
"chunk_index": chunk.ChunkIndex,
|
||||
"content": chunk.Content,
|
||||
"chunk_type": chunk.ChunkType,
|
||||
"knowledge_id": chunk.KnowledgeID,
|
||||
"knowledge_base": chunk.KnowledgeBaseID,
|
||||
},
|
||||
}
|
||||
appendFAQChunkData(formattedChunks[0], chunk)
|
||||
normalizeFAQChunkDataMap(formattedChunks[0], chunk)
|
||||
|
||||
data := map[string]interface{}{
|
||||
"knowledge_id": chunk.KnowledgeID,
|
||||
"knowledge_title": knowledgeTitle,
|
||||
"total_chunks": int64(1),
|
||||
"fetched_chunks": 1,
|
||||
"page": 1,
|
||||
"page_size": 1,
|
||||
"chunks": formattedChunks,
|
||||
"faq_id": chunk.ID,
|
||||
"single_chunk": true,
|
||||
}
|
||||
if q := faqStandardQuestion(chunk); q != "" {
|
||||
data["faq_question"] = q
|
||||
}
|
||||
|
||||
return &types.ToolResult{
|
||||
Success: true,
|
||||
Output: output,
|
||||
Data: data,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// lookupKnowledgeTitle looks up the title of a knowledge document
|
||||
// Uses GetKnowledgeByIDOnly to support cross-tenant shared KB
|
||||
func (t *ListKnowledgeChunksTool) lookupKnowledgeTitle(ctx context.Context, knowledgeID string) string {
|
||||
@@ -315,8 +382,12 @@ func (t *ListKnowledgeChunksTool) buildOutput(
|
||||
}
|
||||
|
||||
for _, c := range chunks {
|
||||
// FAQ entries share the owning knowledge's title, so expose the
|
||||
// standard question as a per-chunk identifier when available.
|
||||
if c.ChunkType == types.ChunkTypeFAQ {
|
||||
writeFAQEntryXML(&b, c)
|
||||
writeChunkImagesXML(&b, c)
|
||||
continue
|
||||
}
|
||||
|
||||
if q := faqStandardQuestion(c); q != "" {
|
||||
fmt.Fprintf(&b, "<chunk chunk_id=\"%s\" chunk_index=\"%d\" type=\"%s\" question=\"%s\">\n",
|
||||
c.ID, c.ChunkIndex, c.ChunkType, xmlEscape(q))
|
||||
@@ -325,27 +396,7 @@ func (t *ListKnowledgeChunksTool) buildOutput(
|
||||
c.ID, c.ChunkIndex, c.ChunkType)
|
||||
}
|
||||
fmt.Fprintf(&b, "<content>%s</content>\n", summarizeContent(c.Content))
|
||||
|
||||
if c.ImageInfo != "" {
|
||||
var imageInfos []types.ImageInfo
|
||||
if err := json.Unmarshal([]byte(c.ImageInfo), &imageInfos); err == nil && len(imageInfos) > 0 {
|
||||
for _, img := range imageInfos {
|
||||
if img.URL != "" {
|
||||
fmt.Fprintf(&b, "<image url=\"%s\">\n", img.URL)
|
||||
} else {
|
||||
b.WriteString("<image>\n")
|
||||
}
|
||||
if img.Caption != "" {
|
||||
fmt.Fprintf(&b, "<image_caption>%s</image_caption>\n", img.Caption)
|
||||
}
|
||||
if img.OCRText != "" {
|
||||
fmt.Fprintf(&b, "<image_ocr>%s</image_ocr>\n", img.OCRText)
|
||||
}
|
||||
b.WriteString("</image>\n")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
writeChunkImagesXML(&b, c)
|
||||
b.WriteString("</chunk>\n")
|
||||
}
|
||||
|
||||
@@ -357,6 +408,30 @@ func (t *ListKnowledgeChunksTool) buildOutput(
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func writeChunkImagesXML(b *strings.Builder, c *types.Chunk) {
|
||||
if c == nil || c.ImageInfo == "" {
|
||||
return
|
||||
}
|
||||
var imageInfos []types.ImageInfo
|
||||
if err := json.Unmarshal([]byte(c.ImageInfo), &imageInfos); err != nil || len(imageInfos) == 0 {
|
||||
return
|
||||
}
|
||||
for _, img := range imageInfos {
|
||||
if img.URL != "" {
|
||||
fmt.Fprintf(b, "<image url=\"%s\">\n", img.URL)
|
||||
} else {
|
||||
b.WriteString("<image>\n")
|
||||
}
|
||||
if img.Caption != "" {
|
||||
fmt.Fprintf(b, "<image_caption>%s</image_caption>\n", img.Caption)
|
||||
}
|
||||
if img.OCRText != "" {
|
||||
fmt.Fprintf(b, "<image_ocr>%s</image_ocr>\n", img.OCRText)
|
||||
}
|
||||
b.WriteString("</image>\n")
|
||||
}
|
||||
}
|
||||
|
||||
// faqStandardQuestion returns the FAQ standard question for an FAQ-type chunk,
|
||||
// or "" for non-FAQ chunks (or when metadata is missing/unparseable). All FAQ
|
||||
// entries inside one knowledge share the same knowledge title, so surfacing the
|
||||
|
||||
Reference in New Issue
Block a user