feat(faq): enhance FAQ handling and localization support

- Updated the agent system prompt to clarify mandatory deep read requirements for FAQ entries, ensuring accurate retrieval and response generation.
- Added new localization strings for FAQ-related labels in English and Chinese, improving user experience and consistency across the application.
- Refactored the document and grep results components to support distinct FAQ identification, enhancing clarity in the output.
- Introduced new tests for FAQ metadata handling, ensuring robust functionality and correctness in the FAQ processing logic.
This commit is contained in:
wizardchen
2026-06-04 20:17:43 +08:00
committed by lyingbug
parent ba8c96d83a
commit 4bb8906520
14 changed files with 1075 additions and 225 deletions
@@ -73,7 +73,7 @@ templates:
### Critical Constraints (ABSOLUTE RULES)
1. **Evidence-Based Facts:** For factual claims about documents or domain knowledge, rely on KB/Web retrieval rather than internal knowledge. However, you MAY answer directly when the user's question is about image content you can see, conversational context, or general interaction.
2. **Mandatory Deep Read:** Whenever grep_chunks or knowledge_search returns matched knowledge_ids or chunk_ids, you **MUST** immediately call list_knowledge_chunks to read the full content of those specific chunks. Do not rely on search snippets alone.
2. **Mandatory Deep Read:** Whenever grep_chunks or knowledge_search returns matches, you **MUST** read full content before answering. For **FAQ** hits use `list_knowledge_chunks(faq_id=...)` or `get_document_info(faq_ids=[...])` with the **faq_id** from the result — do **not** use the parent knowledge_id of the FAQ container. For **documents**, use `list_knowledge_chunks(knowledge_id=...)` or `get_document_info(knowledge_ids=...)`. Do not rely on search snippets alone.
3. **Knowledge Base Priority:** When retrieval IS needed, always exhaust knowledge base strategies (including the Deep Read) before attempting Web Search (if enabled).
4. **Always Re-Retrieve for Each New Question:** You MUST perform fresh knowledge base retrieval for EVERY new user question that requires factual or domain-specific information, even if a similar or identical question was asked earlier in the conversation. NEVER rely on previously retrieved knowledge base content from the conversation history — the knowledge base may have been updated, switched, or had content removed since the last retrieval. Treat each new question as if you have no prior knowledge from previous retrievals.
5. **User-Friendly Communication:** In ALL outputs visible to users (including your thinking/reasoning process), you MUST:
+5
View File
@@ -2701,6 +2701,9 @@ export default {
referenceIconAlt: 'Reference materials icon',
chunkIdLabel: 'Chunk ID:',
documentIdLabel: 'Document ID:',
faqIdLabel: 'FAQ ID:',
faqContainerIdLabel: 'Container ID:',
faqAnswersLabel: 'Answers:',
chunkOrdinal: 'Chunk {index}',
previewContent: 'Preview content',
noPlanSteps: 'No detailed steps provided',
@@ -4570,6 +4573,7 @@ export default {
getDocument: 'Get document: {title}',
document: 'Document',
listChunks: 'View {title}',
listFaqEntry: 'View FAQ: {question}',
deepThinking: 'Deep Thinking',
},
plan: {
@@ -4588,6 +4592,7 @@ export default {
chunkHits: '{count} chunks',
keywordHits: '{count} hits',
titleMatch: 'title',
faqEntry: 'FAQ entry',
},
toolStatus: {
calling: 'Calling {name}...',
+5
View File
@@ -806,6 +806,9 @@ export default {
referenceIconAlt: "参考内容图标",
chunkIdLabel: "片段ID:",
documentIdLabel: "文档ID:",
faqIdLabel: "FAQ ID:",
faqContainerIdLabel: "所属文档ID:",
faqAnswersLabel: "答案:",
chunkOrdinal: "片段 {index}",
previewContent: "预览内容",
noPlanSteps: "未提供具体步骤",
@@ -4564,6 +4567,7 @@ export default {
getDocument: "获取文档:{title}",
document: "文档",
listChunks: "查看 {title}",
listFaqEntry: "查看 FAQ:{question}",
deepThinking: "深度思考",
},
plan: {
@@ -4582,6 +4586,7 @@ export default {
chunkHits: "{count} 片段",
keywordHits: "{count} 次",
titleMatch: "标题匹配",
faqEntry: "FAQ 条目",
},
toolStatus: {
calling: "正在调用 {name}...",
+25 -4
View File
@@ -105,8 +105,14 @@ export interface KnowledgeBaseListData {
// Document info data
export interface DocumentInfoDocument {
knowledge_id: string;
knowledge_id?: string;
faq_id?: string;
chunk_id?: string;
title: string;
faq_question?: string;
faq_answers?: string[];
faq_similar_questions?: string[];
is_faq?: boolean;
description?: string;
type?: string;
source?: string;
@@ -204,13 +210,11 @@ export interface WebFetchResultsData {
count?: number;
}
// Grep knowledge aggregation item
// Grep knowledge aggregation item (legacy, grouped by knowledge_id)
export interface GrepKnowledgeResult {
knowledge_id: string;
knowledge_base_id: string;
knowledge_title: string;
// Standard question of the first matched FAQ entry, used as the row label
// since FAQ entries share the owning document's title.
faq_question?: string;
title_match?: boolean;
chunk_hit_count: number;
@@ -220,11 +224,28 @@ export interface GrepKnowledgeResult {
distinct_patterns: number;
}
// Per-chunk grep hit (preferred for UI — one row per FAQ entry or chunk)
export interface GrepChunkResult {
chunk_id: string;
faq_id?: string;
knowledge_id: string;
knowledge_base_id: string;
knowledge_title: string;
chunk_type?: string;
index?: number;
chunk_index?: number;
faq_question?: string;
title_match?: boolean;
match_snippet?: string;
score?: number;
}
// Grep results data
export interface GrepResultsData {
display_type: 'grep_results';
query?: string;
patterns: string[];
chunk_results?: GrepChunkResult[];
knowledge_results: GrepKnowledgeResult[];
result_count: number;
total_matches: number;
@@ -2080,6 +2080,9 @@ const getToolSummary = (event: any): string => {
return t('agentStream.toolSummary.getDocument', { title: toolData.title });
}
} else if (toolName === 'list_knowledge_chunks') {
if (toolData?.faq_question) {
return t('agentStream.toolSummary.listFaqEntry', { question: toolData.faq_question });
}
if (toolData?.fetched_chunks !== undefined) {
const title = toolData?.knowledge_title || toolData?.knowledge_id || t('agentStream.toolSummary.document');
return t('agentStream.toolSummary.listChunks', { title, fetched: toolData.fetched_chunks, total: toolData.total_chunks ?? '?' });
@@ -2238,7 +2241,8 @@ const getGrepResultsSummary = (toolData: any): string => {
if (!toolData) return '';
const totalChunks = toolData.total_matches || 0;
const docCount = toolData.result_count || 0;
const rowCount =
toolData.chunk_results?.length ?? toolData.result_count ?? toolData.knowledge_results?.length ?? 0;
if (totalChunks === 0) {
return t('agentStream.search.noResults');
@@ -2246,7 +2250,7 @@ const getGrepResultsSummary = (toolData: any): string => {
return t('agentStream.search.grepSummary', {
chunks: `<strong>${totalChunks}</strong>`,
docs: `<strong>${docCount}</strong>`,
docs: `<strong>${rowCount}</strong>`,
});
};
@@ -4,7 +4,7 @@
<div v-if="documents.length" class="documents-list">
<div
v-for="(doc, index) in documents"
:key="doc.knowledge_id || index"
:key="doc.faq_id || doc.knowledge_id || index"
class="result-card document-card"
>
<div class="result-header document-header">
@@ -20,10 +20,23 @@
</div>
<div class="result-content expanded">
<div class="info-section">
<div class="info-field">
<span class="field-label">{{ $t('chat.documentIdLabel') }}</span>
<div class="info-field" v-if="doc.is_faq && doc.faq_id">
<span class="field-label">{{ $t('chat.faqIdLabel') }}</span>
<span class="field-value"><code>{{ doc.faq_id }}</code></span>
</div>
<div class="info-field" v-if="doc.knowledge_id">
<span class="field-label">{{ doc.is_faq ? $t('chat.faqContainerIdLabel') : $t('chat.documentIdLabel') }}</span>
<span class="field-value"><code>{{ doc.knowledge_id }}</code></span>
</div>
<div
v-if="doc.is_faq && doc.faq_answers?.length"
class="info-field info-field--block"
>
<span class="field-label">{{ $t('chat.faqAnswersLabel') }}</span>
<ul class="faq-answers-list">
<li v-for="(ans, aIdx) in doc.faq_answers" :key="aIdx">{{ ans }}</li>
</ul>
</div>
<div class="info-field" v-if="doc.description">
<span class="field-label">{{ $t('chat.documentDescriptionLabel') }}</span>
<span class="field-value">{{ doc.description }}</span>
@@ -205,6 +218,21 @@ const formatMetadataValue = (value: unknown) => {
}
}
.faq-answers-list {
list-style: none;
margin: 4px 0 0;
padding: 0;
display: flex;
flex-direction: column;
gap: 4px;
li {
font-size: 12px;
line-height: 1.5;
color: var(--td-text-color-primary);
}
}
.info-field {
display: flex;
gap: 10px;
@@ -212,6 +240,11 @@ const formatMetadataValue = (value: unknown) => {
font-size: 12px;
line-height: 1.5;
&--block {
flex-direction: column;
gap: 4px;
}
.field-label {
color: var(--td-text-color-secondary);
min-width: 90px;
@@ -1,16 +1,17 @@
<template>
<div class="grep-results">
<div v-if="results.length" class="results-list">
<div v-if="rows.length" class="results-list">
<ResultRow
v-for="(result, index) in results"
:key="result.knowledge_id"
v-for="(result, index) in rows"
:key="result.key"
:index="index + 1"
:title="result.faq_question || result.knowledge_title || $t('knowledge.untitledDocument')"
:meta="formatMeta(result)"
:popup-key="result.knowledge_id || index"
:show-popup="!!cleanedSnippet(result)"
:content="result.match_snippet"
:knowledge-id="result.knowledge_id"
:title="result.title"
:meta="result.meta"
:popup-key="result.key"
:show-popup="!!result.snippet"
:content="result.snippet"
:chunk-id="result.chunkId"
:knowledge-id="result.knowledgeId"
:highlight="searchPattern"
:regex="true"
/>
@@ -27,7 +28,7 @@ import { computed } from 'vue';
import { useI18n } from 'vue-i18n';
import { cleanSnippet } from './contentClean';
import ResultRow from './ResultRow.vue';
import type { GrepKnowledgeResult, GrepResultsData } from '@/types/tool-results';
import type { GrepChunkResult, GrepKnowledgeResult, GrepResultsData } from '@/types/tool-results';
const props = defineProps<{
data: GrepResultsData;
@@ -35,10 +36,18 @@ const props = defineProps<{
const { t } = useI18n();
const results = computed(() => props.data.knowledge_results ?? []);
const searchPattern = computed(() => props.data.query ?? props.data.patterns?.[0] ?? '');
const formatMeta = (result: GrepKnowledgeResult): string => {
type GrepRow = {
key: string;
title: string;
meta: string;
snippet: string;
chunkId?: string;
knowledgeId?: string;
};
const formatKnowledgeMeta = (result: GrepKnowledgeResult): string => {
const parts: string[] = [];
const chunks = result.chunk_hit_count ?? 0;
if (chunks > 0) {
@@ -54,8 +63,46 @@ const formatMeta = (result: GrepKnowledgeResult): string => {
return parts.join(' · ');
};
const cleanedSnippet = (result: GrepKnowledgeResult): string =>
cleanSnippet(result.match_snippet ?? '');
const rowFromChunk = (result: GrepChunkResult): GrepRow => {
const isFAQ = !!result.faq_id || result.chunk_type === 'faq';
const title = result.faq_question || result.knowledge_title || t('knowledge.untitledDocument');
const meta = isFAQ
? t('agentStream.grepResults.faqEntry')
: formatKnowledgeMeta({
knowledge_id: result.knowledge_id,
knowledge_base_id: result.knowledge_base_id,
knowledge_title: result.knowledge_title,
chunk_hit_count: 1,
total_pattern_hits: 1,
distinct_patterns: 1,
pattern_counts: {},
title_match: !!result.title_match,
});
return {
key: result.faq_id || result.chunk_id || String(result.index ?? '') || result.knowledge_id,
title,
meta,
snippet: cleanSnippet(result.match_snippet ?? ''),
chunkId: result.faq_id || result.chunk_id,
knowledgeId: result.knowledge_id,
};
};
const rowFromKnowledge = (result: GrepKnowledgeResult): GrepRow => ({
key: result.knowledge_id,
title: result.faq_question || result.knowledge_title || t('knowledge.untitledDocument'),
meta: formatKnowledgeMeta(result),
snippet: cleanSnippet(result.match_snippet ?? ''),
knowledgeId: result.knowledge_id,
});
const rows = computed((): GrepRow[] => {
const chunkRows = props.data.chunk_results;
if (chunkRows?.length) {
return chunkRows.map(rowFromChunk);
}
return (props.data.knowledge_results ?? []).map(rowFromKnowledge);
});
</script>
<style lang="less" scoped>
+287
View File
@@ -0,0 +1,287 @@
package tools
import (
"fmt"
"regexp"
"strings"
"github.com/Tencent/WeKnora/internal/types"
)
// faqMaxSimilarQuestionsDisplay caps similar_question rows in tool output.
const faqMaxSimilarQuestionsDisplay = 5
func truncateSimilarQuestionsForDisplay(questions []string) (display []string, omitted int) {
if len(questions) == 0 {
return nil, 0
}
if len(questions) <= faqMaxSimilarQuestionsDisplay {
return questions, 0
}
return questions[:faqMaxSimilarQuestionsDisplay], len(questions) - faqMaxSimilarQuestionsDisplay
}
func writeSimilarQuestionsXML(b *strings.Builder, questions []string) {
display, omitted := truncateSimilarQuestionsForDisplay(questions)
for _, sq := range display {
fmt.Fprintf(b, "<similar_question>%s</similar_question>\n", xmlEscape(sq))
}
if omitted > 0 {
fmt.Fprintf(b, "<similar_questions_omitted count=\"%d\" />\n", omitted)
}
}
func appendSimilarQuestionsToChunkData(chunkData map[string]interface{}, questions []string) {
display, omitted := truncateSimilarQuestionsForDisplay(questions)
if len(display) == 0 {
return
}
chunkData["faq_similar_questions"] = display
if omitted > 0 {
chunkData["faq_similar_questions_omitted"] = omitted
}
}
// writeFAQFieldsXML emits question / similar_question / answer children (no wrapper).
func writeFAQFieldsXML(b *strings.Builder, meta *types.FAQChunkMetadata) {
if meta == nil {
return
}
if meta.StandardQuestion != "" {
fmt.Fprintf(b, "<question>%s</question>\n", xmlEscape(meta.StandardQuestion))
}
writeSimilarQuestionsXML(b, meta.SimilarQuestions)
for _, ans := range meta.Answers {
if strings.TrimSpace(ans) == "" {
continue
}
fmt.Fprintf(b, "<answer>%s</answer>\n", xmlEscape(ans))
}
}
func faqFieldsEmpty(meta *types.FAQChunkMetadata) bool {
if meta == nil {
return true
}
return meta.StandardQuestion == "" && len(meta.SimilarQuestions) == 0 && len(meta.Answers) == 0
}
// writeFAQMetadataXML emits a nested <faq> block (used inside knowledge_search <chunk>).
func writeFAQMetadataXML(b *strings.Builder, meta *types.FAQChunkMetadata) {
if faqFieldsEmpty(meta) {
return
}
b.WriteString("<faq>\n")
writeFAQFieldsXML(b, meta)
b.WriteString("</faq>\n")
}
// writeFAQEntryXML emits a top-level FAQ entry for list_knowledge_chunks (not wrapped in <chunk>).
func writeFAQEntryXML(b *strings.Builder, c *types.Chunk) {
if c == nil || c.ChunkType != types.ChunkTypeFAQ {
return
}
meta, err := c.FAQMetadata()
if err != nil {
meta = nil
}
q := faqStandardQuestion(c)
questionAttr := ""
if q != "" {
questionAttr = fmt.Sprintf(" question=\"%s\"", xmlEscape(q))
}
fmt.Fprintf(b, "<faq faq_id=\"%s\" index=\"%d\"%s>\n",
xmlEscape(c.ID),
c.ChunkIndex,
questionAttr,
)
if !faqFieldsEmpty(meta) {
writeFAQFieldsXML(b, meta)
} else if q != "" {
fmt.Fprintf(b, "<question>%s</question>\n", xmlEscape(q))
}
b.WriteString("</faq>\n")
}
// normalizeFAQChunkDataMap uses faq_id / index instead of chunk_id / chunk_index in JSON payloads.
func normalizeFAQChunkDataMap(chunkData map[string]interface{}, c *types.Chunk) {
if c == nil || c.ChunkType != types.ChunkTypeFAQ || chunkData == nil {
return
}
chunkData["faq_id"] = c.ID
chunkData["index"] = c.ChunkIndex
delete(chunkData, "chunk_id")
delete(chunkData, "chunk_index")
}
// appendFAQChunkData adds FAQ metadata fields to structured tool result maps.
func appendFAQChunkData(chunkData map[string]interface{}, c *types.Chunk) {
if c == nil || c.ChunkType != types.ChunkTypeFAQ {
return
}
meta, err := c.FAQMetadata()
if err != nil || meta == nil {
return
}
if q := strings.TrimSpace(meta.StandardQuestion); q != "" {
chunkData["faq_question"] = q
}
appendSimilarQuestionsToChunkData(chunkData, meta.SimilarQuestions)
if len(meta.Answers) > 0 {
chunkData["faq_answers"] = meta.Answers
}
}
// Bounds for retrieval-tool match snippets (grep_chunks, knowledge_search).
const (
snippetContextRunes = 200
snippetMaxMatchRunes = 200
snippetMaxTotalRunes = 800
snippetMaxAnswerRunes = 600
)
// faqMatchSnippetFromQueries builds "Q: … | A: …" for knowledge_search hits.
func faqMatchSnippetFromQueries(meta *types.FAQChunkMetadata, queries []string) string {
if meta == nil {
return ""
}
question := faqMatchedQuestionFromQueries(meta, queries)
if question == "" {
question = strings.TrimSpace(meta.StandardQuestion)
}
return formatFAQMatchSnippet(question, meta.Answers)
}
// faqMatchSnippet builds "Q: … | A: …" for grep_chunks regex hits.
func faqMatchSnippet(chunk *types.Chunk, compiled []*regexp.Regexp) string {
if chunk == nil {
return ""
}
meta, err := chunk.FAQMetadata()
if err != nil || meta == nil {
return ""
}
question := faqMatchedQuestionFromRegex(meta, compiled)
if question == "" {
question = strings.TrimSpace(meta.StandardQuestion)
}
if question == "" {
return ""
}
return formatFAQMatchSnippet(question, meta.Answers)
}
func formatFAQMatchSnippet(question string, answers []string) string {
question = strings.TrimSpace(question)
if question == "" {
return ""
}
var b strings.Builder
b.WriteString("Q: ")
b.WriteString(question)
if answer := faqAnswersForSnippet(answers); answer != "" {
b.WriteString(" | A: ")
b.WriteString(answer)
}
snippet := strings.TrimSpace(b.String())
if len([]rune(snippet)) > snippetMaxTotalRunes {
snippet = truncateRunes(snippet, snippetMaxTotalRunes)
}
return snippet
}
func faqMatchedQuestionFromRegex(meta *types.FAQChunkMetadata, compiled []*regexp.Regexp) string {
if meta == nil {
return ""
}
for _, sq := range meta.SimilarQuestions {
if regexMatchesAny(sq, compiled) {
return sq
}
}
if regexMatchesAny(meta.StandardQuestion, compiled) {
return meta.StandardQuestion
}
return meta.StandardQuestion
}
func faqMatchedQuestionFromQueries(meta *types.FAQChunkMetadata, queries []string) string {
if meta == nil {
return ""
}
tokens := searchQueryTokens(queries)
for _, sq := range meta.SimilarQuestions {
if textMatchesSearchQueries(sq, queries, tokens) {
return sq
}
}
if textMatchesSearchQueries(meta.StandardQuestion, queries, tokens) {
return meta.StandardQuestion
}
return meta.StandardQuestion
}
func faqAnswersForSnippet(answers []string) string {
if len(answers) == 0 {
return ""
}
parts := make([]string, 0, len(answers))
for _, a := range answers {
a = strings.TrimSpace(a)
if a != "" {
parts = append(parts, a)
}
}
if len(parts) == 0 {
return ""
}
return truncateRunes(strings.Join(parts, " | "), snippetMaxAnswerRunes)
}
func textMatchesSearchQueries(text string, queries []string, tokens []string) bool {
if text == "" {
return false
}
lowered := strings.ToLower(text)
for _, q := range queries {
q = strings.ToLower(strings.TrimSpace(q))
if q != "" && strings.Contains(lowered, q) {
return true
}
}
for _, tok := range tokens {
if strings.Contains(lowered, tok) {
return true
}
}
return false
}
func searchQueryTokens(queries []string) []string {
tokens := make([]string, 0, 8)
seen := make(map[string]struct{})
for _, q := range queries {
for _, tok := range strings.FieldsFunc(q, func(r rune) bool {
switch r {
case ' ', '\t', '\n', '\r', ',', '.', ';', ':', '?', '!',
'(', ')', '[', ']', '{', '}', '"', '\'':
return true
}
return false
}) {
tok = strings.ToLower(strings.TrimSpace(tok))
if len([]rune(tok)) < 2 {
continue
}
if _, ok := seen[tok]; ok {
continue
}
seen[tok] = struct{}{}
tokens = append(tokens, tok)
}
}
return tokens
}
+82
View File
@@ -0,0 +1,82 @@
package tools
import (
"fmt"
"strings"
"testing"
"github.com/Tencent/WeKnora/internal/types"
)
func TestWriteFAQMetadataXML_IncludesAnswers(t *testing.T) {
var b strings.Builder
meta := &types.FAQChunkMetadata{
StandardQuestion: "如何创建知识库?",
SimilarQuestions: []string{"怎么创建知识库?"},
Answers: []string{"在控制台点击新建知识库。"},
}
writeFAQMetadataXML(&b, meta)
out := b.String()
for _, want := range []string{
"<faq>", "<question>如何创建知识库?</question>",
"<similar_question>怎么创建知识库?</similar_question>",
"<answer>在控制台点击新建知识库。</answer>", "</faq>",
} {
if !strings.Contains(out, want) {
t.Fatalf("missing %q in:\n%s", want, out)
}
}
}
func TestWriteSimilarQuestionsXML_TruncatesWhenTooMany(t *testing.T) {
var b strings.Builder
questions := make([]string, 8)
for i := range questions {
questions[i] = fmt.Sprintf("相似问%d", i+1)
}
writeSimilarQuestionsXML(&b, questions)
out := b.String()
if strings.Count(out, "<similar_question>") != faqMaxSimilarQuestionsDisplay {
t.Fatalf("want %d similar_question tags, got:\n%s", faqMaxSimilarQuestionsDisplay, out)
}
if !strings.Contains(out, `<similar_questions_omitted count="3"`) {
t.Fatalf("want omitted marker, got:\n%s", out)
}
}
func TestWriteFAQEntryXML_UsesFaqNotChunk(t *testing.T) {
chunk := &types.Chunk{
ID: "faq-chunk-1",
ChunkType: types.ChunkTypeFAQ,
ChunkIndex: 0,
KnowledgeID: "kb-doc-1",
Content: "Q: test\nSimilar Questions:\n- alt",
}
meta := &types.FAQChunkMetadata{
StandardQuestion: "如何创建知识库?",
Answers: []string{"点新建即可。"},
}
if err := chunk.SetFAQMetadata(meta); err != nil {
t.Fatal(err)
}
var b strings.Builder
writeFAQEntryXML(&b, chunk)
out := b.String()
if strings.Contains(out, "<chunk") {
t.Fatalf("FAQ list output must not use <chunk>, got:\n%s", out)
}
for _, want := range []string{
`<faq faq_id="faq-chunk-1"`,
"<question>如何创建知识库?</question>",
"<answer>点新建即可。</answer>",
"</faq>",
} {
if !strings.Contains(out, want) {
t.Fatalf("missing %q in:\n%s", want, out)
}
}
}
+132 -18
View File
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"strings"
"sync"
"github.com/Tencent/WeKnora/internal/types"
@@ -40,13 +41,18 @@ Do not use when:
- Concurrent query for multiple documents provides better performance
- Returns complete document metadata, not just title
- Can check document processing status (parse_status)`,
- Can check document processing status (parse_status)
## IDs
- knowledge_ids: regular documents knowledges
- faq_ids: individual FAQ entries. Returns the standard question and answers, not the container title.`,
schema: utils.GenerateSchema[GetDocumentInfoInput](),
}
// GetDocumentInfoInput defines the input parameters for get document info tool
type GetDocumentInfoInput struct {
KnowledgeIDs []string `json:"knowledge_ids" jsonschema:"Array of document/knowledge IDs, obtained from knowledge_id field in search results, supports concurrent batch queries"`
KnowledgeIDs []string `json:"knowledge_ids" jsonschema:"Document/knowledge IDs for regular documents or FAQ containers"`
FAQIDs []string `json:"faq_ids" jsonschema:"FAQ entry IDs (= chunk_id from grep_chunks). Use instead of knowledge_ids for a single FAQ Q&A."`
}
// GetDocumentInfoTool retrieves detailed information about a document/knowledge
@@ -82,18 +88,19 @@ func (t *GetDocumentInfoTool) Execute(ctx context.Context, args json.RawMessage)
}, err
}
// Extract knowledge_ids array
knowledgeIDs := input.KnowledgeIDs
if len(knowledgeIDs) == 0 {
faqIDs := input.FAQIDs
if len(knowledgeIDs) == 0 && len(faqIDs) == 0 {
return &types.ToolResult{
Success: false,
Error: "knowledge_ids is required and must be a non-empty array",
}, fmt.Errorf("knowledge_ids is required")
Error: "knowledge_ids or faq_ids is required (non-empty array)",
}, fmt.Errorf("missing ids")
}
// Concurrently get info for each knowledge ID
type docInfo struct {
knowledge *types.Knowledge
chunk *types.Chunk
faqMeta *types.FAQChunkMetadata
chunkCount int
err error
}
@@ -102,7 +109,37 @@ func (t *GetDocumentInfoTool) Execute(ctx context.Context, args json.RawMessage)
var mu sync.Mutex
results := make(map[string]*docInfo)
// Concurrently get info for each knowledge ID
for _, faqID := range faqIDs {
faqID = strings.TrimSpace(faqID)
if faqID == "" {
continue
}
wg.Add(1)
go func(id string) {
defer wg.Done()
chunk, err := t.chunkService.GetChunkByIDOnly(ctx, id)
if err != nil || chunk == nil {
mu.Lock()
results["faq:"+id] = &docInfo{err: fmt.Errorf("faq entry not found: %v", err)}
mu.Unlock()
return
}
if !t.searchTargets.ContainsKB(chunk.KnowledgeBaseID) {
mu.Lock()
results["faq:"+id] = &docInfo{err: fmt.Errorf("knowledge base %s is not accessible", chunk.KnowledgeBaseID)}
mu.Unlock()
return
}
var meta *types.FAQChunkMetadata
if chunk.ChunkType == types.ChunkTypeFAQ {
meta, _ = chunk.FAQMetadata()
}
mu.Lock()
results["faq:"+id] = &docInfo{chunk: chunk, faqMeta: meta, chunkCount: 1}
mu.Unlock()
}(faqID)
}
for _, knowledgeID := range knowledgeIDs {
wg.Add(1)
go func(id string) {
@@ -158,18 +195,38 @@ func (t *GetDocumentInfoTool) Execute(ctx context.Context, args json.RawMessage)
wg.Wait()
// Collect successful results and errors
requested := len(knowledgeIDs) + len(faqIDs)
successDocs := make([]*docInfo, 0)
var errors []string
for _, knowledgeID := range knowledgeIDs {
result := results[knowledgeID]
if result == nil {
errors = append(errors, fmt.Sprintf("%s: not found", knowledgeID))
continue
}
if result.err != nil {
errors = append(errors, fmt.Sprintf("%s: %v", knowledgeID, result.err))
} else if result.knowledge != nil {
successDocs = append(successDocs, result)
}
}
for _, faqID := range faqIDs {
faqID = strings.TrimSpace(faqID)
if faqID == "" {
continue
}
result := results["faq:"+faqID]
if result == nil {
errors = append(errors, fmt.Sprintf("faq:%s: not found", faqID))
continue
}
if result.err != nil {
errors = append(errors, fmt.Sprintf("faq:%s: %v", faqID, result.err))
} else if result.chunk != nil {
successDocs = append(successDocs, result)
}
}
if len(successDocs) == 0 {
return &types.ToolResult{
@@ -178,9 +235,8 @@ func (t *GetDocumentInfoTool) Execute(ctx context.Context, args json.RawMessage)
}, fmt.Errorf("all document retrievals failed")
}
// Format output
output := "=== Document Info ===\n\n"
output += fmt.Sprintf("Successfully retrieved %d / %d documents\n\n", len(successDocs), len(knowledgeIDs))
output += fmt.Sprintf("Successfully retrieved %d / %d entries\n\n", len(successDocs), requested)
if len(errors) > 0 {
output += "=== Partial Failures ===\n"
@@ -192,9 +248,15 @@ func (t *GetDocumentInfoTool) Execute(ctx context.Context, args json.RawMessage)
formattedDocs := make([]map[string]interface{}, 0, len(successDocs))
for i, doc := range successDocs {
k := doc.knowledge
output += fmt.Sprintf("[Entry #%d]\n", i+1)
output += fmt.Sprintf("[Document #%d]\n", i+1)
if doc.chunk != nil {
formatted := formatFAQEntryInfo(&output, doc.chunk, doc.faqMeta)
formattedDocs = append(formattedDocs, formatted)
continue
}
k := doc.knowledge
output += fmt.Sprintf(" ID: %s\n", k.ID)
output += fmt.Sprintf(" Title: %s\n", k.Title)
@@ -236,13 +298,15 @@ func (t *GetDocumentInfoTool) Execute(ctx context.Context, args json.RawMessage)
"parse_status": k.ParseStatus,
"chunk_count": doc.chunkCount,
"metadata": k.GetMetadata(),
"is_faq": false,
})
}
// Extract first document title for summary
var firstTitle string
if len(successDocs) > 0 && successDocs[0].knowledge != nil {
firstTitle = successDocs[0].knowledge.Title
if len(formattedDocs) > 0 {
if t, ok := formattedDocs[0]["title"].(string); ok {
firstTitle = t
}
}
return &types.ToolResult{
@@ -251,14 +315,64 @@ func (t *GetDocumentInfoTool) Execute(ctx context.Context, args json.RawMessage)
Data: map[string]interface{}{
"documents": formattedDocs,
"total_docs": len(successDocs),
"requested": len(knowledgeIDs),
"requested": requested,
"errors": errors,
"display_type": "document_info",
"title": firstTitle, // For frontend summary display
"title": firstTitle,
},
}, nil
}
func formatFAQEntryInfo(output *string, chunk *types.Chunk, meta *types.FAQChunkMetadata) map[string]interface{} {
title := faqStandardQuestion(chunk)
if title == "" && meta != nil {
title = strings.TrimSpace(meta.StandardQuestion)
}
if title == "" {
title = "FAQ Entry"
}
*output += fmt.Sprintf(" FAQ ID: %s\n", chunk.ID)
*output += fmt.Sprintf(" Question: %s\n", title)
if chunk.KnowledgeID != "" {
*output += fmt.Sprintf(" Container ID: %s\n", chunk.KnowledgeID)
}
if meta != nil && len(meta.Answers) > 0 {
*output += " Answers:\n"
for _, ans := range meta.Answers {
*output += fmt.Sprintf(" - %s\n", ans)
}
}
if meta != nil && len(meta.SimilarQuestions) > 0 {
display, omitted := truncateSimilarQuestionsForDisplay(meta.SimilarQuestions)
*output += " Similar Questions:\n"
for _, sq := range display {
*output += fmt.Sprintf(" - %s\n", sq)
}
if omitted > 0 {
*output += fmt.Sprintf(" ... and %d more omitted\n", omitted)
}
}
*output += "\n"
entry := map[string]interface{}{
"faq_id": chunk.ID,
"knowledge_id": chunk.KnowledgeID,
"title": title,
"faq_question": title,
"type": "faq",
"is_faq": true,
"chunk_count": 1,
}
if meta != nil {
if len(meta.Answers) > 0 {
entry["faq_answers"] = meta.Answers
}
appendSimilarQuestionsToChunkData(entry, meta.SimilarQuestions)
}
return entry
}
func formatSource(knowledgeType, source string) string {
switch knowledgeType {
case "file":
+115 -46
View File
@@ -27,7 +27,11 @@ Examples:
- Word boundary / anchor: "\\brag\\b" or "^chapter\\s+\\d+"
- Plain text: "engine" (matches literal substring anywhere in chunk content)
IMPORTANT — JSON escaping: every backslash in a regex MUST be written as \\ inside the JSON tool arguments (e.g. to search for literal "C++" write "C\\+\\+", NOT "C\+\+"; for "\d+" write "\\d+"). Plain "\+" / "\d" etc. are invalid JSON escapes and will fail to parse.
Use this to locate candidate chunks by exact identifiers, error codes, product names, or recurring terms. Pair with list_knowledge_chunks afterwards to read the full context around any promising chunk_id.`,
Use this to locate candidate chunks by exact identifiers, error codes, product names, or recurring terms.
## Deep read after grep:
- **FAQ hit** (chunk type faq): call list_knowledge_chunks with **faq_id** from the grep result (NOT the parent knowledge_id).
- **Document hit**: call list_knowledge_chunks with **knowledge_id**, or get_document_info with **knowledge_ids**.`,
schema: json.RawMessage(`{
"type": "object",
"properties": {
@@ -186,7 +190,10 @@ func (t *GrepChunksTool) Execute(ctx context.Context, args json.RawMessage) (*ty
finalResults = finalResults[:limit]
}
// Aggregation by knowledge is still useful for the frontend summary view.
// Per-chunk rows for the UI (FAQ entries are one chunk each — do not fold
// them into the parent knowledge container). Knowledge-level aggregation is
// kept for backward compatibility.
chunkResults := buildGrepChunkResults(finalResults, compiled)
aggregatedResults := t.aggregateByKnowledge(finalResults, queries, compiled)
if len(aggregatedResults) > 20 {
aggregatedResults = aggregatedResults[:20]
@@ -201,8 +208,9 @@ func (t *GrepChunksTool) Execute(ctx context.Context, args json.RawMessage) (*ty
"query": query,
"queries": queries, // legacy alias for older frontends
"patterns": queries, // legacy alias for older frontends
"chunk_results": chunkResults,
"knowledge_results": aggregatedResults,
"result_count": len(aggregatedResults),
"result_count": len(chunkResults),
"total_matches": len(finalResults),
"knowledge_base_ids": kbIDs,
"limit": limit,
@@ -373,60 +381,59 @@ func (t *GrepChunksTool) formatOutput(
for _, r := range results {
counts := countRegexHits(r.Content, compiled, queries)
snippet := extractSnippetRegex(r.Content, compiled)
snippet := extractChunkMatchSnippet(&r.Chunk, compiled)
// FAQ entries share the owning knowledge's title, so expose the
// standard question as a per-chunk identifier when available.
faqAttr := ""
extraAttr := ""
if q := faqStandardQuestion(&r.Chunk); q != "" {
faqAttr = fmt.Sprintf(" faq_question=\"%s\"", xmlEscape(q))
extraAttr = fmt.Sprintf(" faq_question=\"%s\"", xmlEscape(q))
}
isFAQ := r.ChunkType == types.ChunkTypeFAQ
t.mu.Lock()
seen := t.seenChunks[r.ID]
t.seenChunks[r.ID] = true
t.mu.Unlock()
if seen {
b.WriteString(fmt.Sprintf(
"<chunk chunk_id=\"%s\" knowledge_id=\"%s\" knowledge_title=\"%s\"%s chunk_index=\"%d\" score=\"%.3f\" already_seen=\"true\">\n",
xmlEscape(r.ID),
xmlEscape(r.KnowledgeID),
xmlEscape(r.KnowledgeTitle),
faqAttr,
r.ChunkIndex,
r.MatchScore,
))
for _, q := range queries {
if c := counts[q]; c > 0 {
b.WriteString(fmt.Sprintf("<query_hit query=\"%s\" count=\"%d\" />\n",
xmlEscape(q), c))
}
if isFAQ {
if seen {
fmt.Fprintf(&b,
"<faq faq_id=\"%s\" knowledge_title=\"%s\"%s index=\"%d\" score=\"%.3f\" already_seen=\"true\">\n",
xmlEscape(r.ID), xmlEscape(r.KnowledgeTitle),
extraAttr, r.ChunkIndex, r.MatchScore)
} else {
fmt.Fprintf(&b,
"<faq faq_id=\"%s\" knowledge_title=\"%s\"%s index=\"%d\" score=\"%.3f\">\n",
xmlEscape(r.ID), xmlEscape(r.KnowledgeTitle),
extraAttr, r.ChunkIndex, r.MatchScore)
}
b.WriteString("<note>(snippet omitted, already returned in a previous grep_chunks call this session)</note>\n")
b.WriteString("</chunk>\n")
continue
} else if seen {
fmt.Fprintf(&b,
"<chunk chunk_id=\"%s\" knowledge_id=\"%s\" knowledge_title=\"%s\"%s chunk_index=\"%d\" score=\"%.3f\" already_seen=\"true\">\n",
xmlEscape(r.ID), xmlEscape(r.KnowledgeID), xmlEscape(r.KnowledgeTitle),
extraAttr, r.ChunkIndex, r.MatchScore)
} else {
fmt.Fprintf(&b,
"<chunk chunk_id=\"%s\" knowledge_id=\"%s\" knowledge_title=\"%s\"%s chunk_index=\"%d\" score=\"%.3f\">\n",
xmlEscape(r.ID), xmlEscape(r.KnowledgeID), xmlEscape(r.KnowledgeTitle),
extraAttr, r.ChunkIndex, r.MatchScore)
}
b.WriteString(fmt.Sprintf(
"<chunk chunk_id=\"%s\" knowledge_id=\"%s\" knowledge_title=\"%s\"%s chunk_index=\"%d\" score=\"%.3f\">\n",
xmlEscape(r.ID),
xmlEscape(r.KnowledgeID),
xmlEscape(r.KnowledgeTitle),
faqAttr,
r.ChunkIndex,
r.MatchScore,
))
for _, q := range queries {
if c := counts[q]; c > 0 {
b.WriteString(fmt.Sprintf("<query_hit query=\"%s\" count=\"%d\" />\n",
xmlEscape(q), c))
}
}
if snippet != "" {
if seen {
b.WriteString("<note>(snippet omitted, already returned in a previous grep_chunks call this session)</note>\n")
} else if snippet != "" {
b.WriteString(fmt.Sprintf("<match_snippet>%s</match_snippet>\n", xmlEscape(snippet)))
}
b.WriteString("</chunk>\n")
if isFAQ {
b.WriteString("</faq>\n")
} else {
b.WriteString("</chunk>\n")
}
}
b.WriteString("</grep_results>")
@@ -503,7 +510,7 @@ func (t *GrepChunksTool) aggregateByKnowledge(
}
}
if entry.MatchSnippet == "" {
if snippet := extractSnippetRegex(chunk.Content, compiled); snippet != "" {
if snippet := extractChunkMatchSnippet(&chunk.Chunk, compiled); snippet != "" {
entry.MatchSnippet = snippet
}
}
@@ -551,6 +558,49 @@ func (t *GrepChunksTool) aggregateByKnowledge(
return resultSlice
}
type grepChunkResult struct {
ChunkID string `json:"chunk_id,omitempty"`
FAQID string `json:"faq_id,omitempty"`
KnowledgeID string `json:"knowledge_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
KnowledgeTitle string `json:"knowledge_title"`
ChunkType string `json:"chunk_type"`
Index int `json:"index,omitempty"`
ChunkIndex int `json:"chunk_index,omitempty"`
FAQQuestion string `json:"faq_question,omitempty"`
TitleMatch bool `json:"title_match,omitempty"`
MatchSnippet string `json:"match_snippet,omitempty"`
Score float64 `json:"score"`
}
func buildGrepChunkResults(results []chunkWithTitle, compiled []*regexp.Regexp) []grepChunkResult {
if len(results) == 0 {
return nil
}
out := make([]grepChunkResult, 0, len(results))
for _, r := range results {
item := grepChunkResult{
KnowledgeID: r.KnowledgeID,
KnowledgeBaseID: r.KnowledgeBaseID,
KnowledgeTitle: r.KnowledgeTitle,
ChunkType: string(r.ChunkType),
TitleMatch: r.TitleMatch,
MatchSnippet: extractChunkMatchSnippet(&r.Chunk, compiled),
Score: r.MatchScore,
}
if r.ChunkType == types.ChunkTypeFAQ {
item.FAQID = r.ID
item.Index = r.ChunkIndex
item.FAQQuestion = faqStandardQuestion(&r.Chunk)
} else {
item.ChunkID = r.ID
item.ChunkIndex = r.ChunkIndex
}
out = append(out, item)
}
return out
}
// regexMatchesAny reports whether text matches at least one of the compiled
// patterns. Used to flag title hits without counting occurrences.
func regexMatchesAny(text string, compiled []*regexp.Regexp) bool {
@@ -582,6 +632,22 @@ func countRegexHits(content string, compiled []*regexp.Regexp, patterns []string
return counts
}
// extractChunkMatchSnippet returns a preview for tool output. FAQ chunks only
// surface the matched question plus answers from metadata (answers are not
// stored in chunk content for question_only index mode). Other chunk types
// use regex context around the first body match.
func extractChunkMatchSnippet(chunk *types.Chunk, compiled []*regexp.Regexp) string {
if chunk != nil && chunk.ChunkType == types.ChunkTypeFAQ {
if s := faqMatchSnippet(chunk, compiled); s != "" {
return s
}
}
if chunk == nil {
return ""
}
return extractSnippetRegex(chunk.Content, compiled)
}
// extractSnippetRegex returns a short context snippet around the earliest
// regex match across any of the provided compiled patterns. Result is
// compressed to a single line and bounded in length on both sides of the
@@ -614,18 +680,17 @@ func extractSnippetRegex(content string, compiled []*regexp.Regexp) string {
before := content[:earliest]
after := content[earliestEnd:]
const contextRunes = 60
beforeRunes := []rune(before)
if len(beforeRunes) > contextRunes {
beforeRunes = beforeRunes[len(beforeRunes)-contextRunes:]
if len(beforeRunes) > snippetContextRunes {
beforeRunes = beforeRunes[len(beforeRunes)-snippetContextRunes:]
}
afterRunes := []rune(after)
if len(afterRunes) > contextRunes {
afterRunes = afterRunes[:contextRunes]
if len(afterRunes) > snippetContextRunes {
afterRunes = afterRunes[:snippetContextRunes]
}
matchRunes := []rune(matchStr)
if len(matchRunes) > 120 {
matchRunes = append(matchRunes[:120], []rune("...")...)
if len(matchRunes) > snippetMaxMatchRunes {
matchRunes = append(matchRunes[:snippetMaxMatchRunes], []rune("...")...)
}
snippet := string(beforeRunes) + string(matchRunes) + string(afterRunes)
@@ -633,7 +698,11 @@ func extractSnippetRegex(content string, compiled []*regexp.Regexp) string {
for strings.Contains(snippet, " ") {
snippet = strings.ReplaceAll(snippet, " ", " ")
}
return "... " + strings.TrimSpace(snippet) + " ..."
snippet = strings.TrimSpace(snippet)
if len([]rune(snippet)) > snippetMaxTotalRunes {
snippet = string([]rune(snippet)[:snippetMaxTotalRunes]) + "..."
}
return "... " + snippet + " ..."
}
// xmlEscape replaces characters that would break simple XML attribute /
@@ -0,0 +1,106 @@
package tools
import (
"regexp"
"strings"
"testing"
"github.com/Tencent/WeKnora/internal/types"
)
func faqChunkForSnippetTest(t *testing.T) *types.Chunk {
t.Helper()
chunk := &types.Chunk{
ChunkType: types.ChunkTypeFAQ,
Content: `Q: 离职原因是否可以自定义
Similar Questions:
- 离职原因自定义
- 设置离职原因`,
}
meta := &types.FAQChunkMetadata{
StandardQuestion: "离职原因是否可以自定义",
SimilarQuestions: []string{"离职原因自定义", "设置离职原因"},
Answers: []string{"可以,在系统设置中自定义离职原因。"},
}
if err := chunk.SetFAQMetadata(meta); err != nil {
t.Fatalf("SetFAQMetadata: %v", err)
}
return chunk
}
func TestFaqMatchSnippet_StandardQuestionAndMetadataAnswer(t *testing.T) {
chunk := faqChunkForSnippetTest(t)
re := regexp.MustCompile(`(?i)是否可以自定义`)
snippet := faqMatchSnippet(chunk, []*regexp.Regexp{re})
if !strings.Contains(snippet, "Q: 离职原因是否可以自定义") {
t.Fatalf("want standard question in snippet, got: %s", snippet)
}
if !strings.Contains(snippet, "A: 可以,在系统设置中") {
t.Fatalf("want metadata answer in snippet, got: %s", snippet)
}
for _, unwanted := range []string{"Similar Questions", "离职原因自定义", "设置离职原因"} {
if strings.Contains(snippet, unwanted) {
t.Errorf("snippet should not include similar-question noise %q: %s", unwanted, snippet)
}
}
}
func TestFaqMatchSnippetFromQueries_StandardQuestionAndMetadataAnswer(t *testing.T) {
meta := &types.FAQChunkMetadata{
StandardQuestion: "离职原因是否可以自定义",
SimilarQuestions: []string{"离职原因自定义", "设置离职原因"},
Answers: []string{"可以,在系统设置中自定义离职原因。"},
}
snippet := faqMatchSnippetFromQueries(meta, []string{"是否可以自定义"})
if !strings.Contains(snippet, "Q: 离职原因是否可以自定义") {
t.Fatalf("want standard question, got: %s", snippet)
}
if !strings.Contains(snippet, "A: 可以,在系统设置中") {
t.Fatalf("want metadata answer, got: %s", snippet)
}
if strings.Contains(snippet, "Similar Questions") {
t.Fatalf("should not include similar-question list: %s", snippet)
}
}
func TestFaqMatchSnippetFromQueries_SimilarQuestionHitShowsMatchedVariant(t *testing.T) {
meta := &types.FAQChunkMetadata{
StandardQuestion: "离职原因是否可以自定义",
SimilarQuestions: []string{"离职原因自定义", "设置离职原因"},
Answers: []string{"可以自定义。"},
}
snippet := faqMatchSnippetFromQueries(meta, []string{"离职原因自定义"})
if !strings.Contains(snippet, "Q: 离职原因自定义") {
t.Fatalf("want matched similar question, got: %s", snippet)
}
}
func TestFaqMatchSnippet_SimilarQuestionHitShowsMatchedVariant(t *testing.T) {
chunk := faqChunkForSnippetTest(t)
re := regexp.MustCompile(`(?i)离职原因自定义`)
snippet := faqMatchSnippet(chunk, []*regexp.Regexp{re})
if !strings.Contains(snippet, "Q: 离职原因自定义") {
t.Fatalf("want matched similar question, got: %s", snippet)
}
if strings.Contains(snippet, "是否可以自定义") {
t.Fatalf("should not show standard question when similar question matched: %s", snippet)
}
}
func TestExtractChunkMatchSnippet_NonFAQUsesBodyContext(t *testing.T) {
content := strings.Repeat("前置。", 50) + "TARGET" + strings.Repeat("后续。", 50)
chunk := &types.Chunk{ChunkType: types.ChunkTypeText, Content: content}
re := regexp.MustCompile(`TARGET`)
snippet := extractChunkMatchSnippet(chunk, []*regexp.Regexp{re})
if !strings.Contains(snippet, "TARGET") {
t.Fatalf("missing match: %s", snippet)
}
if !strings.Contains(snippet, "前置") || !strings.Contains(snippet, "后续") {
t.Fatalf("expected expanded body context: %s", snippet)
}
}
+79 -77
View File
@@ -1164,36 +1164,74 @@ func (t *KnowledgeSearchTool) formatOutput(
t.seenChunks[result.ID] = true
t.seenMu.Unlock()
isFAQ := faqMeta != nil
if seen {
// Compact rendering for chunks we already returned in a previous
// knowledge_search call during this session. The model has the
// content in context already, so re-emitting it only burns tokens.
ob.WriteString(fmt.Sprintf(
"<chunk rank=\"%d\" chunk_id=\"%s\" chunk_index=\"%d\" knowledge_id=\"%s\" knowledge_base_id=\"%s\" knowledge_title=\"%s\" score=\"%.3f\" source_query=\"%s\" already_seen=\"true\">\n",
i+1,
xmlEscape(result.ID),
result.ChunkIndex,
xmlEscape(result.KnowledgeID),
xmlEscape(result.KnowledgeBaseID),
xmlEscape(result.KnowledgeTitle),
result.Score,
xmlEscape(result.SourceQuery),
))
if isFAQ {
ob.WriteString(fmt.Sprintf(
"<faq rank=\"%d\" faq_id=\"%s\" index=\"%d\" knowledge_base_id=\"%s\" knowledge_title=\"%s\" score=\"%.3f\" source_query=\"%s\" already_seen=\"true\">\n",
i+1,
xmlEscape(result.ID),
result.ChunkIndex,
xmlEscape(result.KnowledgeBaseID),
xmlEscape(result.KnowledgeTitle),
result.Score,
xmlEscape(result.SourceQuery),
))
} else {
ob.WriteString(fmt.Sprintf(
"<chunk rank=\"%d\" chunk_id=\"%s\" chunk_index=\"%d\" knowledge_id=\"%s\" knowledge_base_id=\"%s\" knowledge_title=\"%s\" score=\"%.3f\" source_query=\"%s\" already_seen=\"true\">\n",
i+1,
xmlEscape(result.ID),
result.ChunkIndex,
xmlEscape(result.KnowledgeID),
xmlEscape(result.KnowledgeBaseID),
xmlEscape(result.KnowledgeTitle),
result.Score,
xmlEscape(result.SourceQuery),
))
}
ob.WriteString("<note>(content omitted, already returned in a previous knowledge_search call this session)</note>\n")
ob.WriteString("</chunk>\n")
if isFAQ {
ob.WriteString("</faq>\n")
} else {
ob.WriteString("</chunk>\n")
}
} else {
ob.WriteString(fmt.Sprintf(
"<chunk rank=\"%d\" chunk_id=\"%s\" chunk_index=\"%d\" knowledge_id=\"%s\" knowledge_base_id=\"%s\" knowledge_title=\"%s\" score=\"%.3f\" source_query=\"%s\">\n",
i+1,
xmlEscape(result.ID),
result.ChunkIndex,
xmlEscape(result.KnowledgeID),
xmlEscape(result.KnowledgeBaseID),
xmlEscape(result.KnowledgeTitle),
result.Score,
xmlEscape(result.SourceQuery),
))
if snippet := extractSnippetForQueries(result.Content, queries); snippet != "" {
if isFAQ {
ob.WriteString(fmt.Sprintf(
"<faq rank=\"%d\" faq_id=\"%s\" index=\"%d\" knowledge_base_id=\"%s\" knowledge_title=\"%s\" score=\"%.3f\" source_query=\"%s\">\n",
i+1,
xmlEscape(result.ID),
result.ChunkIndex,
xmlEscape(result.KnowledgeBaseID),
xmlEscape(result.KnowledgeTitle),
result.Score,
xmlEscape(result.SourceQuery),
))
} else {
ob.WriteString(fmt.Sprintf(
"<chunk rank=\"%d\" chunk_id=\"%s\" chunk_index=\"%d\" knowledge_id=\"%s\" knowledge_base_id=\"%s\" knowledge_title=\"%s\" score=\"%.3f\" source_query=\"%s\">\n",
i+1,
xmlEscape(result.ID),
result.ChunkIndex,
xmlEscape(result.KnowledgeID),
xmlEscape(result.KnowledgeBaseID),
xmlEscape(result.KnowledgeTitle),
result.Score,
xmlEscape(result.SourceQuery),
))
}
snippet := ""
if faqMeta != nil {
snippet = faqMatchSnippetFromQueries(faqMeta, queries)
}
if snippet == "" {
snippet = extractSnippetForQueries(result.Content, queries)
}
if snippet != "" {
ob.WriteString(fmt.Sprintf("<match_snippet>%s</match_snippet>\n", xmlEscape(snippet)))
}
ob.WriteString(fmt.Sprintf("<content>%s</content>\n", result.Content))
@@ -1214,30 +1252,16 @@ func (t *KnowledgeSearchTool) formatOutput(
}
}
if faqMeta != nil {
ob.WriteString("<faq>\n")
if faqMeta.StandardQuestion != "" {
ob.WriteString(fmt.Sprintf("<question>%s</question>\n", xmlEscape(faqMeta.StandardQuestion)))
}
if len(faqMeta.SimilarQuestions) > 0 {
for _, sq := range faqMeta.SimilarQuestions {
ob.WriteString(fmt.Sprintf("<similar_question>%s</similar_question>\n", xmlEscape(sq)))
}
}
if len(faqMeta.Answers) > 0 {
for _, ans := range faqMeta.Answers {
ob.WriteString(fmt.Sprintf("<answer>%s</answer>\n", xmlEscape(ans)))
}
}
if isFAQ {
writeFAQFieldsXML(&ob, faqMeta)
ob.WriteString("</faq>\n")
} else {
ob.WriteString("</chunk>\n")
}
ob.WriteString("</chunk>\n")
}
formattedResults = append(formattedResults, map[string]interface{}{
"result_index": i + 1,
"chunk_id": result.ID,
"content": result.Content,
"knowledge_id": result.KnowledgeID,
"knowledge_title": result.KnowledgeTitle,
@@ -1275,15 +1299,18 @@ func (t *KnowledgeSearchTool) formatOutput(
}
if faqMeta != nil {
last["faq_id"] = result.ID
last["index"] = result.ChunkIndex
if faqMeta.StandardQuestion != "" {
last["faq_standard_question"] = faqMeta.StandardQuestion
}
if len(faqMeta.SimilarQuestions) > 0 {
last["faq_similar_questions"] = faqMeta.SimilarQuestions
}
appendSimilarQuestionsToChunkData(last, faqMeta.SimilarQuestions)
if len(faqMeta.Answers) > 0 {
last["faq_answers"] = faqMeta.Answers
}
} else {
last["chunk_id"] = result.ID
last["chunk_index"] = result.ChunkIndex
}
}
@@ -1509,31 +1536,7 @@ func extractSnippetForQueries(content string, queries []string) string {
return ""
}
tokens := make([]string, 0, 8)
seen := make(map[string]struct{})
for _, q := range queries {
for _, tok := range strings.FieldsFunc(q, func(r rune) bool {
// Split on whitespace and common punctuation; keep CJK as whole tokens.
switch r {
case ' ', '\t', '\n', '\r', ',', '.', ';', ':', '?', '!',
'(', ')', '[', ']', '{', '}', '"', '\'':
return true
}
return false
}) {
tok = strings.ToLower(strings.TrimSpace(tok))
// Skip trivially-short stopwords. Three rune floor covers most
// English function words without chopping CJK bigrams.
if len([]rune(tok)) < 2 {
continue
}
if _, ok := seen[tok]; ok {
continue
}
seen[tok] = struct{}{}
tokens = append(tokens, tok)
}
}
tokens := searchQueryTokens(queries)
lowered := strings.ToLower(content)
earliest := -1
@@ -1550,11 +1553,10 @@ func extractSnippetForQueries(content string, queries []string) string {
}
}
const contextRunes = 60
if earliest < 0 {
runes := []rune(content)
if len(runes) > contextRunes*2 {
return strings.TrimSpace(string(runes[:contextRunes*2])) + " ..."
if len(runes) > snippetContextRunes*2 {
return strings.TrimSpace(string(runes[:snippetContextRunes*2])) + " ..."
}
return content
}
@@ -1564,12 +1566,12 @@ func extractSnippetForQueries(content string, queries []string) string {
after := content[earliestEnd:]
beforeRunes := []rune(before)
if len(beforeRunes) > contextRunes {
beforeRunes = beforeRunes[len(beforeRunes)-contextRunes:]
if len(beforeRunes) > snippetContextRunes {
beforeRunes = beforeRunes[len(beforeRunes)-snippetContextRunes:]
}
afterRunes := []rune(after)
if len(afterRunes) > contextRunes {
afterRunes = afterRunes[:contextRunes]
if len(afterRunes) > snippetContextRunes {
afterRunes = afterRunes[:snippetContextRunes]
}
snippet := string(beforeRunes) + matchStr + string(afterRunes)
+135 -60
View File
@@ -13,60 +13,57 @@ import (
var listKnowledgeChunksTool = BaseTool{
name: ToolListKnowledgeChunks,
description: `Retrieve full chunk content for a document by knowledge_id.
description: `Retrieve full chunk content for a document or a single FAQ entry.
## Use After grep_chunks or knowledge_search:
1. grep_chunks(["keyword", "variant"]) → get knowledge_id
2. list_knowledge_chunks(knowledge_id) → read full content
- **FAQ hit** (type faq): list_knowledge_chunks(faq_id="<chunk_id from search>") — reads that one FAQ entry with answers from metadata.
- **Document hit**: list_knowledge_chunks(knowledge_id="<document id>") — pages through all chunks.
## When to Use:
- Need full content of chunks from a known document
- Want to see context around specific chunks
- Check how many chunks a document has
## Parameters:
- knowledge_id (required): Document ID
- limit (optional): Chunks per page (default 20, max 100)
- offset (optional): 0-based position in the chunk list (NOT chunk_index).
Must be < total. Use offset=0 to read from the first chunk, offset=N to
skip the first N chunks. If offset >= total the tool returns an explicit
out-of-range error with the valid total so you can retry.
## Parameters (provide exactly one id target):
- faq_id (optional): FAQ entry ID from grep_chunks / knowledge_search.
- chunk_id (optional): Single non-FAQ chunk ID (do not use for FAQ — use faq_id).
- knowledge_id (optional): Document/knowledge ID to page through all chunks.
- limit / offset: Only for knowledge_id paging (default limit 20, max 100).
## Output:
Full chunk content with chunk_id, chunk_index, and content text.
The response attributes include:
- total: total number of chunks in the document
- fetched: how many chunks this page returned
If fetched=0 with total>0, your offset is past the end — retry with a
smaller offset (e.g. offset = max(0, total - limit)).`,
Full chunk content. FAQ entries include <faq> with <answer> from metadata.`,
schema: json.RawMessage(`{
"type": "object",
"properties": {
"faq_id": {
"type": "string",
"description": "FAQ entry ID (same as chunk_id). Use for FAQ hits instead of knowledge_id."
},
"chunk_id": {
"type": "string",
"description": "Single chunk ID (alias of faq_id)"
},
"knowledge_id": {
"type": "string",
"description": "Document ID to retrieve chunks from"
"description": "Document/knowledge ID to list all chunks"
},
"limit": {
"type": "integer",
"description": "Chunks per page (default 20, max 100)",
"description": "Chunks per page when using knowledge_id (default 20, max 100)",
"default": 20,
"minimum": 1,
"maximum": 100
},
"offset": {
"type": "integer",
"description": "Start position (default 0)",
"description": "Start position when using knowledge_id (default 0)",
"default": 0,
"minimum": 0
}
},
"required": ["knowledge_id"]
}
}`),
}
// ListKnowledgeChunksInput defines the input parameters for list knowledge chunks tool
type ListKnowledgeChunksInput struct {
KnowledgeID string `json:"knowledge_id"`
KnowledgeID string `json:"knowledge_id,omitempty"`
FAQID string `json:"faq_id,omitempty"`
ChunkID string `json:"chunk_id,omitempty"`
Limit int `json:"limit"`
Offset int `json:"offset"`
}
@@ -104,15 +101,21 @@ func (t *ListKnowledgeChunksTool) Execute(ctx context.Context, args json.RawMess
}, err
}
knowledgeID := input.KnowledgeID
ok := knowledgeID != ""
if !ok || strings.TrimSpace(knowledgeID) == "" {
chunkID := strings.TrimSpace(input.FAQID)
if chunkID == "" {
chunkID = strings.TrimSpace(input.ChunkID)
}
if chunkID != "" {
return t.executeByChunkID(ctx, chunkID)
}
knowledgeID := strings.TrimSpace(input.KnowledgeID)
if knowledgeID == "" {
return &types.ToolResult{
Success: false,
Error: "knowledge_id is required",
}, fmt.Errorf("knowledge_id is required")
Error: "one of faq_id, chunk_id, or knowledge_id is required",
}, fmt.Errorf("missing id parameter")
}
knowledgeID = strings.TrimSpace(knowledgeID)
// Get knowledge info without tenant filter to support shared KB
knowledge, err := t.knowledgeService.GetKnowledgeByIDOnly(ctx, knowledgeID)
@@ -230,10 +233,8 @@ func (t *ListKnowledgeChunksTool) Execute(ctx context.Context, args json.RawMess
"parent_chunk_id": c.ParentChunkID,
}
// FAQ 条目共用所属 Knowledge 的标题,补上标准问作为单条身份标识
if q := faqStandardQuestion(c); q != "" {
chunkData["faq_question"] = q
}
appendFAQChunkData(chunkData, c)
normalizeFAQChunkDataMap(chunkData, c)
// 添加图片信息
if c.ImageInfo != "" {
@@ -279,6 +280,72 @@ func (t *ListKnowledgeChunksTool) Execute(ctx context.Context, args json.RawMess
}, nil
}
// executeByChunkID loads one chunk by faq_id / chunk_id (FAQ entry or any chunk).
func (t *ListKnowledgeChunksTool) executeByChunkID(ctx context.Context, chunkID string) (*types.ToolResult, error) {
chunk, err := t.chunkService.GetChunkByIDOnly(ctx, chunkID)
if err != nil || chunk == nil {
return &types.ToolResult{
Success: false,
Error: fmt.Sprintf("chunk not found: %v", err),
}, err
}
if !t.searchTargets.ContainsKB(chunk.KnowledgeBaseID) {
return &types.ToolResult{
Success: false,
Error: fmt.Sprintf("knowledge base %s is not accessible", chunk.KnowledgeBaseID),
}, fmt.Errorf("knowledge base not in search targets")
}
chunks := []*types.Chunk{chunk}
if chunk.ImageInfo == "" {
effectiveTenantID := t.searchTargets.GetTenantIDForKB(chunk.KnowledgeBaseID)
if effectiveTenantID > 0 {
infoMap := searchutil.CollectImageInfoByChunkIDs(ctx, t.chunkService.GetRepository(), effectiveTenantID, []string{chunk.ID})
if merged, ok := infoMap[chunk.ID]; ok {
chunk.ImageInfo = merged
}
}
}
knowledgeTitle := t.lookupKnowledgeTitle(ctx, chunk.KnowledgeID)
output := t.buildOutput(chunk.KnowledgeID, knowledgeTitle, 1, 1, chunks)
formattedChunks := []map[string]interface{}{
{
"seq": 1,
"chunk_id": chunk.ID,
"chunk_index": chunk.ChunkIndex,
"content": chunk.Content,
"chunk_type": chunk.ChunkType,
"knowledge_id": chunk.KnowledgeID,
"knowledge_base": chunk.KnowledgeBaseID,
},
}
appendFAQChunkData(formattedChunks[0], chunk)
normalizeFAQChunkDataMap(formattedChunks[0], chunk)
data := map[string]interface{}{
"knowledge_id": chunk.KnowledgeID,
"knowledge_title": knowledgeTitle,
"total_chunks": int64(1),
"fetched_chunks": 1,
"page": 1,
"page_size": 1,
"chunks": formattedChunks,
"faq_id": chunk.ID,
"single_chunk": true,
}
if q := faqStandardQuestion(chunk); q != "" {
data["faq_question"] = q
}
return &types.ToolResult{
Success: true,
Output: output,
Data: data,
}, nil
}
// lookupKnowledgeTitle looks up the title of a knowledge document
// Uses GetKnowledgeByIDOnly to support cross-tenant shared KB
func (t *ListKnowledgeChunksTool) lookupKnowledgeTitle(ctx context.Context, knowledgeID string) string {
@@ -315,8 +382,12 @@ func (t *ListKnowledgeChunksTool) buildOutput(
}
for _, c := range chunks {
// FAQ entries share the owning knowledge's title, so expose the
// standard question as a per-chunk identifier when available.
if c.ChunkType == types.ChunkTypeFAQ {
writeFAQEntryXML(&b, c)
writeChunkImagesXML(&b, c)
continue
}
if q := faqStandardQuestion(c); q != "" {
fmt.Fprintf(&b, "<chunk chunk_id=\"%s\" chunk_index=\"%d\" type=\"%s\" question=\"%s\">\n",
c.ID, c.ChunkIndex, c.ChunkType, xmlEscape(q))
@@ -325,27 +396,7 @@ func (t *ListKnowledgeChunksTool) buildOutput(
c.ID, c.ChunkIndex, c.ChunkType)
}
fmt.Fprintf(&b, "<content>%s</content>\n", summarizeContent(c.Content))
if c.ImageInfo != "" {
var imageInfos []types.ImageInfo
if err := json.Unmarshal([]byte(c.ImageInfo), &imageInfos); err == nil && len(imageInfos) > 0 {
for _, img := range imageInfos {
if img.URL != "" {
fmt.Fprintf(&b, "<image url=\"%s\">\n", img.URL)
} else {
b.WriteString("<image>\n")
}
if img.Caption != "" {
fmt.Fprintf(&b, "<image_caption>%s</image_caption>\n", img.Caption)
}
if img.OCRText != "" {
fmt.Fprintf(&b, "<image_ocr>%s</image_ocr>\n", img.OCRText)
}
b.WriteString("</image>\n")
}
}
}
writeChunkImagesXML(&b, c)
b.WriteString("</chunk>\n")
}
@@ -357,6 +408,30 @@ func (t *ListKnowledgeChunksTool) buildOutput(
return b.String()
}
func writeChunkImagesXML(b *strings.Builder, c *types.Chunk) {
if c == nil || c.ImageInfo == "" {
return
}
var imageInfos []types.ImageInfo
if err := json.Unmarshal([]byte(c.ImageInfo), &imageInfos); err != nil || len(imageInfos) == 0 {
return
}
for _, img := range imageInfos {
if img.URL != "" {
fmt.Fprintf(b, "<image url=\"%s\">\n", img.URL)
} else {
b.WriteString("<image>\n")
}
if img.Caption != "" {
fmt.Fprintf(b, "<image_caption>%s</image_caption>\n", img.Caption)
}
if img.OCRText != "" {
fmt.Fprintf(b, "<image_ocr>%s</image_ocr>\n", img.OCRText)
}
b.WriteString("</image>\n")
}
}
// faqStandardQuestion returns the FAQ standard question for an FAQ-type chunk,
// or "" for non-FAQ chunks (or when metadata is missing/unparseable). All FAQ
// entries inside one knowledge share the same knowledge title, so surfacing the