052be53c42
- Updated the assistant descriptions in various prompt templates to specify that they are developed by Tencent. - Added new intent-specific system prompt templates for greeting, chitchat, follow-up, image analysis, summarization, and web search unavailability scenarios. - Enhanced citation rules in the system prompts to ensure proper inline citation formatting. - Refactored the rewrite prompt to classify user intent more accurately and output structured JSON responses.
125 lines
3.3 KiB
Go
125 lines
3.3 KiB
Go
package chatpipeline
|
|
|
|
import (
|
|
"testing"
|
|
)
|
|
|
|
func TestCleanPassageForRerank(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
input string
|
|
expect string
|
|
}{
|
|
{
|
|
name: "plain text unchanged",
|
|
input: "这是一段普通的文本内容",
|
|
expect: "这是一段普通的文本内容",
|
|
},
|
|
{
|
|
name: "remove markdown images",
|
|
input: "前文  后文",
|
|
expect: "前文 后文",
|
|
},
|
|
{
|
|
name: "convert markdown links to text",
|
|
input: "请参考 [官方文档](https://docs.example.com) 了解详情",
|
|
expect: "请参考 官方文档 了解详情",
|
|
},
|
|
{
|
|
name: "remove standalone URLs",
|
|
input: "访问 https://example.com/path?q=1&b=2 获取更多信息",
|
|
expect: "访问 获取更多信息",
|
|
},
|
|
{
|
|
name: "remove code blocks",
|
|
input: "示例代码:\n```python\nprint('hello')\n```\n以上是示例",
|
|
expect: "示例代码:\n\n以上是示例",
|
|
},
|
|
{
|
|
name: "remove LaTeX blocks",
|
|
input: "公式如下 $$E=mc^2$$ 其中E是能量",
|
|
expect: "公式如下 其中E是能量",
|
|
},
|
|
{
|
|
name: "remove table separator rows and convert data rows",
|
|
input: "| 名称 | 值 |\n| --- | --- |\n| A | 1 |",
|
|
expect: "名称, 值\n\nA, 1",
|
|
},
|
|
{
|
|
name: "strip heading markers",
|
|
input: "## 第二章 概述\n### 2.1 背景",
|
|
expect: "第二章 概述\n2.1 背景",
|
|
},
|
|
{
|
|
name: "strip blockquote markers",
|
|
input: "> 这是一段引用\n> 第二行引用",
|
|
expect: "这是一段引用\n第二行引用",
|
|
},
|
|
{
|
|
name: "unwrap bold and italic",
|
|
input: "这是 **加粗** 和 *斜体* 以及 ***粗斜体*** 文本",
|
|
expect: "这是 加粗 和 斜体 以及 粗斜体 文本",
|
|
},
|
|
{
|
|
name: "strip list markers",
|
|
input: "- 项目一\n- 项目二\n1. 有序一\n2. 有序二",
|
|
expect: "项目一\n项目二\n有序一\n有序二",
|
|
},
|
|
{
|
|
name: "remove HTML tags",
|
|
input: "文本<br>换行<div class=\"test\">内容</div>结尾",
|
|
expect: "文本换行内容结尾",
|
|
},
|
|
{
|
|
name: "collapse excessive newlines",
|
|
input: "段落一\n\n\n\n\n段落二",
|
|
expect: "段落一\n\n段落二",
|
|
},
|
|
{
|
|
name: "combined real-world passage",
|
|
input: `## 产品介绍
|
|
|
|
这是一个 **重要的** 产品。详见 [产品页面](https://example.com/product)。
|
|
|
|

|
|
|
|
> 用户评价:非常好用
|
|
|
|
- 功能一
|
|
- 功能二
|
|
|
|
` + "```json\n{\"key\": \"value\"}\n```",
|
|
expect: "产品介绍\n\n这是一个 重要的 产品。详见 产品页面。\n\n用户评价:非常好用\n\n功能一\n功能二",
|
|
},
|
|
{
|
|
name: "convert table data rows to plain text",
|
|
input: "| col1 | col2 | col3 |",
|
|
expect: "col1, col2, col3",
|
|
},
|
|
{
|
|
name: "multi-row table fully converted",
|
|
input: "| Header1 | Header2 |\n| --- | --- |\n| data1 | data2 |\n| data3 | data4 |",
|
|
expect: "Header1, Header2\n\ndata1, data2\ndata3, data4",
|
|
},
|
|
{
|
|
name: "table-only passage becomes empty after separator removal",
|
|
input: "| --- | --- |",
|
|
expect: "",
|
|
},
|
|
{
|
|
name: "whitespace-only after cleaning",
|
|
input: " \n\n ",
|
|
expect: "",
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := cleanPassageForRerank(tt.input)
|
|
if got != tt.expect {
|
|
t.Errorf("cleanPassageForRerank():\ngot: %q\nexpect: %q", got, tt.expect)
|
|
}
|
|
})
|
|
}
|
|
}
|