Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 9199938b91 | |||
| 6a598c4317 | |||
| 0eb13a022e | |||
| 148c1be452 | |||
| 1b0b0fb8ed | |||
| 9645de64a1 |
+5
-2
@@ -419,7 +419,9 @@ curl -X POST "https://api.telegram.org/bot<YOUR_BOT_TOKEN>/setWebhook" \
|
|||||||
**第二步:配置机器人**
|
**第二步:配置机器人**
|
||||||
|
|
||||||
1. 在应用详情页 → **机器人** → 配置消息接收模式为 **Stream 模式**
|
1. 在应用详情页 → **机器人** → 配置消息接收模式为 **Stream 模式**
|
||||||
2.(可选)如需流式 AI 卡片效果,在 **互动卡片** 中创建一个 **AI 卡片模板**,记录 **Card Template ID**
|
2. 如需识别用户发送的图片,在钉钉控制台为应用授予机器人消息文件下载所需权限,并在 WeKnora Agent 配置中启用**图片上传/多模态**
|
||||||
|
3.(可选)如需流式 AI 卡片效果,在 **互动卡片** 中创建一个 **AI 卡片模板**,记录 **Card Template ID**
|
||||||
|
4. 如需点赞/点踩和转人工,在卡片模板中配置互动按钮,按钮参数使用 `action=like`、`action=dislike`、`action=unresolved` 或 `action=handoff`;点踩原因可附加 `reason` 参数
|
||||||
|
|
||||||
**第三步:在 WeKnora 中添加 IM 渠道**
|
**第三步:在 WeKnora 中添加 IM 渠道**
|
||||||
|
|
||||||
@@ -431,6 +433,7 @@ curl -X POST "https://api.telegram.org/bot<YOUR_BOT_TOKEN>/setWebhook" \
|
|||||||
- **Client ID**:填入 AppKey
|
- **Client ID**:填入 AppKey
|
||||||
- **Client Secret**:填入 AppSecret
|
- **Client Secret**:填入 AppSecret
|
||||||
- **Card Template ID**(可选):填入 AI 卡片模板 ID,启用后流式回复将以 AI 卡片形式展示
|
- **Card Template ID**(可选):填入 AI 卡片模板 ID,启用后流式回复将以 AI 卡片形式展示
|
||||||
|
- **技术支持群 openConversationId**(可选):配置后,转人工工单会自动推送到该群;可让机器人先加入支持群并从该群消息回调的 `conversationId` 获取
|
||||||
3. 保存
|
3. 保存
|
||||||
|
|
||||||
启动后日志出现以下内容表示连接成功:
|
启动后日志出现以下内容表示连接成功:
|
||||||
@@ -439,7 +442,7 @@ curl -X POST "https://api.telegram.org/bot<YOUR_BOT_TOKEN>/setWebhook" \
|
|||||||
[IM] DingTalk Stream connecting...
|
[IM] DingTalk Stream connecting...
|
||||||
```
|
```
|
||||||
|
|
||||||
> **关于 AI 卡片**:配置 Card Template ID 后,流式回复将通过钉钉 AI 卡片实时展示打字效果;未配置时流式内容将在结束后一次性发送。
|
> **关于 AI 卡片**:配置 Card Template ID 后,流式回复将通过钉钉 AI 卡片实时展示打字效果;未配置时流式内容将在结束后一次性发送。点赞/点踩/转人工回调使用钉钉 Stream 卡片回调,需保持渠道为 WebSocket/Stream 模式。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|||||||
Binary file not shown.
|
Before Width: | Height: | Size: 70 KiB |
@@ -1,34 +0,0 @@
|
|||||||
from pathlib import Path
|
|
||||||
from PIL import Image, ImageDraw, ImageFont
|
|
||||||
|
|
||||||
OUT = Path(__file__).with_name("dingtalk-vision-acceptance.png")
|
|
||||||
FONT_PATH = "/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc"
|
|
||||||
|
|
||||||
image = Image.new("RGB", (1280, 720), "#F7FAFC")
|
|
||||||
draw = ImageDraw.Draw(image)
|
|
||||||
font_title = ImageFont.truetype(FONT_PATH, 46)
|
|
||||||
font_code = ImageFont.truetype(FONT_PATH, 58)
|
|
||||||
font_label = ImageFont.truetype(FONT_PATH, 34)
|
|
||||||
font_question = ImageFont.truetype(FONT_PATH, 38)
|
|
||||||
|
|
||||||
# Header and unique OCR target.
|
|
||||||
draw.rounded_rectangle((50, 40, 1230, 190), radius=24, fill="#E6F4FF", outline="#1677FF", width=4)
|
|
||||||
draw.text((85, 65), "WeKnora 钉钉多模态验收图", font=font_title, fill="#102A43")
|
|
||||||
draw.text((85, 120), "唯一验收码:WK-VISION-7264", font=font_code, fill="#C53030")
|
|
||||||
|
|
||||||
# Three visually distinct objects.
|
|
||||||
draw.ellipse((120, 280, 340, 500), fill="#1677FF", outline="#0B4F9C", width=8)
|
|
||||||
draw.text((160, 520), "蓝色圆形", font=font_label, fill="#0B4F9C")
|
|
||||||
|
|
||||||
draw.polygon([(520, 500), (650, 275), (780, 500)], fill="#FFD43B", outline="#9C7600", width=8)
|
|
||||||
draw.text((548, 520), "黄色三角形", font=font_label, fill="#725500")
|
|
||||||
|
|
||||||
draw.rounded_rectangle((940, 295, 1140, 495), radius=12, fill="#2FB344", outline="#176B2A", width=8)
|
|
||||||
draw.text((958, 520), "绿色方形", font=font_label, fill="#176B2A")
|
|
||||||
|
|
||||||
question = "问题:蓝色圆形右侧紧邻的图形是什么颜色和形状?"
|
|
||||||
draw.rounded_rectangle((80, 610, 1200, 690), radius=18, fill="#FFF4E6", outline="#E67700", width=3)
|
|
||||||
draw.text((110, 625), question, font=font_question, fill="#7C3E00")
|
|
||||||
|
|
||||||
image.save(OUT, format="PNG", optimize=True)
|
|
||||||
print(OUT)
|
|
||||||
@@ -1,170 +0,0 @@
|
|||||||
# 阶段2:钉钉图片传输与多模态识别验收计划
|
|
||||||
|
|
||||||
## 1. 验收目标
|
|
||||||
|
|
||||||
验证图片从钉钉用户消息进入 WeKnora,并由视觉模型真实读取图片内容后返回结果;不得把“图片已上传”“图片作为知识文件入库”或“删除图片后按纯文字回答”判定为多模态通过。
|
|
||||||
|
|
||||||
## 2. 当前自动化验证基线
|
|
||||||
|
|
||||||
测试分支:`feature/dingtalk-image-only`
|
|
||||||
|
|
||||||
生产基线:`4022040e`(不包含阶段3~5提交)
|
|
||||||
|
|
||||||
阶段2提交:待本轮代码审查通过后创建。
|
|
||||||
|
|
||||||
已执行的目标测试:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
GOTOOLCHAIN=local GOPROXY=off GOMAXPROCS=4 \
|
|
||||||
/home/zheromiku/.local/toolchains/go1.26.0/bin/go test -p 1 \
|
|
||||||
./internal/im/dingtalk \
|
|
||||||
-run 'Test(ParseCallbackMessagePicture|AdapterDownloadFileUsesMessageFilesAPI|ValidateDingTalkDownloadURLRejectsUnsafeTargets|ParseDingTalkStreamPayloadPicture|ParseDingTalkStreamPayloadPictureJSONString)$' \
|
|
||||||
-count=1
|
|
||||||
|
|
||||||
GOTOOLCHAIN=local GOPROXY=off GOMAXPROCS=4 \
|
|
||||||
/home/zheromiku/.local/toolchains/go1.26.0/bin/go test -p 1 \
|
|
||||||
./internal/im -run 'Test(PrepareIMImageForQA|IMQAErrorReply|FailClosedIMImageAnswer)' -count=1
|
|
||||||
|
|
||||||
GOTOOLCHAIN=local GOPROXY=off GOMAXPROCS=4 \
|
|
||||||
/home/zheromiku/.local/toolchains/go1.26.0/bin/go test -p 1 \
|
|
||||||
./internal/models/chat \
|
|
||||||
-run '^TestRemoteAPIChat_MultimodalUnsupportedDoesNotStripImages$' \
|
|
||||||
-count=1
|
|
||||||
```
|
|
||||||
|
|
||||||
结果:钉钉图片解析、两段式下载、Token请求禁重定向、DNS解析后固定IP拨号、保留/Teredo/受限6to4地址拒绝、签名URL错误脱敏、所有data URI及图片调试日志全量脱敏、JPEG/PNG格式/大小/维度校验、Worker延迟下载、`QARequest.ImageURLs`传递,以及视觉失败/流式部分输出时禁止删图或伪成功,全部通过。
|
|
||||||
|
|
||||||
已执行相关包回归:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
GOTOOLCHAIN=local GOPROXY=off GOMAXPROCS=4 \
|
|
||||||
/home/zheromiku/.local/toolchains/go1.26.0/bin/go test -p 1 \
|
|
||||||
./internal/agent ./internal/im/dingtalk ./internal/im ./internal/models/chat ./internal/utils ./internal/logger \
|
|
||||||
./internal/application/service/chat_pipeline ./internal/application/service -count=1
|
|
||||||
```
|
|
||||||
|
|
||||||
结果:八个相关包全部通过,八包 `go vet` 同步通过。
|
|
||||||
|
|
||||||
## 3. 真实验收前置条件
|
|
||||||
|
|
||||||
- 采用已授权的 App-only 可回滚部署:先备份当前 App 镜像标签,仅重建 `app`,不重建 PostgreSQL、Redis、前端、DocReader 或 MinerU。
|
|
||||||
- 部署后先检查 `WeKnora-app` 健康状态;失败立即恢复备份镜像并仅重建 `app`。
|
|
||||||
- 钉钉应用已开通机器人消息文件下载权限。
|
|
||||||
- WeKnora Agent 已开启“图片上传/多模态”。
|
|
||||||
- Agent 当前问答模型支持视觉输入,或配置了明确的独立 VLM;VLM不可用或未返回图片描述时必须失败关闭。
|
|
||||||
- 不允许在视觉能力不足时静默删除图片并继续生成答案。
|
|
||||||
|
|
||||||
## 4. 测试数据
|
|
||||||
|
|
||||||
标准测试图:
|
|
||||||
|
|
||||||
```text
|
|
||||||
docs/test-plans/assets/dingtalk-vision-acceptance.png
|
|
||||||
```
|
|
||||||
|
|
||||||
图片包含:
|
|
||||||
|
|
||||||
- 唯一验收码:`WK-VISION-7264`
|
|
||||||
- 蓝色圆形
|
|
||||||
- 黄色三角形
|
|
||||||
- 绿色方形
|
|
||||||
- 明确问题:蓝色圆形右侧紧邻的图形是什么颜色和形状?
|
|
||||||
|
|
||||||
标准答案必须同时包含:
|
|
||||||
|
|
||||||
```text
|
|
||||||
WK-VISION-7264
|
|
||||||
黄色三角形
|
|
||||||
```
|
|
||||||
|
|
||||||
## 5. 人工验收用例
|
|
||||||
|
|
||||||
### DT-IMG-01:PNG图片真实识别
|
|
||||||
|
|
||||||
操作:在钉钉中把标准测试图发送给测试机器人。
|
|
||||||
|
|
||||||
通过标准:
|
|
||||||
|
|
||||||
- 机器人成功回复;
|
|
||||||
- 回复包含 `WK-VISION-7264`;
|
|
||||||
- 回复包含“黄色三角形”;
|
|
||||||
- 回复不是“请用文字描述”“当前不支持图片”或无关答案。
|
|
||||||
|
|
||||||
### DT-IMG-02:普通截图OCR
|
|
||||||
|
|
||||||
操作:发送一张包含中文、英文和数字的实际设备截图。
|
|
||||||
|
|
||||||
通过标准:
|
|
||||||
|
|
||||||
- 能提取截图里的关键文字;
|
|
||||||
- 能结合图片回答具体问题;
|
|
||||||
- 不编造截图中不存在的错误码、型号或参数。
|
|
||||||
|
|
||||||
### DT-IMG-03:纯文字回归
|
|
||||||
|
|
||||||
操作:向同一机器人发送普通文字问题。
|
|
||||||
|
|
||||||
通过标准:正常进入原知识库问答,图片功能不影响文字消息。
|
|
||||||
|
|
||||||
### DT-IMG-04:伪图片拒绝
|
|
||||||
|
|
||||||
操作:将非图片内容伪装为 `.png` 后发送。
|
|
||||||
|
|
||||||
通过标准:拒绝处理,提示图片格式校验失败;伪图片不得进入模型。
|
|
||||||
|
|
||||||
### DT-IMG-05:超限图片拒绝
|
|
||||||
|
|
||||||
操作:发送超过10MB的JPG/PNG。
|
|
||||||
|
|
||||||
通过标准:明确提示10MB限制,不进入模型请求。
|
|
||||||
|
|
||||||
### DT-IMG-06:模型不支持视觉
|
|
||||||
|
|
||||||
操作:测试环境临时选择不支持图片的问答模型后发送标准图。
|
|
||||||
|
|
||||||
通过标准:明确提示视觉模型未配置或不支持;禁止删除图片后继续给出看似正常的答案。
|
|
||||||
|
|
||||||
### DT-IMG-07:钉钉权限缺失
|
|
||||||
|
|
||||||
操作:在没有消息文件下载权限的测试应用中发送图片,或使用接口模拟对应403响应。
|
|
||||||
|
|
||||||
通过标准:日志记录脱敏后的权限错误,用户收到可理解提示;不得记录 access token、downloadCode 或临时下载URL。
|
|
||||||
|
|
||||||
## 6. 日志观察点
|
|
||||||
|
|
||||||
发送标准图后,检查测试环境日志:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
sudo docker logs WeKnora-app --since 5m 2>&1 | \
|
|
||||||
grep -E '\[DingTalk\]|Failed to prepare image|message file|multimodal|vision|image'
|
|
||||||
```
|
|
||||||
|
|
||||||
日志不得出现:
|
|
||||||
|
|
||||||
- access token明文;
|
|
||||||
- `downloadCode`明文;
|
|
||||||
- 完整临时下载URL;
|
|
||||||
- 完整Base64图片。
|
|
||||||
|
|
||||||
## 7. 数据检查
|
|
||||||
|
|
||||||
阶段2只验收图片传输和模型识别。消息审计字段的完整性、Token统计、知识库命中片段和渠道统计属于阶段3,本阶段不提前判定通过。
|
|
||||||
|
|
||||||
本阶段允许确认:
|
|
||||||
|
|
||||||
- 用户消息以默认视觉问题进入原有会话历史;
|
|
||||||
- 图片只作为瞬态 `QARequest.ImageURLs` 传给模型;
|
|
||||||
- 数据库不保存完整Base64、临时下载码和临时签名URL。
|
|
||||||
|
|
||||||
## 8. 通过门槛
|
|
||||||
|
|
||||||
必须同时满足:
|
|
||||||
|
|
||||||
1. 图片自动化测试全部通过;
|
|
||||||
2. 相关IM包回归通过;
|
|
||||||
3. `DT-IMG-01`和`DT-IMG-02`真实钉钉测试通过;
|
|
||||||
4. `DT-IMG-03`文字回归通过;
|
|
||||||
5. `DT-IMG-04`~`DT-IMG-07`至少完成目标验证或提供自动化等价证据;
|
|
||||||
6. 用户确认验收。
|
|
||||||
|
|
||||||
在以上条件完成前,阶段2状态保持“进行中”,不得启动阶段3。
|
|
||||||
@@ -401,6 +401,12 @@
|
|||||||
placeholder="xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.schema" />
|
placeholder="xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.schema" />
|
||||||
<p class="form-desc">{{ $t('agentEditor.im.dingtalkCardTemplateIdHint') }}</p>
|
<p class="form-desc">{{ $t('agentEditor.im.dingtalkCardTemplateIdHint') }}</p>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="form-item">
|
||||||
|
<label class="form-label">{{ $t('agentEditor.im.dingtalkSupportGroupId') }}</label>
|
||||||
|
<t-input v-model="formData.credentials.support_group_open_conversation_id"
|
||||||
|
placeholder="cidxxxxxxxxxxxxxxxx" />
|
||||||
|
<p class="form-desc">{{ $t('agentEditor.im.dingtalkSupportGroupIdHint') }}</p>
|
||||||
|
</div>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<!-- QQBot credentials -->
|
<!-- QQBot credentials -->
|
||||||
|
|||||||
@@ -5375,6 +5375,8 @@ export default {
|
|||||||
dingtalkConsole: 'DingTalk Open Platform',
|
dingtalkConsole: 'DingTalk Open Platform',
|
||||||
dingtalkCardTemplateId: 'Card Template ID (optional)',
|
dingtalkCardTemplateId: 'Card Template ID (optional)',
|
||||||
dingtalkCardTemplateIdHint: 'Create an AI Card template at open-dev.dingtalk.com/fe/card to enable streaming output with typewriter effect',
|
dingtalkCardTemplateIdHint: 'Create an AI Card template at open-dev.dingtalk.com/fe/card to enable streaming output with typewriter effect',
|
||||||
|
dingtalkSupportGroupId: 'Support group openConversationId (optional)',
|
||||||
|
dingtalkSupportGroupIdHint: 'Escalated tickets are pushed to this DingTalk group. Obtain the ID from a callback received in the support group.',
|
||||||
mattermostConsole: 'Mattermost integrations',
|
mattermostConsole: 'Mattermost integrations',
|
||||||
qqbotConsole: 'QQ Open Platform',
|
qqbotConsole: 'QQ Open Platform',
|
||||||
qqbotAPIBaseURLHint: 'Optional. Leave empty to use the default production API endpoint.',
|
qqbotAPIBaseURLHint: 'Optional. Leave empty to use the default production API endpoint.',
|
||||||
|
|||||||
@@ -5316,6 +5316,8 @@ export default {
|
|||||||
dingtalkConsole: "DingTalk 개발 플랫폼",
|
dingtalkConsole: "DingTalk 개발 플랫폼",
|
||||||
dingtalkCardTemplateId: "카드 템플릿 ID (선택 사항)",
|
dingtalkCardTemplateId: "카드 템플릿 ID (선택 사항)",
|
||||||
dingtalkCardTemplateIdHint: "open-dev.dingtalk.com/fe/card에서 AI 카드 템플릿을 만들면 타자기 효과 스트리밍 출력이 활성화됩니다",
|
dingtalkCardTemplateIdHint: "open-dev.dingtalk.com/fe/card에서 AI 카드 템플릿을 만들면 타자기 효과 스트리밍 출력이 활성화됩니다",
|
||||||
|
dingtalkSupportGroupId: "지원 그룹 openConversationId (선택 사항)",
|
||||||
|
dingtalkSupportGroupIdHint: "에스컬레이션된 티켓을 이 DingTalk 그룹으로 전송합니다",
|
||||||
mattermostConsole: "Mattermost 통합",
|
mattermostConsole: "Mattermost 통합",
|
||||||
qqbotConsole: "QQ Open Platform",
|
qqbotConsole: "QQ Open Platform",
|
||||||
qqbotAPIBaseURLHint: "선택 사항. 비워두면 기본 프로덕션 API 엔드포인트를 사용합니다.",
|
qqbotAPIBaseURLHint: "선택 사항. 비워두면 기본 프로덕션 API 엔드포인트를 사용합니다.",
|
||||||
|
|||||||
@@ -4886,6 +4886,8 @@ export default {
|
|||||||
dingtalkConsole: 'Платформа DingTalk',
|
dingtalkConsole: 'Платформа DingTalk',
|
||||||
dingtalkCardTemplateId: 'ID шаблона карточки (необязательно)',
|
dingtalkCardTemplateId: 'ID шаблона карточки (необязательно)',
|
||||||
dingtalkCardTemplateIdHint: 'Создайте шаблон AI-карточки на open-dev.dingtalk.com/fe/card для потоковой передачи с эффектом печатной машинки',
|
dingtalkCardTemplateIdHint: 'Создайте шаблон AI-карточки на open-dev.dingtalk.com/fe/card для потоковой передачи с эффектом печатной машинки',
|
||||||
|
dingtalkSupportGroupId: 'openConversationId группы поддержки (необязательно)',
|
||||||
|
dingtalkSupportGroupIdHint: 'Эскалированные заявки отправляются в эту группу DingTalk',
|
||||||
mattermostConsole: 'Интеграции Mattermost',
|
mattermostConsole: 'Интеграции Mattermost',
|
||||||
qqbotConsole: 'QQ Open Platform',
|
qqbotConsole: 'QQ Open Platform',
|
||||||
qqbotAPIBaseURLHint: 'Необязательно. Оставьте пустым, чтобы использовать стандартный production API endpoint.',
|
qqbotAPIBaseURLHint: 'Необязательно. Оставьте пустым, чтобы использовать стандартный production API endpoint.',
|
||||||
|
|||||||
@@ -5318,6 +5318,8 @@ export default {
|
|||||||
dingtalkConsole: "钉钉开放平台",
|
dingtalkConsole: "钉钉开放平台",
|
||||||
dingtalkCardTemplateId: "卡片模板 ID(可选)",
|
dingtalkCardTemplateId: "卡片模板 ID(可选)",
|
||||||
dingtalkCardTemplateIdHint: "在 open-dev.dingtalk.com/fe/card 创建 AI 卡片模板,启用后支持打字机效果的流式输出",
|
dingtalkCardTemplateIdHint: "在 open-dev.dingtalk.com/fe/card 创建 AI 卡片模板,启用后支持打字机效果的流式输出",
|
||||||
|
dingtalkSupportGroupId: "技术支持群 openConversationId(可选)",
|
||||||
|
dingtalkSupportGroupIdHint: "转人工工单会自动推送到该钉钉群,可从支持群内机器人回调中获取此 ID",
|
||||||
mattermostConsole: "Mattermost 集成说明",
|
mattermostConsole: "Mattermost 集成说明",
|
||||||
qqbotConsole: "QQ 开放平台",
|
qqbotConsole: "QQ 开放平台",
|
||||||
qqbotAPIBaseURLHint: "可选,留空则使用默认正式环境 API 地址。",
|
qqbotAPIBaseURLHint: "可选,留空则使用默认正式环境 API 地址。",
|
||||||
|
|||||||
@@ -57,7 +57,7 @@ export function buildWkcModelConfig(
|
|||||||
truncate_prompt_tokens: 0,
|
truncate_prompt_tokens: 0,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (kind === 'chat' || kind === 'vllm') {
|
if (kind === 'vllm') {
|
||||||
parameters.supports_vision = true
|
parameters.supports_vision = true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -48,6 +48,7 @@ type AgentEngine struct {
|
|||||||
tokenEstimator *agenttoken.Estimator // Token estimator for context window management
|
tokenEstimator *agenttoken.Estimator // Token estimator for context window management
|
||||||
memoryConsolidator *agentmemory.Consolidator // Memory consolidator for LLM-powered summarization (optional)
|
memoryConsolidator *agentmemory.Consolidator // Memory consolidator for LLM-powered summarization (optional)
|
||||||
lastUsage types.TokenUsage // Token usage from the most recent LLM call
|
lastUsage types.TokenUsage // Token usage from the most recent LLM call
|
||||||
|
totalUsage types.TokenUsage // Aggregate token usage across all LLM rounds in this execution
|
||||||
lastSentMsgCount int // Number of messages sent in the most recent LLM call
|
lastSentMsgCount int // Number of messages sent in the most recent LLM call
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -183,6 +184,17 @@ func (e *AgentEngine) estimateCurrentTokens(messages []chat.Message) int {
|
|||||||
return e.tokenEstimator.EstimateMessages(messages)
|
return e.tokenEstimator.EstimateMessages(messages)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (e *AgentEngine) recordTokenUsage(usage types.TokenUsage) {
|
||||||
|
if usage.TotalTokens <= 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
e.lastUsage = usage
|
||||||
|
e.totalUsage.PromptTokens += usage.PromptTokens
|
||||||
|
e.totalUsage.CompletionTokens += usage.CompletionTokens
|
||||||
|
e.totalUsage.TotalTokens += usage.TotalTokens
|
||||||
|
e.totalUsage.CachedTokens += usage.CachedTokens
|
||||||
|
}
|
||||||
|
|
||||||
// Execute executes the agent with conversation history and streaming output
|
// Execute executes the agent with conversation history and streaming output
|
||||||
// All events are emitted to EventBus and handled by subscribers (like Handler layer)
|
// All events are emitted to EventBus and handled by subscribers (like Handler layer)
|
||||||
func (e *AgentEngine) Execute(
|
func (e *AgentEngine) Execute(
|
||||||
@@ -191,6 +203,9 @@ func (e *AgentEngine) Execute(
|
|||||||
llmContext []chat.Message,
|
llmContext []chat.Message,
|
||||||
imageURLs ...[]string,
|
imageURLs ...[]string,
|
||||||
) (*types.AgentState, error) {
|
) (*types.AgentState, error) {
|
||||||
|
e.lastUsage = types.TokenUsage{}
|
||||||
|
e.totalUsage = types.TokenUsage{}
|
||||||
|
e.lastSentMsgCount = 0
|
||||||
logger.Infof(ctx, "[Agent] Starting execution: session=%s, message=%s, query_len=%d, context_msgs=%d",
|
logger.Infof(ctx, "[Agent] Starting execution: session=%s, message=%s, query_len=%d, context_msgs=%d",
|
||||||
sessionID, messageID, len(query), len(llmContext))
|
sessionID, messageID, len(query), len(llmContext))
|
||||||
// Ensure tools are cleaned up after execution
|
// Ensure tools are cleaned up after execution
|
||||||
@@ -269,7 +284,7 @@ func (e *AgentEngine) Execute(
|
|||||||
"tools": toolListStr,
|
"tools": toolListStr,
|
||||||
})
|
})
|
||||||
|
|
||||||
_, err := e.executeLoop(ctx, state, query, messages, tools, sessionID, messageID, imgs)
|
_, err := e.executeLoop(ctx, state, query, messages, tools, sessionID, messageID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Errorf(ctx, "[Agent] Execution failed: %v", err)
|
logger.Errorf(ctx, "[Agent] Execution failed: %v", err)
|
||||||
e.eventBus.Emit(ctx, event.Event{
|
e.eventBus.Emit(ctx, event.Event{
|
||||||
@@ -349,7 +364,6 @@ func (e *AgentEngine) executeLoop(
|
|||||||
tools []chat.Tool,
|
tools []chat.Tool,
|
||||||
sessionID string,
|
sessionID string,
|
||||||
messageID string,
|
messageID string,
|
||||||
imageURLs []string,
|
|
||||||
) (*types.AgentState, error) {
|
) (*types.AgentState, error) {
|
||||||
startTime := time.Now()
|
startTime := time.Now()
|
||||||
common.PipelineInfo(ctx, "Agent", "loop_start", map[string]interface{}{
|
common.PipelineInfo(ctx, "Agent", "loop_start", map[string]interface{}{
|
||||||
@@ -387,7 +401,7 @@ loop:
|
|||||||
if totalTC := countTotalToolCalls(state.RoundSteps); totalTC > 0 {
|
if totalTC := countTotalToolCalls(state.RoundSteps); totalTC > 0 {
|
||||||
logger.Infof(ctx, "[Agent] Synthesizing final answer from %d existing tool results",
|
logger.Infof(ctx, "[Agent] Synthesizing final answer from %d existing tool results",
|
||||||
totalTC)
|
totalTC)
|
||||||
_ = e.streamFinalAnswerToEventBus(ctx, query, state, sessionID, imageURLs)
|
_ = e.streamFinalAnswerToEventBus(ctx, query, state, sessionID)
|
||||||
state.IsComplete = true
|
state.IsComplete = true
|
||||||
}
|
}
|
||||||
return state, ctx.Err()
|
return state, ctx.Err()
|
||||||
@@ -420,7 +434,7 @@ loop:
|
|||||||
// complete answer." message, which then leaks to the UI as the final
|
// complete answer." message, which then leaks to the UI as the final
|
||||||
// answer for a conversation the user deliberately stopped.
|
// answer for a conversation the user deliberately stopped.
|
||||||
if !state.IsComplete && ctx.Err() == nil {
|
if !state.IsComplete && ctx.Err() == nil {
|
||||||
e.handleMaxIterations(ctx, query, state, sessionID, imageURLs)
|
e.handleMaxIterations(ctx, query, state, sessionID)
|
||||||
}
|
}
|
||||||
|
|
||||||
return state, nil
|
return state, nil
|
||||||
@@ -541,7 +555,7 @@ func (e *AgentEngine) runReActIteration(
|
|||||||
}
|
}
|
||||||
response = resp
|
response = resp
|
||||||
if response.Usage.TotalTokens > 0 {
|
if response.Usage.TotalTokens > 0 {
|
||||||
e.lastUsage = response.Usage
|
e.recordTokenUsage(response.Usage)
|
||||||
logger.Debugf(ctx, "[Agent][Round-%d] Usage: prompt=%d, completion=%d, total=%d",
|
logger.Debugf(ctx, "[Agent][Round-%d] Usage: prompt=%d, completion=%d, total=%d",
|
||||||
round, response.Usage.PromptTokens,
|
round, response.Usage.PromptTokens,
|
||||||
response.Usage.CompletionTokens, response.Usage.TotalTokens)
|
response.Usage.CompletionTokens, response.Usage.TotalTokens)
|
||||||
|
|||||||
@@ -120,7 +120,7 @@ func TestExecuteLoop_EmptyContentWithStop_ShouldNotCompleteWithEmpty(t *testing.
|
|||||||
state := &types.AgentState{}
|
state := &types.AgentState{}
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
||||||
_, err := engine.executeLoop(ctx, state, "test query", emptyMessages(), emptyTools(), "sess-1", "msg-1", nil)
|
_, err := engine.executeLoop(ctx, state, "test query", emptyMessages(), emptyTools(), "sess-1", "msg-1")
|
||||||
|
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
assert.True(t, state.IsComplete)
|
assert.True(t, state.IsComplete)
|
||||||
@@ -146,7 +146,7 @@ func TestExecuteLoop_NonEmptyContentWithStop_ShouldComplete(t *testing.T) {
|
|||||||
state := &types.AgentState{}
|
state := &types.AgentState{}
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
||||||
_, err := engine.executeLoop(ctx, state, "test query", emptyMessages(), emptyTools(), "sess-1", "msg-1", nil)
|
_, err := engine.executeLoop(ctx, state, "test query", emptyMessages(), emptyTools(), "sess-1", "msg-1")
|
||||||
|
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
assert.True(t, state.IsComplete)
|
assert.True(t, state.IsComplete)
|
||||||
@@ -173,7 +173,7 @@ func TestExecuteLoop_EmptyThenNonEmpty_ShouldRetryAndComplete(t *testing.T) {
|
|||||||
state := &types.AgentState{}
|
state := &types.AgentState{}
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
||||||
_, err := engine.executeLoop(ctx, state, "test query", emptyMessages(), emptyTools(), "sess-1", "msg-1", nil)
|
_, err := engine.executeLoop(ctx, state, "test query", emptyMessages(), emptyTools(), "sess-1", "msg-1")
|
||||||
|
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
assert.True(t, state.IsComplete)
|
assert.True(t, state.IsComplete)
|
||||||
@@ -328,7 +328,7 @@ func TestExecuteLoop_NaturalStop_DoesNotDuplicateAnswer(t *testing.T) {
|
|||||||
|
|
||||||
state := &types.AgentState{}
|
state := &types.AgentState{}
|
||||||
_, err := engine.executeLoop(context.Background(), state, "test query",
|
_, err := engine.executeLoop(context.Background(), state, "test query",
|
||||||
emptyMessages(), emptyTools(), "sess-1", "msg-1", nil)
|
emptyMessages(), emptyTools(), "sess-1", "msg-1")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
assert.True(t, state.IsComplete)
|
assert.True(t, state.IsComplete)
|
||||||
@@ -358,7 +358,7 @@ func TestStreamFinalAnswerToEventBus_EmitsDoneWhenProviderEndsWithEmptyChunk(t *
|
|||||||
})
|
})
|
||||||
|
|
||||||
state := &types.AgentState{}
|
state := &types.AgentState{}
|
||||||
err := engine.streamFinalAnswerToEventBus(context.Background(), "test query", state, "sess-1", nil)
|
err := engine.streamFinalAnswerToEventBus(context.Background(), "test query", state, "sess-1")
|
||||||
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Len(t, finalAnswerEvents, 2)
|
require.Len(t, finalAnswerEvents, 2)
|
||||||
|
|||||||
@@ -19,7 +19,6 @@ func (e *AgentEngine) streamFinalAnswerToEventBus(
|
|||||||
query string,
|
query string,
|
||||||
state *types.AgentState,
|
state *types.AgentState,
|
||||||
sessionID string,
|
sessionID string,
|
||||||
imageURLs []string,
|
|
||||||
) error {
|
) error {
|
||||||
totalToolCalls := countTotalToolCalls(state.RoundSteps)
|
totalToolCalls := countTotalToolCalls(state.RoundSteps)
|
||||||
logger.Infof(ctx, "[Agent][FinalAnswer] Synthesizing from %d steps, %d tool calls",
|
logger.Infof(ctx, "[Agent][FinalAnswer] Synthesizing from %d steps, %d tool calls",
|
||||||
@@ -37,7 +36,7 @@ func (e *AgentEngine) streamFinalAnswerToEventBus(
|
|||||||
|
|
||||||
messages := []chat.Message{
|
messages := []chat.Message{
|
||||||
{Role: "system", Content: systemPrompt},
|
{Role: "system", Content: systemPrompt},
|
||||||
{Role: "user", Content: userTurn, Images: append([]string(nil), imageURLs...)},
|
{Role: "user", Content: userTurn},
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add all tool call results as context
|
// Add all tool call results as context
|
||||||
@@ -139,23 +138,10 @@ Now generate the final answer:`, query)
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func collectAgentImageURLs(messages []chat.Message) []string {
|
|
||||||
var images []string
|
|
||||||
for _, message := range messages {
|
|
||||||
images = append(images, message.Images...)
|
|
||||||
for _, part := range message.MultiContent {
|
|
||||||
if part.Type == "image_url" && part.ImageURL != nil && part.ImageURL.URL != "" {
|
|
||||||
images = append(images, part.ImageURL.URL)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return images
|
|
||||||
}
|
|
||||||
|
|
||||||
// handleMaxIterations generates a final answer when the agent loop exhausted all iterations
|
// handleMaxIterations generates a final answer when the agent loop exhausted all iterations
|
||||||
// without the LLM producing a natural stop. It marks state.IsComplete = true.
|
// without the LLM producing a natural stop. It marks state.IsComplete = true.
|
||||||
func (e *AgentEngine) handleMaxIterations(
|
func (e *AgentEngine) handleMaxIterations(
|
||||||
ctx context.Context, query string, state *types.AgentState, sessionID string, imageURLs []string,
|
ctx context.Context, query string, state *types.AgentState, sessionID string,
|
||||||
) {
|
) {
|
||||||
logger.Info(ctx, "Reached max iterations, generating final answer")
|
logger.Info(ctx, "Reached max iterations, generating final answer")
|
||||||
common.PipelineWarn(ctx, "Agent", "max_iterations_reached", map[string]interface{}{
|
common.PipelineWarn(ctx, "Agent", "max_iterations_reached", map[string]interface{}{
|
||||||
@@ -164,7 +150,7 @@ func (e *AgentEngine) handleMaxIterations(
|
|||||||
})
|
})
|
||||||
|
|
||||||
// Stream final answer generation through EventBus
|
// Stream final answer generation through EventBus
|
||||||
if err := e.streamFinalAnswerToEventBus(ctx, query, state, sessionID, imageURLs); err != nil {
|
if err := e.streamFinalAnswerToEventBus(ctx, query, state, sessionID); err != nil {
|
||||||
logger.Errorf(ctx, "Failed to synthesize final answer: %v", err)
|
logger.Errorf(ctx, "Failed to synthesize final answer: %v", err)
|
||||||
common.PipelineError(ctx, "Agent", "final_answer_failed", map[string]interface{}{
|
common.PipelineError(ctx, "Agent", "final_answer_failed", map[string]interface{}{
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
@@ -183,6 +169,11 @@ func (e *AgentEngine) emitCompletionEvent(
|
|||||||
for _, ref := range state.KnowledgeRefs {
|
for _, ref := range state.KnowledgeRefs {
|
||||||
knowledgeRefsInterface = append(knowledgeRefsInterface, ref)
|
knowledgeRefsInterface = append(knowledgeRefsInterface, ref)
|
||||||
}
|
}
|
||||||
|
var tokenUsage *types.TokenUsage
|
||||||
|
if e.totalUsage.TotalTokens > 0 {
|
||||||
|
usage := e.totalUsage
|
||||||
|
tokenUsage = &usage
|
||||||
|
}
|
||||||
|
|
||||||
e.eventBus.Emit(ctx, event.Event{
|
e.eventBus.Emit(ctx, event.Event{
|
||||||
ID: generateEventID("complete"),
|
ID: generateEventID("complete"),
|
||||||
@@ -195,6 +186,7 @@ func (e *AgentEngine) emitCompletionEvent(
|
|||||||
TotalSteps: len(state.RoundSteps),
|
TotalSteps: len(state.RoundSteps),
|
||||||
TotalDurationMs: time.Since(startTime).Milliseconds(),
|
TotalDurationMs: time.Since(startTime).Milliseconds(),
|
||||||
MessageID: messageID, // Include message ID for proper message update
|
MessageID: messageID, // Include message ID for proper message update
|
||||||
|
TokenUsage: tokenUsage,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|||||||
@@ -1,18 +0,0 @@
|
|||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/Tencent/WeKnora/internal/models/chat"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestCollectAgentImageURLsForFinalSynthesis(t *testing.T) {
|
|
||||||
messages := []chat.Message{
|
|
||||||
{Role: "system", Content: "system"},
|
|
||||||
{Role: "user", Content: "question", Images: []string{"data:image/png;base64,YWJj"}},
|
|
||||||
}
|
|
||||||
got := collectAgentImageURLs(messages)
|
|
||||||
if len(got) != 1 || got[0] != "data:image/png;base64,YWJj" {
|
|
||||||
t.Fatalf("collectAgentImageURLs() = %#v", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+6
-10
@@ -2,7 +2,6 @@ package agent
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
@@ -61,7 +60,7 @@ func (e *AgentEngine) streamLLMToEventBus(
|
|||||||
// as if they were part of the LLM answer.
|
// as if they were part of the LLM answer.
|
||||||
if chunk.ResponseType == types.ResponseTypeError {
|
if chunk.ResponseType == types.ResponseTypeError {
|
||||||
result.StreamError = chunk.Content
|
result.StreamError = chunk.Content
|
||||||
break
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if chunk.Content != "" {
|
if chunk.Content != "" {
|
||||||
@@ -90,9 +89,6 @@ func (e *AgentEngine) streamLLMToEventBus(
|
|||||||
if emitFunc != nil {
|
if emitFunc != nil {
|
||||||
emitFunc(&chunk, result.Content)
|
emitFunc(&chunk, result.Content)
|
||||||
}
|
}
|
||||||
if chunk.Done && chunk.ResponseType == types.ResponseTypeAnswer {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stream diagnostic summary: helps identify non-streaming patterns
|
// Stream diagnostic summary: helps identify non-streaming patterns
|
||||||
@@ -105,10 +101,10 @@ func (e *AgentEngine) streamLLMToEventBus(
|
|||||||
chunkCount, len(result.Content), len(result.ToolCalls),
|
chunkCount, len(result.Content), len(result.ToolCalls),
|
||||||
streamDuration.Milliseconds(), responseTypeCounts)
|
streamDuration.Milliseconds(), responseTypeCounts)
|
||||||
|
|
||||||
// A provider error invalidates the entire stream, even if partial content or
|
// If the stream produced an error and no usable content/tool calls,
|
||||||
// tool calls arrived first. The caller must replace/discard that partial data.
|
// surface it as a Go error so the caller can retry or degrade gracefully.
|
||||||
if result.StreamError != "" {
|
if result.StreamError != "" && result.Content == "" && len(result.ToolCalls) == 0 {
|
||||||
return result, errors.New("LLM stream failed")
|
return result, fmt.Errorf("LLM stream error: %s", result.StreamError)
|
||||||
}
|
}
|
||||||
|
|
||||||
return result, nil
|
return result, nil
|
||||||
@@ -410,7 +406,7 @@ func (e *AgentEngine) callLLMWithRetry(
|
|||||||
"steps": len(state.RoundSteps),
|
"steps": len(state.RoundSteps),
|
||||||
"tool_calls": totalTC,
|
"tool_calls": totalTC,
|
||||||
})
|
})
|
||||||
if synthErr := e.streamFinalAnswerToEventBus(ctx, query, state, sessionID, collectAgentImageURLs(messages)); synthErr != nil {
|
if synthErr := e.streamFinalAnswerToEventBus(ctx, query, state, sessionID); synthErr != nil {
|
||||||
logger.Errorf(ctx, "[Agent] Final answer synthesis also failed: %v", synthErr)
|
logger.Errorf(ctx, "[Agent] Final answer synthesis also failed: %v", synthErr)
|
||||||
return nil, fmt.Errorf("LLM call failed: %w (synthesis also failed: %v)", err, synthErr)
|
return nil, fmt.Errorf("LLM call failed: %w (synthesis also failed: %v)", err, synthErr)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,26 +0,0 @@
|
|||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/Tencent/WeKnora/internal/models/chat"
|
|
||||||
"github.com/Tencent/WeKnora/internal/types"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestStreamLLMToEventBusRejectsPartialContentAfterStreamError(t *testing.T) {
|
|
||||||
model := &mockChat{responses: []mockResponse{{chunks: []types.StreamResponse{
|
|
||||||
{ResponseType: types.ResponseTypeAnswer, Content: "unverified partial"},
|
|
||||||
{ResponseType: types.ResponseTypeError, Content: "provider failed", Done: true},
|
|
||||||
{ResponseType: types.ResponseTypeAnswer, Content: "must be ignored", Done: true},
|
|
||||||
}}}}
|
|
||||||
engine := &AgentEngine{config: &types.AgentConfig{}, chatModel: model}
|
|
||||||
|
|
||||||
result, err := engine.streamLLMToEventBus(context.Background(), []chat.Message{{Role: "user", Content: "inspect image"}}, nil, nil)
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected stream error after partial content")
|
|
||||||
}
|
|
||||||
if result == nil || result.Content != "unverified partial" {
|
|
||||||
t.Fatalf("result = %#v; caller needs partial content identified for replacement", result)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/Tencent/WeKnora/internal/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestRecordTokenUsageAggregatesAgentRounds(t *testing.T) {
|
||||||
|
engine := &AgentEngine{}
|
||||||
|
engine.recordTokenUsage(types.TokenUsage{PromptTokens: 100, CompletionTokens: 20, TotalTokens: 120, CachedTokens: 40})
|
||||||
|
engine.recordTokenUsage(types.TokenUsage{PromptTokens: 140, CompletionTokens: 30, TotalTokens: 170, CachedTokens: 60})
|
||||||
|
|
||||||
|
wantTotal := types.TokenUsage{PromptTokens: 240, CompletionTokens: 50, TotalTokens: 290, CachedTokens: 100}
|
||||||
|
if engine.totalUsage != wantTotal {
|
||||||
|
t.Fatalf("totalUsage = %#v, want %#v", engine.totalUsage, wantTotal)
|
||||||
|
}
|
||||||
|
wantLast := types.TokenUsage{PromptTokens: 140, CompletionTokens: 30, TotalTokens: 170, CachedTokens: 60}
|
||||||
|
if engine.lastUsage != wantLast {
|
||||||
|
t.Fatalf("lastUsage = %#v, want %#v", engine.lastUsage, wantLast)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRecordTokenUsageIgnoresEmptyUsage(t *testing.T) {
|
||||||
|
engine := &AgentEngine{}
|
||||||
|
engine.recordTokenUsage(types.TokenUsage{})
|
||||||
|
if engine.totalUsage != (types.TokenUsage{}) || engine.lastUsage != (types.TokenUsage{}) {
|
||||||
|
t.Fatalf("empty usage changed state: total=%#v last=%#v", engine.totalUsage, engine.lastUsage)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -79,10 +79,8 @@ func (p *PluginChatCompletionStream) OnEvent(ctx context.Context,
|
|||||||
pipelineInfo(ctx, "Stream", "model_call", map[string]interface{}{
|
pipelineInfo(ctx, "Stream", "model_call", map[string]interface{}{
|
||||||
"chat_model": chatManage.ChatModelID,
|
"chat_model": chatManage.ChatModelID,
|
||||||
})
|
})
|
||||||
streamCtx, streamCancel := context.WithCancel(ctx)
|
responseChan, err := chatModel.ChatStream(ctx, chatMessages, opt)
|
||||||
responseChan, err := chatModel.ChatStream(streamCtx, chatMessages, opt)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
streamCancel()
|
|
||||||
pipelineError(ctx, "Stream", "model_call", map[string]interface{}{
|
pipelineError(ctx, "Stream", "model_call", map[string]interface{}{
|
||||||
"chat_model": chatManage.ChatModelID,
|
"chat_model": chatManage.ChatModelID,
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
@@ -90,7 +88,6 @@ func (p *PluginChatCompletionStream) OnEvent(ctx context.Context,
|
|||||||
return ErrModelCall.WithError(err)
|
return ErrModelCall.WithError(err)
|
||||||
}
|
}
|
||||||
if responseChan == nil {
|
if responseChan == nil {
|
||||||
streamCancel()
|
|
||||||
pipelineError(ctx, "Stream", "model_call", map[string]interface{}{
|
pipelineError(ctx, "Stream", "model_call", map[string]interface{}{
|
||||||
"chat_model": chatManage.ChatModelID,
|
"chat_model": chatManage.ChatModelID,
|
||||||
"error": "nil_channel",
|
"error": "nil_channel",
|
||||||
@@ -108,7 +105,6 @@ func (p *PluginChatCompletionStream) OnEvent(ctx context.Context,
|
|||||||
// The goroutine monitors ctx.Done() to avoid leaking when the context is cancelled
|
// The goroutine monitors ctx.Done() to avoid leaking when the context is cancelled
|
||||||
// and the upstream channel is not closed promptly.
|
// and the upstream channel is not closed promptly.
|
||||||
go func() {
|
go func() {
|
||||||
defer streamCancel()
|
|
||||||
thinkingID := fmt.Sprintf("%s-thinking", uuid.New().String()[:8])
|
thinkingID := fmt.Sprintf("%s-thinking", uuid.New().String()[:8])
|
||||||
answerID := fmt.Sprintf("%s-answer", uuid.New().String()[:8])
|
answerID := fmt.Sprintf("%s-answer", uuid.New().String()[:8])
|
||||||
thinkingOpen := false
|
thinkingOpen := false
|
||||||
@@ -149,20 +145,19 @@ func (p *PluginChatCompletionStream) OnEvent(ctx context.Context,
|
|||||||
if response.ResponseType == types.ResponseTypeError {
|
if response.ResponseType == types.ResponseTypeError {
|
||||||
pipelineError(ctx, "Stream", "stream_error", map[string]interface{}{
|
pipelineError(ctx, "Stream", "stream_error", map[string]interface{}{
|
||||||
"session_id": chatManage.SessionID,
|
"session_id": chatManage.SessionID,
|
||||||
"error": "chat completion stream failed",
|
"error": response.Content,
|
||||||
})
|
})
|
||||||
eventBus.Emit(ctx, types.Event{
|
eventBus.Emit(ctx, types.Event{
|
||||||
ID: fmt.Sprintf("%s-error", uuid.New().String()[:8]),
|
ID: fmt.Sprintf("%s-error", uuid.New().String()[:8]),
|
||||||
Type: types.EventType(event.EventError),
|
Type: types.EventType(event.EventError),
|
||||||
SessionID: chatManage.SessionID,
|
SessionID: chatManage.SessionID,
|
||||||
Data: event.ErrorData{
|
Data: event.ErrorData{
|
||||||
Error: "chat completion stream failed",
|
Error: response.Content,
|
||||||
Stage: "chat_completion_stream",
|
Stage: "chat_completion_stream",
|
||||||
SessionID: chatManage.SessionID,
|
SessionID: chatManage.SessionID,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
closeThinking()
|
continue
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if response.ResponseType == types.ResponseTypeThinking {
|
if response.ResponseType == types.ResponseTypeThinking {
|
||||||
@@ -191,13 +186,11 @@ func (p *PluginChatCompletionStream) OnEvent(ctx context.Context,
|
|||||||
Type: types.EventType(event.EventAgentFinalAnswer),
|
Type: types.EventType(event.EventAgentFinalAnswer),
|
||||||
SessionID: chatManage.SessionID,
|
SessionID: chatManage.SessionID,
|
||||||
Data: event.AgentFinalAnswerData{
|
Data: event.AgentFinalAnswerData{
|
||||||
Content: response.Content,
|
Content: response.Content,
|
||||||
Done: response.Done,
|
Done: response.Done,
|
||||||
|
TokenUsage: response.Usage,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if response.Done {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -91,14 +91,7 @@ func (p *PluginQueryUnderstand) OnEvent(ctx context.Context,
|
|||||||
}
|
}
|
||||||
|
|
||||||
// --- Select the appropriate model ---
|
// --- Select the appropriate model ---
|
||||||
rewriteModel, useImages, selectErr := p.selectModel(ctx, chatManage, hasImages)
|
rewriteModel, useImages := p.selectModel(ctx, chatManage, hasImages)
|
||||||
if selectErr != nil {
|
|
||||||
pipelineError(ctx, "QueryUnderstand", "get_visual_model", map[string]interface{}{
|
|
||||||
"session_id": chatManage.SessionID,
|
|
||||||
"error": selectErr.Error(),
|
|
||||||
})
|
|
||||||
return &PluginError{Err: selectErr, Description: "Visual model is unavailable", ErrorType: "image_understanding_failed"}
|
|
||||||
}
|
|
||||||
if rewriteModel == nil {
|
if rewriteModel == nil {
|
||||||
pipelineError(ctx, "QueryUnderstand", "get_model", map[string]interface{}{
|
pipelineError(ctx, "QueryUnderstand", "get_model", map[string]interface{}{
|
||||||
"session_id": chatManage.SessionID,
|
"session_id": chatManage.SessionID,
|
||||||
@@ -134,18 +127,11 @@ func (p *PluginQueryUnderstand) OnEvent(ctx context.Context,
|
|||||||
"session_id": chatManage.SessionID,
|
"session_id": chatManage.SessionID,
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
})
|
})
|
||||||
if hasImages {
|
|
||||||
return &PluginError{Err: err, Description: "Visual model request failed", ErrorType: "image_understanding_failed"}
|
|
||||||
}
|
|
||||||
return next()
|
return next()
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- Parse structured output ---
|
// --- Parse structured output ---
|
||||||
p.parseOutput(chatManage, response.Content)
|
p.parseOutput(chatManage, response.Content)
|
||||||
if hasImages && !chatManage.ChatModelSupportsVision && strings.TrimSpace(chatManage.ImageDescription) == "" {
|
|
||||||
err := fmt.Errorf("visual model returned no image description")
|
|
||||||
return &PluginError{Err: err, Description: "Visual model returned no usable image description", ErrorType: "image_understanding_failed"}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Persist image description asynchronously — this DB write does not affect
|
// Persist image description asynchronously — this DB write does not affect
|
||||||
// the current pipeline result, so it can run in the background.
|
// the current pipeline result, so it can run in the background.
|
||||||
@@ -236,43 +222,33 @@ func (p *PluginQueryUnderstand) loadHistory(ctx context.Context, chatManage *typ
|
|||||||
}
|
}
|
||||||
|
|
||||||
// selectModel picks the model for query understanding. When images are present
|
// selectModel picks the model for query understanding. When images are present
|
||||||
// it requires a vision-capable model and never falls back to text-only processing.
|
// it prefers a vision-capable model. Returns (model, useImages).
|
||||||
func (p *PluginQueryUnderstand) selectModel(ctx context.Context, chatManage *types.ChatManage, hasImages bool) (chat.Chat, bool, error) {
|
func (p *PluginQueryUnderstand) selectModel(ctx context.Context, chatManage *types.ChatManage, hasImages bool) (chat.Chat, bool) {
|
||||||
if hasImages {
|
if hasImages {
|
||||||
var lastErr error
|
|
||||||
if chatManage.ChatModelSupportsVision {
|
if chatManage.ChatModelSupportsVision {
|
||||||
m, err := p.modelService.GetChatModel(ctx, chatManage.ChatModelID)
|
m, err := p.modelService.GetChatModel(ctx, chatManage.ChatModelID)
|
||||||
if err == nil && chat.SupportsImageInput(m) {
|
|
||||||
return m, true, nil
|
|
||||||
}
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
err = fmt.Errorf("configured Chat transport does not support image input")
|
return m, true
|
||||||
}
|
}
|
||||||
lastErr = err
|
pipelineWarn(ctx, "QueryUnderstand", "vision_model_fallback", map[string]interface{}{
|
||||||
pipelineWarn(ctx, "QueryUnderstand", "vision_model_unavailable", map[string]interface{}{
|
|
||||||
"session_id": chatManage.SessionID,
|
"session_id": chatManage.SessionID,
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
if chatManage.VLMModelID != "" {
|
if chatManage.VLMModelID != "" {
|
||||||
m, err := p.modelService.GetChatModel(ctx, chatManage.VLMModelID)
|
m, err := p.modelService.GetChatModel(ctx, chatManage.VLMModelID)
|
||||||
if err == nil && chat.SupportsImageInput(m) {
|
|
||||||
return m, true, nil
|
|
||||||
}
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
err = fmt.Errorf("configured VLM transport does not support image input")
|
return m, true
|
||||||
}
|
}
|
||||||
lastErr = err
|
pipelineWarn(ctx, "QueryUnderstand", "vlm_model_fallback", map[string]interface{}{
|
||||||
pipelineWarn(ctx, "QueryUnderstand", "vlm_model_unavailable", map[string]interface{}{
|
|
||||||
"session_id": chatManage.SessionID,
|
"session_id": chatManage.SessionID,
|
||||||
"vlm_model_id": chatManage.VLMModelID,
|
"vlm_model_id": chatManage.VLMModelID,
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
if lastErr != nil {
|
pipelineWarn(ctx, "QueryUnderstand", "no_vision_model", map[string]interface{}{
|
||||||
return nil, false, fmt.Errorf("no usable visual model: %w", lastErr)
|
"session_id": chatManage.SessionID,
|
||||||
}
|
})
|
||||||
return nil, false, fmt.Errorf("no visual model is configured")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
textModelID := chatManage.ChatModelID
|
textModelID := chatManage.ChatModelID
|
||||||
@@ -290,7 +266,7 @@ func (p *PluginQueryUnderstand) selectModel(ctx context.Context, chatManage *typ
|
|||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
})
|
})
|
||||||
if fallback, fbErr := p.modelService.GetChatModel(ctx, chatManage.ChatModelID); fbErr == nil {
|
if fallback, fbErr := p.modelService.GetChatModel(ctx, chatManage.ChatModelID); fbErr == nil {
|
||||||
return fallback, false, nil
|
return fallback, false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
pipelineError(ctx, "QueryUnderstand", "get_model", map[string]interface{}{
|
pipelineError(ctx, "QueryUnderstand", "get_model", map[string]interface{}{
|
||||||
@@ -298,9 +274,9 @@ func (p *PluginQueryUnderstand) selectModel(ctx context.Context, chatManage *typ
|
|||||||
"chat_model_id": textModelID,
|
"chat_model_id": textModelID,
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
})
|
})
|
||||||
return nil, false, nil
|
return nil, false
|
||||||
}
|
}
|
||||||
return m, false, nil
|
return m, false
|
||||||
}
|
}
|
||||||
|
|
||||||
// buildPrompts constructs system and user prompts with placeholder replacement.
|
// buildPrompts constructs system and user prompts with placeholder replacement.
|
||||||
|
|||||||
@@ -1,54 +0,0 @@
|
|||||||
package chatpipeline
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
modelchat "github.com/Tencent/WeKnora/internal/models/chat"
|
|
||||||
"github.com/Tencent/WeKnora/internal/types"
|
|
||||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
|
||||||
)
|
|
||||||
|
|
||||||
type visionModelStub struct{}
|
|
||||||
|
|
||||||
func (visionModelStub) Chat(context.Context, []modelchat.Message, *modelchat.ChatOptions) (*types.ChatResponse, error) {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
func (visionModelStub) ChatStream(context.Context, []modelchat.Message, *modelchat.ChatOptions) (<-chan types.StreamResponse, error) {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
func (visionModelStub) GetModelName() string { return "test-vlm" }
|
|
||||||
func (visionModelStub) GetModelID() string { return "test-vlm" }
|
|
||||||
func (visionModelStub) SupportsImageInput() bool { return true }
|
|
||||||
|
|
||||||
type modelServiceStub struct {
|
|
||||||
interfaces.ModelService
|
|
||||||
model modelchat.Chat
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s modelServiceStub) GetChatModel(context.Context, string) (modelchat.Chat, error) {
|
|
||||||
return s.model, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSelectModelFailsClosedWhenImageHasNoVisualModel(t *testing.T) {
|
|
||||||
plugin := &PluginQueryUnderstand{}
|
|
||||||
manage := &types.ChatManage{PipelineRequest: types.PipelineRequest{ChatModelSupportsVision: false}}
|
|
||||||
model, useImages, err := plugin.selectModel(context.Background(), manage, true)
|
|
||||||
if err == nil || model != nil || useImages {
|
|
||||||
t.Fatalf("selectModel = model %v useImages %v err %v; want fail closed", model, useImages, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSelectModelUsesVLMWhenChatTransportIsNotVisual(t *testing.T) {
|
|
||||||
vlm := visionModelStub{}
|
|
||||||
plugin := &PluginQueryUnderstand{modelService: modelServiceStub{model: vlm}}
|
|
||||||
manage := &types.ChatManage{PipelineRequest: types.PipelineRequest{
|
|
||||||
ChatModelSupportsVision: false,
|
|
||||||
VLMModelID: "document-parser-vlm",
|
|
||||||
}}
|
|
||||||
|
|
||||||
model, useImages, err := plugin.selectModel(context.Background(), manage, true)
|
|
||||||
if err != nil || model == nil || !useImages {
|
|
||||||
t.Fatalf("selectModel = model %v useImages %v err %v; want visual VLM fallback", model, useImages, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,63 +0,0 @@
|
|||||||
package service
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/base64"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/Tencent/WeKnora/internal/types"
|
|
||||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
|
||||||
)
|
|
||||||
|
|
||||||
const maxAgentFallbackImageBytes = 10 << 20
|
|
||||||
|
|
||||||
// prepareAgentImageInput routes image input according to the resolved primary
|
|
||||||
// transport capability. A configured VLM is required only when the actual
|
|
||||||
// primary Chat transport cannot carry images directly.
|
|
||||||
func prepareAgentImageInput(
|
|
||||||
ctx context.Context,
|
|
||||||
req *types.QARequest,
|
|
||||||
primarySupportsVision bool,
|
|
||||||
modelService interfaces.ModelService,
|
|
||||||
) (string, []string, error) {
|
|
||||||
if req == nil {
|
|
||||||
return "", nil, errors.New("QA request is required")
|
|
||||||
}
|
|
||||||
if len(req.ImageURLs) == 0 || primarySupportsVision || strings.TrimSpace(req.ImageDescription) != "" {
|
|
||||||
return routeAgentImageInput(req, primarySupportsVision)
|
|
||||||
}
|
|
||||||
if req.CustomAgent == nil || strings.TrimSpace(req.CustomAgent.Config.VLMModelID) == "" {
|
|
||||||
return "", nil, errors.New("agent model does not support visual input and no VLM is configured")
|
|
||||||
}
|
|
||||||
if modelService == nil {
|
|
||||||
return "", nil, errors.New("agent VLM service is unavailable")
|
|
||||||
}
|
|
||||||
|
|
||||||
images := make([][]byte, 0, len(req.ImageURLs))
|
|
||||||
for _, raw := range req.ImageURLs {
|
|
||||||
comma := strings.IndexByte(raw, ',')
|
|
||||||
if comma <= 0 || !strings.HasPrefix(strings.ToLower(raw[:comma]), "data:image/") ||
|
|
||||||
!strings.HasSuffix(strings.ToLower(raw[:comma]), ";base64") {
|
|
||||||
return "", nil, errors.New("agent image payload is invalid")
|
|
||||||
}
|
|
||||||
decoded, err := base64.StdEncoding.Strict().DecodeString(raw[comma+1:])
|
|
||||||
if err != nil || len(decoded) == 0 || len(decoded) > maxAgentFallbackImageBytes {
|
|
||||||
return "", nil, errors.New("agent image payload is invalid")
|
|
||||||
}
|
|
||||||
images = append(images, decoded)
|
|
||||||
}
|
|
||||||
|
|
||||||
model, err := modelService.GetVLMModel(ctx, req.CustomAgent.Config.VLMModelID)
|
|
||||||
if err != nil || model == nil {
|
|
||||||
return "", nil, errors.New("agent VLM is unavailable")
|
|
||||||
}
|
|
||||||
prompt := fmt.Sprintf("Analyze the uploaded image(s) for the user's question. Return a concise factual description only.\nQuestion: %s", req.Query)
|
|
||||||
description, err := model.Predict(ctx, images, prompt)
|
|
||||||
if err != nil || strings.TrimSpace(description) == "" {
|
|
||||||
return "", nil, errors.New("agent VLM analysis failed")
|
|
||||||
}
|
|
||||||
req.ImageDescription = strings.TrimSpace(description)
|
|
||||||
return routeAgentImageInput(req, false)
|
|
||||||
}
|
|
||||||
@@ -1,26 +0,0 @@
|
|||||||
package service
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/Tencent/WeKnora/internal/types"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestRouteAgentImageInputFailsClosedWithoutVision(t *testing.T) {
|
|
||||||
req := &types.QARequest{Query: "what is this", ImageURLs: []string{"data:image/png;base64,abc"}}
|
|
||||||
if _, images, err := routeAgentImageInput(req, false); err == nil || len(images) != 0 {
|
|
||||||
t.Fatalf("routeAgentImageInput should reject unsupported visual input: images=%v err=%v", images, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
query, images, err := routeAgentImageInput(req, true)
|
|
||||||
if err != nil || len(images) != 1 || query != req.Query {
|
|
||||||
t.Fatalf("vision route = query %q images %d err %v", query, len(images), err)
|
|
||||||
}
|
|
||||||
|
|
||||||
req.ImageDescription = "a blue robot"
|
|
||||||
query, images, err = routeAgentImageInput(req, false)
|
|
||||||
if err != nil || len(images) != 0 || !strings.Contains(query, req.ImageDescription) {
|
|
||||||
t.Fatalf("description fallback = query %q images %d err %v", query, len(images), err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -6,7 +6,6 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/Tencent/WeKnora/internal/agent/tools"
|
"github.com/Tencent/WeKnora/internal/agent/tools"
|
||||||
"github.com/Tencent/WeKnora/internal/event"
|
"github.com/Tencent/WeKnora/internal/event"
|
||||||
@@ -157,18 +156,21 @@ func (s *sessionService) AgentQA(
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// The resolved concrete transport is authoritative. Stored model metadata can
|
// Route image data based on agent model's vision capability
|
||||||
// be stale (notably older WeKnora Cloud records), while unknown transports
|
var agentModelSupportsVision bool
|
||||||
// still fail closed through SupportsImageInput.
|
if effectiveModelID != "" {
|
||||||
agentModelSupportsVision := modelSupportsDirectImages(summaryModel)
|
if modelInfo, err := s.modelService.GetModelByID(ctx, effectiveModelID); err == nil && modelInfo != nil {
|
||||||
|
agentModelSupportsVision = modelInfo.Parameters.SupportsVision
|
||||||
agentQuery, agentImageURLs, err := prepareAgentImageInput(ctx, req, agentModelSupportsVision, s.modelService)
|
}
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
}
|
||||||
if len(agentImageURLs) > 0 {
|
|
||||||
|
agentQuery := req.Query
|
||||||
|
var agentImageURLs []string
|
||||||
|
if agentModelSupportsVision && len(req.ImageURLs) > 0 {
|
||||||
|
agentImageURLs = req.ImageURLs
|
||||||
logger.Infof(ctx, "Agent model supports vision, passing %d image(s) directly", len(agentImageURLs))
|
logger.Infof(ctx, "Agent model supports vision, passing %d image(s) directly", len(agentImageURLs))
|
||||||
} else if req.ImageDescription != "" {
|
} else if req.ImageDescription != "" {
|
||||||
|
agentQuery = req.Query + "\n\n[用户上传图片内容]\n" + req.ImageDescription
|
||||||
logger.Infof(ctx, "Agent model does not support vision, appending image description (%d chars)", len(req.ImageDescription))
|
logger.Infof(ctx, "Agent model does not support vision, appending image description (%d chars)", len(req.ImageDescription))
|
||||||
}
|
}
|
||||||
if req.QuotedContext != "" {
|
if req.QuotedContext != "" {
|
||||||
@@ -206,22 +208,6 @@ func (s *sessionService) AgentQA(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func routeAgentImageInput(req *types.QARequest, modelSupportsVision bool) (string, []string, error) {
|
|
||||||
if req == nil {
|
|
||||||
return "", nil, errors.New("QA request is required")
|
|
||||||
}
|
|
||||||
if len(req.ImageURLs) == 0 {
|
|
||||||
return req.Query, nil, nil
|
|
||||||
}
|
|
||||||
if modelSupportsVision {
|
|
||||||
return req.Query, req.ImageURLs, nil
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(req.ImageDescription) != "" {
|
|
||||||
return req.Query + "\n\n[用户上传图片内容]\n" + req.ImageDescription, nil, nil
|
|
||||||
}
|
|
||||||
return "", nil, errors.New("agent model does not support visual input and no image description is available")
|
|
||||||
}
|
|
||||||
|
|
||||||
// buildAgentConfig creates a runtime AgentConfig from the QARequest's custom agent configuration,
|
// buildAgentConfig creates a runtime AgentConfig from the QARequest's custom agent configuration,
|
||||||
// tenant info, and resolved knowledge bases / search targets.
|
// tenant info, and resolved knowledge bases / search targets.
|
||||||
func (s *sessionService) buildAgentConfig(
|
func (s *sessionService) buildAgentConfig(
|
||||||
|
|||||||
@@ -1,75 +0,0 @@
|
|||||||
package service
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/Tencent/WeKnora/internal/models/vlm"
|
|
||||||
"github.com/Tencent/WeKnora/internal/types"
|
|
||||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
|
||||||
)
|
|
||||||
|
|
||||||
type agentImageVLMStub struct {
|
|
||||||
calls int
|
|
||||||
images [][]byte
|
|
||||||
result string
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *agentImageVLMStub) Predict(_ context.Context, images [][]byte, _ string) (string, error) {
|
|
||||||
s.calls++
|
|
||||||
s.images = images
|
|
||||||
return s.result, s.err
|
|
||||||
}
|
|
||||||
func (*agentImageVLMStub) GetModelName() string { return "vlm-stub" }
|
|
||||||
func (*agentImageVLMStub) GetModelID() string { return "vlm-1" }
|
|
||||||
|
|
||||||
type agentImageModelServiceStub struct {
|
|
||||||
interfaces.ModelService
|
|
||||||
model vlm.VLM
|
|
||||||
err error
|
|
||||||
calls int
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *agentImageModelServiceStub) GetVLMModel(context.Context, string) (vlm.VLM, error) {
|
|
||||||
s.calls++
|
|
||||||
return s.model, s.err
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPrepareAgentImageInputUsesVLMOnlyForNonVisualPrimary(t *testing.T) {
|
|
||||||
visual := &agentImageVLMStub{result: "机械臂报警E01"}
|
|
||||||
models := &agentImageModelServiceStub{model: visual}
|
|
||||||
req := &types.QARequest{
|
|
||||||
Query: "什么故障?",
|
|
||||||
ImageURLs: []string{"data:image/png;base64,YWJj"},
|
|
||||||
CustomAgent: &types.CustomAgent{Config: types.CustomAgentConfig{VLMModelID: "vlm-1"}},
|
|
||||||
}
|
|
||||||
query, images, err := prepareAgentImageInput(context.Background(), req, false, models)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("prepareAgentImageInput: %v", err)
|
|
||||||
}
|
|
||||||
if len(images) != 0 || query == req.Query || req.ImageDescription != visual.result {
|
|
||||||
t.Fatalf("query=%q images=%v description=%q", query, images, req.ImageDescription)
|
|
||||||
}
|
|
||||||
if models.calls != 1 || visual.calls != 1 || len(visual.images) != 1 {
|
|
||||||
t.Fatalf("model calls=%d VLM calls=%d images=%d", models.calls, visual.calls, len(visual.images))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPrepareAgentImageInputSkipsBrokenVLMForVisualPrimary(t *testing.T) {
|
|
||||||
models := &agentImageModelServiceStub{err: errors.New("stale VLM")}
|
|
||||||
req := &types.QARequest{
|
|
||||||
Query: "describe",
|
|
||||||
ImageURLs: []string{"data:image/png;base64,YWJj"},
|
|
||||||
CustomAgent: &types.CustomAgent{Config: types.CustomAgentConfig{VLMModelID: "broken"}},
|
|
||||||
}
|
|
||||||
primary := &captureChatModel{supportsImage: true}
|
|
||||||
query, images, err := prepareAgentImageInput(context.Background(), req, modelSupportsDirectImages(primary), models)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("visual primary should bypass VLM: %v", err)
|
|
||||||
}
|
|
||||||
if query != req.Query || len(images) != 1 || models.calls != 0 {
|
|
||||||
t.Fatalf("query=%q images=%v model_calls=%d", query, images, models.calls)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -16,12 +16,6 @@ import (
|
|||||||
"github.com/Tencent/WeKnora/internal/types"
|
"github.com/Tencent/WeKnora/internal/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
// modelSupportsDirectImages asks the resolved concrete Chat transport. Unknown
|
|
||||||
// transports fail closed because chat.SupportsImageInput returns false.
|
|
||||||
func modelSupportsDirectImages(model chat.Chat) bool {
|
|
||||||
return chat.SupportsImageInput(model)
|
|
||||||
}
|
|
||||||
|
|
||||||
// KnowledgeQA performs knowledge base question answering with LLM summarization
|
// KnowledgeQA performs knowledge base question answering with LLM summarization
|
||||||
// Events are emitted through eventBus (references, answer chunks, completion)
|
// Events are emitted through eventBus (references, answer chunks, completion)
|
||||||
// customAgent is optional - if provided, uses custom agent configuration for multiTurnEnabled and historyTurns
|
// customAgent is optional - if provided, uses custom agent configuration for multiTurnEnabled and historyTurns
|
||||||
@@ -74,17 +68,12 @@ func (s *sessionService) KnowledgeQA(
|
|||||||
logger.Infof(ctx, "Fallback strategy not set, using default: %v", fallbackStrategy)
|
logger.Infof(ctx, "Fallback strategy not set, using default: %v", fallbackStrategy)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Resolve the concrete Chat transport for image routing. Runtime transport
|
// Resolve chat model vision capability and VLM model ID for image routing
|
||||||
// capability is authoritative; stored metadata may be stale for existing
|
|
||||||
// WeKnora Cloud records.
|
|
||||||
var chatModelSupportsVision bool
|
var chatModelSupportsVision bool
|
||||||
var vlmModelID string
|
var vlmModelID string
|
||||||
if chatModelID != "" && len(req.ImageURLs) > 0 {
|
if chatModelID != "" {
|
||||||
chatModel, modelErr := s.modelService.GetChatModel(ctx, chatModelID)
|
if chatModelInfo, err := s.modelService.GetModelByID(ctx, chatModelID); err == nil && chatModelInfo != nil {
|
||||||
if modelErr != nil {
|
chatModelSupportsVision = chatModelInfo.Parameters.SupportsVision
|
||||||
logger.Warnf(ctx, "Failed to initialize Chat model %s for image routing: %v", chatModelID, modelErr)
|
|
||||||
} else {
|
|
||||||
chatModelSupportsVision = modelSupportsDirectImages(chatModel)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if req.CustomAgent != nil {
|
if req.CustomAgent != nil {
|
||||||
@@ -147,8 +136,7 @@ func (s *sessionService) KnowledgeQA(
|
|||||||
VLMModelID: vlmModelID,
|
VLMModelID: vlmModelID,
|
||||||
ChatModelSupportsVision: chatModelSupportsVision,
|
ChatModelSupportsVision: chatModelSupportsVision,
|
||||||
Attachments: req.Attachments,
|
Attachments: req.Attachments,
|
||||||
|
Language: types.LanguageNameFromContext(ctx),
|
||||||
Language: types.LanguageNameFromContext(ctx),
|
|
||||||
},
|
},
|
||||||
PipelineState: types.PipelineState{
|
PipelineState: types.PipelineState{
|
||||||
RewriteQuery: req.Query,
|
RewriteQuery: req.Query,
|
||||||
@@ -170,7 +158,6 @@ func (s *sessionService) KnowledgeQA(
|
|||||||
hasKB := len(knowledgeBaseIDs) > 0 || len(knowledgeIDs) > 0
|
hasKB := len(knowledgeBaseIDs) > 0 || len(knowledgeIDs) > 0
|
||||||
needsRAG := hasKB || req.WebSearchEnabled
|
needsRAG := hasKB || req.WebSearchEnabled
|
||||||
hasHistory := chatManage.MaxRounds > 0
|
hasHistory := chatManage.MaxRounds > 0
|
||||||
needsVisualDescription := len(req.ImageURLs) > 0 && !chatModelSupportsVision
|
|
||||||
|
|
||||||
var pipeline []types.EventType
|
var pipeline []types.EventType
|
||||||
if !needsRAG {
|
if !needsRAG {
|
||||||
@@ -191,8 +178,6 @@ func (s *sessionService) KnowledgeQA(
|
|||||||
pipeline = types.NewPipelineBuilder().
|
pipeline = types.NewPipelineBuilder().
|
||||||
AddIf(hasHistory, types.LOAD_HISTORY).
|
AddIf(hasHistory, types.LOAD_HISTORY).
|
||||||
AddIf(chatManage.EnableMemory, types.MEMORY_RETRIEVAL).
|
AddIf(chatManage.EnableMemory, types.MEMORY_RETRIEVAL).
|
||||||
AddIf(needsVisualDescription, types.QUERY_UNDERSTAND).
|
|
||||||
AddIf(needsVisualDescription, types.INTO_CHAT_MESSAGE).
|
|
||||||
Add(types.CHAT_COMPLETION_STREAM).
|
Add(types.CHAT_COMPLETION_STREAM).
|
||||||
AddIf(chatManage.EnableMemory, types.MEMORY_STORAGE).
|
AddIf(chatManage.EnableMemory, types.MEMORY_STORAGE).
|
||||||
Build()
|
Build()
|
||||||
|
|||||||
@@ -16,19 +16,15 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type captureChatModel struct {
|
type captureChatModel struct {
|
||||||
lastMessages []chat.Message
|
lastMessages []chat.Message
|
||||||
supportsImage bool
|
|
||||||
chatResponse *types.ChatResponse
|
|
||||||
chatErr error
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *captureChatModel) Chat(
|
func (m *captureChatModel) Chat(
|
||||||
_ context.Context,
|
context.Context,
|
||||||
messages []chat.Message,
|
[]chat.Message,
|
||||||
_ *chat.ChatOptions,
|
*chat.ChatOptions,
|
||||||
) (*types.ChatResponse, error) {
|
) (*types.ChatResponse, error) {
|
||||||
m.lastMessages = append([]chat.Message(nil), messages...)
|
return nil, nil
|
||||||
return m.chatResponse, m.chatErr
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *captureChatModel) ChatStream(
|
func (m *captureChatModel) ChatStream(
|
||||||
@@ -48,14 +44,8 @@ func (m *captureChatModel) ChatStream(
|
|||||||
return ch, nil
|
return ch, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *captureChatModel) GetModelName() string { return "capture" }
|
func (m *captureChatModel) GetModelName() string { return "capture" }
|
||||||
func (m *captureChatModel) GetModelID() string { return "capture" }
|
func (m *captureChatModel) GetModelID() string { return "capture" }
|
||||||
func (m *captureChatModel) SupportsImageInput() bool { return m.supportsImage }
|
|
||||||
|
|
||||||
func TestModelSupportsDirectImagesUsesResolvedTransportAuthority(t *testing.T) {
|
|
||||||
assert.False(t, modelSupportsDirectImages(&captureChatModel{supportsImage: false}))
|
|
||||||
assert.True(t, modelSupportsDirectImages(&captureChatModel{supportsImage: true}))
|
|
||||||
}
|
|
||||||
|
|
||||||
type stubModelService struct {
|
type stubModelService struct {
|
||||||
chatModel chat.Chat
|
chatModel chat.Chat
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
package event
|
package event
|
||||||
|
|
||||||
|
import "github.com/Tencent/WeKnora/internal/types"
|
||||||
|
|
||||||
// EventData contains common event data structures for different stages
|
// EventData contains common event data structures for different stages
|
||||||
|
|
||||||
// QueryData represents query-related event data
|
// QueryData represents query-related event data
|
||||||
@@ -142,6 +144,7 @@ type AgentCompleteData struct {
|
|||||||
TotalDurationMs int64 `json:"total_duration_ms"`
|
TotalDurationMs int64 `json:"total_duration_ms"`
|
||||||
MessageID string `json:"message_id,omitempty"` // Assistant message ID
|
MessageID string `json:"message_id,omitempty"` // Assistant message ID
|
||||||
RequestID string `json:"request_id,omitempty"`
|
RequestID string `json:"request_id,omitempty"`
|
||||||
|
TokenUsage *types.TokenUsage `json:"token_usage,omitempty"`
|
||||||
Extra map[string]interface{} `json:"extra,omitempty"`
|
Extra map[string]interface{} `json:"extra,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -184,9 +187,10 @@ type AgentReferencesData struct {
|
|||||||
|
|
||||||
// AgentFinalAnswerData represents final answer streaming data
|
// AgentFinalAnswerData represents final answer streaming data
|
||||||
type AgentFinalAnswerData struct {
|
type AgentFinalAnswerData struct {
|
||||||
Content string `json:"content"`
|
Content string `json:"content"`
|
||||||
Done bool `json:"done"`
|
Done bool `json:"done"`
|
||||||
IsFallback bool `json:"is_fallback,omitempty"` // True when response is a fallback (no knowledge base match)
|
IsFallback bool `json:"is_fallback,omitempty"` // True when response is a fallback (no knowledge base match)
|
||||||
|
TokenUsage *types.TokenUsage `json:"token_usage,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// AgentReflectionData represents agent reflection data
|
// AgentReflectionData represents agent reflection data
|
||||||
|
|||||||
@@ -475,6 +475,10 @@ func (h *AgentStreamHandler) handleFinalAnswer(ctx context.Context, evt event.Ev
|
|||||||
if data.IsFallback {
|
if data.IsFallback {
|
||||||
h.assistantMessage.IsFallback = true
|
h.assistantMessage.IsFallback = true
|
||||||
}
|
}
|
||||||
|
if data.TokenUsage != nil {
|
||||||
|
usage := *data.TokenUsage
|
||||||
|
h.assistantMessage.TokenUsage = &usage
|
||||||
|
}
|
||||||
|
|
||||||
// Calculate duration if done
|
// Calculate duration if done
|
||||||
var metadata map[string]interface{}
|
var metadata map[string]interface{}
|
||||||
@@ -604,6 +608,10 @@ func (h *AgentStreamHandler) handleComplete(ctx context.Context, evt event.Event
|
|||||||
// h.assistantMessage.Content = data.FinalAnswer
|
// h.assistantMessage.Content = data.FinalAnswer
|
||||||
h.assistantMessage.IsCompleted = true
|
h.assistantMessage.IsCompleted = true
|
||||||
h.assistantMessage.AgentDurationMs = data.TotalDurationMs
|
h.assistantMessage.AgentDurationMs = data.TotalDurationMs
|
||||||
|
if data.TokenUsage != nil {
|
||||||
|
usage := *data.TokenUsage
|
||||||
|
h.assistantMessage.TokenUsage = &usage
|
||||||
|
}
|
||||||
|
|
||||||
// Update knowledge references if provided
|
// Update knowledge references if provided
|
||||||
if len(data.KnowledgeRefs) > 0 {
|
if len(data.KnowledgeRefs) > 0 {
|
||||||
|
|||||||
@@ -718,6 +718,10 @@ func (h *Handler) executeQA(reqCtx *qaRequestContext, mode qaMode, generateTitle
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
streamCtx.assistantMessage.Content += data.Content
|
streamCtx.assistantMessage.Content += data.Content
|
||||||
|
if data.TokenUsage != nil {
|
||||||
|
usage := *data.TokenUsage
|
||||||
|
streamCtx.assistantMessage.TokenUsage = &usage
|
||||||
|
}
|
||||||
if data.IsFallback {
|
if data.IsFallback {
|
||||||
streamCtx.assistantMessage.IsFallback = true
|
streamCtx.assistantMessage.IsFallback = true
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"io"
|
"io"
|
||||||
|
|
||||||
|
"github.com/Tencent/WeKnora/internal/types"
|
||||||
"github.com/gin-gonic/gin"
|
"github.com/gin-gonic/gin"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -66,6 +67,8 @@ type IncomingMessage struct {
|
|||||||
FileSize int64
|
FileSize int64
|
||||||
// ImageURLs carries transient model-ready image inputs (usually data URIs). Never log these values.
|
// ImageURLs carries transient model-ready image inputs (usually data URIs). Never log these values.
|
||||||
ImageURLs []string
|
ImageURLs []string
|
||||||
|
// ImageReferences carries durable, non-secret source references persisted on the user message.
|
||||||
|
ImageReferences types.MessageImages
|
||||||
// ThreadID is the platform-specific thread identifier.
|
// ThreadID is the platform-specific thread identifier.
|
||||||
// - Slack: thread_ts (top-level message uses its own timestamp)
|
// - Slack: thread_ts (top-level message uses its own timestamp)
|
||||||
// - Mattermost: root_id, or post_id if top-level
|
// - Mattermost: root_id, or post_id if top-level
|
||||||
|
|||||||
+43
-207
@@ -12,7 +12,6 @@ import (
|
|||||||
"mime"
|
"mime"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/netip"
|
|
||||||
"net/url"
|
"net/url"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strconv"
|
"strconv"
|
||||||
@@ -25,7 +24,6 @@ import (
|
|||||||
|
|
||||||
"github.com/Tencent/WeKnora/internal/im"
|
"github.com/Tencent/WeKnora/internal/im"
|
||||||
"github.com/Tencent/WeKnora/internal/logger"
|
"github.com/Tencent/WeKnora/internal/logger"
|
||||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// httpClient is a shared HTTP client with a reasonable timeout for DingTalk API calls.
|
// httpClient is a shared HTTP client with a reasonable timeout for DingTalk API calls.
|
||||||
@@ -176,14 +174,23 @@ func parseCallbackMessage(msg *callbackMessage) *im.IncomingMessage {
|
|||||||
userID = msg.SenderID
|
userID = msg.SenderID
|
||||||
}
|
}
|
||||||
|
|
||||||
text := ""
|
content := ""
|
||||||
if msg.Text != nil {
|
if msg.Text != nil {
|
||||||
text = msg.Text.Content
|
content = strings.TrimSpace(msg.Text.Content)
|
||||||
}
|
}
|
||||||
messageType, content, fileKey := parseDingTalkStreamPayload(msg.Msgtype, msg.Content, text)
|
messageType := im.MessageTypeText
|
||||||
|
fileKey := ""
|
||||||
fileName := ""
|
fileName := ""
|
||||||
if messageType == im.MessageTypeImage {
|
if strings.EqualFold(msg.Msgtype, "picture") {
|
||||||
fileName = msg.MsgID + ".png"
|
var picture struct {
|
||||||
|
DownloadCode string `json:"downloadCode"`
|
||||||
|
}
|
||||||
|
if json.Unmarshal(msg.Content, &picture) == nil && strings.TrimSpace(picture.DownloadCode) != "" {
|
||||||
|
messageType = im.MessageTypeImage
|
||||||
|
fileKey = strings.TrimSpace(picture.DownloadCode)
|
||||||
|
fileName = msg.MsgID + ".png"
|
||||||
|
content = ""
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return &im.IncomingMessage{
|
return &im.IncomingMessage{
|
||||||
@@ -203,39 +210,7 @@ func parseCallbackMessage(msg *callbackMessage) *im.IncomingMessage {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const (
|
const dingtalkMessageFileDownloadURL = "https://api.dingtalk.com/v1.0/robot/messageFiles/download"
|
||||||
dingtalkMessageFileDownloadURL = "https://api.dingtalk.com/v1.0/robot/messageFiles/download"
|
|
||||||
maxDingTalkAPIResponseBytes = 1 << 20
|
|
||||||
)
|
|
||||||
|
|
||||||
func (a *Adapter) newDingTalkAPIClient(base *http.Client) *http.Client {
|
|
||||||
client := *base
|
|
||||||
client.CheckRedirect = func(_ *http.Request, _ []*http.Request) error {
|
|
||||||
return http.ErrUseLastResponse
|
|
||||||
}
|
|
||||||
if !a.allowInsecureDownload {
|
|
||||||
transport := http.DefaultTransport.(*http.Transport).Clone()
|
|
||||||
transport.Proxy = nil
|
|
||||||
dialer := &net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second}
|
|
||||||
transport.DialContext = newDingTalkSafeDialContext(net.DefaultResolver.LookupIPAddr, dialer.DialContext)
|
|
||||||
client.Transport = transport
|
|
||||||
}
|
|
||||||
return &client
|
|
||||||
}
|
|
||||||
|
|
||||||
func readBoundedDingTalkAPIResponse(resp *http.Response) ([]byte, error) {
|
|
||||||
if resp.ContentLength > maxDingTalkAPIResponseBytes {
|
|
||||||
return nil, fmt.Errorf("dingtalk API response exceeds size limit")
|
|
||||||
}
|
|
||||||
payload, err := io.ReadAll(io.LimitReader(resp.Body, maxDingTalkAPIResponseBytes+1))
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("read dingtalk API response: %w", err)
|
|
||||||
}
|
|
||||||
if len(payload) > maxDingTalkAPIResponseBytes {
|
|
||||||
return nil, fmt.Errorf("dingtalk API response exceeds size limit")
|
|
||||||
}
|
|
||||||
return payload, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *Adapter) DownloadFile(ctx context.Context, msg *im.IncomingMessage) (io.ReadCloser, string, error) {
|
func (a *Adapter) DownloadFile(ctx context.Context, msg *im.IncomingMessage) (io.ReadCloser, string, error) {
|
||||||
if msg == nil || strings.TrimSpace(msg.FileKey) == "" {
|
if msg == nil || strings.TrimSpace(msg.FileKey) == "" {
|
||||||
@@ -262,8 +237,7 @@ func (a *Adapter) DownloadFile(ctx context.Context, msg *im.IncomingMessage) (io
|
|||||||
}
|
}
|
||||||
req.Header.Set("Content-Type", "application/json")
|
req.Header.Set("Content-Type", "application/json")
|
||||||
req.Header.Set("x-acs-dingtalk-access-token", token)
|
req.Header.Set("x-acs-dingtalk-access-token", token)
|
||||||
apiClient := a.newDingTalkAPIClient(httpClient)
|
resp, err := httpClient.Do(req)
|
||||||
resp, err := apiClient.Do(req)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, "", fmt.Errorf("request message file URL: %w", err)
|
return nil, "", fmt.Errorf("request message file URL: %w", err)
|
||||||
}
|
}
|
||||||
@@ -271,22 +245,12 @@ func (a *Adapter) DownloadFile(ctx context.Context, msg *im.IncomingMessage) (io
|
|||||||
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
|
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
|
||||||
return nil, "", fmt.Errorf("dingtalk message file API returned %d", resp.StatusCode)
|
return nil, "", fmt.Errorf("dingtalk message file API returned %d", resp.StatusCode)
|
||||||
}
|
}
|
||||||
payload, err := readBoundedDingTalkAPIResponse(resp)
|
|
||||||
if err != nil {
|
|
||||||
return nil, "", err
|
|
||||||
}
|
|
||||||
var result struct {
|
var result struct {
|
||||||
DownloadURL string `json:"downloadUrl"`
|
DownloadURL string `json:"downloadUrl"`
|
||||||
}
|
}
|
||||||
if err := json.Unmarshal(payload, &result); err != nil {
|
if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&result); err != nil {
|
||||||
return nil, "", fmt.Errorf("decode message file response: %w", err)
|
return nil, "", fmt.Errorf("decode message file response: %w", err)
|
||||||
}
|
}
|
||||||
if !a.allowInsecureDownload {
|
|
||||||
result.DownloadURL, err = normalizeDingTalkInitialDownloadURL(result.DownloadURL)
|
|
||||||
if err != nil {
|
|
||||||
return nil, "", err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := validateDingTalkDownloadURL(result.DownloadURL, a.allowInsecureDownload); err != nil {
|
if err := validateDingTalkDownloadURL(result.DownloadURL, a.allowInsecureDownload); err != nil {
|
||||||
return nil, "", err
|
return nil, "", err
|
||||||
}
|
}
|
||||||
@@ -296,133 +260,25 @@ func (a *Adapter) DownloadFile(ctx context.Context, msg *im.IncomingMessage) (io
|
|||||||
return nil, "", fmt.Errorf("create image download request: %w", err)
|
return nil, "", fmt.Errorf("create image download request: %w", err)
|
||||||
}
|
}
|
||||||
downloadClient := *httpClient
|
downloadClient := *httpClient
|
||||||
if !a.allowInsecureDownload {
|
downloadClient.CheckRedirect = func(req *http.Request, _ []*http.Request) error {
|
||||||
transport := http.DefaultTransport.(*http.Transport).Clone()
|
return validateDingTalkDownloadURL(req.URL.String(), a.allowInsecureDownload)
|
||||||
transport.Proxy = nil
|
|
||||||
dialer := &net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second}
|
|
||||||
transport.DialContext = newDingTalkSafeDialContext(net.DefaultResolver.LookupIPAddr, dialer.DialContext)
|
|
||||||
downloadClient.Transport = transport
|
|
||||||
}
|
|
||||||
downloadClient.CheckRedirect = func(req *http.Request, via []*http.Request) error {
|
|
||||||
return validateDingTalkRedirect(req, via, a.allowInsecureDownload)
|
|
||||||
}
|
}
|
||||||
downloadResp, err := downloadClient.Do(downloadReq)
|
downloadResp, err := downloadClient.Do(downloadReq)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if ctx.Err() != nil {
|
return nil, "", fmt.Errorf("download dingtalk image: %w", err)
|
||||||
return nil, "", ctx.Err()
|
|
||||||
}
|
|
||||||
return nil, "", fmt.Errorf("download dingtalk image request failed")
|
|
||||||
}
|
}
|
||||||
if downloadResp.StatusCode < http.StatusOK || downloadResp.StatusCode >= http.StatusMultipleChoices {
|
if downloadResp.StatusCode < http.StatusOK || downloadResp.StatusCode >= http.StatusMultipleChoices {
|
||||||
downloadResp.Body.Close()
|
downloadResp.Body.Close()
|
||||||
return nil, "", fmt.Errorf("dingtalk image download returned %d", downloadResp.StatusCode)
|
return nil, "", fmt.Errorf("dingtalk image download returned %d", downloadResp.StatusCode)
|
||||||
}
|
}
|
||||||
contentType := strings.ToLower(strings.TrimSpace(strings.Split(downloadResp.Header.Get("Content-Type"), ";")[0]))
|
contentType := strings.ToLower(strings.TrimSpace(strings.Split(downloadResp.Header.Get("Content-Type"), ";")[0]))
|
||||||
if msg.MessageType == im.MessageTypeImage && contentType != "" && contentType != "application/octet-stream" && !strings.HasPrefix(contentType, "image/") {
|
if msg.MessageType == im.MessageTypeImage && !strings.HasPrefix(contentType, "image/") {
|
||||||
downloadResp.Body.Close()
|
downloadResp.Body.Close()
|
||||||
return nil, "", fmt.Errorf("dingtalk picture response is not an image")
|
return nil, "", fmt.Errorf("dingtalk picture response is not an image")
|
||||||
}
|
}
|
||||||
return downloadResp.Body, dingtalkDownloadedFileName(downloadResp, msg, contentType), nil
|
return downloadResp.Body, dingtalkDownloadedFileName(downloadResp, msg, contentType), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
var dingtalkBlockedPublicRanges = []netip.Prefix{
|
|
||||||
netip.MustParsePrefix("100.64.0.0/10"),
|
|
||||||
netip.MustParsePrefix("192.0.0.0/24"),
|
|
||||||
netip.MustParsePrefix("192.0.2.0/24"),
|
|
||||||
netip.MustParsePrefix("198.18.0.0/15"),
|
|
||||||
netip.MustParsePrefix("198.51.100.0/24"),
|
|
||||||
netip.MustParsePrefix("203.0.113.0/24"),
|
|
||||||
netip.MustParsePrefix("2001:db8::/32"),
|
|
||||||
}
|
|
||||||
|
|
||||||
func isSafeDingTalkDownloadIP(ip net.IP) bool {
|
|
||||||
if ip == nil || !secutils.IsPublicIP(ip) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
addr, ok := netip.AddrFromSlice(ip)
|
|
||||||
if !ok {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
addr = addr.Unmap()
|
|
||||||
if !addr.IsGlobalUnicast() || addr.IsPrivate() || addr.IsLoopback() || addr.IsLinkLocalUnicast() || addr.IsUnspecified() || addr.IsMulticast() {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
for _, prefix := range dingtalkBlockedPublicRanges {
|
|
||||||
if prefix.Contains(addr) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
func newDingTalkSafeDialContext(
|
|
||||||
lookup func(context.Context, string) ([]net.IPAddr, error),
|
|
||||||
dial func(context.Context, string, string) (net.Conn, error),
|
|
||||||
) func(context.Context, string, string) (net.Conn, error) {
|
|
||||||
return func(ctx context.Context, network, addr string) (net.Conn, error) {
|
|
||||||
host, port, err := net.SplitHostPort(addr)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("invalid dingtalk media address")
|
|
||||||
}
|
|
||||||
var ips []net.IPAddr
|
|
||||||
if literal := net.ParseIP(host); literal != nil {
|
|
||||||
ips = []net.IPAddr{{IP: literal}}
|
|
||||||
} else {
|
|
||||||
ips, err = lookup(ctx, host)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("resolve dingtalk media host failed")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(ips) == 0 {
|
|
||||||
return nil, fmt.Errorf("dingtalk media host resolved without addresses")
|
|
||||||
}
|
|
||||||
for _, resolved := range ips {
|
|
||||||
if !isSafeDingTalkDownloadIP(resolved.IP) {
|
|
||||||
return nil, fmt.Errorf("dingtalk media host resolved to a blocked address")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var lastErr error
|
|
||||||
for _, resolved := range ips {
|
|
||||||
conn, dialErr := dial(ctx, network, net.JoinHostPort(resolved.IP.String(), port))
|
|
||||||
if dialErr == nil {
|
|
||||||
return conn, nil
|
|
||||||
}
|
|
||||||
lastErr = dialErr
|
|
||||||
}
|
|
||||||
return nil, fmt.Errorf("connect to validated dingtalk media address: %w", lastErr)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateDingTalkRedirect(req *http.Request, via []*http.Request, allowInsecure bool) error {
|
|
||||||
if len(via) >= 10 {
|
|
||||||
return fmt.Errorf("stopped after 10 dingtalk image redirects")
|
|
||||||
}
|
|
||||||
// Go may synthesize Referer from the credential-bearing source URL and copy
|
|
||||||
// headers across redirects. Media redirects need no inherited headers.
|
|
||||||
req.Header = make(http.Header)
|
|
||||||
return validateDingTalkDownloadURL(req.URL.String(), allowInsecure)
|
|
||||||
}
|
|
||||||
|
|
||||||
// normalizeDingTalkInitialDownloadURL upgrades the one-time media URL returned
|
|
||||||
// directly by DingTalk's authenticated message-file API. Some DingTalk media
|
|
||||||
// backends still return an http scheme even though the same signed resource is
|
|
||||||
// available over TLS. Only this initial trusted API response is upgraded;
|
|
||||||
// redirects remain HTTPS-only and are validated independently.
|
|
||||||
func normalizeDingTalkInitialDownloadURL(rawURL string) (string, error) {
|
|
||||||
u, err := url.Parse(strings.TrimSpace(rawURL))
|
|
||||||
if err != nil || u.Hostname() == "" || u.User != nil {
|
|
||||||
return "", fmt.Errorf("invalid dingtalk download URL")
|
|
||||||
}
|
|
||||||
switch u.Scheme {
|
|
||||||
case "http":
|
|
||||||
u.Scheme = "https"
|
|
||||||
case "https":
|
|
||||||
default:
|
|
||||||
return "", fmt.Errorf("invalid dingtalk download URL scheme")
|
|
||||||
}
|
|
||||||
return u.String(), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateDingTalkDownloadURL(rawURL string, allowInsecure bool) error {
|
func validateDingTalkDownloadURL(rawURL string, allowInsecure bool) error {
|
||||||
u, err := url.Parse(strings.TrimSpace(rawURL))
|
u, err := url.Parse(strings.TrimSpace(rawURL))
|
||||||
if err != nil || u.Hostname() == "" || u.User != nil {
|
if err != nil || u.Hostname() == "" || u.User != nil {
|
||||||
@@ -440,9 +296,6 @@ func validateDingTalkDownloadURL(rawURL string, allowInsecure bool) error {
|
|||||||
if ip := net.ParseIP(u.Hostname()); ip != nil && (ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsUnspecified()) {
|
if ip := net.ParseIP(u.Hostname()); ip != nil && (ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsUnspecified()) {
|
||||||
return fmt.Errorf("dingtalk download URL points to a private address")
|
return fmt.Errorf("dingtalk download URL points to a private address")
|
||||||
}
|
}
|
||||||
if err := secutils.ValidateURLForSSRF(u.String()); err != nil {
|
|
||||||
return fmt.Errorf("dingtalk download URL failed SSRF validation: %w", err)
|
|
||||||
}
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -482,9 +335,6 @@ func (a *Adapter) SendReply(ctx context.Context, incoming *im.IncomingMessage, r
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (a *Adapter) replyViaSessionWebhook(ctx context.Context, webhookURL, content string) error {
|
func (a *Adapter) replyViaSessionWebhook(ctx context.Context, webhookURL, content string) error {
|
||||||
if err := validateDingTalkDownloadURL(webhookURL, a.allowInsecureDownload); err != nil {
|
|
||||||
return fmt.Errorf("invalid DingTalk session webhook")
|
|
||||||
}
|
|
||||||
body := map[string]interface{}{
|
body := map[string]interface{}{
|
||||||
"msgtype": "markdown",
|
"msgtype": "markdown",
|
||||||
"markdown": map[string]string{
|
"markdown": map[string]string{
|
||||||
@@ -500,22 +350,19 @@ func (a *Adapter) replyViaSessionWebhook(ctx context.Context, webhookURL, conten
|
|||||||
|
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, webhookURL, bytes.NewReader(jsonBody))
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, webhookURL, bytes.NewReader(jsonBody))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("create DingTalk session webhook request")
|
return fmt.Errorf("create request: %w", err)
|
||||||
}
|
}
|
||||||
req.Header.Set("Content-Type", "application/json")
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
|
||||||
client := a.newDingTalkAPIClient(httpClient)
|
resp, err := httpClient.Do(req)
|
||||||
resp, err := client.Do(req)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("send DingTalk session webhook failed")
|
return fmt.Errorf("send reply: %w", err)
|
||||||
}
|
}
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
if resp.StatusCode != http.StatusOK {
|
||||||
return fmt.Errorf("dingtalk session webhook returned %d", resp.StatusCode)
|
respBody, _ := io.ReadAll(resp.Body)
|
||||||
}
|
return fmt.Errorf("dingtalk sessionWebhook returned %d: %s", resp.StatusCode, string(respBody))
|
||||||
if _, err := readBoundedDingTalkAPIResponse(resp); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -558,18 +405,15 @@ func (a *Adapter) replyViaOpenAPI(ctx context.Context, incoming *im.IncomingMess
|
|||||||
req.Header.Set("Content-Type", "application/json")
|
req.Header.Set("Content-Type", "application/json")
|
||||||
req.Header.Set("x-acs-dingtalk-access-token", token)
|
req.Header.Set("x-acs-dingtalk-access-token", token)
|
||||||
|
|
||||||
apiClient := a.newDingTalkAPIClient(httpClient)
|
resp, err := httpClient.Do(req)
|
||||||
resp, err := apiClient.Do(req)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("send reply failed")
|
return fmt.Errorf("send reply: %w", err)
|
||||||
}
|
}
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
if resp.StatusCode != http.StatusOK {
|
||||||
return fmt.Errorf("dingtalk OpenAPI returned %d", resp.StatusCode)
|
respBody, _ := io.ReadAll(resp.Body)
|
||||||
}
|
return fmt.Errorf("dingtalk OpenAPI returned %d: %s", resp.StatusCode, string(respBody))
|
||||||
if _, err := readBoundedDingTalkAPIResponse(resp); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -605,26 +449,22 @@ func (a *Adapter) getAccessToken(ctx context.Context) (string, error) {
|
|||||||
}
|
}
|
||||||
req.Header.Set("Content-Type", "application/json")
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
|
||||||
apiClient := a.newDingTalkAPIClient(httpClient)
|
resp, err := httpClient.Do(req)
|
||||||
resp, err := apiClient.Do(req)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", fmt.Errorf("dingtalk access token request failed")
|
return "", err
|
||||||
}
|
}
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
if resp.StatusCode != http.StatusOK {
|
||||||
return "", fmt.Errorf("dingtalk access token returned %d", resp.StatusCode)
|
respBody, _ := io.ReadAll(resp.Body)
|
||||||
}
|
return "", fmt.Errorf("dingtalk accessToken returned %d: %s", resp.StatusCode, string(respBody))
|
||||||
payload, err := readBoundedDingTalkAPIResponse(resp)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var result struct {
|
var result struct {
|
||||||
AccessToken string `json:"accessToken"`
|
AccessToken string `json:"accessToken"`
|
||||||
ExpireIn int64 `json:"expireIn"`
|
ExpireIn int64 `json:"expireIn"`
|
||||||
}
|
}
|
||||||
if err := json.Unmarshal(payload, &result); err != nil {
|
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||||
return "", fmt.Errorf("decode token response: %w", err)
|
return "", fmt.Errorf("decode token response: %w", err)
|
||||||
}
|
}
|
||||||
if result.AccessToken == "" {
|
if result.AccessToken == "" {
|
||||||
@@ -658,19 +498,15 @@ func (a *Adapter) dingtalkAPI(ctx context.Context, method, path string, body int
|
|||||||
req.Header.Set("Content-Type", "application/json")
|
req.Header.Set("Content-Type", "application/json")
|
||||||
req.Header.Set("x-acs-dingtalk-access-token", token)
|
req.Header.Set("x-acs-dingtalk-access-token", token)
|
||||||
|
|
||||||
apiClient := a.newDingTalkAPIClient(httpClient)
|
resp, err := httpClient.Do(req)
|
||||||
resp, err := apiClient.Do(req)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("dingtalk API request failed")
|
return nil, fmt.Errorf("do request: %w", err)
|
||||||
}
|
}
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
|
|
||||||
respBody, readErr := readBoundedDingTalkAPIResponse(resp)
|
respBody, _ := io.ReadAll(resp.Body)
|
||||||
if readErr != nil {
|
|
||||||
return nil, readErr
|
|
||||||
}
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
if resp.StatusCode != http.StatusOK {
|
||||||
return nil, fmt.Errorf("dingtalk API %s returned %d", path, resp.StatusCode)
|
return nil, fmt.Errorf("dingtalk API %s returned %d: %s", path, resp.StatusCode, string(respBody))
|
||||||
}
|
}
|
||||||
return respBody, nil
|
return respBody, nil
|
||||||
}
|
}
|
||||||
@@ -805,6 +641,10 @@ func (a *Adapter) StartStream(ctx context.Context, incoming *im.IncomingMessage)
|
|||||||
logger.Warnf(ctx, "[DingTalk] Failed to create AI card, falling back to sessionWebhook: %v", err)
|
logger.Warnf(ctx, "[DingTalk] Failed to create AI card, falling back to sessionWebhook: %v", err)
|
||||||
} else {
|
} else {
|
||||||
state.outTrackID = outTrackID
|
state.outTrackID = outTrackID
|
||||||
|
if incoming.Extra == nil {
|
||||||
|
incoming.Extra = make(map[string]string)
|
||||||
|
}
|
||||||
|
incoming.Extra["dingtalk_out_track_id"] = outTrackID
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -869,7 +709,6 @@ func (a *Adapter) FinalizeStream(ctx context.Context, incoming *im.IncomingMessa
|
|||||||
if outTrackID != "" {
|
if outTrackID != "" {
|
||||||
if err := a.streamingUpdateCard(ctx, outTrackID, finalContent, false); err != nil {
|
if err := a.streamingUpdateCard(ctx, outTrackID, finalContent, false); err != nil {
|
||||||
logger.Warnf(ctx, "[DingTalk] Failed to finalize card stream: %v", err)
|
logger.Warnf(ctx, "[DingTalk] Failed to finalize card stream: %v", err)
|
||||||
return fmt.Errorf("finalize DingTalk card stream: %w", err)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
@@ -898,17 +737,14 @@ func (a *Adapter) EndStream(ctx context.Context, incoming *im.IncomingMessage, s
|
|||||||
if outTrackID != "" {
|
if outTrackID != "" {
|
||||||
if err := a.streamingUpdateCard(ctx, outTrackID, fullContent, true); err != nil {
|
if err := a.streamingUpdateCard(ctx, outTrackID, fullContent, true); err != nil {
|
||||||
logger.Warnf(ctx, "[DingTalk] Failed to finalize card stream: %v", err)
|
logger.Warnf(ctx, "[DingTalk] Failed to finalize card stream: %v", err)
|
||||||
return fmt.Errorf("end DingTalk card stream: %w", err)
|
|
||||||
}
|
}
|
||||||
} else if sessionWebhook != "" {
|
} else if sessionWebhook != "" {
|
||||||
if err := a.replyViaSessionWebhook(ctx, sessionWebhook, fullContent); err != nil {
|
if err := a.replyViaSessionWebhook(ctx, sessionWebhook, fullContent); err != nil {
|
||||||
logger.Warnf(ctx, "[DingTalk] Failed to end stream: %v", err)
|
logger.Warnf(ctx, "[DingTalk] Failed to end stream: %v", err)
|
||||||
return fmt.Errorf("end DingTalk session webhook stream: %w", err)
|
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
if err := a.replyViaOpenAPI(ctx, incoming, fullContent); err != nil {
|
if err := a.replyViaOpenAPI(ctx, incoming, fullContent); err != nil {
|
||||||
logger.Warnf(ctx, "[DingTalk] Failed to end stream via OpenAPI: %v", err)
|
logger.Warnf(ctx, "[DingTalk] Failed to end stream via OpenAPI: %v", err)
|
||||||
return fmt.Errorf("end DingTalk OpenAPI stream: %w", err)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,45 +0,0 @@
|
|||||||
package dingtalk
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestDingTalkRedirectClearsRefererAndInheritedHeaders(t *testing.T) {
|
|
||||||
req := httptest.NewRequest(http.MethodGet, "https://cdn-two.example/image.png", nil)
|
|
||||||
req.Header.Set("Referer", "https://cdn-one.example/image.png?signature=TOPSECRET")
|
|
||||||
req.Header.Set("Authorization", "Bearer TOPSECRET")
|
|
||||||
if err := validateDingTalkRedirect(req, nil, true); err != nil {
|
|
||||||
t.Fatalf("validateDingTalkRedirect: %v", err)
|
|
||||||
}
|
|
||||||
if len(req.Header) != 0 {
|
|
||||||
t.Fatalf("redirect headers = %#v, want empty", req.Header)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReplyViaSessionWebhookRejectsRedirectWithoutLeakingURL(t *testing.T) {
|
|
||||||
var reached bool
|
|
||||||
target := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { reached = true }))
|
|
||||||
defer target.Close()
|
|
||||||
source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.Header().Set("Location", target.URL)
|
|
||||||
w.WriteHeader(http.StatusTemporaryRedirect)
|
|
||||||
}))
|
|
||||||
defer source.Close()
|
|
||||||
|
|
||||||
a := &Adapter{allowInsecureDownload: true}
|
|
||||||
signedURL := source.URL + "/hook?access_token=TOPSECRET"
|
|
||||||
err := a.replyViaSessionWebhook(context.Background(), signedURL, "reply")
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected webhook redirect rejection")
|
|
||||||
}
|
|
||||||
if reached {
|
|
||||||
t.Fatal("session webhook followed redirect")
|
|
||||||
}
|
|
||||||
if strings.Contains(err.Error(), "TOPSECRET") || strings.Contains(err.Error(), signedURL) {
|
|
||||||
t.Fatalf("credential URL leaked in error: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -2,9 +2,7 @@ package dingtalk
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"io"
|
"io"
|
||||||
"net"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -41,28 +39,6 @@ func TestParseCallbackMessagePicture(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseCallbackMessageRichTextWithPictureAndText(t *testing.T) {
|
|
||||||
msg := &callbackMessage{
|
|
||||||
MsgID: "msg-richtext-1",
|
|
||||||
Msgtype: "richText",
|
|
||||||
Content: []byte(`{"richText":[{"text":" 请描述这张图片 "},{"type":"picture","downloadCode":"rich-webhook-code"}]}`),
|
|
||||||
}
|
|
||||||
|
|
||||||
got := parseCallbackMessage(msg)
|
|
||||||
if got.MessageType != im.MessageTypeImage {
|
|
||||||
t.Fatalf("MessageType = %q, want image", got.MessageType)
|
|
||||||
}
|
|
||||||
if got.Content != "请描述这张图片" {
|
|
||||||
t.Fatalf("Content = %q", got.Content)
|
|
||||||
}
|
|
||||||
if got.FileKey != "rich-webhook-code" {
|
|
||||||
t.Fatalf("FileKey = %q", got.FileKey)
|
|
||||||
}
|
|
||||||
if got.FileName != "msg-richtext-1.png" {
|
|
||||||
t.Fatalf("FileName = %q", got.FileName)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAdapterDownloadFileUsesMessageFilesAPI(t *testing.T) {
|
func TestAdapterDownloadFileUsesMessageFilesAPI(t *testing.T) {
|
||||||
var sawToken bool
|
var sawToken bool
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
@@ -115,184 +91,3 @@ func TestAdapterDownloadFileUsesMessageFilesAPI(t *testing.T) {
|
|||||||
func serverURLForRequest(r *http.Request) string {
|
func serverURLForRequest(r *http.Request) string {
|
||||||
return "http://" + r.Host
|
return "http://" + r.Host
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNormalizeDingTalkInitialDownloadURLUpgradesHTTPToHTTPS(t *testing.T) {
|
|
||||||
rawURL := "http://media.example.com/path/image.png?signature=abc%2F123&name=a+b.png"
|
|
||||||
got, err := normalizeDingTalkInitialDownloadURL(rawURL)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("normalizeDingTalkInitialDownloadURL: %v", err)
|
|
||||||
}
|
|
||||||
want := "https://media.example.com/path/image.png?signature=abc%2F123&name=a+b.png"
|
|
||||||
if got != want {
|
|
||||||
t.Fatalf("normalized URL = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidateDingTalkDownloadURLRejectsUnsafeTargets(t *testing.T) {
|
|
||||||
for _, rawURL := range []string{
|
|
||||||
"http://example.com/image.png",
|
|
||||||
"https://user:pass@example.com/image.png",
|
|
||||||
"https://127.0.0.1/image.png",
|
|
||||||
"https://10.0.0.8/image.png",
|
|
||||||
} {
|
|
||||||
t.Run(rawURL, func(t *testing.T) {
|
|
||||||
if err := validateDingTalkDownloadURL(rawURL, false); err == nil {
|
|
||||||
t.Fatalf("unsafe URL accepted: %s", rawURL)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseCallbackMessagePictureWithoutDownloadCodeStaysImage(t *testing.T) {
|
|
||||||
for _, content := range [][]byte{[]byte(`{}`), []byte(`not-json`)} {
|
|
||||||
msg := parseCallbackMessage(&callbackMessage{MsgID: "bad-picture", Msgtype: "picture", Content: content})
|
|
||||||
if msg.MessageType != im.MessageTypeImage {
|
|
||||||
t.Fatalf("MessageType = %q, want image", msg.MessageType)
|
|
||||||
}
|
|
||||||
if msg.FileKey != "" {
|
|
||||||
t.Fatalf("FileKey = %q, want empty", msg.FileKey)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAdapterDownloadFileDoesNotFollowCredentialedAPIRedirect(t *testing.T) {
|
|
||||||
var redirectedToken string
|
|
||||||
target := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {
|
|
||||||
redirectedToken = r.Header.Get("x-acs-dingtalk-access-token")
|
|
||||||
}))
|
|
||||||
defer target.Close()
|
|
||||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.Header().Set("Location", target.URL)
|
|
||||||
w.WriteHeader(http.StatusTemporaryRedirect)
|
|
||||||
}))
|
|
||||||
defer api.Close()
|
|
||||||
|
|
||||||
adapter := NewAdapter("robot-code-1", "secret", "")
|
|
||||||
adapter.token = "cached-access-token"
|
|
||||||
adapter.tokenExpAt = time.Now().Add(time.Hour)
|
|
||||||
adapter.messageFileDownloadURL = api.URL
|
|
||||||
adapter.allowInsecureDownload = true
|
|
||||||
_, _, err := adapter.DownloadFile(context.Background(), &im.IncomingMessage{MessageType: im.MessageTypeImage, FileKey: "code"})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected redirect rejection")
|
|
||||||
}
|
|
||||||
if redirectedToken != "" {
|
|
||||||
t.Fatal("access token leaked across redirect")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAdapterDownloadFileAcceptsGenericMediaContentType(t *testing.T) {
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.URL.Path == "/api" {
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
_, _ = w.Write([]byte(`{"downloadUrl":"` + serverURLForRequest(r) + `/media"}`))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.Header().Set("Content-Type", "application/octet-stream")
|
|
||||||
_, _ = w.Write([]byte("PNGDATA"))
|
|
||||||
}))
|
|
||||||
defer server.Close()
|
|
||||||
adapter := NewAdapter("robot", "secret", "")
|
|
||||||
adapter.token = "token"
|
|
||||||
adapter.tokenExpAt = time.Now().Add(time.Hour)
|
|
||||||
adapter.messageFileDownloadURL = server.URL + "/api"
|
|
||||||
adapter.allowInsecureDownload = true
|
|
||||||
body, _, err := adapter.DownloadFile(context.Background(), &im.IncomingMessage{MessageType: im.MessageTypeImage, FileKey: "code"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("DownloadFile: %v", err)
|
|
||||||
}
|
|
||||||
body.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDingTalkAPIClientDisablesProxyAndRedirects(t *testing.T) {
|
|
||||||
a := &Adapter{}
|
|
||||||
client := a.newDingTalkAPIClient(httpClient)
|
|
||||||
transport, ok := client.Transport.(*http.Transport)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("transport type = %T, want *http.Transport", client.Transport)
|
|
||||||
}
|
|
||||||
if transport.Proxy != nil {
|
|
||||||
t.Fatal("DingTalk API client must ignore environment proxies")
|
|
||||||
}
|
|
||||||
req := httptest.NewRequest(http.MethodPost, "https://api.dingtalk.com/redirect", nil)
|
|
||||||
if err := client.CheckRedirect(req, nil); !errors.Is(err, http.ErrUseLastResponse) {
|
|
||||||
t.Fatalf("redirect error = %v, want http.ErrUseLastResponse", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDingTalkRedirectLimitFailsClosed(t *testing.T) {
|
|
||||||
req := httptest.NewRequest(http.MethodGet, "https://example.com/image.png", nil)
|
|
||||||
via := make([]*http.Request, 10)
|
|
||||||
if err := validateDingTalkRedirect(req, via, false); err == nil {
|
|
||||||
t.Fatal("validateDingTalkRedirect should reject more than 10 requests")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDingTalkSafeDialUsesValidatedConcreteIP(t *testing.T) {
|
|
||||||
stop := errors.New("stop after address capture")
|
|
||||||
var dialed string
|
|
||||||
dial := newDingTalkSafeDialContext(
|
|
||||||
func(context.Context, string) ([]net.IPAddr, error) {
|
|
||||||
return []net.IPAddr{{IP: net.ParseIP("1.1.1.1")}}, nil
|
|
||||||
},
|
|
||||||
func(_ context.Context, _ string, addr string) (net.Conn, error) {
|
|
||||||
dialed = addr
|
|
||||||
return nil, stop
|
|
||||||
},
|
|
||||||
)
|
|
||||||
_, err := dial(context.Background(), "tcp", "cdn.example.test:443")
|
|
||||||
if !errors.Is(err, stop) {
|
|
||||||
t.Fatalf("dial error = %v, want sentinel", err)
|
|
||||||
}
|
|
||||||
if dialed != "1.1.1.1:443" {
|
|
||||||
t.Fatalf("dialed address = %q, want validated concrete IP", dialed)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDingTalkSafeDialRejectsEveryRestrictedResolvedIP(t *testing.T) {
|
|
||||||
for _, rawIP := range []string{
|
|
||||||
"0.0.0.1",
|
|
||||||
"240.0.0.1",
|
|
||||||
"2001:0000::1",
|
|
||||||
"2002:7f00:0001::1",
|
|
||||||
} {
|
|
||||||
t.Run(rawIP, func(t *testing.T) {
|
|
||||||
dialCalled := false
|
|
||||||
dial := newDingTalkSafeDialContext(
|
|
||||||
func(context.Context, string) ([]net.IPAddr, error) {
|
|
||||||
return []net.IPAddr{{IP: net.ParseIP(rawIP)}}, nil
|
|
||||||
},
|
|
||||||
func(context.Context, string, string) (net.Conn, error) {
|
|
||||||
dialCalled = true
|
|
||||||
return nil, errors.New("must not dial")
|
|
||||||
},
|
|
||||||
)
|
|
||||||
if _, err := dial(context.Background(), "tcp", "cdn.example.test:443"); err == nil {
|
|
||||||
t.Fatal("restricted resolved IP was accepted")
|
|
||||||
}
|
|
||||||
if dialCalled {
|
|
||||||
t.Fatal("restricted resolved IP reached network dial")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAdapterDownloadFileRedactsSignedURLFromErrors(t *testing.T) {
|
|
||||||
api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
_, _ = w.Write([]byte(`{"downloadUrl":"http://127.0.0.1:1/media?X-Amz-Signature=TOPSECRET"}`))
|
|
||||||
}))
|
|
||||||
defer api.Close()
|
|
||||||
adapter := NewAdapter("robot", "secret", "")
|
|
||||||
adapter.token = "token"
|
|
||||||
adapter.tokenExpAt = time.Now().Add(time.Hour)
|
|
||||||
adapter.messageFileDownloadURL = api.URL
|
|
||||||
adapter.allowInsecureDownload = true
|
|
||||||
_, _, err := adapter.DownloadFile(context.Background(), &im.IncomingMessage{MessageType: im.MessageTypeImage, FileKey: "code"})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected download error")
|
|
||||||
}
|
|
||||||
if strings.Contains(err.Error(), "TOPSECRET") || strings.Contains(err.Error(), "X-Amz-Signature") {
|
|
||||||
t.Fatalf("signed URL leaked in error: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,43 +0,0 @@
|
|||||||
package dingtalk
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"net/http"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/Tencent/WeKnora/internal/im"
|
|
||||||
)
|
|
||||||
|
|
||||||
type failingCardRoundTripper struct{}
|
|
||||||
|
|
||||||
func (failingCardRoundTripper) RoundTrip(*http.Request) (*http.Response, error) {
|
|
||||||
return nil, errors.New("simulated delivery failure")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFinalizeStreamPropagatesCardDeliveryFailure(t *testing.T) {
|
|
||||||
oldClient := httpClient
|
|
||||||
httpClient = &http.Client{Transport: failingCardRoundTripper{}, Timeout: time.Second}
|
|
||||||
t.Cleanup(func() { httpClient = oldClient })
|
|
||||||
|
|
||||||
adapter := &Adapter{
|
|
||||||
clientID: "robot",
|
|
||||||
token: "cached-token",
|
|
||||||
tokenExpAt: time.Now().Add(time.Hour),
|
|
||||||
}
|
|
||||||
streamID := "dt:fail-closed-test"
|
|
||||||
streamsMu.Lock()
|
|
||||||
dStreams[streamID] = &streamState{outTrackID: "track-id", createdAt: time.Now()}
|
|
||||||
streamsMu.Unlock()
|
|
||||||
t.Cleanup(func() {
|
|
||||||
streamsMu.Lock()
|
|
||||||
delete(dStreams, streamID)
|
|
||||||
streamsMu.Unlock()
|
|
||||||
})
|
|
||||||
|
|
||||||
err := adapter.FinalizeStream(context.Background(), &im.IncomingMessage{}, streamID, "image recognition failed")
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected final card delivery failure to propagate")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -102,11 +102,9 @@ func (c *LongConnClient) onChatBotMessage(ctx context.Context, data *chatbot.Bot
|
|||||||
}
|
}
|
||||||
|
|
||||||
func parseDingTalkStreamPayload(msgType string, rawContent interface{}, text string) (im.MessageType, string, string) {
|
func parseDingTalkStreamPayload(msgType string, rawContent interface{}, text string) (im.MessageType, string, string) {
|
||||||
normalizedType := strings.ToLower(strings.TrimSpace(msgType))
|
if !strings.EqualFold(strings.TrimSpace(msgType), "picture") {
|
||||||
if normalizedType != "picture" && normalizedType != "richtext" {
|
|
||||||
return im.MessageTypeText, strings.TrimSpace(text), ""
|
return im.MessageTypeText, strings.TrimSpace(text), ""
|
||||||
}
|
}
|
||||||
|
|
||||||
var payload []byte
|
var payload []byte
|
||||||
switch value := rawContent.(type) {
|
switch value := rawContent.(type) {
|
||||||
case string:
|
case string:
|
||||||
@@ -122,41 +120,6 @@ func parseDingTalkStreamPayload(msgType string, rawContent interface{}, text str
|
|||||||
return im.MessageTypeImage, "", ""
|
return im.MessageTypeImage, "", ""
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if normalizedType == "richtext" {
|
|
||||||
var rich struct {
|
|
||||||
Items []struct {
|
|
||||||
Text string `json:"text"`
|
|
||||||
Type string `json:"type"`
|
|
||||||
DownloadCode string `json:"downloadCode"`
|
|
||||||
} `json:"richText"`
|
|
||||||
}
|
|
||||||
if json.Unmarshal(payload, &rich) != nil {
|
|
||||||
return im.MessageTypeText, strings.TrimSpace(text), ""
|
|
||||||
}
|
|
||||||
|
|
||||||
var textParts []string
|
|
||||||
fileKey := ""
|
|
||||||
hasPicture := false
|
|
||||||
for _, item := range rich.Items {
|
|
||||||
if itemText := strings.TrimSpace(item.Text); itemText != "" {
|
|
||||||
textParts = append(textParts, itemText)
|
|
||||||
}
|
|
||||||
if strings.EqualFold(strings.TrimSpace(item.Type), "picture") {
|
|
||||||
hasPicture = true
|
|
||||||
if fileKey == "" {
|
|
||||||
fileKey = strings.TrimSpace(item.DownloadCode)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
content := strings.Join(textParts, "\n")
|
|
||||||
if hasPicture {
|
|
||||||
return im.MessageTypeImage, content, fileKey
|
|
||||||
}
|
|
||||||
return im.MessageTypeText, content, ""
|
|
||||||
}
|
|
||||||
|
|
||||||
var picture struct {
|
var picture struct {
|
||||||
DownloadCode string `json:"downloadCode"`
|
DownloadCode string `json:"downloadCode"`
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -35,58 +35,3 @@ func TestParseDingTalkStreamPayloadPictureJSONString(t *testing.T) {
|
|||||||
t.Fatalf("got type=%q fileKey=%q", messageType, fileKey)
|
t.Fatalf("got type=%q fileKey=%q", messageType, fileKey)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseDingTalkStreamPayloadRichTextWithPictureAndText(t *testing.T) {
|
|
||||||
messageType, content, fileKey := parseDingTalkStreamPayload("richText", map[string]interface{}{
|
|
||||||
"richText": []interface{}{
|
|
||||||
map[string]interface{}{"text": " 请回答图片中的问题 "},
|
|
||||||
map[string]interface{}{"type": "picture", "downloadCode": "rich-stream-code"},
|
|
||||||
},
|
|
||||||
}, "")
|
|
||||||
|
|
||||||
if messageType != im.MessageTypeImage {
|
|
||||||
t.Fatalf("MessageType = %q, want image", messageType)
|
|
||||||
}
|
|
||||||
if content != "请回答图片中的问题" {
|
|
||||||
t.Fatalf("content = %q", content)
|
|
||||||
}
|
|
||||||
if fileKey != "rich-stream-code" {
|
|
||||||
t.Fatalf("fileKey = %q", fileKey)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseDingTalkStreamPayloadRichTextJoinsTextParts(t *testing.T) {
|
|
||||||
messageType, content, fileKey := parseDingTalkStreamPayload("richText", map[string]interface{}{
|
|
||||||
"richText": []interface{}{
|
|
||||||
map[string]interface{}{"text": "第一段"},
|
|
||||||
map[string]interface{}{"type": "picture", "downloadCode": "rich-code"},
|
|
||||||
map[string]interface{}{"text": "第二段"},
|
|
||||||
},
|
|
||||||
}, "")
|
|
||||||
if messageType != im.MessageTypeImage || content != "第一段\n第二段" || fileKey != "rich-code" {
|
|
||||||
t.Fatalf("got type=%q content=%q fileKey=%q", messageType, content, fileKey)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseDingTalkStreamPayloadRichTextWithoutPictureStaysText(t *testing.T) {
|
|
||||||
messageType, content, fileKey := parseDingTalkStreamPayload("richText", map[string]interface{}{
|
|
||||||
"richText": []interface{}{
|
|
||||||
map[string]interface{}{"text": " 只有文字 "},
|
|
||||||
},
|
|
||||||
}, "")
|
|
||||||
if messageType != im.MessageTypeText || content != "只有文字" || fileKey != "" {
|
|
||||||
t.Fatalf("got type=%q content=%q fileKey=%q", messageType, content, fileKey)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseDingTalkStreamPayloadRichTextPictureWithoutDownloadCodeStaysImage(t *testing.T) {
|
|
||||||
messageType, content, fileKey := parseDingTalkStreamPayload("richText", map[string]interface{}{
|
|
||||||
"richText": []interface{}{
|
|
||||||
map[string]interface{}{"text": "识别图片"},
|
|
||||||
map[string]interface{}{"type": "picture"},
|
|
||||||
},
|
|
||||||
}, "")
|
|
||||||
if messageType != im.MessageTypeImage || content != "识别图片" || fileKey != "" {
|
|
||||||
t.Fatalf("got type=%q content=%q fileKey=%q", messageType, content, fileKey)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,12 +1,7 @@
|
|||||||
package im
|
package im
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"image"
|
|
||||||
"image/color"
|
|
||||||
"image/png"
|
|
||||||
"io"
|
"io"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -23,19 +18,8 @@ func (d staticImageDownloader) DownloadFile(context.Context, *IncomingMessage) (
|
|||||||
return io.NopCloser(strings.NewReader(d.body)), d.fileName, nil
|
return io.NopCloser(strings.NewReader(d.body)), d.fileName, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func testPNG(t *testing.T, width, height int) string {
|
|
||||||
t.Helper()
|
|
||||||
img := image.NewNRGBA(image.Rect(0, 0, width, height))
|
|
||||||
img.SetNRGBA(0, 0, color.NRGBA{R: 20, G: 120, B: 240, A: 255})
|
|
||||||
var buf bytes.Buffer
|
|
||||||
if err := png.Encode(&buf, img); err != nil {
|
|
||||||
t.Fatalf("encode png: %v", err)
|
|
||||||
}
|
|
||||||
return buf.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPrepareIMImageForQAAndBuildRequest(t *testing.T) {
|
func TestPrepareIMImageForQAAndBuildRequest(t *testing.T) {
|
||||||
png := testPNG(t, 32, 32)
|
png := string([]byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'}) + strings.Repeat("x", 600)
|
||||||
msg := &IncomingMessage{
|
msg := &IncomingMessage{
|
||||||
Platform: PlatformDingtalk,
|
Platform: PlatformDingtalk,
|
||||||
MessageType: MessageTypeImage,
|
MessageType: MessageTypeImage,
|
||||||
@@ -49,6 +33,12 @@ func TestPrepareIMImageForQAAndBuildRequest(t *testing.T) {
|
|||||||
if len(msg.ImageURLs) != 1 || !strings.HasPrefix(msg.ImageURLs[0], "data:image/png;base64,") {
|
if len(msg.ImageURLs) != 1 || !strings.HasPrefix(msg.ImageURLs[0], "data:image/png;base64,") {
|
||||||
t.Fatalf("ImageURLs = %#v", msg.ImageURLs)
|
t.Fatalf("ImageURLs = %#v", msg.ImageURLs)
|
||||||
}
|
}
|
||||||
|
if len(msg.ImageReferences) != 1 {
|
||||||
|
t.Fatalf("ImageReferences = %#v", msg.ImageReferences)
|
||||||
|
}
|
||||||
|
if got := msg.ImageReferences[0]; got.URL != "dingtalk://message/picture-message-1" || got.Caption != "question.png" {
|
||||||
|
t.Fatalf("image reference = %#v", got)
|
||||||
|
}
|
||||||
if msg.Content == "" {
|
if msg.Content == "" {
|
||||||
t.Fatal("image-only message should receive a default visual question")
|
t.Fatal("image-only message should receive a default visual question")
|
||||||
}
|
}
|
||||||
@@ -58,6 +48,15 @@ func TestPrepareIMImageForQAAndBuildRequest(t *testing.T) {
|
|||||||
t.Fatalf("request ImageURLs = %#v", req.ImageURLs)
|
t.Fatalf("request ImageURLs = %#v", req.ImageURLs)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
audit := newIMMessageAuditContext(msg, "channel-1", nil)
|
||||||
|
userMessage := createIMUserMessagePayload("session-1", msg.Content, "request-1", audit)
|
||||||
|
assistantMessage := createIMAssistantMessagePayload("session-1", "request-1", audit)
|
||||||
|
if len(userMessage.Images) != 1 || userMessage.Images[0].URL != "dingtalk://message/picture-message-1" {
|
||||||
|
t.Fatalf("user message Images = %#v", userMessage.Images)
|
||||||
|
}
|
||||||
|
if len(assistantMessage.Images) != 0 {
|
||||||
|
t.Fatalf("assistant message should not duplicate input image: %#v", assistantMessage.Images)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPrepareIMImageForQARejectsInvalidOrOversizedPayload(t *testing.T) {
|
func TestPrepareIMImageForQARejectsInvalidOrOversizedPayload(t *testing.T) {
|
||||||
@@ -81,52 +80,4 @@ func TestPrepareIMImageForQARejectsInvalidOrOversizedPayload(t *testing.T) {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
t.Run("dimensions", func(t *testing.T) {
|
|
||||||
msg := &IncomingMessage{MessageType: MessageTypeImage}
|
|
||||||
err := prepareIMImageForQA(context.Background(), msg, staticImageDownloader{body: testPNG(t, maxIMImageDimension+1, 1), fileName: "wide.png"})
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "dimensions") {
|
|
||||||
t.Fatalf("error = %v, want dimensions rejection", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("animated formats rejected", func(t *testing.T) {
|
|
||||||
msg := &IncomingMessage{MessageType: MessageTypeImage}
|
|
||||||
err := prepareIMImageForQA(context.Background(), msg, staticImageDownloader{body: "GIF89a" + strings.Repeat("x", 600), fileName: "animated.gif"})
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "only JPEG and PNG") {
|
|
||||||
t.Fatalf("error = %v, want JPEG/PNG rejection", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIMQAErrorReplyDistinguishesImageFailures(t *testing.T) {
|
|
||||||
imageReply := imQAErrorReply([]string{"data:image/png;base64,abc"})
|
|
||||||
if !strings.Contains(imageReply, "图片识别失败") || !strings.Contains(imageReply, "视觉模型") {
|
|
||||||
t.Fatalf("image error reply = %q", imageReply)
|
|
||||||
}
|
|
||||||
if got := imQAErrorReply(nil); got != "抱歉,处理您的问题时出现了异常,请稍后再试。" {
|
|
||||||
t.Fatalf("text error reply = %q", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestImageStreamDoesNotPublishPartialContent(t *testing.T) {
|
|
||||||
if allowIncrementalIMFlush(&IncomingMessage{ImageURLs: []string{"data:image/png;base64,YWJj"}}) {
|
|
||||||
t.Fatal("image streams must buffer until successful completion")
|
|
||||||
}
|
|
||||||
if !allowIncrementalIMFlush(&IncomingMessage{Content: "hello"}) {
|
|
||||||
t.Fatal("text streams must preserve incremental updates")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFailClosedIMImageAnswerDiscardsPartialOutput(t *testing.T) {
|
|
||||||
partial := "I think the image might show"
|
|
||||||
err := errors.New("multimodal request rejected")
|
|
||||||
got, overridden := failClosedIMImageAnswer(partial, err, []string{"data:image/png;base64,abc"})
|
|
||||||
if !overridden || strings.Contains(got, partial) || !strings.Contains(got, "图片识别失败") {
|
|
||||||
t.Fatalf("image override = %q overridden=%v", got, overridden)
|
|
||||||
}
|
|
||||||
got, overridden = failClosedIMImageAnswer(partial, err, nil)
|
|
||||||
if overridden || got != partial {
|
|
||||||
t.Fatalf("text answer changed = %q overridden=%v", got, overridden)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,93 @@
|
|||||||
|
package im
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/Tencent/WeKnora/internal/types"
|
||||||
|
"github.com/gin-gonic/gin"
|
||||||
|
)
|
||||||
|
|
||||||
|
type recordingReplyAdapter struct {
|
||||||
|
incoming *IncomingMessage
|
||||||
|
reply *ReplyMessage
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *recordingReplyAdapter) Platform() Platform { return PlatformDingtalk }
|
||||||
|
func (a *recordingReplyAdapter) VerifyCallback(*gin.Context) error { return nil }
|
||||||
|
func (a *recordingReplyAdapter) ParseCallback(*gin.Context) (*IncomingMessage, error) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
func (a *recordingReplyAdapter) SendReply(_ context.Context, incoming *IncomingMessage, reply *ReplyMessage) error {
|
||||||
|
a.incoming = incoming
|
||||||
|
a.reply = reply
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func (a *recordingReplyAdapter) HandleURLVerification(*gin.Context) bool { return false }
|
||||||
|
|
||||||
|
func TestSendReplyToSource(t *testing.T) {
|
||||||
|
adapter := &recordingReplyAdapter{}
|
||||||
|
service := &Service{channels: map[string]*channelState{
|
||||||
|
"channel-1": {
|
||||||
|
Adapter: adapter,
|
||||||
|
Channel: &IMChannel{ID: "channel-1", Platform: "dingtalk", Enabled: true},
|
||||||
|
},
|
||||||
|
}}
|
||||||
|
|
||||||
|
err := service.SendReplyToSource(context.Background(), ReplyTarget{
|
||||||
|
ChannelID: "channel-1",
|
||||||
|
UserID: "staff-1",
|
||||||
|
ChatID: "conversation-1",
|
||||||
|
ChatType: ChatTypeGroup,
|
||||||
|
}, "技术支持回复内容")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("SendReplyToSource: %v", err)
|
||||||
|
}
|
||||||
|
if adapter.incoming == nil || adapter.incoming.UserID != "staff-1" || adapter.incoming.ChatID != "conversation-1" || adapter.incoming.ChatType != ChatTypeGroup {
|
||||||
|
t.Fatalf("incoming = %#v", adapter.incoming)
|
||||||
|
}
|
||||||
|
if adapter.reply == nil || adapter.reply.Content != "技术支持回复内容" || !adapter.reply.IsFinal {
|
||||||
|
t.Fatalf("reply = %#v", adapter.reply)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSendReplyToSourceRejectsUnavailableChannel(t *testing.T) {
|
||||||
|
service := &Service{channels: map[string]*channelState{}}
|
||||||
|
if err := service.SendReplyToSource(context.Background(), ReplyTarget{ChannelID: "missing"}, "reply"); err == nil {
|
||||||
|
t.Fatal("expected missing channel error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSendSupportNotificationUsesConfiguredGroup(t *testing.T) {
|
||||||
|
adapter := &recordingReplyAdapter{}
|
||||||
|
service := &Service{channels: map[string]*channelState{
|
||||||
|
"channel-1": {
|
||||||
|
Adapter: adapter,
|
||||||
|
Channel: &IMChannel{
|
||||||
|
ID: "channel-1",
|
||||||
|
Platform: "dingtalk",
|
||||||
|
Enabled: true,
|
||||||
|
Credentials: types.JSON(`{"support_group_open_conversation_id":"support-group-1"}`),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}}
|
||||||
|
|
||||||
|
if err := service.SendSupportNotification(context.Background(), "channel-1", "新工单:客户未解决"); err != nil {
|
||||||
|
t.Fatalf("SendSupportNotification: %v", err)
|
||||||
|
}
|
||||||
|
if adapter.incoming == nil || adapter.incoming.ChatType != ChatTypeGroup || adapter.incoming.ChatID != "support-group-1" {
|
||||||
|
t.Fatalf("support target = %#v", adapter.incoming)
|
||||||
|
}
|
||||||
|
if adapter.reply == nil || adapter.reply.Content != "新工单:客户未解决" {
|
||||||
|
t.Fatalf("support reply = %#v", adapter.reply)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSendSupportNotificationRequiresConfiguredGroup(t *testing.T) {
|
||||||
|
service := &Service{channels: map[string]*channelState{
|
||||||
|
"channel-1": {Adapter: &recordingReplyAdapter{}, Channel: &IMChannel{ID: "channel-1", Platform: "dingtalk", Enabled: true}},
|
||||||
|
}}
|
||||||
|
if err := service.SendSupportNotification(context.Background(), "channel-1", "ticket"); err == nil {
|
||||||
|
t.Fatal("expected missing support group configuration error")
|
||||||
|
}
|
||||||
|
}
|
||||||
+200
-116
@@ -6,13 +6,11 @@ import (
|
|||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"image"
|
|
||||||
_ "image/jpeg"
|
|
||||||
_ "image/png"
|
|
||||||
"io"
|
"io"
|
||||||
"mime/multipart"
|
"mime/multipart"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/textproto"
|
"net/textproto"
|
||||||
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -512,11 +510,7 @@ func buildIMQARequest(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const (
|
const maxIMImageBytes = 10 << 20
|
||||||
maxIMImageBytes = 10 << 20
|
|
||||||
maxIMImageDimension = 16_384
|
|
||||||
maxIMImagePixels = 40_000_000
|
|
||||||
)
|
|
||||||
|
|
||||||
func prepareIMImageForQA(ctx context.Context, msg *IncomingMessage, downloader FileDownloader) error {
|
func prepareIMImageForQA(ctx context.Context, msg *IncomingMessage, downloader FileDownloader) error {
|
||||||
if msg == nil || downloader == nil {
|
if msg == nil || downloader == nil {
|
||||||
@@ -540,44 +534,32 @@ func prepareIMImageForQA(ctx context.Context, msg *IncomingMessage, downloader F
|
|||||||
}
|
}
|
||||||
contentType := strings.ToLower(http.DetectContentType(data))
|
contentType := strings.ToLower(http.DetectContentType(data))
|
||||||
switch contentType {
|
switch contentType {
|
||||||
case "image/jpeg", "image/png":
|
case "image/jpeg", "image/png", "image/gif", "image/webp":
|
||||||
case "image/gif", "image/webp":
|
|
||||||
return fmt.Errorf("only JPEG and PNG images are supported")
|
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("unsupported image content type %q", contentType)
|
return fmt.Errorf("unsupported image content type %q", contentType)
|
||||||
}
|
}
|
||||||
config, _, err := image.DecodeConfig(bytes.NewReader(data))
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("decode image metadata: %w", err)
|
|
||||||
}
|
|
||||||
if config.Width <= 0 || config.Height <= 0 || config.Width > maxIMImageDimension || config.Height > maxIMImageDimension || int64(config.Width)*int64(config.Height) > maxIMImagePixels {
|
|
||||||
return fmt.Errorf("image dimensions %dx%d exceed safety limits", config.Width, config.Height)
|
|
||||||
}
|
|
||||||
|
|
||||||
msg.FileName = fileName
|
msg.FileName = fileName
|
||||||
msg.ImageURLs = []string{"data:" + contentType + ";base64," + base64.StdEncoding.EncodeToString(data)}
|
msg.ImageURLs = []string{"data:" + contentType + ";base64," + base64.StdEncoding.EncodeToString(data)}
|
||||||
|
referenceID := strings.TrimSpace(msg.MessageID)
|
||||||
|
if referenceID == "" {
|
||||||
|
referenceID = "unknown"
|
||||||
|
}
|
||||||
|
msg.ImageReferences = types.MessageImages{{
|
||||||
|
URL: string(msg.Platform) + "://message/" + url.PathEscape(referenceID),
|
||||||
|
Caption: fileName,
|
||||||
|
}}
|
||||||
if strings.TrimSpace(msg.Content) == "" {
|
if strings.TrimSpace(msg.Content) == "" {
|
||||||
msg.Content = "请识别这张图片的内容,并回答图片中的问题。"
|
msg.Content = "请识别这张图片的内容,并回答图片中的问题。"
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func imQAErrorReply(imageURLs []string) string {
|
func auditImageURLs(audit *imMessageAuditContext) []string {
|
||||||
if len(imageURLs) > 0 {
|
if audit == nil {
|
||||||
return "图片识别失败:当前模型可能不支持视觉输入,请检查智能体的图片上传开关和视觉模型配置。"
|
return nil
|
||||||
}
|
}
|
||||||
return "抱歉,处理您的问题时出现了异常,请稍后再试。"
|
return append([]string(nil), audit.imageURLs...)
|
||||||
}
|
|
||||||
|
|
||||||
func allowIncrementalIMFlush(msg *IncomingMessage) bool {
|
|
||||||
return msg == nil || len(msg.ImageURLs) == 0
|
|
||||||
}
|
|
||||||
|
|
||||||
func failClosedIMImageAnswer(answer string, err error, imageURLs []string) (string, bool) {
|
|
||||||
if err != nil && len(imageURLs) > 0 {
|
|
||||||
return imQAErrorReply(imageURLs), true
|
|
||||||
}
|
|
||||||
return answer, false
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildIMLastRequestState(agentID string, customAgent *types.CustomAgent, kbIDs []string) *types.SessionLastRequestState {
|
func buildIMLastRequestState(agentID string, customAgent *types.CustomAgent, kbIDs []string) *types.SessionLastRequestState {
|
||||||
@@ -600,8 +582,56 @@ func buildIMLastRequestState(agentID string, customAgent *types.CustomAgent, kbI
|
|||||||
return state
|
return state
|
||||||
}
|
}
|
||||||
|
|
||||||
func createIMUserMessagePayload(sessionID, content, requestID string) *types.Message {
|
type imMessageAuditContext struct {
|
||||||
return &types.Message{
|
platform string
|
||||||
|
channelID string
|
||||||
|
messageID string
|
||||||
|
deliveryID string
|
||||||
|
userID string
|
||||||
|
chatID string
|
||||||
|
threadID string
|
||||||
|
knowledgeBaseIDs types.MessageKnowledgeBaseIDs
|
||||||
|
imageReferences types.MessageImages
|
||||||
|
imageURLs []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func newIMMessageAuditContext(msg *IncomingMessage, channelID string, kbIDs []string) *imMessageAuditContext {
|
||||||
|
audit := &imMessageAuditContext{
|
||||||
|
channelID: channelID,
|
||||||
|
knowledgeBaseIDs: append(types.MessageKnowledgeBaseIDs(nil), kbIDs...),
|
||||||
|
}
|
||||||
|
if msg == nil {
|
||||||
|
return audit
|
||||||
|
}
|
||||||
|
audit.platform = string(msg.Platform)
|
||||||
|
audit.messageID = msg.MessageID
|
||||||
|
if msg.Extra != nil {
|
||||||
|
audit.deliveryID = msg.Extra["dingtalk_out_track_id"]
|
||||||
|
}
|
||||||
|
audit.userID = msg.UserID
|
||||||
|
audit.chatID = msg.ChatID
|
||||||
|
audit.threadID = msg.ThreadID
|
||||||
|
audit.imageReferences = append(types.MessageImages(nil), msg.ImageReferences...)
|
||||||
|
audit.imageURLs = append([]string(nil), msg.ImageURLs...)
|
||||||
|
return audit
|
||||||
|
}
|
||||||
|
|
||||||
|
func applyIMMessageAudit(message *types.Message, audit *imMessageAuditContext) {
|
||||||
|
if message == nil || audit == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
message.SourcePlatform = audit.platform
|
||||||
|
message.SourceChannelID = audit.channelID
|
||||||
|
message.SourceMessageID = audit.messageID
|
||||||
|
message.SourceDeliveryID = audit.deliveryID
|
||||||
|
message.SourceUserID = audit.userID
|
||||||
|
message.SourceChatID = audit.chatID
|
||||||
|
message.SourceThreadID = audit.threadID
|
||||||
|
message.SearchedKnowledgeBaseIDs = append(types.MessageKnowledgeBaseIDs(nil), audit.knowledgeBaseIDs...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func createIMUserMessagePayload(sessionID, content, requestID string, audit *imMessageAuditContext) *types.Message {
|
||||||
|
message := &types.Message{
|
||||||
SessionID: sessionID,
|
SessionID: sessionID,
|
||||||
Role: "user",
|
Role: "user",
|
||||||
Content: content,
|
Content: content,
|
||||||
@@ -610,10 +640,15 @@ func createIMUserMessagePayload(sessionID, content, requestID string) *types.Mes
|
|||||||
IsCompleted: true,
|
IsCompleted: true,
|
||||||
Channel: "im",
|
Channel: "im",
|
||||||
}
|
}
|
||||||
|
applyIMMessageAudit(message, audit)
|
||||||
|
if audit != nil {
|
||||||
|
message.Images = append(types.MessageImages(nil), audit.imageReferences...)
|
||||||
|
}
|
||||||
|
return message
|
||||||
}
|
}
|
||||||
|
|
||||||
func createIMAssistantMessagePayload(sessionID, requestID string) *types.Message {
|
func createIMAssistantMessagePayload(sessionID, requestID string, audit *imMessageAuditContext) *types.Message {
|
||||||
return &types.Message{
|
message := &types.Message{
|
||||||
SessionID: sessionID,
|
SessionID: sessionID,
|
||||||
Role: "assistant",
|
Role: "assistant",
|
||||||
RequestID: requestID,
|
RequestID: requestID,
|
||||||
@@ -621,6 +656,8 @@ func createIMAssistantMessagePayload(sessionID, requestID string) *types.Message
|
|||||||
IsCompleted: false,
|
IsCompleted: false,
|
||||||
Channel: "im",
|
Channel: "im",
|
||||||
}
|
}
|
||||||
|
applyIMMessageAudit(message, audit)
|
||||||
|
return message
|
||||||
}
|
}
|
||||||
|
|
||||||
func collectIMKnowledgeReferences(dst *[]*types.SearchResult, refs interface{}) {
|
func collectIMKnowledgeReferences(dst *[]*types.SearchResult, refs interface{}) {
|
||||||
@@ -656,6 +693,10 @@ func applyIMCompleteDataToMessage(msg *types.Message, data event.AgentCompleteDa
|
|||||||
}
|
}
|
||||||
msg.IsCompleted = true
|
msg.IsCompleted = true
|
||||||
msg.AgentDurationMs = data.TotalDurationMs
|
msg.AgentDurationMs = data.TotalDurationMs
|
||||||
|
if data.TokenUsage != nil {
|
||||||
|
usage := *data.TokenUsage
|
||||||
|
msg.TokenUsage = &usage
|
||||||
|
}
|
||||||
if len(data.KnowledgeRefs) > 0 {
|
if len(data.KnowledgeRefs) > 0 {
|
||||||
refs := make([]*types.SearchResult, 0, len(data.KnowledgeRefs))
|
refs := make([]*types.SearchResult, 0, len(data.KnowledgeRefs))
|
||||||
collectIMKnowledgeReferences(&refs, data.KnowledgeRefs)
|
collectIMKnowledgeReferences(&refs, data.KnowledgeRefs)
|
||||||
@@ -668,6 +709,14 @@ func applyIMCompleteDataToMessage(msg *types.Message, data event.AgentCompleteDa
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func applyIMFinalAnswerDataToMessage(msg *types.Message, data event.AgentFinalAnswerData) {
|
||||||
|
if msg == nil || data.TokenUsage == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
usage := *data.TokenUsage
|
||||||
|
msg.TokenUsage = &usage
|
||||||
|
}
|
||||||
|
|
||||||
// waitForIMAgentComplete blocks until EventAgentComplete, ctx cancellation, or timeout.
|
// waitForIMAgentComplete blocks until EventAgentComplete, ctx cancellation, or timeout.
|
||||||
func waitForIMAgentComplete(ctx context.Context, completeDone <-chan struct{}, sessionID string) {
|
func waitForIMAgentComplete(ctx context.Context, completeDone <-chan struct{}, sessionID string) {
|
||||||
timer := time.NewTimer(agentCompleteWaitTimeout)
|
timer := time.NewTimer(agentCompleteWaitTimeout)
|
||||||
@@ -1204,6 +1253,70 @@ func (s *Service) GetChannelAdapter(channelID string) (Adapter, *IMChannel, bool
|
|||||||
return cs.Adapter, cs.Channel, true
|
return cs.Adapter, cs.Channel, true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ReplyTarget identifies the original customer conversation for a proactive reply.
|
||||||
|
type ReplyTarget struct {
|
||||||
|
ChannelID string
|
||||||
|
UserID string
|
||||||
|
ChatID string
|
||||||
|
ChatType ChatType
|
||||||
|
ThreadID string
|
||||||
|
}
|
||||||
|
|
||||||
|
// SendReplyToSource delivers a support reply through the configured platform adapter.
|
||||||
|
// It intentionally does not depend on a short-lived session webhook; adapters can use
|
||||||
|
// their authenticated proactive-message API when Extra is empty.
|
||||||
|
func (s *Service) SendReplyToSource(ctx context.Context, target ReplyTarget, content string) error {
|
||||||
|
if strings.TrimSpace(target.ChannelID) == "" {
|
||||||
|
return fmt.Errorf("channel ID is required")
|
||||||
|
}
|
||||||
|
content = strings.TrimSpace(content)
|
||||||
|
if content == "" {
|
||||||
|
return fmt.Errorf("reply content is required")
|
||||||
|
}
|
||||||
|
adapter, channel, ok := s.GetChannelAdapter(target.ChannelID)
|
||||||
|
if !ok || adapter == nil || channel == nil {
|
||||||
|
return fmt.Errorf("IM channel %s is not running", target.ChannelID)
|
||||||
|
}
|
||||||
|
if !channel.Enabled {
|
||||||
|
return fmt.Errorf("IM channel %s is disabled", target.ChannelID)
|
||||||
|
}
|
||||||
|
chatType := target.ChatType
|
||||||
|
if chatType == "" {
|
||||||
|
chatType = ChatTypeDirect
|
||||||
|
}
|
||||||
|
incoming := &IncomingMessage{
|
||||||
|
Platform: Platform(channel.Platform),
|
||||||
|
UserID: strings.TrimSpace(target.UserID),
|
||||||
|
ChatID: strings.TrimSpace(target.ChatID),
|
||||||
|
ChatType: chatType,
|
||||||
|
ThreadID: strings.TrimSpace(target.ThreadID),
|
||||||
|
MessageID: "support:" + uuid.New().String(),
|
||||||
|
}
|
||||||
|
return adapter.SendReply(ctx, incoming, &ReplyMessage{Content: content, IsFinal: true})
|
||||||
|
}
|
||||||
|
|
||||||
|
// SendSupportNotification pushes a newly escalated ticket to the support group
|
||||||
|
// configured in the source channel credentials. The group identifier is never logged.
|
||||||
|
func (s *Service) SendSupportNotification(ctx context.Context, channelID, content string) error {
|
||||||
|
_, channel, ok := s.GetChannelAdapter(channelID)
|
||||||
|
if !ok || channel == nil {
|
||||||
|
return fmt.Errorf("IM channel %s is not running", channelID)
|
||||||
|
}
|
||||||
|
credentials, err := ParseCredentials(channel.Credentials)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("parse support group configuration: %w", err)
|
||||||
|
}
|
||||||
|
supportGroupID := strings.TrimSpace(GetString(credentials, "support_group_open_conversation_id"))
|
||||||
|
if supportGroupID == "" {
|
||||||
|
return fmt.Errorf("support group is not configured for IM channel %s", channelID)
|
||||||
|
}
|
||||||
|
return s.SendReplyToSource(ctx, ReplyTarget{
|
||||||
|
ChannelID: channelID,
|
||||||
|
ChatID: supportGroupID,
|
||||||
|
ChatType: ChatTypeGroup,
|
||||||
|
}, content)
|
||||||
|
}
|
||||||
|
|
||||||
// GetChannelByID loads a channel from the database.
|
// GetChannelByID loads a channel from the database.
|
||||||
func (s *Service) GetChannelByID(channelID string) (*IMChannel, error) {
|
func (s *Service) GetChannelByID(channelID string) (*IMChannel, error) {
|
||||||
var ch IMChannel
|
var ch IMChannel
|
||||||
@@ -1311,16 +1424,20 @@ func (s *Service) HandleMessage(ctx context.Context, msg *IncomingMessage, chann
|
|||||||
|
|
||||||
logger.Infof(ctx, "[IM] HandleMessage: channel=%s platform=%s user=%s chat=%s msgtype=%s content_len=%d",
|
logger.Infof(ctx, "[IM] HandleMessage: channel=%s platform=%s user=%s chat=%s msgtype=%s content_len=%d",
|
||||||
channelID, msg.Platform, msg.UserID, msg.ChatID, msg.MessageType, len(msg.Content))
|
channelID, msg.Platform, msg.UserID, msg.ChatID, msg.MessageType, len(msg.Content))
|
||||||
logger.Debugf(ctx, "[IM] HandleMessage detail: msgid=%s has_filekey=%t filename=%s",
|
logger.Debugf(ctx, "[IM] HandleMessage detail: msgid=%s filekey=%s filename=%s",
|
||||||
msg.MessageID, strings.TrimSpace(msg.FileKey) != "", msg.FileName)
|
msg.MessageID, msg.FileKey, msg.FileName)
|
||||||
|
|
||||||
// Files keep the legacy knowledge-base ingestion path. Images are routed
|
// ── File/Image message shortcut ──
|
||||||
// below either to multimodal QA or, when vision is disabled, to this path.
|
// If the message is a file or image and the channel has a knowledge_base_id configured,
|
||||||
|
// handle it separately without entering the QA pipeline.
|
||||||
if msg.MessageType == MessageTypeFile && channel.KnowledgeBaseID != "" {
|
if msg.MessageType == MessageTypeFile && channel.KnowledgeBaseID != "" {
|
||||||
return s.handleFileMessage(ctx, msg, adapter, channel)
|
return s.handleFileMessage(ctx, msg, adapter, channel)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Non-text files without a configured knowledge base still cannot enter QA.
|
// ── Non-text message without text content ──
|
||||||
|
// If the message is an image/file/video but has no text content, the QA pipeline
|
||||||
|
// cannot do anything useful (no vision support in IM yet). Sending an empty query
|
||||||
|
// to KB retrieval would return irrelevant results and cause hallucination.
|
||||||
if msg.Content == "" && msg.MessageType == MessageTypeFile {
|
if msg.Content == "" && msg.MessageType == MessageTypeFile {
|
||||||
logger.Infof(ctx, "[IM] Skipping QA for non-text message without content: type=%s", msg.MessageType)
|
logger.Infof(ctx, "[IM] Skipping QA for non-text message without content: type=%s", msg.MessageType)
|
||||||
if err := adapter.SendReply(ctx, msg, &ReplyMessage{
|
if err := adapter.SendReply(ctx, msg, &ReplyMessage{
|
||||||
@@ -1359,19 +1476,21 @@ func (s *Service) HandleMessage(ctx context.Context, msg *IncomingMessage, chann
|
|||||||
|
|
||||||
if msg.MessageType == MessageTypeImage {
|
if msg.MessageType == MessageTypeImage {
|
||||||
if customAgent == nil || !customAgent.Config.ImageUploadEnabled {
|
if customAgent == nil || !customAgent.Config.ImageUploadEnabled {
|
||||||
|
if channel.KnowledgeBaseID != "" {
|
||||||
|
return s.handleFileMessage(ctx, msg, adapter, channel)
|
||||||
|
}
|
||||||
return adapter.SendReply(ctx, msg, &ReplyMessage{
|
return adapter.SendReply(ctx, msg, &ReplyMessage{
|
||||||
Content: "当前机器人未启用图片识别,请在智能体设置中开启图片上传并配置视觉模型。",
|
Content: "当前机器人未启用图片识别,请在智能体设置中开启图片上传并配置视觉模型。",
|
||||||
IsFinal: true,
|
IsFinal: true,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(msg.FileKey) == "" {
|
downloader, ok := adapter.(FileDownloader)
|
||||||
return adapter.SendReply(ctx, msg, &ReplyMessage{Content: "图片消息缺少有效下载凭证,请重新发送图片。", IsFinal: true})
|
if !ok {
|
||||||
}
|
|
||||||
if _, ok := adapter.(FileDownloader); !ok {
|
|
||||||
return adapter.SendReply(ctx, msg, &ReplyMessage{Content: "当前钉钉渠道暂不支持图片下载。", IsFinal: true})
|
return adapter.SendReply(ctx, msg, &ReplyMessage{Content: "当前钉钉渠道暂不支持图片下载。", IsFinal: true})
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(msg.Content) == "" {
|
if err := prepareIMImageForQA(sessionCtx, msg, downloader); err != nil {
|
||||||
msg.Content = "请识别这张图片的内容,并回答图片中的问题。"
|
logger.Warnf(ctx, "[IM] Failed to prepare image for QA: %v", err)
|
||||||
|
return adapter.SendReply(ctx, msg, &ReplyMessage{Content: "图片下载或格式校验失败,请发送10MB以内的 JPG、PNG、GIF 或 WebP 图片。", IsFinal: true})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1508,23 +1627,13 @@ func (s *Service) executeQARequest(req *qaRequest) {
|
|||||||
// runQA after the assistant message is created (that's when we have the
|
// runQA after the assistant message is created (that's when we have the
|
||||||
// sessionID + messageID needed to poll StreamManager).
|
// sessionID + messageID needed to poll StreamManager).
|
||||||
|
|
||||||
// Keep large image bytes out of the bounded queue. Download, validate and
|
|
||||||
// encode only after this worker owns a concurrency slot.
|
|
||||||
if req.msg.MessageType == MessageTypeImage && len(req.msg.ImageURLs) == 0 {
|
|
||||||
downloader, ok := req.adapter.(FileDownloader)
|
|
||||||
if !ok {
|
|
||||||
_ = req.adapter.SendReply(ctx, req.msg, &ReplyMessage{Content: "当前钉钉渠道暂不支持图片下载。", IsFinal: true})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err := prepareIMImageForQA(ctx, req.msg, downloader); err != nil {
|
|
||||||
logger.Warnf(ctx, "[IM] Failed to prepare image for QA: %v", err)
|
|
||||||
_ = req.adapter.SendReply(ctx, req.msg, &ReplyMessage{Content: "图片下载或格式校验失败,请发送10MB以内的 JPG 或 PNG 图片。", IsFinal: true})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// kbIDs is left empty so the QA pipeline resolves them from the agent config.
|
// kbIDs is left empty so the QA pipeline resolves them from the agent config.
|
||||||
var kbIDs []string
|
var kbIDs []string
|
||||||
|
auditedKBIDs := kbIDs
|
||||||
|
if len(auditedKBIDs) == 0 && req.agent != nil {
|
||||||
|
auditedKBIDs = req.agent.Config.KnowledgeBases
|
||||||
|
}
|
||||||
|
audit := newIMMessageAuditContext(req.msg, req.channelID, auditedKBIDs)
|
||||||
|
|
||||||
// Determine output mode from channel config.
|
// Determine output mode from channel config.
|
||||||
streamDisabled := req.channel.OutputMode == "full"
|
streamDisabled := req.channel.OutputMode == "full"
|
||||||
@@ -1532,7 +1641,7 @@ func (s *Service) executeQARequest(req *qaRequest) {
|
|||||||
// If the adapter supports streaming and output is not "full", use streaming.
|
// If the adapter supports streaming and output is not "full", use streaming.
|
||||||
if !streamDisabled {
|
if !streamDisabled {
|
||||||
if streamer, ok := req.adapter.(StreamSender); ok {
|
if streamer, ok := req.adapter.(StreamSender); ok {
|
||||||
if err := s.handleMessageStream(ctx, req.msg, req.session, req.agent, kbIDs, streamer, req.adapter, req.userKey, req.tenant); err != nil {
|
if err := s.handleMessageStream(ctx, req.msg, req.session, req.agent, kbIDs, streamer, req.adapter, req.userKey, req.tenant, audit); err != nil {
|
||||||
logger.Errorf(ctx, "[IM] Stream QA failed: %v", err)
|
logger.Errorf(ctx, "[IM] Stream QA failed: %v", err)
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
@@ -1540,10 +1649,10 @@ func (s *Service) executeQARequest(req *qaRequest) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Non-streaming fallback: collect full answer then send.
|
// Non-streaming fallback: collect full answer then send.
|
||||||
answer, err := s.runQA(ctx, req.session, req.msg.Content, req.agent, kbIDs, req.userKey, req.msg.Quote, req.msg.ImageURLs)
|
answer, err := s.runQA(ctx, req.session, req.msg.Content, req.agent, kbIDs, req.userKey, req.msg.Quote, audit)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Errorf(ctx, "[IM] QA failed: %v, sending fallback reply", err)
|
logger.Errorf(ctx, "[IM] QA failed: %v, sending fallback reply", err)
|
||||||
answer = imQAErrorReply(req.msg.ImageURLs)
|
answer = "抱歉,处理您的问题时出现了异常,请稍后再试。"
|
||||||
}
|
}
|
||||||
|
|
||||||
reply := &ReplyMessage{
|
reply := &ReplyMessage{
|
||||||
@@ -1963,12 +2072,12 @@ func briefToolSummary(output string) string {
|
|||||||
// handleMessageStream runs the QA pipeline and streams answer chunks to the IM platform
|
// handleMessageStream runs the QA pipeline and streams answer chunks to the IM platform
|
||||||
// in real-time via the StreamSender interface. Chunks are batched at streamFlushInterval
|
// in real-time via the StreamSender interface. Chunks are batched at streamFlushInterval
|
||||||
// to avoid API rate-limiting.
|
// to avoid API rate-limiting.
|
||||||
func (s *Service) handleMessageStream(ctx context.Context, msg *IncomingMessage, session *types.Session, customAgent *types.CustomAgent, kbIDs []string, streamer StreamSender, adapter Adapter, userKey string, tenant *types.Tenant) error {
|
func (s *Service) handleMessageStream(ctx context.Context, msg *IncomingMessage, session *types.Session, customAgent *types.CustomAgent, kbIDs []string, streamer StreamSender, adapter Adapter, userKey string, tenant *types.Tenant, audit *imMessageAuditContext) error {
|
||||||
// Start the stream on the IM platform (e.g., create Feishu streaming card)
|
// Start the stream on the IM platform (e.g., create Feishu streaming card)
|
||||||
streamID, err := streamer.StartStream(ctx, msg)
|
streamID, err := streamer.StartStream(ctx, msg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Warnf(ctx, "[IM] StartStream failed, falling back to non-streaming: %v", err)
|
logger.Warnf(ctx, "[IM] StartStream failed, falling back to non-streaming: %v", err)
|
||||||
return s.fallbackNonStream(ctx, msg, session, customAgent, kbIDs, adapter, userKey, tenant)
|
return s.fallbackNonStream(ctx, msg, session, customAgent, kbIDs, adapter, userKey, tenant, audit)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Prepare the QA pipeline
|
// Prepare the QA pipeline
|
||||||
@@ -2059,6 +2168,7 @@ func (s *Service) handleMessageStream(ctx context.Context, msg *IncomingMessage,
|
|||||||
}
|
}
|
||||||
|
|
||||||
bufMu.Lock()
|
bufMu.Lock()
|
||||||
|
applyIMFinalAnswerDataToMessage(assistantMsg, data)
|
||||||
if useAgent && !agentDone {
|
if useAgent && !agentDone {
|
||||||
if data.Content != "" {
|
if data.Content != "" {
|
||||||
agentLiveAnswer.WriteString(data.Content)
|
agentLiveAnswer.WriteString(data.Content)
|
||||||
@@ -2212,13 +2322,13 @@ func (s *Service) handleMessageStream(ctx context.Context, msg *IncomingMessage,
|
|||||||
requestID := uuid.New().String()
|
requestID := uuid.New().String()
|
||||||
|
|
||||||
// Create user message
|
// Create user message
|
||||||
userMsg, err := s.messageService.CreateMessage(qaCtx, createIMUserMessagePayload(session.ID, msg.Content, requestID))
|
userMsg, err := s.messageService.CreateMessage(qaCtx, createIMUserMessagePayload(session.ID, msg.Content, requestID, audit))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("create user message: %w", err)
|
return fmt.Errorf("create user message: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create placeholder assistant message
|
// Create placeholder assistant message
|
||||||
assistantMsg, err = s.messageService.CreateMessage(qaCtx, createIMAssistantMessagePayload(session.ID, requestID))
|
assistantMsg, err = s.messageService.CreateMessage(qaCtx, createIMAssistantMessagePayload(session.ID, requestID, audit))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("create assistant message: %w", err)
|
return fmt.Errorf("create assistant message: %w", err)
|
||||||
}
|
}
|
||||||
@@ -2266,9 +2376,6 @@ func (s *Service) handleMessageStream(ctx context.Context, msg *IncomingMessage,
|
|||||||
defer ticker.Stop()
|
defer ticker.Stop()
|
||||||
|
|
||||||
flush := func() {
|
flush := func() {
|
||||||
if !allowIncrementalIMFlush(msg) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
bufMu.Lock()
|
bufMu.Lock()
|
||||||
parts := getStreamParts()
|
parts := getStreamParts()
|
||||||
agentRunning := useAgent && !agentDone
|
agentRunning := useAgent && !agentDone
|
||||||
@@ -2322,13 +2429,10 @@ loop:
|
|||||||
bufMu.Unlock()
|
bufMu.Unlock()
|
||||||
|
|
||||||
finalDisplay := cleanIMContent(ctx, FormatIMFinalFromParts(parts), tenant, s.defaultFileSvc)
|
finalDisplay := cleanIMContent(ctx, FormatIMFinalFromParts(parts), tenant, s.defaultFileSvc)
|
||||||
if replacement, override := failClosedIMImageAnswer(answer, finalErr, msg.ImageURLs); override {
|
if noVisibleContent || finalDisplay == "" {
|
||||||
answer = replacement
|
|
||||||
finalDisplay = replacement
|
|
||||||
} else if noVisibleContent || finalDisplay == "" {
|
|
||||||
fallback := "抱歉,我暂时无法回答这个问题。"
|
fallback := "抱歉,我暂时无法回答这个问题。"
|
||||||
if finalErr != nil {
|
if finalErr != nil {
|
||||||
fallback = imQAErrorReply(msg.ImageURLs)
|
fallback = "抱歉,处理您的问题时出现了异常,请稍后再试。"
|
||||||
}
|
}
|
||||||
finalDisplay = fallback
|
finalDisplay = fallback
|
||||||
if answer == "" {
|
if answer == "" {
|
||||||
@@ -2336,7 +2440,14 @@ loop:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
deliveryErr := finalizeAndEndIMStream(ctx, streamer, msg, streamID, finalDisplay)
|
if err := streamer.FinalizeStream(ctx, msg, streamID, finalDisplay); err != nil {
|
||||||
|
logger.Warnf(ctx, "[IM] FinalizeStream failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// End the stream
|
||||||
|
if err := streamer.EndStream(ctx, msg, streamID); err != nil {
|
||||||
|
logger.Warnf(ctx, "[IM] EndStream failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
if answer == "" {
|
if answer == "" {
|
||||||
answer = "抱歉,我暂时无法回答这个问题。"
|
answer = "抱歉,我暂时无法回答这个问题。"
|
||||||
@@ -2349,53 +2460,33 @@ loop:
|
|||||||
}
|
}
|
||||||
|
|
||||||
logger.Infof(ctx, "[IM] Stream reply sent: platform=%s user=%s answer_len=%d", msg.Platform, msg.UserID, len(answer))
|
logger.Infof(ctx, "[IM] Stream reply sent: platform=%s user=%s answer_len=%d", msg.Platform, msg.UserID, len(answer))
|
||||||
return deliveryErr
|
return nil
|
||||||
}
|
|
||||||
|
|
||||||
func finalizeAndEndIMStream(ctx context.Context, streamer StreamSender, msg *IncomingMessage, streamID, finalDisplay string) error {
|
|
||||||
var deliveryErr error
|
|
||||||
if err := streamer.FinalizeStream(ctx, msg, streamID, finalDisplay); err != nil {
|
|
||||||
logger.Warnf(ctx, "[IM] FinalizeStream failed: %v", err)
|
|
||||||
deliveryErr = fmt.Errorf("finalize IM stream: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// End the stream even when finalization failed so adapter resources are released.
|
|
||||||
if err := streamer.EndStream(ctx, msg, streamID); err != nil {
|
|
||||||
logger.Warnf(ctx, "[IM] EndStream failed: %v", err)
|
|
||||||
if deliveryErr == nil {
|
|
||||||
deliveryErr = fmt.Errorf("end IM stream: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return deliveryErr
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// fallbackNonStream is used when streaming initialization fails.
|
// fallbackNonStream is used when streaming initialization fails.
|
||||||
func (s *Service) fallbackNonStream(ctx context.Context, msg *IncomingMessage, session *types.Session, customAgent *types.CustomAgent, kbIDs []string, adapter Adapter, userKey string, tenant *types.Tenant) error {
|
func (s *Service) fallbackNonStream(ctx context.Context, msg *IncomingMessage, session *types.Session, customAgent *types.CustomAgent, kbIDs []string, adapter Adapter, userKey string, tenant *types.Tenant, audit *imMessageAuditContext) error {
|
||||||
answer, err := s.runQA(ctx, session, msg.Content, customAgent, kbIDs, userKey, msg.Quote, msg.ImageURLs)
|
answer, err := s.runQA(ctx, session, msg.Content, customAgent, kbIDs, userKey, msg.Quote, audit)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Errorf(ctx, "[IM] QA fallback failed: %v", err)
|
logger.Errorf(ctx, "[IM] QA fallback failed: %v", err)
|
||||||
answer = imQAErrorReply(msg.ImageURLs)
|
answer = "抱歉,处理您的问题时出现了异常,请稍后再试。"
|
||||||
}
|
}
|
||||||
|
|
||||||
return adapter.SendReply(ctx, msg, &ReplyMessage{Content: formatIMOutboundAnswer(ctx, answer, tenant, s.defaultFileSvc), IsFinal: true})
|
return adapter.SendReply(ctx, msg, &ReplyMessage{Content: formatIMOutboundAnswer(ctx, answer, tenant, s.defaultFileSvc), IsFinal: true})
|
||||||
}
|
}
|
||||||
|
|
||||||
// runQA executes the WeKnora QA pipeline and returns the full answer text.
|
// runQA executes the WeKnora QA pipeline and returns the full answer text.
|
||||||
func (s *Service) runQA(ctx context.Context, session *types.Session, query string, customAgent *types.CustomAgent, kbIDs []string, userKey string, quote *QuotedMessage, imageURLs ...[]string) (string, error) {
|
func (s *Service) runQA(ctx context.Context, session *types.Session, query string, customAgent *types.CustomAgent, kbIDs []string, userKey string, quote *QuotedMessage, audit *imMessageAuditContext) (string, error) {
|
||||||
// Cancellable context (no hard deadline): each agent round has its own
|
// Cancellable context (no hard deadline): each agent round has its own
|
||||||
// LLMCallTimeout. The context can still be cancelled by /stop.
|
// LLMCallTimeout. The context can still be cancelled by /stop.
|
||||||
ctx, cancel := context.WithCancel(ctx)
|
ctx, cancel := context.WithCancel(ctx)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
eventBus := event.NewEventBus()
|
eventBus := event.NewEventBus()
|
||||||
var images []string
|
|
||||||
if len(imageURLs) > 0 {
|
|
||||||
images = imageURLs[0]
|
|
||||||
}
|
|
||||||
|
|
||||||
// Thread-safe answer collection
|
// Thread-safe answer collection
|
||||||
var answerMu sync.Mutex
|
var answerMu sync.Mutex
|
||||||
var answerBuilder strings.Builder
|
var answerBuilder strings.Builder
|
||||||
|
var assistantMsg *types.Message
|
||||||
var qaErr error
|
var qaErr error
|
||||||
done := make(chan struct{})
|
done := make(chan struct{})
|
||||||
completeDone := make(chan struct{})
|
completeDone := make(chan struct{})
|
||||||
@@ -2410,6 +2501,7 @@ func (s *Service) runQA(ctx context.Context, session *types.Session, query strin
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
answerMu.Lock()
|
answerMu.Lock()
|
||||||
|
applyIMFinalAnswerDataToMessage(assistantMsg, data)
|
||||||
answerBuilder.WriteString(data.Content)
|
answerBuilder.WriteString(data.Content)
|
||||||
answerMu.Unlock()
|
answerMu.Unlock()
|
||||||
if data.Done {
|
if data.Done {
|
||||||
@@ -2439,13 +2531,13 @@ func (s *Service) runQA(ctx context.Context, session *types.Session, query strin
|
|||||||
requestID := uuid.New().String()
|
requestID := uuid.New().String()
|
||||||
|
|
||||||
// Create user message so it appears in conversation history
|
// Create user message so it appears in conversation history
|
||||||
userMsg, err := s.messageService.CreateMessage(ctx, createIMUserMessagePayload(session.ID, query, requestID))
|
userMsg, err := s.messageService.CreateMessage(ctx, createIMUserMessagePayload(session.ID, query, requestID, audit))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", fmt.Errorf("create user message: %w", err)
|
return "", fmt.Errorf("create user message: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create a placeholder assistant message
|
// Create a placeholder assistant message
|
||||||
assistantMsg, err := s.messageService.CreateMessage(ctx, createIMAssistantMessagePayload(session.ID, requestID))
|
assistantMsg, err = s.messageService.CreateMessage(ctx, createIMAssistantMessagePayload(session.ID, requestID, audit))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", fmt.Errorf("create assistant message: %w", err)
|
return "", fmt.Errorf("create assistant message: %w", err)
|
||||||
}
|
}
|
||||||
@@ -2493,7 +2585,7 @@ func (s *Service) runQA(ctx context.Context, session *types.Session, query strin
|
|||||||
// Run QA async
|
// Run QA async
|
||||||
go func() {
|
go func() {
|
||||||
var err error
|
var err error
|
||||||
req := buildIMQARequest(session, query, assistantMsg.ID, userMsg.ID, customAgent, kbIDs, quote, images)
|
req := buildIMQARequest(session, query, assistantMsg.ID, userMsg.ID, customAgent, kbIDs, quote, auditImageURLs(audit))
|
||||||
if req.QuotedContext != "" {
|
if req.QuotedContext != "" {
|
||||||
logger.Debugf(ctx, "[IM] QuotedContext set: length=%d", len(req.QuotedContext))
|
logger.Debugf(ctx, "[IM] QuotedContext set: length=%d", len(req.QuotedContext))
|
||||||
}
|
}
|
||||||
@@ -2534,14 +2626,6 @@ func (s *Service) runQA(ctx context.Context, session *types.Session, query strin
|
|||||||
qaError := qaErr
|
qaError := qaErr
|
||||||
answerMu.Unlock()
|
answerMu.Unlock()
|
||||||
|
|
||||||
if replacement, override := failClosedIMImageAnswer(answer, qaError, images); override {
|
|
||||||
assistantMsg.Content = replacement
|
|
||||||
assistantMsg.IsCompleted = true
|
|
||||||
if updateErr := s.messageService.UpdateMessage(ctx, assistantMsg); updateErr != nil {
|
|
||||||
logger.Warnf(ctx, "[IM] Failed to persist image QA error: %v", updateErr)
|
|
||||||
}
|
|
||||||
return "", qaError
|
|
||||||
}
|
|
||||||
if answer == "" && qaError != nil {
|
if answer == "" && qaError != nil {
|
||||||
return "", qaError
|
return "", qaError
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -202,8 +202,17 @@ func TestBuildIMLastRequestStateKeepsExplicitKBs(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestCreateIMMessagePayloadsShareRequestShape(t *testing.T) {
|
func TestCreateIMMessagePayloadsShareRequestShape(t *testing.T) {
|
||||||
userMsg := createIMUserMessagePayload("session-1", "hello", "request-1")
|
incoming := &IncomingMessage{
|
||||||
assistantMsg := createIMAssistantMessagePayload("session-1", "request-1")
|
Platform: PlatformDingtalk,
|
||||||
|
UserID: "platform-user-1",
|
||||||
|
ChatID: "platform-chat-1",
|
||||||
|
ThreadID: "platform-thread-1",
|
||||||
|
MessageID: "platform-message-1",
|
||||||
|
Extra: map[string]string{"dingtalk_out_track_id": "card-track-1"},
|
||||||
|
}
|
||||||
|
audit := newIMMessageAuditContext(incoming, "channel-1", []string{"kb-1", "kb-2"})
|
||||||
|
userMsg := createIMUserMessagePayload("session-1", "hello", "request-1", audit)
|
||||||
|
assistantMsg := createIMAssistantMessagePayload("session-1", "request-1", audit)
|
||||||
|
|
||||||
if userMsg.SessionID != "session-1" || assistantMsg.SessionID != "session-1" {
|
if userMsg.SessionID != "session-1" || assistantMsg.SessionID != "session-1" {
|
||||||
t.Fatalf("SessionID mismatch: user=%q assistant=%q", userMsg.SessionID, assistantMsg.SessionID)
|
t.Fatalf("SessionID mismatch: user=%q assistant=%q", userMsg.SessionID, assistantMsg.SessionID)
|
||||||
@@ -229,6 +238,20 @@ func TestCreateIMMessagePayloadsShareRequestShape(t *testing.T) {
|
|||||||
if assistantMsg.Content != "" {
|
if assistantMsg.Content != "" {
|
||||||
t.Fatalf("assistant placeholder content = %q, want empty", assistantMsg.Content)
|
t.Fatalf("assistant placeholder content = %q, want empty", assistantMsg.Content)
|
||||||
}
|
}
|
||||||
|
for _, message := range []*types.Message{userMsg, assistantMsg} {
|
||||||
|
if message.SourcePlatform != "dingtalk" || message.SourceChannelID != "channel-1" {
|
||||||
|
t.Fatalf("source channel metadata = platform %q channel %q", message.SourcePlatform, message.SourceChannelID)
|
||||||
|
}
|
||||||
|
if message.SourceMessageID != "platform-message-1" || message.SourceUserID != "platform-user-1" || message.SourceDeliveryID != "card-track-1" {
|
||||||
|
t.Fatalf("source actor metadata = message %q user %q", message.SourceMessageID, message.SourceUserID)
|
||||||
|
}
|
||||||
|
if message.SourceChatID != "platform-chat-1" || message.SourceThreadID != "platform-thread-1" {
|
||||||
|
t.Fatalf("source conversation metadata = chat %q thread %q", message.SourceChatID, message.SourceThreadID)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(message.SearchedKnowledgeBaseIDs, types.MessageKnowledgeBaseIDs{"kb-1", "kb-2"}) {
|
||||||
|
t.Fatalf("searched knowledge bases = %#v", message.SearchedKnowledgeBaseIDs)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestApplyIMCompleteDataToMessage(t *testing.T) {
|
func TestApplyIMCompleteDataToMessage(t *testing.T) {
|
||||||
@@ -239,11 +262,13 @@ func TestApplyIMCompleteDataToMessage(t *testing.T) {
|
|||||||
ReasoningContent: "thinking",
|
ReasoningContent: "thinking",
|
||||||
}}
|
}}
|
||||||
|
|
||||||
|
usage := &types.TokenUsage{PromptTokens: 100, CompletionTokens: 20, TotalTokens: 120, CachedTokens: 40}
|
||||||
applyIMCompleteDataToMessage(msg, event.AgentCompleteData{
|
applyIMCompleteDataToMessage(msg, event.AgentCompleteData{
|
||||||
MessageID: "assistant-1",
|
MessageID: "assistant-1",
|
||||||
TotalDurationMs: 1234,
|
TotalDurationMs: 1234,
|
||||||
KnowledgeRefs: []interface{}{ref},
|
KnowledgeRefs: []interface{}{ref},
|
||||||
AgentSteps: steps,
|
AgentSteps: steps,
|
||||||
|
TokenUsage: usage,
|
||||||
})
|
})
|
||||||
|
|
||||||
if !msg.IsCompleted {
|
if !msg.IsCompleted {
|
||||||
@@ -258,6 +283,20 @@ func TestApplyIMCompleteDataToMessage(t *testing.T) {
|
|||||||
if len(msg.AgentSteps) != 1 || msg.AgentSteps[0].ReasoningContent != "thinking" {
|
if len(msg.AgentSteps) != 1 || msg.AgentSteps[0].ReasoningContent != "thinking" {
|
||||||
t.Fatalf("AgentSteps = %#v, want one thinking step", msg.AgentSteps)
|
t.Fatalf("AgentSteps = %#v, want one thinking step", msg.AgentSteps)
|
||||||
}
|
}
|
||||||
|
if !reflect.DeepEqual(msg.TokenUsage, usage) {
|
||||||
|
t.Fatalf("TokenUsage = %#v, want %#v", msg.TokenUsage, usage)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyIMFinalAnswerUsageToMessage(t *testing.T) {
|
||||||
|
msg := &types.Message{ID: "assistant-1", Role: "assistant"}
|
||||||
|
usage := &types.TokenUsage{PromptTokens: 80, CompletionTokens: 16, TotalTokens: 96}
|
||||||
|
|
||||||
|
applyIMFinalAnswerDataToMessage(msg, event.AgentFinalAnswerData{Done: true, TokenUsage: usage})
|
||||||
|
|
||||||
|
if !reflect.DeepEqual(msg.TokenUsage, usage) {
|
||||||
|
t.Fatalf("TokenUsage = %#v, want %#v", msg.TokenUsage, usage)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPickIMStoredAnswerPrefersFirstNonEmpty(t *testing.T) {
|
func TestPickIMStoredAnswerPrefersFirstNonEmpty(t *testing.T) {
|
||||||
|
|||||||
@@ -1,42 +0,0 @@
|
|||||||
package im
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
type failingFinalStreamSender struct {
|
|
||||||
finalizeErr error
|
|
||||||
endErr error
|
|
||||||
ended bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func (*failingFinalStreamSender) StartStream(context.Context, *IncomingMessage) (string, error) {
|
|
||||||
return "stream", nil
|
|
||||||
}
|
|
||||||
func (*failingFinalStreamSender) UpdateStreamContent(context.Context, *IncomingMessage, string, string) error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
func (s *failingFinalStreamSender) FinalizeStream(context.Context, *IncomingMessage, string, string) error {
|
|
||||||
return s.finalizeErr
|
|
||||||
}
|
|
||||||
func (s *failingFinalStreamSender) EndStream(context.Context, *IncomingMessage, string) error {
|
|
||||||
s.ended = true
|
|
||||||
return s.endErr
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFinalizeAndEndIMStreamPropagatesDeliveryFailure(t *testing.T) {
|
|
||||||
sender := &failingFinalStreamSender{
|
|
||||||
finalizeErr: errors.New("simulated finalize failure"),
|
|
||||||
endErr: errors.New("simulated end failure"),
|
|
||||||
}
|
|
||||||
err := finalizeAndEndIMStream(context.Background(), sender, &IncomingMessage{}, "stream", "final")
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "finalize IM stream") {
|
|
||||||
t.Fatalf("error = %v; want finalize delivery error", err)
|
|
||||||
}
|
|
||||||
if !sender.ended {
|
|
||||||
t.Fatal("EndStream must run even after FinalizeStream fails")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -8,8 +8,6 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var llmDebug struct {
|
var llmDebug struct {
|
||||||
@@ -38,14 +36,10 @@ func configureLLMDebugLog() {
|
|||||||
dir = "llm_debug"
|
dir = "llm_debug"
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||||
fmt.Fprintf(os.Stderr, "llm_debug: failed to create dir %s: %v\n", dir, err)
|
fmt.Fprintf(os.Stderr, "llm_debug: failed to create dir %s: %v\n", dir, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err := os.Chmod(dir, 0o700); err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "llm_debug: failed to secure dir %s: %v\n", dir, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
llmDebug.dir = dir
|
llmDebug.dir = dir
|
||||||
llmDebug.enabled = true
|
llmDebug.enabled = true
|
||||||
@@ -107,16 +101,12 @@ func LLMDebugLog(ctx context.Context, record *LLMCallRecord) {
|
|||||||
defer llmDebug.mu.Unlock()
|
defer llmDebug.mu.Unlock()
|
||||||
|
|
||||||
path := filepath.Join(llmDebug.dir, filename)
|
path := filepath.Join(llmDebug.dir, filename)
|
||||||
f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600)
|
f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o644)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintf(os.Stderr, "llm_debug: open %s: %v\n", path, err)
|
fmt.Fprintf(os.Stderr, "llm_debug: open %s: %v\n", path, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
defer f.Close()
|
defer f.Close()
|
||||||
if err := f.Chmod(0o600); err != nil {
|
|
||||||
fmt.Fprintf(os.Stderr, "llm_debug: secure %s: %v\n", path, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
_, _ = f.WriteString(text)
|
_, _ = f.WriteString(text)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -142,17 +132,16 @@ func formatRecord(r *LLMCallRecord) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, s := range r.Sections {
|
for _, s := range r.Sections {
|
||||||
content := secutils.RedactDataURLsForLog(s.Content)
|
|
||||||
b.WriteString(fmt.Sprintf("\n---------- %s ----------\n", s.Title))
|
b.WriteString(fmt.Sprintf("\n---------- %s ----------\n", s.Title))
|
||||||
b.WriteString(content)
|
b.WriteString(s.Content)
|
||||||
if !strings.HasSuffix(content, "\n") {
|
if !strings.HasSuffix(s.Content, "\n") {
|
||||||
b.WriteString("\n")
|
b.WriteString("\n")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if r.Error != "" {
|
if r.Error != "" {
|
||||||
b.WriteString("\n---------- Error ----------\n")
|
b.WriteString("\n---------- Error ----------\n")
|
||||||
b.WriteString(secutils.RedactDataURLsForLog(r.Error))
|
b.WriteString(r.Error)
|
||||||
b.WriteString("\n")
|
b.WriteString("\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -174,7 +163,7 @@ func extractRequestID(ctx context.Context) string {
|
|||||||
|
|
||||||
// LLMCallRecord holds all information for one model API call.
|
// LLMCallRecord holds all information for one model API call.
|
||||||
type LLMCallRecord struct {
|
type LLMCallRecord struct {
|
||||||
CallType string // "Chat", "Chat Stream", "Embedding", "Rerank", "VLM"
|
CallType string // "Chat", "Chat Stream", "Embedding", "Rerank", "VLM"
|
||||||
Model string
|
Model string
|
||||||
Duration time.Duration
|
Duration time.Duration
|
||||||
Sections []RecordSection
|
Sections []RecordSection
|
||||||
@@ -224,7 +213,11 @@ func FormatMessages(messages []LLMMessage) string {
|
|||||||
b.WriteString("\n")
|
b.WriteString("\n")
|
||||||
}
|
}
|
||||||
for _, img := range m.Images {
|
for _, img := range m.Images {
|
||||||
b.WriteString(fmt.Sprintf("[image: redacted (%d bytes)]\n", len(img)))
|
if len([]rune(img)) > 80 {
|
||||||
|
b.WriteString(fmt.Sprintf("[image: %s (%d bytes)]\n", TruncateRunes(img, 80), len(img)))
|
||||||
|
} else {
|
||||||
|
b.WriteString(fmt.Sprintf("[image: %s]\n", img))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
for _, tc := range m.ToolCalls {
|
for _, tc := range m.ToolCalls {
|
||||||
b.WriteString(fmt.Sprintf(" -> tool_call: id=%s, func=%s, args=%s\n", tc.ID, tc.FuncName, tc.Arguments))
|
b.WriteString(fmt.Sprintf(" -> tool_call: id=%s, func=%s, args=%s\n", tc.ID, tc.FuncName, tc.Arguments))
|
||||||
|
|||||||
@@ -17,32 +17,6 @@ func newEntry(level logrus.Level, msg string, data logrus.Fields) *logrus.Entry
|
|||||||
return e
|
return e
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestLLMCallRecordFormatRedactsDataURIInSectionsAndErrors(t *testing.T) {
|
|
||||||
secret := "TOPSECRETBASE64"
|
|
||||||
record := &LLMCallRecord{
|
|
||||||
Sections: []RecordSection{{Title: "Response", Content: "data:image/png;base64," + secret}},
|
|
||||||
Error: "provider echoed data:text/plain," + secret,
|
|
||||||
}
|
|
||||||
got := formatRecord(record)
|
|
||||||
if strings.Contains(got, secret) || strings.Contains(strings.ToLower(got), "data:") {
|
|
||||||
t.Fatalf("record leaked data URI: %q", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFormatMessagesRedactsImageValues(t *testing.T) {
|
|
||||||
secret := "TOPSECRETBASE64"
|
|
||||||
got := FormatMessages([]LLMMessage{{
|
|
||||||
Role: "user",
|
|
||||||
Images: []string{"data:image/png;base64," + secret},
|
|
||||||
}})
|
|
||||||
if strings.Contains(got, secret) || strings.Contains(strings.ToLower(got), "data:image") {
|
|
||||||
t.Fatalf("FormatMessages leaked image value: %q", got)
|
|
||||||
}
|
|
||||||
if !strings.Contains(got, "redacted") {
|
|
||||||
t.Fatalf("FormatMessages missing redaction marker: %q", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFormat_DefaultModeUnchanged(t *testing.T) {
|
func TestFormat_DefaultModeUnchanged(t *testing.T) {
|
||||||
f := &CustomFormatter{} // no template, no color
|
f := &CustomFormatter{} // no template, no color
|
||||||
entry := newEntry(logrus.InfoLevel, "hello", logrus.Fields{
|
entry := newEntry(logrus.InfoLevel, "hello", logrus.Fields{
|
||||||
|
|||||||
@@ -3,9 +3,7 @@ package chat
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"encoding/base64"
|
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
@@ -17,37 +15,19 @@ import (
|
|||||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const anthropicVersion = "2023-06-01"
|
||||||
anthropicVersion = "2023-06-01"
|
|
||||||
anthropicResponseBodyLimit = 1 << 20
|
|
||||||
)
|
|
||||||
|
|
||||||
type AnthropicChat struct {
|
type AnthropicChat struct {
|
||||||
modelName string
|
modelName string
|
||||||
modelID string
|
modelID string
|
||||||
baseURL string
|
baseURL string
|
||||||
apiKey string
|
apiKey string
|
||||||
supportsVision bool
|
customHeaders map[string]string
|
||||||
customHeaders map[string]string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *AnthropicChat) SupportsImageInput() bool { return c.supportsVision }
|
|
||||||
|
|
||||||
type anthropicMessage struct {
|
type anthropicMessage struct {
|
||||||
Role string `json:"role"`
|
Role string `json:"role"`
|
||||||
Content any `json:"content"`
|
Content string `json:"content"`
|
||||||
}
|
|
||||||
|
|
||||||
type anthropicContentBlock struct {
|
|
||||||
Type string `json:"type"`
|
|
||||||
Text string `json:"text,omitempty"`
|
|
||||||
Source *anthropicImageSource `json:"source,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type anthropicImageSource struct {
|
|
||||||
Type string `json:"type"`
|
|
||||||
MediaType string `json:"media_type"`
|
|
||||||
Data string `json:"data"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type anthropicRequest struct {
|
type anthropicRequest struct {
|
||||||
@@ -81,7 +61,6 @@ type anthropicResponse struct {
|
|||||||
|
|
||||||
type anthropicStreamEvent struct {
|
type anthropicStreamEvent struct {
|
||||||
Type string `json:"type"`
|
Type string `json:"type"`
|
||||||
Index *int `json:"index,omitempty"`
|
|
||||||
Message *struct {
|
Message *struct {
|
||||||
Usage struct {
|
Usage struct {
|
||||||
InputTokens int `json:"input_tokens"`
|
InputTokens int `json:"input_tokens"`
|
||||||
@@ -90,9 +69,6 @@ type anthropicStreamEvent struct {
|
|||||||
CacheReadInputTokens int `json:"cache_read_input_tokens"`
|
CacheReadInputTokens int `json:"cache_read_input_tokens"`
|
||||||
} `json:"usage"`
|
} `json:"usage"`
|
||||||
} `json:"message,omitempty"`
|
} `json:"message,omitempty"`
|
||||||
ContentBlock *struct {
|
|
||||||
Type string `json:"type"`
|
|
||||||
} `json:"content_block,omitempty"`
|
|
||||||
Delta *struct {
|
Delta *struct {
|
||||||
Type string `json:"type"`
|
Type string `json:"type"`
|
||||||
Text string `json:"text"`
|
Text string `json:"text"`
|
||||||
@@ -110,117 +86,6 @@ type anthropicStreamEvent struct {
|
|||||||
} `json:"error,omitempty"`
|
} `json:"error,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type anthropicStreamProtocolState struct {
|
|
||||||
messageStarted bool
|
|
||||||
finishReason string
|
|
||||||
messageStopped bool
|
|
||||||
activeBlocks map[int]string
|
|
||||||
completedBlocks map[int]struct{}
|
|
||||||
nextBlockIndex int
|
|
||||||
}
|
|
||||||
|
|
||||||
func validAnthropicStopReason(reason string) bool {
|
|
||||||
switch reason {
|
|
||||||
case "end_turn", "max_tokens", "stop_sequence", "tool_use", "pause_turn", "refusal", "model_context_window_exceeded":
|
|
||||||
return true
|
|
||||||
default:
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func anthropicDeltaMatchesBlock(blockType, deltaType string) bool {
|
|
||||||
switch deltaType {
|
|
||||||
case "text_delta", "citations_delta":
|
|
||||||
return blockType == "text"
|
|
||||||
case "thinking_delta", "signature_delta":
|
|
||||||
return blockType == "thinking"
|
|
||||||
case "input_json_delta":
|
|
||||||
return blockType == "tool_use" || blockType == "server_tool_use" || blockType == "mcp_tool_use"
|
|
||||||
default:
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *anthropicStreamProtocolState) observe(event *anthropicStreamEvent) error {
|
|
||||||
if event == nil {
|
|
||||||
return errors.New("anthropic stream event is missing")
|
|
||||||
}
|
|
||||||
if event.Type == "error" || event.Error != nil {
|
|
||||||
return errors.New("anthropic stream failed")
|
|
||||||
}
|
|
||||||
if s.messageStopped {
|
|
||||||
if event.Type == "ping" {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return errors.New("anthropic event followed message_stop")
|
|
||||||
}
|
|
||||||
if event.Delta != nil && event.Delta.StopReason != "" && event.Type != "message_delta" {
|
|
||||||
return errors.New("anthropic stop reason appeared on an invalid event")
|
|
||||||
}
|
|
||||||
|
|
||||||
switch event.Type {
|
|
||||||
case "message_start":
|
|
||||||
if s.messageStarted || s.finishReason != "" {
|
|
||||||
return errors.New("anthropic message_start is out of order")
|
|
||||||
}
|
|
||||||
s.messageStarted = true
|
|
||||||
s.activeBlocks = make(map[int]string)
|
|
||||||
s.completedBlocks = make(map[int]struct{})
|
|
||||||
case "message_delta":
|
|
||||||
if !s.messageStarted || s.finishReason != "" || len(s.activeBlocks) != 0 || event.Delta == nil || !validAnthropicStopReason(event.Delta.StopReason) {
|
|
||||||
return errors.New("anthropic message_delta has an invalid stop reason")
|
|
||||||
}
|
|
||||||
s.finishReason = event.Delta.StopReason
|
|
||||||
case "message_stop":
|
|
||||||
if !s.messageStarted || s.finishReason == "" || len(s.activeBlocks) != 0 {
|
|
||||||
return errors.New("anthropic message_stop preceded valid completion")
|
|
||||||
}
|
|
||||||
s.messageStopped = true
|
|
||||||
case "content_block_start":
|
|
||||||
if !s.messageStarted || s.finishReason != "" || event.Index == nil || *event.Index < 0 || event.ContentBlock == nil || strings.TrimSpace(event.ContentBlock.Type) == "" {
|
|
||||||
return errors.New("anthropic content_block_start is invalid")
|
|
||||||
}
|
|
||||||
index := *event.Index
|
|
||||||
if len(s.activeBlocks) != 0 || index != s.nextBlockIndex {
|
|
||||||
return errors.New("anthropic content block index is out of order")
|
|
||||||
}
|
|
||||||
if _, active := s.activeBlocks[index]; active {
|
|
||||||
return errors.New("anthropic content block started twice")
|
|
||||||
}
|
|
||||||
if _, completed := s.completedBlocks[index]; completed {
|
|
||||||
return errors.New("anthropic completed content block restarted")
|
|
||||||
}
|
|
||||||
s.activeBlocks[index] = event.ContentBlock.Type
|
|
||||||
case "content_block_delta":
|
|
||||||
if !s.messageStarted || s.finishReason != "" || event.Index == nil || event.Delta == nil {
|
|
||||||
return errors.New("anthropic content_block_delta is invalid")
|
|
||||||
}
|
|
||||||
blockType, active := s.activeBlocks[*event.Index]
|
|
||||||
if !active || !anthropicDeltaMatchesBlock(blockType, event.Delta.Type) {
|
|
||||||
return errors.New("anthropic content block delta is out of order")
|
|
||||||
}
|
|
||||||
case "content_block_stop":
|
|
||||||
if !s.messageStarted || s.finishReason != "" || event.Index == nil {
|
|
||||||
return errors.New("anthropic content_block_stop is invalid")
|
|
||||||
}
|
|
||||||
index := *event.Index
|
|
||||||
if _, active := s.activeBlocks[index]; !active {
|
|
||||||
return errors.New("anthropic content block stop is out of order")
|
|
||||||
}
|
|
||||||
delete(s.activeBlocks, index)
|
|
||||||
s.completedBlocks[index] = struct{}{}
|
|
||||||
s.nextBlockIndex++
|
|
||||||
case "ping":
|
|
||||||
default:
|
|
||||||
return errors.New("anthropic stream event type is invalid")
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *anthropicStreamProtocolState) complete() bool {
|
|
||||||
return s.messageStarted && s.messageStopped && len(s.activeBlocks) == 0 && validAnthropicStopReason(s.finishReason)
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewAnthropicChat(config *ChatConfig) (*AnthropicChat, error) {
|
func NewAnthropicChat(config *ChatConfig) (*AnthropicChat, error) {
|
||||||
if config.BaseURL != "" {
|
if config.BaseURL != "" {
|
||||||
if err := secutils.ValidateURLForSSRF(config.BaseURL); err != nil {
|
if err := secutils.ValidateURLForSSRF(config.BaseURL); err != nil {
|
||||||
@@ -237,31 +102,16 @@ func NewAnthropicChat(config *ChatConfig) (*AnthropicChat, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return &AnthropicChat{
|
return &AnthropicChat{
|
||||||
modelName: config.ModelName,
|
modelName: config.ModelName,
|
||||||
modelID: config.ModelID,
|
modelID: config.ModelID,
|
||||||
baseURL: baseURL,
|
baseURL: baseURL,
|
||||||
apiKey: config.APIKey,
|
apiKey: config.APIKey,
|
||||||
supportsVision: config.SupportsVision,
|
customHeaders: config.CustomHeaders,
|
||||||
customHeaders: config.CustomHeaders,
|
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func readAnthropicResponseBody(body io.ReadCloser) ([]byte, error) {
|
|
||||||
data, err := io.ReadAll(io.LimitReader(body, anthropicResponseBodyLimit+1))
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.New("read Anthropic response failed")
|
|
||||||
}
|
|
||||||
if len(data) > anthropicResponseBodyLimit {
|
|
||||||
return nil, errors.New("Anthropic response body too large")
|
|
||||||
}
|
|
||||||
return data, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *AnthropicChat) Chat(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) {
|
func (c *AnthropicChat) Chat(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) {
|
||||||
reqBody, err := c.buildRequest(messages, opts)
|
reqBody := c.buildRequest(messages, opts)
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
jsonData, err := json.Marshal(reqBody)
|
jsonData, err := json.Marshal(reqBody)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("marshal request: %w", err)
|
return nil, fmt.Errorf("marshal request: %w", err)
|
||||||
@@ -286,13 +136,13 @@ func (c *AnthropicChat) Chat(ctx context.Context, messages []Message, opts *Chat
|
|||||||
|
|
||||||
resp, err := rawHTTPClient.Do(httpReq)
|
resp, err := rawHTTPClient.Do(httpReq)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.New("send Anthropic request failed")
|
return nil, fmt.Errorf("send request: %w", err)
|
||||||
}
|
}
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
|
|
||||||
body, err := readAnthropicResponseBody(resp.Body)
|
body, err := io.ReadAll(resp.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, fmt.Errorf("read response: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if strings.Contains(strings.ToLower(resp.Header.Get("Content-Type")), "text/event-stream") {
|
if strings.Contains(strings.ToLower(resp.Header.Get("Content-Type")), "text/event-stream") {
|
||||||
@@ -301,7 +151,7 @@ func (c *AnthropicChat) Chat(ctx context.Context, messages []Message, opts *Chat
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
|
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
|
||||||
return nil, remoteAPIStatusError(resp.StatusCode, body)
|
return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, chatResp.Content)
|
||||||
}
|
}
|
||||||
logUsage(ctx, c.modelName, &chatResp.Usage)
|
logUsage(ctx, c.modelName, &chatResp.Usage)
|
||||||
return chatResp, nil
|
return chatResp, nil
|
||||||
@@ -309,10 +159,13 @@ func (c *AnthropicChat) Chat(ctx context.Context, messages []Message, opts *Chat
|
|||||||
|
|
||||||
var chatResp anthropicResponse
|
var chatResp anthropicResponse
|
||||||
if err := json.Unmarshal(body, &chatResp); err != nil {
|
if err := json.Unmarshal(body, &chatResp); err != nil {
|
||||||
return nil, errors.New("decode Anthropic response failed")
|
return nil, fmt.Errorf("decode response: %w", err)
|
||||||
}
|
}
|
||||||
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
|
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
|
||||||
return nil, remoteAPIStatusError(resp.StatusCode, body)
|
if chatResp.Error != nil && chatResp.Error.Message != "" {
|
||||||
|
return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, chatResp.Error.Message)
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
|
||||||
}
|
}
|
||||||
|
|
||||||
result := c.parseResponse(&chatResp)
|
result := c.parseResponse(&chatResp)
|
||||||
@@ -321,10 +174,7 @@ func (c *AnthropicChat) Chat(ctx context.Context, messages []Message, opts *Chat
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (c *AnthropicChat) ChatStream(ctx context.Context, messages []Message, opts *ChatOptions) (<-chan types.StreamResponse, error) {
|
func (c *AnthropicChat) ChatStream(ctx context.Context, messages []Message, opts *ChatOptions) (<-chan types.StreamResponse, error) {
|
||||||
reqBody, err := c.buildRequest(messages, opts)
|
reqBody := c.buildRequest(messages, opts)
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
reqBody.Stream = true
|
reqBody.Stream = true
|
||||||
jsonData, err := json.Marshal(reqBody)
|
jsonData, err := json.Marshal(reqBody)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -348,15 +198,12 @@ func (c *AnthropicChat) ChatStream(ctx context.Context, messages []Message, opts
|
|||||||
|
|
||||||
resp, err := rawHTTPClient.Do(httpReq)
|
resp, err := rawHTTPClient.Do(httpReq)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.New("send Anthropic stream request failed")
|
return nil, fmt.Errorf("send request: %w", err)
|
||||||
}
|
}
|
||||||
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
|
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
|
||||||
body, readErr := readAnthropicResponseBody(resp.Body)
|
body, _ := io.ReadAll(resp.Body)
|
||||||
resp.Body.Close()
|
resp.Body.Close()
|
||||||
if readErr != nil {
|
return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
|
||||||
return nil, readErr
|
|
||||||
}
|
|
||||||
return nil, remoteAPIStatusError(resp.StatusCode, body)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
streamChan := make(chan types.StreamResponse)
|
streamChan := make(chan types.StreamResponse)
|
||||||
@@ -401,7 +248,7 @@ func isAnthropicVersionedBaseURL(baseURL string) bool {
|
|||||||
return strings.HasSuffix(path, "/v1") || strings.HasSuffix(path, "/v1beta")
|
return strings.HasSuffix(path, "/v1") || strings.HasSuffix(path, "/v1beta")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *AnthropicChat) buildRequest(messages []Message, opts *ChatOptions) (anthropicRequest, error) {
|
func (c *AnthropicChat) buildRequest(messages []Message, opts *ChatOptions) anthropicRequest {
|
||||||
req := anthropicRequest{
|
req := anthropicRequest{
|
||||||
Model: c.modelName,
|
Model: c.modelName,
|
||||||
MaxTokens: 1024,
|
MaxTokens: 1024,
|
||||||
@@ -429,75 +276,22 @@ func (c *AnthropicChat) buildRequest(messages []Message, opts *ChatOptions) (ant
|
|||||||
if content == "" {
|
if content == "" {
|
||||||
content = textFromMultiContent(msg.MultiContent)
|
content = textFromMultiContent(msg.MultiContent)
|
||||||
}
|
}
|
||||||
images := append([]string(nil), msg.Images...)
|
if content == "" {
|
||||||
for _, part := range msg.MultiContent {
|
|
||||||
if part.Type == "image_url" && part.ImageURL != nil && strings.TrimSpace(part.ImageURL.URL) != "" {
|
|
||||||
images = append(images, part.ImageURL.URL)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(images) == 0 && content == "" {
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
switch msg.Role {
|
switch msg.Role {
|
||||||
case "system":
|
case "system":
|
||||||
if len(images) > 0 {
|
|
||||||
return anthropicRequest{}, fmt.Errorf("anthropic system messages cannot contain images")
|
|
||||||
}
|
|
||||||
systemParts = append(systemParts, content)
|
systemParts = append(systemParts, content)
|
||||||
case "assistant":
|
case "assistant":
|
||||||
if len(images) > 0 {
|
|
||||||
return anthropicRequest{}, fmt.Errorf("anthropic assistant messages cannot contain images")
|
|
||||||
}
|
|
||||||
req.Messages = append(req.Messages, anthropicMessage{Role: "assistant", Content: content})
|
req.Messages = append(req.Messages, anthropicMessage{Role: "assistant", Content: content})
|
||||||
|
case "user":
|
||||||
|
req.Messages = append(req.Messages, anthropicMessage{Role: "user", Content: content})
|
||||||
default:
|
default:
|
||||||
if len(images) == 0 {
|
req.Messages = append(req.Messages, anthropicMessage{Role: "user", Content: content})
|
||||||
req.Messages = append(req.Messages, anthropicMessage{Role: "user", Content: content})
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
blocks := make([]anthropicContentBlock, 0, len(images)+1)
|
|
||||||
if content != "" {
|
|
||||||
blocks = append(blocks, anthropicContentBlock{Type: "text", Text: content})
|
|
||||||
}
|
|
||||||
for _, imageURL := range images {
|
|
||||||
block, err := anthropicImageBlock(imageURL)
|
|
||||||
if err != nil {
|
|
||||||
return anthropicRequest{}, err
|
|
||||||
}
|
|
||||||
blocks = append(blocks, block)
|
|
||||||
}
|
|
||||||
req.Messages = append(req.Messages, anthropicMessage{Role: "user", Content: blocks})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
req.System = strings.Join(systemParts, "\n\n")
|
req.System = strings.Join(systemParts, "\n\n")
|
||||||
return req, nil
|
return req
|
||||||
}
|
|
||||||
|
|
||||||
func anthropicImageBlock(raw string) (anthropicContentBlock, error) {
|
|
||||||
metadata, payload, ok := strings.Cut(strings.TrimSpace(raw), ",")
|
|
||||||
if !ok {
|
|
||||||
return anthropicContentBlock{}, fmt.Errorf("anthropic image input must be a JPEG or PNG data URI")
|
|
||||||
}
|
|
||||||
metadata = strings.ToLower(metadata)
|
|
||||||
var mediaType string
|
|
||||||
switch metadata {
|
|
||||||
case "data:image/jpeg;base64", "data:image/jpg;base64":
|
|
||||||
mediaType = "image/jpeg"
|
|
||||||
case "data:image/png;base64":
|
|
||||||
mediaType = "image/png"
|
|
||||||
default:
|
|
||||||
return anthropicContentBlock{}, fmt.Errorf("anthropic image input must be a JPEG or PNG data URI")
|
|
||||||
}
|
|
||||||
if _, err := base64.StdEncoding.DecodeString(payload); err != nil {
|
|
||||||
return anthropicContentBlock{}, fmt.Errorf("anthropic image data is invalid")
|
|
||||||
}
|
|
||||||
return anthropicContentBlock{
|
|
||||||
Type: "image",
|
|
||||||
Source: &anthropicImageSource{
|
|
||||||
Type: "base64",
|
|
||||||
MediaType: mediaType,
|
|
||||||
Data: payload,
|
|
||||||
},
|
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func textFromMultiContent(parts []MessageContentPart) string {
|
func textFromMultiContent(parts []MessageContentPart) string {
|
||||||
@@ -536,7 +330,7 @@ func (c *AnthropicChat) parseResponse(resp *anthropicResponse) *types.ChatRespon
|
|||||||
func parseAnthropicSSE(reader io.Reader) (*types.ChatResponse, error) {
|
func parseAnthropicSSE(reader io.Reader) (*types.ChatResponse, error) {
|
||||||
sseReader := NewSSEReader(reader)
|
sseReader := NewSSEReader(reader)
|
||||||
var contentParts []string
|
var contentParts []string
|
||||||
var protocol anthropicStreamProtocolState
|
var finishReason string
|
||||||
var inputTokens int
|
var inputTokens int
|
||||||
var outputTokens int
|
var outputTokens int
|
||||||
|
|
||||||
@@ -559,8 +353,8 @@ func parseAnthropicSSE(reader io.Reader) (*types.ChatResponse, error) {
|
|||||||
if err := json.Unmarshal(event.Data, &streamEvent); err != nil {
|
if err := json.Unmarshal(event.Data, &streamEvent); err != nil {
|
||||||
return nil, fmt.Errorf("decode SSE response: %w", err)
|
return nil, fmt.Errorf("decode SSE response: %w", err)
|
||||||
}
|
}
|
||||||
if err := protocol.observe(&streamEvent); err != nil {
|
if streamEvent.Error != nil && streamEvent.Error.Message != "" {
|
||||||
return nil, errors.New("anthropic stream protocol failed")
|
return nil, fmt.Errorf("API stream error: %s", streamEvent.Error.Message)
|
||||||
}
|
}
|
||||||
if streamEvent.Message != nil {
|
if streamEvent.Message != nil {
|
||||||
inputTokens = max(inputTokens, streamEvent.Message.Usage.InputTokens)
|
inputTokens = max(inputTokens, streamEvent.Message.Usage.InputTokens)
|
||||||
@@ -570,19 +364,19 @@ func parseAnthropicSSE(reader io.Reader) (*types.ChatResponse, error) {
|
|||||||
if streamEvent.Delta.Type == "text_delta" && streamEvent.Delta.Text != "" {
|
if streamEvent.Delta.Type == "text_delta" && streamEvent.Delta.Text != "" {
|
||||||
contentParts = append(contentParts, streamEvent.Delta.Text)
|
contentParts = append(contentParts, streamEvent.Delta.Text)
|
||||||
}
|
}
|
||||||
|
if streamEvent.Delta.StopReason != "" {
|
||||||
|
finishReason = streamEvent.Delta.StopReason
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if streamEvent.Usage != nil {
|
if streamEvent.Usage != nil {
|
||||||
inputTokens = max(inputTokens, streamEvent.Usage.InputTokens)
|
inputTokens = max(inputTokens, streamEvent.Usage.InputTokens)
|
||||||
outputTokens = max(outputTokens, streamEvent.Usage.OutputTokens)
|
outputTokens = max(outputTokens, streamEvent.Usage.OutputTokens)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !protocol.complete() {
|
|
||||||
return nil, errors.New("anthropic stream ended before completion")
|
|
||||||
}
|
|
||||||
|
|
||||||
return &types.ChatResponse{
|
return &types.ChatResponse{
|
||||||
Content: strings.Join(contentParts, ""),
|
Content: strings.Join(contentParts, ""),
|
||||||
FinishReason: protocol.finishReason,
|
FinishReason: finishReason,
|
||||||
Usage: types.TokenUsage{
|
Usage: types.TokenUsage{
|
||||||
PromptTokens: inputTokens,
|
PromptTokens: inputTokens,
|
||||||
CompletionTokens: outputTokens,
|
CompletionTokens: outputTokens,
|
||||||
@@ -597,43 +391,38 @@ func processAnthropicStream(ctx context.Context, model string, resp *http.Respon
|
|||||||
|
|
||||||
sseReader := NewSSEReader(resp.Body)
|
sseReader := NewSSEReader(resp.Body)
|
||||||
var usage *types.TokenUsage
|
var usage *types.TokenUsage
|
||||||
var protocol anthropicStreamProtocolState
|
var finishReason string
|
||||||
|
|
||||||
emitTerminal := func() {
|
|
||||||
if !protocol.complete() {
|
|
||||||
streamChan <- types.StreamResponse{
|
|
||||||
ResponseType: types.ResponseTypeError,
|
|
||||||
Content: "anthropic stream ended before completion",
|
|
||||||
Done: true,
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
logUsage(ctx, model, usage)
|
|
||||||
streamChan <- types.StreamResponse{
|
|
||||||
ResponseType: types.ResponseTypeAnswer,
|
|
||||||
Content: "",
|
|
||||||
Done: true,
|
|
||||||
Usage: usage,
|
|
||||||
FinishReason: protocol.finishReason,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for {
|
for {
|
||||||
event, err := sseReader.ReadEvent()
|
event, err := sseReader.ReadEvent()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if err == io.EOF {
|
if err == io.EOF {
|
||||||
emitTerminal()
|
logUsage(ctx, model, usage)
|
||||||
|
streamChan <- types.StreamResponse{
|
||||||
|
ResponseType: types.ResponseTypeAnswer,
|
||||||
|
Content: "",
|
||||||
|
Done: true,
|
||||||
|
Usage: usage,
|
||||||
|
FinishReason: finishReason,
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
streamChan <- types.StreamResponse{
|
streamChan <- types.StreamResponse{
|
||||||
ResponseType: types.ResponseTypeError,
|
ResponseType: types.ResponseTypeError,
|
||||||
Content: "anthropic stream read failed",
|
Content: err.Error(),
|
||||||
Done: true,
|
Done: true,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if event.Done {
|
if event.Done {
|
||||||
emitTerminal()
|
logUsage(ctx, model, usage)
|
||||||
|
streamChan <- types.StreamResponse{
|
||||||
|
ResponseType: types.ResponseTypeAnswer,
|
||||||
|
Content: "",
|
||||||
|
Done: true,
|
||||||
|
Usage: usage,
|
||||||
|
FinishReason: finishReason,
|
||||||
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if len(event.Data) == 0 {
|
if len(event.Data) == 0 {
|
||||||
@@ -644,15 +433,15 @@ func processAnthropicStream(ctx context.Context, model string, resp *http.Respon
|
|||||||
if err := json.Unmarshal(event.Data, &streamEvent); err != nil {
|
if err := json.Unmarshal(event.Data, &streamEvent); err != nil {
|
||||||
streamChan <- types.StreamResponse{
|
streamChan <- types.StreamResponse{
|
||||||
ResponseType: types.ResponseTypeError,
|
ResponseType: types.ResponseTypeError,
|
||||||
Content: "anthropic stream response was invalid",
|
Content: fmt.Sprintf("decode SSE response: %v", err),
|
||||||
Done: true,
|
Done: true,
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err := protocol.observe(&streamEvent); err != nil {
|
if streamEvent.Error != nil && streamEvent.Error.Message != "" {
|
||||||
streamChan <- types.StreamResponse{
|
streamChan <- types.StreamResponse{
|
||||||
ResponseType: types.ResponseTypeError,
|
ResponseType: types.ResponseTypeError,
|
||||||
Content: "anthropic stream protocol failed",
|
Content: streamEvent.Error.Message,
|
||||||
Done: true,
|
Done: true,
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
@@ -661,6 +450,9 @@ func processAnthropicStream(ctx context.Context, model string, resp *http.Respon
|
|||||||
usage = mergeAnthropicUsage(usage, streamEvent.Message.Usage.InputTokens, streamEvent.Message.Usage.OutputTokens)
|
usage = mergeAnthropicUsage(usage, streamEvent.Message.Usage.InputTokens, streamEvent.Message.Usage.OutputTokens)
|
||||||
}
|
}
|
||||||
if streamEvent.Delta != nil {
|
if streamEvent.Delta != nil {
|
||||||
|
if streamEvent.Delta.StopReason != "" {
|
||||||
|
finishReason = streamEvent.Delta.StopReason
|
||||||
|
}
|
||||||
if streamEvent.Delta.Type == "text_delta" && streamEvent.Delta.Text != "" {
|
if streamEvent.Delta.Type == "text_delta" && streamEvent.Delta.Text != "" {
|
||||||
streamChan <- types.StreamResponse{
|
streamChan <- types.StreamResponse{
|
||||||
ResponseType: types.ResponseTypeAnswer,
|
ResponseType: types.ResponseTypeAnswer,
|
||||||
|
|||||||
@@ -1,38 +0,0 @@
|
|||||||
package chat
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/Tencent/WeKnora/internal/types"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestAnthropicProviderErrorDoesNotExposeImagePayload(t *testing.T) {
|
|
||||||
secret := "TOPSECRETIMAGEPAYLOAD"
|
|
||||||
payload := "data: {\"type\":\"error\",\"error\":{\"message\":\"data:image/png;base64," + secret + "\"}}\n\n"
|
|
||||||
|
|
||||||
_, err := parseAnthropicSSE(strings.NewReader(payload))
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected Anthropic SSE error")
|
|
||||||
}
|
|
||||||
if strings.Contains(err.Error(), secret) || strings.Contains(strings.ToLower(err.Error()), "data:image") {
|
|
||||||
t.Fatalf("non-stream error leaked provider payload: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
resp := &http.Response{Body: io.NopCloser(strings.NewReader(payload))}
|
|
||||||
stream := make(chan types.StreamResponse)
|
|
||||||
go processAnthropicStream(context.Background(), "vision-test", resp, stream)
|
|
||||||
var terminal types.StreamResponse
|
|
||||||
for chunk := range stream {
|
|
||||||
terminal = chunk
|
|
||||||
}
|
|
||||||
if terminal.ResponseType != types.ResponseTypeError {
|
|
||||||
t.Fatalf("terminal response = %#v; want error", terminal)
|
|
||||||
}
|
|
||||||
if strings.Contains(terminal.Content, secret) || strings.Contains(strings.ToLower(terminal.Content), "data:image") {
|
|
||||||
t.Fatalf("stream error leaked provider payload: %q", terminal.Content)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,223 +0,0 @@
|
|||||||
package chat
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/Tencent/WeKnora/internal/types"
|
|
||||||
)
|
|
||||||
|
|
||||||
const anthropicTestMessageStart = "data: {\"type\":\"message_start\",\"message\":{\"usage\":{\"input_tokens\":1,\"output_tokens\":0}}}\n\n"
|
|
||||||
const anthropicTestBlockStart0 = "data: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\n\n"
|
|
||||||
const anthropicTestBlockDelta0 = "data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"x\"}}\n\n"
|
|
||||||
const anthropicTestBlockStop0 = "data: {\"type\":\"content_block_stop\",\"index\":0}\n\n"
|
|
||||||
|
|
||||||
func invalidAnthropicBlockPayloads() map[string]string {
|
|
||||||
return map[string]string{
|
|
||||||
"stop before start": anthropicTestMessageStart + anthropicTestBlockStop0,
|
|
||||||
"delta before start": anthropicTestMessageStart + anthropicTestBlockDelta0,
|
|
||||||
"delta after stop": anthropicTestMessageStart + anthropicTestBlockStart0 + anthropicTestBlockStop0 + anthropicTestBlockDelta0,
|
|
||||||
"duplicate start": anthropicTestMessageStart + anthropicTestBlockStart0 + anthropicTestBlockStart0,
|
|
||||||
"duplicate stop": anthropicTestMessageStart + anthropicTestBlockStart0 + anthropicTestBlockStop0 + anthropicTestBlockStop0,
|
|
||||||
"mismatched index": anthropicTestMessageStart + anthropicTestBlockStart0 + "data: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\"x\"}}\n\n",
|
|
||||||
"index gap": anthropicTestMessageStart + "data: {\"type\":\"content_block_start\",\"index\":1,\"content_block\":{\"type\":\"text\"}}\n\n",
|
|
||||||
"concurrent block start": anthropicTestMessageStart + anthropicTestBlockStart0 + "data: {\"type\":\"content_block_start\",\"index\":1,\"content_block\":{\"type\":\"text\"}}\n\n",
|
|
||||||
"mismatched delta type": anthropicTestMessageStart + anthropicTestBlockStart0 + "data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\"}}\n\n",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAnthropicContentBlockLifecycleRejectedByBothParsers(t *testing.T) {
|
|
||||||
for name, payload := range invalidAnthropicBlockPayloads() {
|
|
||||||
t.Run(name+"/aggregate", func(t *testing.T) {
|
|
||||||
if _, err := parseAnthropicSSE(strings.NewReader(payload)); err == nil {
|
|
||||||
t.Fatal("aggregate parser accepted invalid content-block lifecycle")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
t.Run(name+"/stream", func(t *testing.T) {
|
|
||||||
resp := &http.Response{Body: io.NopCloser(strings.NewReader(payload))}
|
|
||||||
stream := make(chan types.StreamResponse, 8)
|
|
||||||
go processAnthropicStream(context.Background(), "test", resp, stream)
|
|
||||||
sawError := false
|
|
||||||
for item := range stream {
|
|
||||||
if item.Done && item.ResponseType == types.ResponseTypeAnswer {
|
|
||||||
t.Fatalf("stream accepted invalid content-block lifecycle: %#v", item)
|
|
||||||
}
|
|
||||||
if item.Done && item.ResponseType == types.ResponseTypeError {
|
|
||||||
sawError = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !sawError {
|
|
||||||
t.Fatal("stream did not emit an error terminal")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAnthropicContentBlockLifecycleAcceptsSequentialBlocks(t *testing.T) {
|
|
||||||
payload := anthropicTestMessageStart + anthropicTestBlockStart0 + anthropicTestBlockDelta0 + anthropicTestBlockStop0 +
|
|
||||||
"data: {\"type\":\"content_block_start\",\"index\":1,\"content_block\":{\"type\":\"thinking\",\"thinking\":\"\"}}\n\n" +
|
|
||||||
"data: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"thinking_delta\"}}\n\n" +
|
|
||||||
"data: {\"type\":\"content_block_stop\",\"index\":1}\n\n" +
|
|
||||||
"data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\n" +
|
|
||||||
"data: {\"type\":\"message_stop\"}\n\n"
|
|
||||||
response, err := parseAnthropicSSE(strings.NewReader(payload))
|
|
||||||
if err != nil || response.FinishReason != "end_turn" || response.Content != "x" {
|
|
||||||
t.Fatalf("valid sequential blocks rejected: response=%#v err=%v", response, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
resp := &http.Response{Body: io.NopCloser(strings.NewReader(payload))}
|
|
||||||
stream := make(chan types.StreamResponse, 8)
|
|
||||||
go processAnthropicStream(context.Background(), "test", resp, stream)
|
|
||||||
successTerminals := 0
|
|
||||||
for item := range stream {
|
|
||||||
if item.Done && item.ResponseType == types.ResponseTypeError {
|
|
||||||
t.Fatalf("valid stream rejected: %#v", item)
|
|
||||||
}
|
|
||||||
if item.Done && item.ResponseType == types.ResponseTypeAnswer {
|
|
||||||
successTerminals++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if successTerminals != 1 {
|
|
||||||
t.Fatalf("success terminals=%d, want 1", successTerminals)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAnthropicProtocolAcceptsOfficialStopReasons(t *testing.T) {
|
|
||||||
reasons := []string{"end_turn", "max_tokens", "stop_sequence", "tool_use", "pause_turn", "refusal", "model_context_window_exceeded"}
|
|
||||||
for _, reason := range reasons {
|
|
||||||
t.Run(reason, func(t *testing.T) {
|
|
||||||
var state anthropicStreamProtocolState
|
|
||||||
start := anthropicStreamEvent{Type: "message_start"}
|
|
||||||
if err := state.observe(&start); err != nil {
|
|
||||||
t.Fatalf("message_start: %v", err)
|
|
||||||
}
|
|
||||||
delta := anthropicStreamEvent{Type: "message_delta"}
|
|
||||||
delta.Delta = &struct {
|
|
||||||
Type string `json:"type"`
|
|
||||||
Text string `json:"text"`
|
|
||||||
StopReason string `json:"stop_reason"`
|
|
||||||
}{StopReason: reason}
|
|
||||||
if err := state.observe(&delta); err != nil {
|
|
||||||
t.Fatalf("message_delta: %v", err)
|
|
||||||
}
|
|
||||||
stop := anthropicStreamEvent{Type: "message_stop"}
|
|
||||||
if err := state.observe(&stop); err != nil || !state.complete() {
|
|
||||||
t.Fatalf("message_stop err=%v complete=%v", err, state.complete())
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseAnthropicSSERejectsPrematureEOF(t *testing.T) {
|
|
||||||
payload := anthropicTestMessageStart + anthropicTestBlockStart0 + "event: content_block_delta\n" +
|
|
||||||
"data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"partial\"}}\n\n"
|
|
||||||
if _, err := parseAnthropicSSE(strings.NewReader(payload)); err == nil {
|
|
||||||
t.Fatal("expected premature Anthropic EOF to fail closed")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseAnthropicSSERejectsMalformedPacketAfterFinishReason(t *testing.T) {
|
|
||||||
payload := anthropicTestMessageStart + "event: message_delta\n" +
|
|
||||||
"data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\n" +
|
|
||||||
"event: broken\n" +
|
|
||||||
"data: {not-json}\n\n"
|
|
||||||
if _, err := parseAnthropicSSE(strings.NewReader(payload)); err == nil {
|
|
||||||
t.Fatal("expected trailing malformed packet to override finish reason")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseAnthropicSSERejectsInvalidOrOutOfOrderCompletion(t *testing.T) {
|
|
||||||
cases := map[string]string{
|
|
||||||
"invalid stop reason": anthropicTestMessageStart + "data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"made_up\"}}\n\n" +
|
|
||||||
"data: {\"type\":\"message_stop\"}\n\n",
|
|
||||||
"message stop first": "data: {\"type\":\"message_stop\"}\n\n" +
|
|
||||||
"data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\n",
|
|
||||||
"stop reason wrong event": anthropicTestMessageStart + "data: {\"type\":\"content_block_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\n" +
|
|
||||||
"data: {\"type\":\"message_stop\"}\n\n",
|
|
||||||
"empty provider error after completion": anthropicTestMessageStart + "data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\n" +
|
|
||||||
"data: {\"type\":\"message_stop\"}\n\n" +
|
|
||||||
"data: {\"type\":\"error\",\"error\":{\"type\":\"api_error\",\"message\":\"\"}}\n\n",
|
|
||||||
}
|
|
||||||
for name, payload := range cases {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
if _, err := parseAnthropicSSE(strings.NewReader(payload)); err == nil {
|
|
||||||
t.Fatal("expected invalid Anthropic completion protocol to fail")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProcessAnthropicStreamRejectsPrematureDone(t *testing.T) {
|
|
||||||
payload := anthropicTestMessageStart + anthropicTestBlockStart0 + anthropicTestBlockDelta0 +
|
|
||||||
"data: [DONE]\n\n"
|
|
||||||
resp := &http.Response{Body: io.NopCloser(strings.NewReader(payload))}
|
|
||||||
stream := make(chan types.StreamResponse, 4)
|
|
||||||
go processAnthropicStream(context.Background(), "test", resp, stream)
|
|
||||||
for item := range stream {
|
|
||||||
if item.Done && item.ResponseType == types.ResponseTypeAnswer {
|
|
||||||
t.Fatalf("premature [DONE] produced success: %#v", item)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProcessAnthropicStreamRejectsEmptyProviderErrorAfterCompletion(t *testing.T) {
|
|
||||||
payload := anthropicTestMessageStart + "data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\n" +
|
|
||||||
"data: {\"type\":\"message_stop\"}\n\n" +
|
|
||||||
"data: {\"type\":\"error\",\"error\":{\"type\":\"api_error\",\"message\":\"\"}}\n\n"
|
|
||||||
resp := &http.Response{Body: io.NopCloser(strings.NewReader(payload))}
|
|
||||||
stream := make(chan types.StreamResponse, 4)
|
|
||||||
go processAnthropicStream(context.Background(), "test", resp, stream)
|
|
||||||
for item := range stream {
|
|
||||||
if item.Done && item.ResponseType == types.ResponseTypeAnswer {
|
|
||||||
t.Fatalf("empty-message provider error was ignored: %#v", item)
|
|
||||||
}
|
|
||||||
if item.Done && item.ResponseType == types.ResponseTypeError {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
t.Fatal("expected provider error terminal")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProcessAnthropicStreamRejectsPrematureEOF(t *testing.T) {
|
|
||||||
payload := anthropicTestMessageStart + anthropicTestBlockStart0 + "event: content_block_delta\n" +
|
|
||||||
"data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"partial\"}}\n\n"
|
|
||||||
resp := &http.Response{Body: io.NopCloser(strings.NewReader(payload))}
|
|
||||||
stream := make(chan types.StreamResponse, 4)
|
|
||||||
go processAnthropicStream(context.Background(), "test", resp, stream)
|
|
||||||
|
|
||||||
successTerminals, errorTerminals := 0, 0
|
|
||||||
for item := range stream {
|
|
||||||
if item.Done && item.ResponseType == types.ResponseTypeAnswer {
|
|
||||||
successTerminals++
|
|
||||||
}
|
|
||||||
if item.Done && item.ResponseType == types.ResponseTypeError {
|
|
||||||
errorTerminals++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if successTerminals != 0 || errorTerminals != 1 {
|
|
||||||
t.Fatalf("terminal counts success=%d error=%d; want 0/1", successTerminals, errorTerminals)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProcessAnthropicStreamRejectsMalformedPacketAfterFinishReason(t *testing.T) {
|
|
||||||
payload := anthropicTestMessageStart + "event: message_delta\n" +
|
|
||||||
"data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\n" +
|
|
||||||
"event: broken\n" +
|
|
||||||
"data: {not-json}\n\n"
|
|
||||||
resp := &http.Response{Body: io.NopCloser(strings.NewReader(payload))}
|
|
||||||
stream := make(chan types.StreamResponse, 4)
|
|
||||||
go processAnthropicStream(context.Background(), "test", resp, stream)
|
|
||||||
|
|
||||||
for item := range stream {
|
|
||||||
if item.Done && item.ResponseType == types.ResponseTypeAnswer {
|
|
||||||
t.Fatalf("unexpected success terminal before trailing malformed packet: %#v", item)
|
|
||||||
}
|
|
||||||
if item.ResponseType == types.ResponseTypeError {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
t.Fatal("expected terminal stream error")
|
|
||||||
}
|
|
||||||
@@ -13,33 +13,6 @@ import (
|
|||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestAnthropicBuildRequestPreservesImageDataURI(t *testing.T) {
|
|
||||||
chat := &AnthropicChat{modelName: "claude-test"}
|
|
||||||
req, err := chat.buildRequest([]Message{{
|
|
||||||
Role: "user",
|
|
||||||
Content: "describe this image",
|
|
||||||
Images: []string{"data:image/png;base64,YWJj"},
|
|
||||||
}}, nil)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, req.Messages, 1)
|
|
||||||
blocks, ok := req.Messages[0].Content.([]anthropicContentBlock)
|
|
||||||
require.True(t, ok, "content type = %T", req.Messages[0].Content)
|
|
||||||
require.Len(t, blocks, 2)
|
|
||||||
assert.Equal(t, "text", blocks[0].Type)
|
|
||||||
assert.Equal(t, "describe this image", blocks[0].Text)
|
|
||||||
assert.Equal(t, "image", blocks[1].Type)
|
|
||||||
require.NotNil(t, blocks[1].Source)
|
|
||||||
assert.Equal(t, "base64", blocks[1].Source.Type)
|
|
||||||
assert.Equal(t, "image/png", blocks[1].Source.MediaType)
|
|
||||||
assert.Equal(t, "YWJj", blocks[1].Source.Data)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAnthropicBuildRequestRejectsMalformedImage(t *testing.T) {
|
|
||||||
chat := &AnthropicChat{modelName: "claude-test"}
|
|
||||||
_, err := chat.buildRequest([]Message{{Role: "user", Content: "describe", Images: []string{"not-an-image"}}}, nil)
|
|
||||||
require.Error(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAnthropicChat(t *testing.T) {
|
func TestAnthropicChat(t *testing.T) {
|
||||||
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
|
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
|
||||||
|
|
||||||
@@ -174,14 +147,8 @@ func TestAnthropicChat_SSEResponse(t *testing.T) {
|
|||||||
_, _ = w.Write([]byte(`event: message_start
|
_, _ = w.Write([]byte(`event: message_start
|
||||||
data: {"type":"message_start","message":{"usage":{"input_tokens":0,"output_tokens":0}}}
|
data: {"type":"message_start","message":{"usage":{"input_tokens":0,"output_tokens":0}}}
|
||||||
|
|
||||||
event: content_block_start
|
|
||||||
data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}
|
|
||||||
|
|
||||||
event: content_block_delta
|
event: content_block_delta
|
||||||
data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"pong"}}
|
data: {"type":"content_block_delta","delta":{"type":"text_delta","text":"pong"}}
|
||||||
|
|
||||||
event: content_block_stop
|
|
||||||
data: {"type":"content_block_stop","index":0}
|
|
||||||
|
|
||||||
event: message_delta
|
event: message_delta
|
||||||
data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"input_tokens":114,"output_tokens":5}}
|
data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"input_tokens":114,"output_tokens":5}}
|
||||||
@@ -224,14 +191,8 @@ func TestAnthropicChat_ChatStream(t *testing.T) {
|
|||||||
_, _ = w.Write([]byte(`event: message_start
|
_, _ = w.Write([]byte(`event: message_start
|
||||||
data: {"type":"message_start","message":{"usage":{"input_tokens":0,"output_tokens":0}}}
|
data: {"type":"message_start","message":{"usage":{"input_tokens":0,"output_tokens":0}}}
|
||||||
|
|
||||||
event: content_block_start
|
|
||||||
data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}
|
|
||||||
|
|
||||||
event: content_block_delta
|
event: content_block_delta
|
||||||
data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"pong"}}
|
data: {"type":"content_block_delta","delta":{"type":"text_delta","text":"pong"}}
|
||||||
|
|
||||||
event: content_block_stop
|
|
||||||
data: {"type":"content_block_stop","index":0}
|
|
||||||
|
|
||||||
event: message_delta
|
event: message_delta
|
||||||
data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"input_tokens":114,"output_tokens":5}}
|
data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"input_tokens":114,"output_tokens":5}}
|
||||||
|
|||||||
@@ -98,27 +98,14 @@ type Chat interface {
|
|||||||
GetModelID() string
|
GetModelID() string
|
||||||
}
|
}
|
||||||
|
|
||||||
type imageInputCapable interface {
|
|
||||||
SupportsImageInput() bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// SupportsImageInput reports whether the concrete Chat transport can preserve
|
|
||||||
// image input. Unknown implementations fail closed instead of trusting model
|
|
||||||
// metadata alone.
|
|
||||||
func SupportsImageInput(model Chat) bool {
|
|
||||||
capable, ok := model.(imageInputCapable)
|
|
||||||
return ok && capable.SupportsImageInput()
|
|
||||||
}
|
|
||||||
|
|
||||||
type ChatConfig struct {
|
type ChatConfig struct {
|
||||||
Source types.ModelSource
|
Source types.ModelSource
|
||||||
BaseURL string
|
BaseURL string
|
||||||
ModelName string
|
ModelName string
|
||||||
APIKey string
|
APIKey string
|
||||||
ModelID string
|
ModelID string
|
||||||
Provider string
|
Provider string
|
||||||
SupportsVision bool
|
ExtraConfig map[string]string
|
||||||
ExtraConfig map[string]string
|
|
||||||
// CustomHeaders 允许在调用远程 OpenAI 兼容 API 时附加自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。
|
// CustomHeaders 允许在调用远程 OpenAI 兼容 API 时附加自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。
|
||||||
CustomHeaders map[string]string
|
CustomHeaders map[string]string
|
||||||
AppID string
|
AppID string
|
||||||
@@ -133,23 +120,17 @@ func ConfigFromModel(m *types.Model, appID, appSecret string) *ChatConfig {
|
|||||||
if m == nil {
|
if m == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
supportsVision := m.Parameters.SupportsVision
|
|
||||||
if provider.ProviderName(m.Parameters.Provider) == provider.ProviderWeKnoraCloud &&
|
|
||||||
m.Type == types.ModelTypeKnowledgeQA && strings.EqualFold(strings.TrimSpace(m.Name), "chat") {
|
|
||||||
supportsVision = true
|
|
||||||
}
|
|
||||||
return &ChatConfig{
|
return &ChatConfig{
|
||||||
ModelID: m.ID,
|
ModelID: m.ID,
|
||||||
APIKey: m.Parameters.APIKey,
|
APIKey: m.Parameters.APIKey,
|
||||||
BaseURL: m.Parameters.BaseURL,
|
BaseURL: m.Parameters.BaseURL,
|
||||||
ModelName: m.Name,
|
ModelName: m.Name,
|
||||||
Source: m.Source,
|
Source: m.Source,
|
||||||
Provider: m.Parameters.Provider,
|
Provider: m.Parameters.Provider,
|
||||||
SupportsVision: supportsVision,
|
ExtraConfig: m.Parameters.ExtraConfig,
|
||||||
ExtraConfig: m.Parameters.ExtraConfig,
|
CustomHeaders: m.Parameters.CustomHeaders,
|
||||||
CustomHeaders: m.Parameters.CustomHeaders,
|
AppID: appID,
|
||||||
AppID: appID,
|
AppSecret: appSecret,
|
||||||
AppSecret: appSecret,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -74,3 +74,24 @@ func readLocalStorageBytes(storagePath string) []byte {
|
|||||||
}
|
}
|
||||||
return data
|
return data
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// isMultimodalNotSupportedError checks if an error indicates the model does not
|
||||||
|
// support multimodal/image input.
|
||||||
|
func isMultimodalNotSupportedError(err error) bool {
|
||||||
|
if err == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
msg := strings.ToLower(err.Error())
|
||||||
|
return (strings.Contains(msg, "multimodal") || strings.Contains(msg, "image") || strings.Contains(msg, "vision")) &&
|
||||||
|
(strings.Contains(msg, "not support") || strings.Contains(msg, "unsupported") || strings.Contains(msg, "400"))
|
||||||
|
}
|
||||||
|
|
||||||
|
// stripImagesFromMessages returns a copy of messages with all image data removed.
|
||||||
|
func stripImagesFromMessages(messages []Message) []Message {
|
||||||
|
cleaned := make([]Message, len(messages))
|
||||||
|
for i, msg := range messages {
|
||||||
|
cleaned[i] = msg
|
||||||
|
cleaned[i].Images = nil
|
||||||
|
}
|
||||||
|
return cleaned
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,26 +0,0 @@
|
|||||||
package chat
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestBuildLangfuseMessagesRedactsMultiContentImage(t *testing.T) {
|
|
||||||
secret := "TOPSECRETBASE64"
|
|
||||||
msgs := buildLangfuseMessages([]Message{{
|
|
||||||
Role: "user",
|
|
||||||
MultiContent: []MessageContentPart{
|
|
||||||
{Type: "image_url", ImageURL: &ImageURL{URL: "data:image/png;base64," + secret}},
|
|
||||||
{Type: "text", Text: "describe"},
|
|
||||||
},
|
|
||||||
}})
|
|
||||||
encoded, err := json.Marshal(msgs)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
serialized := string(encoded)
|
|
||||||
if strings.Contains(serialized, secret) || strings.Contains(strings.ToLower(serialized), "data:image") {
|
|
||||||
t.Fatalf("Langfuse input leaked image payload: %s", serialized)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -2,12 +2,10 @@ package chat
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
|
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
|
||||||
"github.com/Tencent/WeKnora/internal/types"
|
"github.com/Tencent/WeKnora/internal/types"
|
||||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// langfuseChat wraps a Chat implementation and emits a Langfuse generation
|
// langfuseChat wraps a Chat implementation and emits a Langfuse generation
|
||||||
@@ -18,9 +16,8 @@ type langfuseChat struct {
|
|||||||
inner Chat
|
inner Chat
|
||||||
}
|
}
|
||||||
|
|
||||||
func (l *langfuseChat) GetModelName() string { return l.inner.GetModelName() }
|
func (l *langfuseChat) GetModelName() string { return l.inner.GetModelName() }
|
||||||
func (l *langfuseChat) GetModelID() string { return l.inner.GetModelID() }
|
func (l *langfuseChat) GetModelID() string { return l.inner.GetModelID() }
|
||||||
func (l *langfuseChat) SupportsImageInput() bool { return SupportsImageInput(l.inner) }
|
|
||||||
|
|
||||||
func (l *langfuseChat) Chat(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) {
|
func (l *langfuseChat) Chat(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) {
|
||||||
mgr := langfuse.GetManager()
|
mgr := langfuse.GetManager()
|
||||||
@@ -50,7 +47,7 @@ func (l *langfuseChat) Chat(ctx context.Context, messages []Message, opts *ChatO
|
|||||||
resp.Content, resp.ReasoningContent, resp.FinishReason, resp.ToolCalls,
|
resp.Content, resp.ReasoningContent, resp.FinishReason, resp.ToolCalls,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
gen.Finish(output, usage, sanitizeLangfuseError(err))
|
gen.Finish(output, usage, err)
|
||||||
return resp, err
|
return resp, err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -74,7 +71,7 @@ func (l *langfuseChat) ChatStream(ctx context.Context, messages []Message, opts
|
|||||||
|
|
||||||
ch, err := l.inner.ChatStream(genCtx, messages, opts)
|
ch, err := l.inner.ChatStream(genCtx, messages, opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
gen.Finish(nil, nil, sanitizeLangfuseError(err))
|
gen.Finish(nil, nil, err)
|
||||||
return ch, err
|
return ch, err
|
||||||
}
|
}
|
||||||
if ch == nil {
|
if ch == nil {
|
||||||
@@ -134,21 +131,10 @@ func buildLangfuseMessages(messages []Message) []map[string]interface{} {
|
|||||||
"role": m.Role,
|
"role": m.Role,
|
||||||
}
|
}
|
||||||
if m.Content != "" {
|
if m.Content != "" {
|
||||||
entry["content"] = secutils.RedactDataURLsForLog(m.Content)
|
entry["content"] = m.Content
|
||||||
}
|
}
|
||||||
if len(m.MultiContent) > 0 {
|
if len(m.MultiContent) > 0 {
|
||||||
parts := make([]MessageContentPart, len(m.MultiContent))
|
entry["content"] = m.MultiContent
|
||||||
copy(parts, m.MultiContent)
|
|
||||||
for i := range parts {
|
|
||||||
parts[i].Text = secutils.RedactDataURLsForLog(parts[i].Text)
|
|
||||||
if parts[i].ImageURL == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
imageURL := *parts[i].ImageURL
|
|
||||||
imageURL.URL = "[image-url-redacted]"
|
|
||||||
parts[i].ImageURL = &imageURL
|
|
||||||
}
|
|
||||||
entry["content"] = parts
|
|
||||||
}
|
}
|
||||||
if m.Name != "" {
|
if m.Name != "" {
|
||||||
entry["name"] = m.Name
|
entry["name"] = m.Name
|
||||||
@@ -157,15 +143,10 @@ func buildLangfuseMessages(messages []Message) []map[string]interface{} {
|
|||||||
entry["tool_call_id"] = m.ToolCallID
|
entry["tool_call_id"] = m.ToolCallID
|
||||||
}
|
}
|
||||||
if len(m.ToolCalls) > 0 {
|
if len(m.ToolCalls) > 0 {
|
||||||
calls := make([]ToolCall, len(m.ToolCalls))
|
entry["tool_calls"] = m.ToolCalls
|
||||||
copy(calls, m.ToolCalls)
|
|
||||||
for i := range calls {
|
|
||||||
calls[i].Function.Arguments = secutils.RedactDataURLsForLog(calls[i].Function.Arguments)
|
|
||||||
}
|
|
||||||
entry["tool_calls"] = calls
|
|
||||||
}
|
}
|
||||||
if m.ReasoningContent != "" {
|
if m.ReasoningContent != "" {
|
||||||
entry["reasoning_content"] = secutils.RedactDataURLsForLog(m.ReasoningContent)
|
entry["reasoning_content"] = m.ReasoningContent
|
||||||
}
|
}
|
||||||
out = append(out, entry)
|
out = append(out, entry)
|
||||||
}
|
}
|
||||||
@@ -176,32 +157,17 @@ func buildLangfuseGenerationOutput(
|
|||||||
content, reasoningContent, finishReason string,
|
content, reasoningContent, finishReason string,
|
||||||
toolCalls []types.LLMToolCall,
|
toolCalls []types.LLMToolCall,
|
||||||
) map[string]interface{} {
|
) map[string]interface{} {
|
||||||
var safeToolCalls []types.LLMToolCall
|
|
||||||
if toolCalls != nil {
|
|
||||||
safeToolCalls = make([]types.LLMToolCall, len(toolCalls))
|
|
||||||
copy(safeToolCalls, toolCalls)
|
|
||||||
}
|
|
||||||
for i := range safeToolCalls {
|
|
||||||
safeToolCalls[i].Function.Arguments = secutils.RedactDataURLsForLog(safeToolCalls[i].Function.Arguments)
|
|
||||||
}
|
|
||||||
output := map[string]interface{}{
|
output := map[string]interface{}{
|
||||||
"content": secutils.RedactDataURLsForLog(content),
|
"content": content,
|
||||||
"tool_calls": safeToolCalls,
|
"tool_calls": toolCalls,
|
||||||
"finish_reason": finishReason,
|
"finish_reason": finishReason,
|
||||||
}
|
}
|
||||||
if reasoningContent != "" {
|
if reasoningContent != "" {
|
||||||
output["reasoning_content"] = secutils.RedactDataURLsForLog(reasoningContent)
|
output["reasoning_content"] = reasoningContent
|
||||||
}
|
}
|
||||||
return output
|
return output
|
||||||
}
|
}
|
||||||
|
|
||||||
func sanitizeLangfuseError(err error) error {
|
|
||||||
if err == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return errors.New(secutils.RedactDataURLsForLog(err.Error()))
|
|
||||||
}
|
|
||||||
|
|
||||||
func buildLangfuseModelParams(opts *ChatOptions) map[string]interface{} {
|
func buildLangfuseModelParams(opts *ChatOptions) map[string]interface{} {
|
||||||
if opts == nil {
|
if opts == nil {
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ func buildLLMMessages(messages []Message) []logger.LLMMessage {
|
|||||||
parts = append(parts, mc.Text)
|
parts = append(parts, mc.Text)
|
||||||
case "image_url":
|
case "image_url":
|
||||||
if mc.ImageURL != nil {
|
if mc.ImageURL != nil {
|
||||||
parts = append(parts, fmt.Sprintf("[image_url: redacted (%d bytes)]", len(mc.ImageURL.URL)))
|
parts = append(parts, fmt.Sprintf("[image_url: %s]", truncateForDebug(mc.ImageURL.URL, 120)))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,27 +0,0 @@
|
|||||||
package chat
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestBuildLLMMessagesRedactsMultiContentImageURL(t *testing.T) {
|
|
||||||
secret := "TOPSECRETBASE64"
|
|
||||||
messages := buildLLMMessages([]Message{{
|
|
||||||
Role: "user",
|
|
||||||
MultiContent: []MessageContentPart{{
|
|
||||||
Type: "image_url",
|
|
||||||
ImageURL: &ImageURL{URL: "data:image/png;base64," + secret},
|
|
||||||
}},
|
|
||||||
}})
|
|
||||||
if len(messages) != 1 {
|
|
||||||
t.Fatalf("messages length = %d", len(messages))
|
|
||||||
}
|
|
||||||
got := messages[0].Content
|
|
||||||
if strings.Contains(got, secret) || strings.Contains(strings.ToLower(got), "data:image") {
|
|
||||||
t.Fatalf("buildLLMMessages leaked image URL: %q", got)
|
|
||||||
}
|
|
||||||
if !strings.Contains(got, "redacted") {
|
|
||||||
t.Fatalf("buildLLMMessages missing redaction marker: %q", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -16,9 +16,8 @@ type debugChat struct {
|
|||||||
inner Chat
|
inner Chat
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *debugChat) GetModelName() string { return d.inner.GetModelName() }
|
func (d *debugChat) GetModelName() string { return d.inner.GetModelName() }
|
||||||
func (d *debugChat) GetModelID() string { return d.inner.GetModelID() }
|
func (d *debugChat) GetModelID() string { return d.inner.GetModelID() }
|
||||||
func (d *debugChat) SupportsImageInput() bool { return SupportsImageInput(d.inner) }
|
|
||||||
|
|
||||||
func (d *debugChat) Chat(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) {
|
func (d *debugChat) Chat(ctx context.Context, messages []Message, opts *ChatOptions) (*types.ChatResponse, error) {
|
||||||
callStart := time.Now()
|
callStart := time.Now()
|
||||||
|
|||||||
@@ -1,29 +0,0 @@
|
|||||||
package chat
|
|
||||||
|
|
||||||
import (
|
|
||||||
"io"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestOllamaBuildChatRequestRejectsAnyUnresolvableImage(t *testing.T) {
|
|
||||||
c := &OllamaChat{supportsVision: true}
|
|
||||||
_, err := c.buildChatRequest([]Message{{
|
|
||||||
Role: "user",
|
|
||||||
Content: "describe",
|
|
||||||
Images: []string{"data:image/png;base64,%%%INVALID%%%"},
|
|
||||||
}}, nil, false)
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected malformed image to fail closed")
|
|
||||||
}
|
|
||||||
if strings.Contains(err.Error(), "%%%INVALID%%%") {
|
|
||||||
t.Fatalf("image value leaked in error: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReadAnthropicResponseBodyRejectsOversize(t *testing.T) {
|
|
||||||
_, err := readAnthropicResponseBody(io.NopCloser(strings.NewReader(strings.Repeat("x", anthropicResponseBodyLimit+1))))
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected oversized Anthropic response rejection")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -3,7 +3,7 @@ package chat
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strconv"
|
"strconv"
|
||||||
@@ -13,33 +13,27 @@ import (
|
|||||||
"github.com/Tencent/WeKnora/internal/logger"
|
"github.com/Tencent/WeKnora/internal/logger"
|
||||||
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
|
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
|
||||||
"github.com/Tencent/WeKnora/internal/types"
|
"github.com/Tencent/WeKnora/internal/types"
|
||||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
|
||||||
ollamaapi "github.com/ollama/ollama/api"
|
ollamaapi "github.com/ollama/ollama/api"
|
||||||
)
|
)
|
||||||
|
|
||||||
// OllamaChat 实现了基于 Ollama 的聊天
|
// OllamaChat 实现了基于 Ollama 的聊天
|
||||||
type OllamaChat struct {
|
type OllamaChat struct {
|
||||||
modelName string
|
modelName string
|
||||||
modelID string
|
modelID string
|
||||||
supportsVision bool
|
ollamaService *ollama.OllamaService
|
||||||
ollamaService *ollama.OllamaService
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *OllamaChat) SupportsImageInput() bool { return c.supportsVision }
|
|
||||||
|
|
||||||
// NewOllamaChat 创建 Ollama 聊天实例
|
// NewOllamaChat 创建 Ollama 聊天实例
|
||||||
func NewOllamaChat(config *ChatConfig, ollamaService *ollama.OllamaService) (*OllamaChat, error) {
|
func NewOllamaChat(config *ChatConfig, ollamaService *ollama.OllamaService) (*OllamaChat, error) {
|
||||||
return &OllamaChat{
|
return &OllamaChat{
|
||||||
modelName: config.ModelName,
|
modelName: config.ModelName,
|
||||||
modelID: config.ModelID,
|
modelID: config.ModelID,
|
||||||
supportsVision: config.SupportsVision,
|
ollamaService: ollamaService,
|
||||||
ollamaService: ollamaService,
|
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// convertMessages converts messages and fails closed if any declared image
|
// convertMessages 转换消息格式为Ollama API格式
|
||||||
// cannot be resolved. Silently dropping one image would change the user's input.
|
func (c *OllamaChat) convertMessages(messages []Message) []ollamaapi.Message {
|
||||||
func (c *OllamaChat) convertMessages(messages []Message) ([]ollamaapi.Message, error) {
|
|
||||||
ollamaMessages := make([]ollamaapi.Message, 0, len(messages))
|
ollamaMessages := make([]ollamaapi.Message, 0, len(messages))
|
||||||
for _, msg := range messages {
|
for _, msg := range messages {
|
||||||
msgOllama := ollamaapi.Message{
|
msgOllama := ollamaapi.Message{
|
||||||
@@ -52,69 +46,47 @@ func (c *OllamaChat) convertMessages(messages []Message) ([]ollamaapi.Message, e
|
|||||||
}
|
}
|
||||||
if len(msg.Images) > 0 && msg.Role == "user" {
|
if len(msg.Images) > 0 && msg.Role == "user" {
|
||||||
for _, imgURL := range msg.Images {
|
for _, imgURL := range msg.Images {
|
||||||
imgData, err := resolveImageForOllama(imgURL)
|
if imgData := resolveImageForOllama(imgURL); imgData != nil {
|
||||||
if err != nil {
|
msgOllama.Images = append(msgOllama.Images, imgData)
|
||||||
return nil, err
|
|
||||||
}
|
}
|
||||||
msgOllama.Images = append(msgOllama.Images, imgData)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
ollamaMessages = append(ollamaMessages, msgOllama)
|
ollamaMessages = append(ollamaMessages, msgOllama)
|
||||||
}
|
}
|
||||||
return ollamaMessages, nil
|
return ollamaMessages
|
||||||
}
|
}
|
||||||
|
|
||||||
// resolveImageForOllama resolves an image URL into raw bytes for Ollama.
|
// resolveImageForOllama resolves an image URL into raw bytes for Ollama.
|
||||||
// Handles local serving paths (/files/...), data URIs, and remote HTTP URLs.
|
// Handles local serving paths (/files/...), data URIs, and remote HTTP URLs.
|
||||||
func resolveImageForOllama(imageURL string) (ollamaapi.ImageData, error) {
|
func resolveImageForOllama(imageURL string) ollamaapi.ImageData {
|
||||||
if data := resolveImageURLForOllama(imageURL); data != nil {
|
if data := resolveImageURLForOllama(imageURL); data != nil {
|
||||||
return data, nil
|
return data
|
||||||
}
|
}
|
||||||
if strings.HasPrefix(imageURL, "http://") || strings.HasPrefix(imageURL, "https://") {
|
if strings.HasPrefix(imageURL, "http://") || strings.HasPrefix(imageURL, "https://") {
|
||||||
if err := secutils.ValidateURLForSSRF(imageURL); err != nil {
|
client := &http.Client{Timeout: 30 * time.Second}
|
||||||
return nil, errors.New("ollama image URL is not allowed")
|
|
||||||
}
|
|
||||||
client := &http.Client{
|
|
||||||
Timeout: 30 * time.Second,
|
|
||||||
Transport: &http.Transport{
|
|
||||||
Proxy: nil,
|
|
||||||
DialContext: secutils.SSRFSafeDialContext,
|
|
||||||
},
|
|
||||||
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
|
|
||||||
}
|
|
||||||
resp, err := client.Get(imageURL)
|
resp, err := client.Get(imageURL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.New("ollama image download failed")
|
return nil
|
||||||
}
|
}
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
|
data, err := io.ReadAll(io.LimitReader(resp.Body, 20*1024*1024))
|
||||||
return nil, errors.New("ollama image download failed")
|
|
||||||
}
|
|
||||||
const maxOllamaImageBytes = 20 * 1024 * 1024
|
|
||||||
data, err := io.ReadAll(io.LimitReader(resp.Body, maxOllamaImageBytes+1))
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.New("ollama image download failed")
|
return nil
|
||||||
}
|
}
|
||||||
if len(data) == 0 || len(data) > maxOllamaImageBytes {
|
return data
|
||||||
return nil, errors.New("ollama image payload is invalid")
|
|
||||||
}
|
|
||||||
return data, nil
|
|
||||||
}
|
}
|
||||||
return nil, errors.New("ollama image payload is invalid")
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// buildChatRequest 构建聊天请求参数
|
// buildChatRequest 构建聊天请求参数
|
||||||
func (c *OllamaChat) buildChatRequest(messages []Message, opts *ChatOptions, isStream bool) (*ollamaapi.ChatRequest, error) {
|
func (c *OllamaChat) buildChatRequest(messages []Message, opts *ChatOptions, isStream bool) *ollamaapi.ChatRequest {
|
||||||
// 设置流式标志
|
// 设置流式标志
|
||||||
streamFlag := isStream
|
streamFlag := isStream
|
||||||
|
|
||||||
convertedMessages, err := c.convertMessages(messages)
|
// 构建请求参数
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
chatReq := &ollamaapi.ChatRequest{
|
chatReq := &ollamaapi.ChatRequest{
|
||||||
Model: c.modelName,
|
Model: c.modelName,
|
||||||
Messages: convertedMessages,
|
Messages: c.convertMessages(messages),
|
||||||
Stream: &streamFlag,
|
Stream: &streamFlag,
|
||||||
Options: make(map[string]interface{}),
|
Options: make(map[string]interface{}),
|
||||||
}
|
}
|
||||||
@@ -141,7 +113,7 @@ func (c *OllamaChat) buildChatRequest(messages []Message, opts *ChatOptions, isS
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return chatReq, nil
|
return chatReq
|
||||||
}
|
}
|
||||||
|
|
||||||
// Chat 进行非流式聊天
|
// Chat 进行非流式聊天
|
||||||
@@ -152,10 +124,7 @@ func (c *OllamaChat) Chat(ctx context.Context, messages []Message, opts *ChatOpt
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 构建请求参数
|
// 构建请求参数
|
||||||
chatReq, err := c.buildChatRequest(messages, opts, false)
|
chatReq := c.buildChatRequest(messages, opts, false)
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// 记录请求日志
|
// 记录请求日志
|
||||||
logger.GetLogger(ctx).Infof("发送聊天请求到模型 %s", c.modelName)
|
logger.GetLogger(ctx).Infof("发送聊天请求到模型 %s", c.modelName)
|
||||||
@@ -165,7 +134,7 @@ func (c *OllamaChat) Chat(ctx context.Context, messages []Message, opts *ChatOpt
|
|||||||
var promptTokens, completionTokens int
|
var promptTokens, completionTokens int
|
||||||
|
|
||||||
// 使用 Ollama 客户端发送请求
|
// 使用 Ollama 客户端发送请求
|
||||||
err = c.ollamaService.Chat(ctx, chatReq, func(resp ollamaapi.ChatResponse) error {
|
err := c.ollamaService.Chat(ctx, chatReq, func(resp ollamaapi.ChatResponse) error {
|
||||||
responseContent = resp.Message.Content
|
responseContent = resp.Message.Content
|
||||||
// 当 Content 为空但 Thinking 有内容时(如推理模型未正确配置 thinking 参数),使用 Thinking 作为兜底
|
// 当 Content 为空但 Thinking 有内容时(如推理模型未正确配置 thinking 参数),使用 Thinking 作为兜底
|
||||||
if responseContent == "" && resp.Message.Thinking != "" {
|
if responseContent == "" && resp.Message.Thinking != "" {
|
||||||
@@ -182,7 +151,7 @@ func (c *OllamaChat) Chat(ctx context.Context, messages []Message, opts *ChatOpt
|
|||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.New("ollama chat failed")
|
return nil, fmt.Errorf("聊天请求失败: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
usage := types.TokenUsage{
|
usage := types.TokenUsage{
|
||||||
@@ -211,10 +180,7 @@ func (c *OllamaChat) ChatStream(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 构建请求参数
|
// 构建请求参数
|
||||||
chatReq, err := c.buildChatRequest(messages, opts, true)
|
chatReq := c.buildChatRequest(messages, opts, true)
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// 记录请求日志
|
// 记录请求日志
|
||||||
logger.GetLogger(ctx).Infof("发送流式聊天请求到模型 %s", c.modelName)
|
logger.GetLogger(ctx).Infof("发送流式聊天请求到模型 %s", c.modelName)
|
||||||
@@ -227,8 +193,6 @@ func (c *OllamaChat) ChatStream(
|
|||||||
defer close(streamChan)
|
defer close(streamChan)
|
||||||
|
|
||||||
var thinking thinkingEmitter
|
var thinking thinkingEmitter
|
||||||
var terminalUsage *types.TokenUsage
|
|
||||||
providerDone := false
|
|
||||||
err := c.ollamaService.Chat(ctx, chatReq, func(resp ollamaapi.ChatResponse) error {
|
err := c.ollamaService.Chat(ctx, chatReq, func(resp ollamaapi.ChatResponse) error {
|
||||||
// 发送思考内容(支持 Qwen3、DeepSeek 等推理模型)
|
// 发送思考内容(支持 Qwen3、DeepSeek 等推理模型)
|
||||||
if resp.Message.Thinking != "" {
|
if resp.Message.Thinking != "" {
|
||||||
@@ -283,32 +247,32 @@ func (c *OllamaChat) ChatStream(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if resp.Done {
|
if resp.Done {
|
||||||
providerDone = true
|
var usage *types.TokenUsage
|
||||||
if resp.PromptEvalCount > 0 || resp.EvalCount > 0 {
|
if resp.PromptEvalCount > 0 || resp.EvalCount > 0 {
|
||||||
terminalUsage = &types.TokenUsage{
|
usage = &types.TokenUsage{
|
||||||
PromptTokens: resp.PromptEvalCount,
|
PromptTokens: resp.PromptEvalCount,
|
||||||
CompletionTokens: resp.EvalCount,
|
CompletionTokens: resp.EvalCount,
|
||||||
TotalTokens: resp.PromptEvalCount + resp.EvalCount,
|
TotalTokens: resp.PromptEvalCount + resp.EvalCount,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
logUsage(ctx, c.modelName, usage)
|
||||||
|
streamChan <- types.StreamResponse{
|
||||||
|
ResponseType: types.ResponseTypeAnswer,
|
||||||
|
Done: true,
|
||||||
|
Usage: usage,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
if err != nil || !providerDone {
|
if err != nil {
|
||||||
logger.GetLogger(ctx).Error("流式聊天请求失败")
|
logger.GetLogger(ctx).Errorf("流式聊天请求失败: %v", err)
|
||||||
|
// 发送错误响应
|
||||||
streamChan <- types.StreamResponse{
|
streamChan <- types.StreamResponse{
|
||||||
ResponseType: types.ResponseTypeError,
|
ResponseType: types.ResponseTypeError,
|
||||||
Content: "ollama stream failed",
|
Content: err.Error(),
|
||||||
Done: true,
|
Done: true,
|
||||||
}
|
}
|
||||||
return
|
|
||||||
}
|
|
||||||
logUsage(ctx, c.modelName, terminalUsage)
|
|
||||||
streamChan <- types.StreamResponse{
|
|
||||||
ResponseType: types.ResponseTypeAnswer,
|
|
||||||
Done: true,
|
|
||||||
Usage: terminalUsage,
|
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
|||||||
@@ -122,7 +122,7 @@ func (c *RemoteAPIChat) processStream(
|
|||||||
for {
|
for {
|
||||||
response, err := stream.Recv()
|
response, err := stream.Recv()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if err == io.EOF && state.lastFinishReason != "" {
|
if err == io.EOF {
|
||||||
logUsage(ctx, c.modelName, state.usage)
|
logUsage(ctx, c.modelName, state.usage)
|
||||||
toolCalls := state.buildOrderedToolCalls()
|
toolCalls := state.buildOrderedToolCalls()
|
||||||
streamChan <- types.StreamResponse{
|
streamChan <- types.StreamResponse{
|
||||||
@@ -134,10 +134,9 @@ func (c *RemoteAPIChat) processStream(
|
|||||||
FinishReason: state.lastFinishReason,
|
FinishReason: state.lastFinishReason,
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
logger.Warnf(ctx, "Chat completion stream ended before finish reason")
|
|
||||||
streamChan <- types.StreamResponse{
|
streamChan <- types.StreamResponse{
|
||||||
ResponseType: types.ResponseTypeError,
|
ResponseType: types.ResponseTypeError,
|
||||||
Content: "chat completion stream failed",
|
Content: err.Error(),
|
||||||
Done: true,
|
Done: true,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -179,26 +178,23 @@ func (c *RemoteAPIChat) processRawHTTPStream(
|
|||||||
for {
|
for {
|
||||||
event, err := reader.ReadEvent()
|
event, err := reader.ReadEvent()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if err == io.EOF && state.lastFinishReason != "" {
|
if err == io.EOF {
|
||||||
logUsage(ctx, c.modelName, state.usage)
|
logUsage(ctx, c.modelName, state.usage)
|
||||||
|
toolCalls := state.buildOrderedToolCalls()
|
||||||
streamChan <- types.StreamResponse{
|
streamChan <- types.StreamResponse{
|
||||||
ResponseType: types.ResponseTypeAnswer,
|
ResponseType: types.ResponseTypeAnswer,
|
||||||
|
Content: "",
|
||||||
Done: true,
|
Done: true,
|
||||||
ToolCalls: state.buildOrderedToolCalls(),
|
ToolCalls: toolCalls,
|
||||||
Usage: state.usage,
|
Usage: state.usage,
|
||||||
FinishReason: state.lastFinishReason,
|
|
||||||
}
|
}
|
||||||
return
|
|
||||||
}
|
|
||||||
if err != io.EOF {
|
|
||||||
logger.Warnf(ctx, "Chat stream read failed")
|
|
||||||
} else {
|
} else {
|
||||||
logger.Warnf(ctx, "Chat stream ended before finish reason")
|
logger.Errorf(ctx, "Stream read error: %v", err)
|
||||||
}
|
streamChan <- types.StreamResponse{
|
||||||
streamChan <- types.StreamResponse{
|
ResponseType: types.ResponseTypeError,
|
||||||
ResponseType: types.ResponseTypeError,
|
Content: err.Error(),
|
||||||
Content: "chat stream ended before completion",
|
Done: true,
|
||||||
Done: true,
|
}
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -208,14 +204,6 @@ func (c *RemoteAPIChat) processRawHTTPStream(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if event.Done {
|
if event.Done {
|
||||||
if state.lastFinishReason == "" {
|
|
||||||
streamChan <- types.StreamResponse{
|
|
||||||
ResponseType: types.ResponseTypeError,
|
|
||||||
Content: "chat stream ended before completion",
|
|
||||||
Done: true,
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
logUsage(ctx, c.modelName, state.usage)
|
logUsage(ctx, c.modelName, state.usage)
|
||||||
toolCalls := state.buildOrderedToolCalls()
|
toolCalls := state.buildOrderedToolCalls()
|
||||||
streamChan <- types.StreamResponse{
|
streamChan <- types.StreamResponse{
|
||||||
@@ -224,7 +212,6 @@ func (c *RemoteAPIChat) processRawHTTPStream(
|
|||||||
Done: true,
|
Done: true,
|
||||||
ToolCalls: toolCalls,
|
ToolCalls: toolCalls,
|
||||||
Usage: state.usage,
|
Usage: state.usage,
|
||||||
FinishReason: state.lastFinishReason,
|
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -254,13 +241,8 @@ func (c *RemoteAPIChat) processRawHTTPStream(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if err := json.Unmarshal(event.Data, &streamResp); err != nil {
|
if err := json.Unmarshal(event.Data, &streamResp); err != nil {
|
||||||
logger.Warnf(ctx, "Failed to parse chat stream response")
|
logger.Errorf(ctx, "Failed to parse stream response: %v", err)
|
||||||
streamChan <- types.StreamResponse{
|
continue
|
||||||
ResponseType: types.ResponseTypeError,
|
|
||||||
Content: "chat stream response was invalid",
|
|
||||||
Done: true,
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if streamResp.Usage != nil {
|
if streamResp.Usage != nil {
|
||||||
@@ -471,18 +453,38 @@ func (c *RemoteAPIChat) processStreamDelta(
|
|||||||
streamChan <- types.StreamResponse{
|
streamChan <- types.StreamResponse{
|
||||||
ResponseType: types.ResponseTypeAnswer,
|
ResponseType: types.ResponseTypeAnswer,
|
||||||
Content: delta.Content,
|
Content: delta.Content,
|
||||||
Done: false,
|
Done: isDone,
|
||||||
ToolCalls: state.buildOrderedToolCalls(),
|
ToolCalls: state.buildOrderedToolCalls(),
|
||||||
FinishReason: string(choice.FinishReason),
|
FinishReason: string(choice.FinishReason),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Ensure thinking done is sent when the provider supplied a finish reason.
|
if isDone && len(state.toolCallMap) > 0 {
|
||||||
// The outer stream reader emits the sole terminal answer after it confirms
|
streamChan <- types.StreamResponse{
|
||||||
// the provider ended successfully.
|
ResponseType: types.ResponseTypeAnswer,
|
||||||
|
Content: "",
|
||||||
|
Done: true,
|
||||||
|
ToolCalls: state.buildOrderedToolCalls(),
|
||||||
|
FinishReason: string(choice.FinishReason),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensure thinking done is sent when stream finishes without any answer content
|
||||||
|
// (e.g., model only produced reasoning then hit finish_reason with empty content).
|
||||||
if isDone {
|
if isDone {
|
||||||
state.finish(streamChan)
|
state.finish(streamChan)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Catch-all: isDone but none of the above branches sent a response with
|
||||||
|
// FinishReason (empty content, no tool calls, no thinking). This prevents
|
||||||
|
// the finish_reason from being lost in the streaming pipeline.
|
||||||
|
if isDone && delta.Content == "" && len(state.toolCallMap) == 0 {
|
||||||
|
streamChan <- types.StreamResponse{
|
||||||
|
ResponseType: types.ResponseTypeAnswer,
|
||||||
|
Done: true,
|
||||||
|
FinishReason: string(choice.FinishReason),
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// processToolCallsDelta 处理 tool calls 的增量更新
|
// processToolCallsDelta 处理 tool calls 的增量更新
|
||||||
|
|||||||
@@ -1,129 +0,0 @@
|
|||||||
package chat
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/Tencent/WeKnora/internal/types"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestRemoteAPIProviderErrorDoesNotExposeImagePayload(t *testing.T) {
|
|
||||||
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
|
|
||||||
secret := "TOPSECRETIMAGEPAYLOAD"
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
w.WriteHeader(http.StatusBadRequest)
|
|
||||||
_, _ = w.Write([]byte(`{"error":{"message":"data:image/png;base64,` + secret + `"}}`))
|
|
||||||
}))
|
|
||||||
defer server.Close()
|
|
||||||
|
|
||||||
model, err := NewRemoteAPIChat(&ChatConfig{
|
|
||||||
Source: types.ModelSourceRemote,
|
|
||||||
BaseURL: server.URL + "/v1",
|
|
||||||
ModelName: "vision-test",
|
|
||||||
APIKey: "test-key",
|
|
||||||
Provider: "openai",
|
|
||||||
SupportsVision: true,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
_, err = model.Chat(context.Background(), []Message{{
|
|
||||||
Role: "user", Content: "inspect", Images: []string{"data:image/png;base64," + secret},
|
|
||||||
}}, nil)
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("expected provider error")
|
|
||||||
}
|
|
||||||
if strings.Contains(err.Error(), secret) || strings.Contains(strings.ToLower(err.Error()), "data:image") {
|
|
||||||
t.Fatalf("provider error leaked image payload: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProcessRawHTTPStreamPrematureEOFFailsClosed(t *testing.T) {
|
|
||||||
body := "data: {\"choices\":[{\"delta\":{\"content\":\"partial\"}}]}\n\n"
|
|
||||||
resp := &http.Response{Body: io.NopCloser(strings.NewReader(body))}
|
|
||||||
stream := make(chan types.StreamResponse)
|
|
||||||
model := &RemoteAPIChat{modelName: "cloud-vision"}
|
|
||||||
go model.processRawHTTPStream(context.Background(), resp, stream, nil)
|
|
||||||
|
|
||||||
var got []types.StreamResponse
|
|
||||||
for chunk := range stream {
|
|
||||||
got = append(got, chunk)
|
|
||||||
}
|
|
||||||
if len(got) == 0 || got[len(got)-1].ResponseType != types.ResponseTypeError {
|
|
||||||
t.Fatalf("stream = %#v; want terminal error on premature EOF", got)
|
|
||||||
}
|
|
||||||
if got[len(got)-1].Content != "chat stream ended before completion" {
|
|
||||||
t.Fatalf("terminal error = %q", got[len(got)-1].Content)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProcessRawHTTPStreamFinishReasonThenEOFCompletes(t *testing.T) {
|
|
||||||
body := "data: {\"choices\":[{\"delta\":{\"content\":\"answer\"},\"finish_reason\":\"stop\"}]}\n\n"
|
|
||||||
resp := &http.Response{Body: io.NopCloser(strings.NewReader(body))}
|
|
||||||
stream := make(chan types.StreamResponse)
|
|
||||||
model := &RemoteAPIChat{modelName: "cloud-vision"}
|
|
||||||
go model.processRawHTTPStream(context.Background(), resp, stream, nil)
|
|
||||||
|
|
||||||
var got []types.StreamResponse
|
|
||||||
for chunk := range stream {
|
|
||||||
got = append(got, chunk)
|
|
||||||
}
|
|
||||||
if len(got) == 0 {
|
|
||||||
t.Fatal("expected stream chunks")
|
|
||||||
}
|
|
||||||
last := got[len(got)-1]
|
|
||||||
if last.ResponseType == types.ResponseTypeError || !last.Done || last.FinishReason != "stop" {
|
|
||||||
t.Fatalf("stream = %#v; want successful stop completion", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProcessRawHTTPStreamFinishReasonThenMalformedPacketFailsWithoutSuccessTerminal(t *testing.T) {
|
|
||||||
body := "data: {\"choices\":[{\"delta\":{\"content\":\"partial\"}}]}\n\n" +
|
|
||||||
"data: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n" +
|
|
||||||
"data: {not-json}\n\n"
|
|
||||||
resp := &http.Response{Body: io.NopCloser(strings.NewReader(body))}
|
|
||||||
stream := make(chan types.StreamResponse, 8)
|
|
||||||
model := &RemoteAPIChat{modelName: "stream-test"}
|
|
||||||
go model.processRawHTTPStream(context.Background(), resp, stream, nil)
|
|
||||||
|
|
||||||
successTerminals, errorTerminals := 0, 0
|
|
||||||
for item := range stream {
|
|
||||||
if item.Done && item.ResponseType == types.ResponseTypeAnswer {
|
|
||||||
successTerminals++
|
|
||||||
}
|
|
||||||
if item.Done && item.ResponseType == types.ResponseTypeError {
|
|
||||||
errorTerminals++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if successTerminals != 0 || errorTerminals != 1 {
|
|
||||||
t.Fatalf("terminal counts success=%d error=%d; want 0/1", successTerminals, errorTerminals)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProcessRawHTTPStreamEmitsExactlyOneSuccessTerminal(t *testing.T) {
|
|
||||||
body := "data: {\"choices\":[{\"delta\":{\"content\":\"ok\"}}]}\n\n" +
|
|
||||||
"data: {\"choices\":[{\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n" +
|
|
||||||
"data: [DONE]\n\n"
|
|
||||||
resp := &http.Response{Body: io.NopCloser(strings.NewReader(body))}
|
|
||||||
stream := make(chan types.StreamResponse, 8)
|
|
||||||
model := &RemoteAPIChat{modelName: "stream-test"}
|
|
||||||
go model.processRawHTTPStream(context.Background(), resp, stream, nil)
|
|
||||||
|
|
||||||
successTerminals := 0
|
|
||||||
for item := range stream {
|
|
||||||
if item.Done && item.ResponseType == types.ResponseTypeAnswer {
|
|
||||||
successTerminals++
|
|
||||||
}
|
|
||||||
if item.ResponseType == types.ResponseTypeError {
|
|
||||||
t.Fatalf("unexpected stream error: %#v", item)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if successTerminals != 1 {
|
|
||||||
t.Fatalf("success terminals = %d, want 1", successTerminals)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -21,15 +21,14 @@ import (
|
|||||||
// providerAdapter(见 provider.go),thinking 编码委托给 ThinkingStrategy
|
// providerAdapter(见 provider.go),thinking 编码委托给 ThinkingStrategy
|
||||||
// (见 thinking.go)。
|
// (见 thinking.go)。
|
||||||
type RemoteAPIChat struct {
|
type RemoteAPIChat struct {
|
||||||
modelName string
|
modelName string
|
||||||
client *openai.Client
|
client *openai.Client
|
||||||
modelID string
|
modelID string
|
||||||
baseURL string
|
baseURL string
|
||||||
apiKey string
|
apiKey string
|
||||||
provider provider.ProviderName
|
provider provider.ProviderName
|
||||||
supportsVision bool
|
appID string
|
||||||
appID string
|
appSecret string
|
||||||
appSecret string
|
|
||||||
// customHeaders 为用户在模型配置中指定的自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。
|
// customHeaders 为用户在模型配置中指定的自定义 HTTP 请求头(类似 OpenAI Python SDK 的 extra_headers)。
|
||||||
customHeaders map[string]string
|
customHeaders map[string]string
|
||||||
|
|
||||||
@@ -39,8 +38,6 @@ type RemoteAPIChat struct {
|
|||||||
thinkingOverride ThinkingStrategy
|
thinkingOverride ThinkingStrategy
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *RemoteAPIChat) SupportsImageInput() bool { return c.supportsVision }
|
|
||||||
|
|
||||||
// NewRemoteAPIChat 创建远程 API 聊天实例
|
// NewRemoteAPIChat 创建远程 API 聊天实例
|
||||||
func NewRemoteAPIChat(chatConfig *ChatConfig) (*RemoteAPIChat, error) {
|
func NewRemoteAPIChat(chatConfig *ChatConfig) (*RemoteAPIChat, error) {
|
||||||
if chatConfig.BaseURL != "" {
|
if chatConfig.BaseURL != "" {
|
||||||
@@ -106,7 +103,6 @@ func NewRemoteAPIChat(chatConfig *ChatConfig) (*RemoteAPIChat, error) {
|
|||||||
baseURL: chatConfig.BaseURL,
|
baseURL: chatConfig.BaseURL,
|
||||||
apiKey: apiKey,
|
apiKey: apiKey,
|
||||||
provider: providerName,
|
provider: providerName,
|
||||||
supportsVision: chatConfig.SupportsVision,
|
|
||||||
appID: chatConfig.AppID,
|
appID: chatConfig.AppID,
|
||||||
appSecret: chatConfig.AppSecret,
|
appSecret: chatConfig.AppSecret,
|
||||||
customHeaders: chatConfig.CustomHeaders,
|
customHeaders: chatConfig.CustomHeaders,
|
||||||
@@ -124,11 +120,7 @@ func (c *RemoteAPIChat) authCreds() authCreds {
|
|||||||
// transform and parameter shaping (but not thinking, which may wrap the body).
|
// transform and parameter shaping (but not thinking, which may wrap the body).
|
||||||
func (c *RemoteAPIChat) shapedRequest(messages []Message, opts *ChatOptions, isStream bool) openai.ChatCompletionRequest {
|
func (c *RemoteAPIChat) shapedRequest(messages []Message, opts *ChatOptions, isStream bool) openai.ChatCompletionRequest {
|
||||||
req := c.BuildChatCompletionRequest(messages, opts, isStream)
|
req := c.BuildChatCompletionRequest(messages, opts, isStream)
|
||||||
// WeKnora Cloud text models retain the legacy plain-text downgrade, while
|
req.Messages = c.adapter.TransformMessages(req.Messages)
|
||||||
// vision-capable Chat models must keep the OpenAI-compatible image parts.
|
|
||||||
if c.provider != provider.ProviderWeKnoraCloud || !c.supportsVision {
|
|
||||||
req.Messages = c.adapter.TransformMessages(req.Messages)
|
|
||||||
}
|
|
||||||
c.adapter.ShapeRequest(&req, opts, isStream)
|
c.adapter.ShapeRequest(&req, opts, isStream)
|
||||||
return req
|
return req
|
||||||
}
|
}
|
||||||
@@ -188,7 +180,15 @@ func (c *RemoteAPIChat) Chat(ctx context.Context, messages []Message, opts *Chat
|
|||||||
c.logRequest(timeoutCtx, req, false)
|
c.logRequest(timeoutCtx, req, false)
|
||||||
resp, err := c.client.CreateChatCompletion(timeoutCtx, req)
|
resp, err := c.client.CreateChatCompletion(timeoutCtx, req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("create chat completion failed")
|
if isMultimodalNotSupportedError(err) {
|
||||||
|
logger.Warnf(timeoutCtx, "[LLM Request] Model %s does not support multimodal, retrying without images", c.modelName)
|
||||||
|
cleaned := stripImagesFromMessages(messages)
|
||||||
|
req = c.shapedRequest(cleaned, opts, false)
|
||||||
|
resp, err = c.client.CreateChatCompletion(timeoutCtx, req)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("create chat completion: %w", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
result, err := c.parseCompletionResponse(&resp)
|
result, err := c.parseCompletionResponse(&resp)
|
||||||
@@ -231,13 +231,13 @@ func (c *RemoteAPIChat) chatWithRawHTTP(ctx context.Context, endpoint string, cu
|
|||||||
|
|
||||||
resp, err := rawHTTPClient.Do(httpReq)
|
resp, err := rawHTTPClient.Do(httpReq)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("send chat request failed")
|
return nil, fmt.Errorf("send request: %w", err)
|
||||||
}
|
}
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
if resp.StatusCode != http.StatusOK {
|
||||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
body, _ := io.ReadAll(resp.Body)
|
||||||
return nil, remoteAPIStatusError(resp.StatusCode, body)
|
return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
|
||||||
}
|
}
|
||||||
|
|
||||||
body, err := io.ReadAll(resp.Body)
|
body, err := io.ReadAll(resp.Body)
|
||||||
@@ -259,10 +259,6 @@ func (c *RemoteAPIChat) chatWithRawHTTP(ctx context.Context, endpoint string, cu
|
|||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func remoteAPIStatusError(status int, _ []byte) error {
|
|
||||||
return fmt.Errorf("API request failed with status %d", status)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ChatStream 进行流式聊天
|
// ChatStream 进行流式聊天
|
||||||
func (c *RemoteAPIChat) ChatStream(ctx context.Context, messages []Message, opts *ChatOptions) (<-chan types.StreamResponse, error) {
|
func (c *RemoteAPIChat) ChatStream(ctx context.Context, messages []Message, opts *ChatOptions) (<-chan types.StreamResponse, error) {
|
||||||
// 仅在调用方未设置 deadline 时附加兜底超时;流式调用默认超时更长,
|
// 仅在调用方未设置 deadline 时附加兜底超时;流式调用默认超时更长,
|
||||||
@@ -291,9 +287,17 @@ func (c *RemoteAPIChat) ChatStream(ctx context.Context, messages []Message, opts
|
|||||||
|
|
||||||
stream, err := c.client.CreateChatCompletionStream(timeoutCtx, req)
|
stream, err := c.client.CreateChatCompletionStream(timeoutCtx, req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
cancel()
|
if isMultimodalNotSupportedError(err) {
|
||||||
close(streamChan)
|
logger.Warnf(timeoutCtx, "[LLM Stream] Model %s does not support multimodal, retrying without images", c.modelName)
|
||||||
return nil, fmt.Errorf("create chat completion stream failed")
|
cleaned := stripImagesFromMessages(messages)
|
||||||
|
req = c.shapedRequest(cleaned, opts, true)
|
||||||
|
stream, err = c.client.CreateChatCompletionStream(timeoutCtx, req)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
cancel()
|
||||||
|
close(streamChan)
|
||||||
|
return nil, fmt.Errorf("create chat completion stream: %w", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
@@ -359,13 +363,13 @@ func (c *RemoteAPIChat) chatStreamWithRawHTTP(ctx context.Context, endpoint stri
|
|||||||
|
|
||||||
resp, err := rawHTTPClient.Do(httpReq)
|
resp, err := rawHTTPClient.Do(httpReq)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("send chat request failed")
|
return nil, fmt.Errorf("send request: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
if resp.StatusCode != http.StatusOK {
|
||||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
body, _ := io.ReadAll(resp.Body)
|
||||||
resp.Body.Close()
|
resp.Body.Close()
|
||||||
return nil, remoteAPIStatusError(resp.StatusCode, body)
|
return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
|
||||||
}
|
}
|
||||||
|
|
||||||
streamChan := make(chan types.StreamResponse)
|
streamChan := make(chan types.StreamResponse)
|
||||||
|
|||||||
@@ -3,11 +3,7 @@ package chat
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"os"
|
"os"
|
||||||
"strings"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -17,17 +13,6 @@ import (
|
|||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestRemoteAPIStatusErrorDoesNotExposeResponseBody(t *testing.T) {
|
|
||||||
secret := "data:image/png;base64,TOPSECRET"
|
|
||||||
err := remoteAPIStatusError(http.StatusBadRequest, []byte(secret))
|
|
||||||
if strings.Contains(err.Error(), secret) || strings.Contains(err.Error(), "TOPSECRET") {
|
|
||||||
t.Fatalf("remote API error leaked response body: %v", err)
|
|
||||||
}
|
|
||||||
if !strings.Contains(err.Error(), "400") {
|
|
||||||
t.Fatalf("remote API error = %v, want status code", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func newTestRemoteChat(t *testing.T) *RemoteAPIChat {
|
func newTestRemoteChat(t *testing.T) *RemoteAPIChat {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -548,44 +533,3 @@ func TestTokenUsage_CachedTokensJSONOmitempty(t *testing.T) {
|
|||||||
assert.Contains(t, string(b), `"cached_tokens":7`)
|
assert.Contains(t, string(b), `"cached_tokens":7`)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func newMultimodalUnsupportedTestChat(t *testing.T, calls *atomic.Int32) *RemoteAPIChat {
|
|
||||||
t.Helper()
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
calls.Add(1)
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
w.WriteHeader(http.StatusBadRequest)
|
|
||||||
_, _ = w.Write([]byte(`{"error":{"message":"image input is not supported","type":"invalid_request_error","code":"unsupported"}}`))
|
|
||||||
}))
|
|
||||||
t.Cleanup(server.Close)
|
|
||||||
|
|
||||||
chat := newTestRemoteChat(t)
|
|
||||||
cfg := openai.DefaultConfig("test-key")
|
|
||||||
cfg.BaseURL = server.URL
|
|
||||||
chat.client = openai.NewClientWithConfig(cfg)
|
|
||||||
return chat
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRemoteAPIChat_MultimodalUnsupportedDoesNotStripImages(t *testing.T) {
|
|
||||||
messages := []Message{{
|
|
||||||
Role: "user",
|
|
||||||
Content: "read this image",
|
|
||||||
Images: []string{"data:image/png;base64,iVBORw0KGgo="},
|
|
||||||
}}
|
|
||||||
|
|
||||||
t.Run("non-streaming", func(t *testing.T) {
|
|
||||||
var calls atomic.Int32
|
|
||||||
chat := newMultimodalUnsupportedTestChat(t, &calls)
|
|
||||||
_, err := chat.Chat(context.Background(), messages, &ChatOptions{})
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.EqualValues(t, 1, calls.Load(), "image input must not be removed and retried as text")
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("streaming", func(t *testing.T) {
|
|
||||||
var calls atomic.Int32
|
|
||||||
chat := newMultimodalUnsupportedTestChat(t, &calls)
|
|
||||||
_, err := chat.ChatStream(context.Background(), messages, &ChatOptions{})
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.EqualValues(t, 1, calls.Load(), "image input must not be removed and retried as text")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -8,8 +8,6 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// streamRawDumpDir returns the directory for per-stream raw packet dumps.
|
// streamRawDumpDir returns the directory for per-stream raw packet dumps.
|
||||||
@@ -45,10 +43,7 @@ func newStreamPacketDumper(modelName string, request any) *streamPacketDumper {
|
|||||||
if dir == "" {
|
if dir == "" {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if err := os.Chmod(dir, 0o700); err != nil {
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -75,14 +70,10 @@ func newStreamPacketDumper(modelName string, request any) *streamPacketDumper {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
|
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if err := f.Chmod(0o600); err != nil {
|
|
||||||
_ = f.Close()
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
d := &streamPacketDumper{file: f, path: path, model: modelName}
|
d := &streamPacketDumper{file: f, path: path, model: modelName}
|
||||||
_ = d.writeRequest(request)
|
_ = d.writeRequest(request)
|
||||||
@@ -90,19 +81,11 @@ func newStreamPacketDumper(modelName string, request any) *streamPacketDumper {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (d *streamPacketDumper) writeRequest(request any) error {
|
func (d *streamPacketDumper) writeRequest(request any) error {
|
||||||
raw, err := json.Marshal(request)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
var redacted any
|
|
||||||
if err := json.Unmarshal([]byte(secutils.CompactImageDataURLForLog(string(raw))), &redacted); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
line, err := json.Marshal(map[string]any{
|
line, err := json.Marshal(map[string]any{
|
||||||
"type": "request",
|
"type": "request",
|
||||||
"model": d.model,
|
"model": d.model,
|
||||||
"timestamp": time.Now().UTC().Format(time.RFC3339Nano),
|
"timestamp": time.Now().UTC().Format(time.RFC3339Nano),
|
||||||
"data": redacted,
|
"data": request,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -129,7 +112,6 @@ func (d *streamPacketDumper) WritePacketRaw(raw []byte) {
|
|||||||
if len(raw) == 0 {
|
if len(raw) == 0 {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
raw = []byte(secutils.CompactImageDataURLForLog(string(raw)))
|
|
||||||
|
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
defer d.mu.Unlock()
|
defer d.mu.Unlock()
|
||||||
@@ -157,7 +139,6 @@ func (d *streamPacketDumper) WriteError(message string) {
|
|||||||
if d == nil || d.file == nil || strings.TrimSpace(message) == "" {
|
if d == nil || d.file == nil || strings.TrimSpace(message) == "" {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
message = secutils.CompactImageDataURLForLog(message)
|
|
||||||
line, err := json.Marshal(map[string]any{
|
line, err := json.Marshal(map[string]any{
|
||||||
"type": "error",
|
"type": "error",
|
||||||
"model": d.model,
|
"model": d.model,
|
||||||
@@ -182,7 +163,7 @@ func (d *streamPacketDumper) WriteHTTPError(statusCode int, body []byte) {
|
|||||||
"status_code": statusCode,
|
"status_code": statusCode,
|
||||||
}
|
}
|
||||||
if len(body) > 0 {
|
if len(body) > 0 {
|
||||||
trimmed := []byte(secutils.CompactImageDataURLForLog(string(bytesTrimSpace(body))))
|
trimmed := bytesTrimSpace(body)
|
||||||
if json.Valid(trimmed) {
|
if json.Valid(trimmed) {
|
||||||
var parsed any
|
var parsed any
|
||||||
if json.Unmarshal(trimmed, &parsed) == nil {
|
if json.Unmarshal(trimmed, &parsed) == nil {
|
||||||
|
|||||||
@@ -1,51 +0,0 @@
|
|||||||
package chat
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestStreamPacketDumperRedactsImageRequestAndUsesPrivatePermissions(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
if err := os.Chmod(dir, 0o755); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Setenv("WEKNORA_LLM_STREAM_RAW_DUMP_DIR", dir)
|
|
||||||
secret := strings.Repeat("TOPSECRET", 30)
|
|
||||||
dumper := newStreamPacketDumper("vision-model", map[string]any{
|
|
||||||
"messages": []any{map[string]any{
|
|
||||||
"role": "user",
|
|
||||||
"image_url": map[string]any{"url": "data:image/png;base64," + secret},
|
|
||||||
}},
|
|
||||||
})
|
|
||||||
if dumper == nil {
|
|
||||||
t.Fatal("expected dumper")
|
|
||||||
}
|
|
||||||
path := dumper.Path()
|
|
||||||
dumper.Close()
|
|
||||||
data, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("read dump: %v", err)
|
|
||||||
}
|
|
||||||
if strings.Contains(string(data), secret) || strings.Contains(string(data), "data:image/") {
|
|
||||||
t.Fatalf("image data leaked to dump: %s", data)
|
|
||||||
}
|
|
||||||
if !strings.Contains(string(data), "[image-data-redacted]") {
|
|
||||||
t.Fatalf("missing redaction marker: %s", data)
|
|
||||||
}
|
|
||||||
info, err := os.Stat(path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("stat dump: %v", err)
|
|
||||||
}
|
|
||||||
if got := info.Mode().Perm(); got != 0o600 {
|
|
||||||
t.Fatalf("dump permissions = %o, want 600", got)
|
|
||||||
}
|
|
||||||
dirInfo, err := os.Stat(dir)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("stat dump directory: %v", err)
|
|
||||||
}
|
|
||||||
if got := dirInfo.Mode().Perm(); got != 0o700 {
|
|
||||||
t.Fatalf("dump directory permissions = %o, want 700", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,80 +0,0 @@
|
|||||||
package chat
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/Tencent/WeKnora/internal/models/provider"
|
|
||||||
"github.com/Tencent/WeKnora/internal/types"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestRemoteAPIChat_WeKnoraCloudTransformsImagesByCapability(t *testing.T) {
|
|
||||||
messages := []Message{{
|
|
||||||
Role: "user",
|
|
||||||
Content: "请识别图片",
|
|
||||||
Images: []string{"data:image/png;base64,iVBORw0KGgo="},
|
|
||||||
}}
|
|
||||||
|
|
||||||
build := func(t *testing.T, supportsVision bool) *RemoteAPIChat {
|
|
||||||
t.Helper()
|
|
||||||
model, err := NewRemoteAPIChat(&ChatConfig{
|
|
||||||
Source: types.ModelSourceRemote,
|
|
||||||
ModelName: "cloud-chat",
|
|
||||||
ModelID: "cloud-chat",
|
|
||||||
Provider: string(provider.ProviderWeKnoraCloud),
|
|
||||||
AppID: "test-app",
|
|
||||||
AppSecret: "test-secret",
|
|
||||||
SupportsVision: supportsVision,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
return model
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Run("vision chat preserves image parts", func(t *testing.T) {
|
|
||||||
req := build(t, true).shapedRequest(messages, &ChatOptions{}, false)
|
|
||||||
require.Len(t, req.Messages, 1)
|
|
||||||
require.Len(t, req.Messages[0].MultiContent, 2)
|
|
||||||
require.NotNil(t, req.Messages[0].MultiContent[0].ImageURL)
|
|
||||||
require.Equal(t, messages[0].Images[0], req.Messages[0].MultiContent[0].ImageURL.URL)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("text chat keeps legacy downgrade", func(t *testing.T) {
|
|
||||||
req := build(t, false).shapedRequest(messages, &ChatOptions{}, false)
|
|
||||||
require.Len(t, req.Messages, 1)
|
|
||||||
require.Empty(t, req.Messages[0].MultiContent)
|
|
||||||
require.Equal(t, messages[0].Content, req.Messages[0].Content)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConfigFromModelRecognizesExistingWeKnoraCloudChatAsVision(t *testing.T) {
|
|
||||||
cloudChat := &types.Model{
|
|
||||||
Name: "chat",
|
|
||||||
Type: types.ModelTypeKnowledgeQA,
|
|
||||||
Source: types.ModelSourceRemote,
|
|
||||||
Parameters: types.ModelParameters{
|
|
||||||
Provider: string(provider.ProviderWeKnoraCloud),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
genericChat := &types.Model{
|
|
||||||
Name: "chat",
|
|
||||||
Type: types.ModelTypeKnowledgeQA,
|
|
||||||
Source: types.ModelSourceRemote,
|
|
||||||
}
|
|
||||||
|
|
||||||
require.True(t, ConfigFromModel(cloudChat, "", "").SupportsVision)
|
|
||||||
require.False(t, ConfigFromModel(genericChat, "", "").SupportsVision)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSupportsImageInputUsesRuntimeTransportCapability(t *testing.T) {
|
|
||||||
visionRemote := &RemoteAPIChat{supportsVision: true}
|
|
||||||
textRemote := &RemoteAPIChat{supportsVision: false}
|
|
||||||
anthropicText := &AnthropicChat{}
|
|
||||||
anthropicVision := &AnthropicChat{supportsVision: true}
|
|
||||||
|
|
||||||
require.True(t, SupportsImageInput(visionRemote))
|
|
||||||
require.False(t, SupportsImageInput(textRemote))
|
|
||||||
require.False(t, SupportsImageInput(anthropicText))
|
|
||||||
require.True(t, SupportsImageInput(anthropicVision))
|
|
||||||
require.True(t, SupportsImageInput(&debugChat{inner: visionRemote}))
|
|
||||||
require.True(t, SupportsImageInput(&langfuseChat{inner: visionRemote}))
|
|
||||||
}
|
|
||||||
@@ -221,16 +221,15 @@ func (c *ChatManage) Clone() *ChatManage {
|
|||||||
Images: append([]string(nil), c.Images...),
|
Images: append([]string(nil), c.Images...),
|
||||||
VLMModelID: c.VLMModelID,
|
VLMModelID: c.VLMModelID,
|
||||||
ChatModelSupportsVision: c.ChatModelSupportsVision,
|
ChatModelSupportsVision: c.ChatModelSupportsVision,
|
||||||
|
Attachments: append(MessageAttachments(nil), c.Attachments...),
|
||||||
Attachments: append(MessageAttachments(nil), c.Attachments...),
|
TenantID: c.TenantID,
|
||||||
TenantID: c.TenantID,
|
WebSearchEnabled: c.WebSearchEnabled,
|
||||||
WebSearchEnabled: c.WebSearchEnabled,
|
WebSearchProviderID: c.WebSearchProviderID,
|
||||||
WebSearchProviderID: c.WebSearchProviderID,
|
WebSearchMaxResults: c.WebSearchMaxResults,
|
||||||
WebSearchMaxResults: c.WebSearchMaxResults,
|
WebFetchEnabled: c.WebFetchEnabled,
|
||||||
WebFetchEnabled: c.WebFetchEnabled,
|
WebFetchTopN: c.WebFetchTopN,
|
||||||
WebFetchTopN: c.WebFetchTopN,
|
Language: c.Language,
|
||||||
Language: c.Language,
|
IntentPromptOverrides: maps.Clone(c.IntentPromptOverrides),
|
||||||
IntentPromptOverrides: maps.Clone(c.IntentPromptOverrides),
|
|
||||||
},
|
},
|
||||||
PipelineState: PipelineState{
|
PipelineState: PipelineState{
|
||||||
RewriteQuery: c.RewriteQuery,
|
RewriteQuery: c.RewriteQuery,
|
||||||
|
|||||||
@@ -150,6 +150,10 @@ func (m *MessageAttachments) Scan(value interface{}) error {
|
|||||||
// MentionedItems is a slice of MentionedItem for database storage
|
// MentionedItems is a slice of MentionedItem for database storage
|
||||||
type MentionedItems []MentionedItem
|
type MentionedItems []MentionedItem
|
||||||
|
|
||||||
|
// MessageKnowledgeBaseIDs stores the knowledge-base scope searched for a message.
|
||||||
|
// It is kept even when retrieval returns no references so the request remains auditable.
|
||||||
|
type MessageKnowledgeBaseIDs []string
|
||||||
|
|
||||||
// Value implements the driver.Valuer interface for database serialization
|
// Value implements the driver.Valuer interface for database serialization
|
||||||
func (m MentionedItems) Value() (driver.Value, error) {
|
func (m MentionedItems) Value() (driver.Value, error) {
|
||||||
if m == nil {
|
if m == nil {
|
||||||
@@ -216,6 +220,24 @@ type Message struct {
|
|||||||
RenderedContent string `json:"-" gorm:"type:text;column:rendered_content;default:''"`
|
RenderedContent string `json:"-" gorm:"type:text;column:rendered_content;default:''"`
|
||||||
// Channel indicates the source channel of this message (e.g., "web", "api", "im")
|
// Channel indicates the source channel of this message (e.g., "web", "api", "im")
|
||||||
Channel string `json:"channel,omitempty" gorm:"type:varchar(50);default:''"`
|
Channel string `json:"channel,omitempty" gorm:"type:varchar(50);default:''"`
|
||||||
|
// SourcePlatform is the concrete external platform for IM messages (for example, dingtalk).
|
||||||
|
SourcePlatform string `json:"source_platform,omitempty" gorm:"type:varchar(20);default:''"`
|
||||||
|
// SourceChannelID identifies the configured WeKnora IM channel that received the message.
|
||||||
|
SourceChannelID string `json:"source_channel_id,omitempty" gorm:"type:varchar(36);default:''"`
|
||||||
|
// SourceMessageID is the platform message identifier used for end-to-end correlation.
|
||||||
|
SourceMessageID string `json:"source_message_id,omitempty" gorm:"type:varchar(128);default:''"`
|
||||||
|
// SourceDeliveryID correlates outbound cards/messages with later platform callbacks.
|
||||||
|
SourceDeliveryID string `json:"source_delivery_id,omitempty" gorm:"type:varchar(128);default:''"`
|
||||||
|
// SourceUserID records the platform user for this individual message.
|
||||||
|
SourceUserID string `json:"source_user_id,omitempty" gorm:"type:varchar(128);default:''"`
|
||||||
|
// SourceChatID records the platform conversation or group identifier.
|
||||||
|
SourceChatID string `json:"source_chat_id,omitempty" gorm:"type:varchar(128);default:''"`
|
||||||
|
// SourceThreadID records the platform thread identifier when present.
|
||||||
|
SourceThreadID string `json:"source_thread_id,omitempty" gorm:"type:varchar(128);default:''"`
|
||||||
|
// SearchedKnowledgeBaseIDs records the request scope even when retrieval has zero hits.
|
||||||
|
SearchedKnowledgeBaseIDs MessageKnowledgeBaseIDs `json:"searched_knowledge_base_ids,omitempty" gorm:"serializer:json;type:jsonb;column:searched_knowledge_base_ids"`
|
||||||
|
// TokenUsage records answer-generation model usage for the assistant message.
|
||||||
|
TokenUsage *TokenUsage `json:"token_usage,omitempty" gorm:"serializer:json;type:jsonb;column:token_usage"`
|
||||||
// KnowledgeID links this message to a Knowledge entry in the chat history knowledge base
|
// KnowledgeID links this message to a Knowledge entry in the chat history knowledge base
|
||||||
// Used for vector search indexing: when set, the message content has been indexed as a Knowledge passage
|
// Used for vector search indexing: when set, the message content has been indexed as a Knowledge passage
|
||||||
KnowledgeID string `json:"knowledge_id,omitempty" gorm:"type:varchar(36);index"`
|
KnowledgeID string `json:"knowledge_id,omitempty" gorm:"type:varchar(36);index"`
|
||||||
|
|||||||
@@ -0,0 +1,58 @@
|
|||||||
|
package types
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMessageAuditMigrationCoversModelFields(t *testing.T) {
|
||||||
|
upPath := "../../migrations/versioned/000064_message_audit.up.sql"
|
||||||
|
downPath := "../../migrations/versioned/000064_message_audit.down.sql"
|
||||||
|
|
||||||
|
up, err := os.ReadFile(upPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read up migration: %v", err)
|
||||||
|
}
|
||||||
|
down, err := os.ReadFile(downPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read down migration: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
columns := []string{
|
||||||
|
"source_platform",
|
||||||
|
"source_channel_id",
|
||||||
|
"source_message_id",
|
||||||
|
"source_delivery_id",
|
||||||
|
"source_user_id",
|
||||||
|
"source_chat_id",
|
||||||
|
"source_thread_id",
|
||||||
|
"searched_knowledge_base_ids",
|
||||||
|
"token_usage",
|
||||||
|
}
|
||||||
|
upSQL := strings.ToLower(string(up))
|
||||||
|
downSQL := strings.ToLower(string(down))
|
||||||
|
for _, column := range columns {
|
||||||
|
if !strings.Contains(upSQL, column) {
|
||||||
|
t.Errorf("up migration missing column %q", column)
|
||||||
|
}
|
||||||
|
if !strings.Contains(downSQL, column) {
|
||||||
|
t.Errorf("down migration missing column %q", column)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
indexes := []string{
|
||||||
|
"idx_messages_request_role",
|
||||||
|
"idx_messages_source_channel",
|
||||||
|
"idx_messages_source_message",
|
||||||
|
"idx_messages_source_user",
|
||||||
|
}
|
||||||
|
for _, index := range indexes {
|
||||||
|
if !strings.Contains(upSQL, index) {
|
||||||
|
t.Errorf("up migration missing index %q", index)
|
||||||
|
}
|
||||||
|
if !strings.Contains(downSQL, index) {
|
||||||
|
t.Errorf("down migration missing index %q", index)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
package types
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMessageCarriesAuditMetadata(t *testing.T) {
|
||||||
|
original := Message{
|
||||||
|
ID: "message-1",
|
||||||
|
SourcePlatform: "dingtalk",
|
||||||
|
SourceChannelID: "channel-1",
|
||||||
|
SourceMessageID: "platform-message-1",
|
||||||
|
SourceDeliveryID: "card-track-1",
|
||||||
|
SourceUserID: "user-1",
|
||||||
|
SourceChatID: "chat-1",
|
||||||
|
SourceThreadID: "thread-1",
|
||||||
|
SearchedKnowledgeBaseIDs: MessageKnowledgeBaseIDs{"kb-1", "kb-2"},
|
||||||
|
TokenUsage: &TokenUsage{
|
||||||
|
PromptTokens: 100,
|
||||||
|
CompletionTokens: 20,
|
||||||
|
TotalTokens: 120,
|
||||||
|
CachedTokens: 40,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
payload, err := json.Marshal(original)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("json.Marshal() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var decoded Message
|
||||||
|
if err := json.Unmarshal(payload, &decoded); err != nil {
|
||||||
|
t.Fatalf("json.Unmarshal() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if decoded.SourcePlatform != original.SourcePlatform ||
|
||||||
|
decoded.SourceChannelID != original.SourceChannelID ||
|
||||||
|
decoded.SourceMessageID != original.SourceMessageID ||
|
||||||
|
decoded.SourceDeliveryID != original.SourceDeliveryID ||
|
||||||
|
decoded.SourceUserID != original.SourceUserID ||
|
||||||
|
decoded.SourceChatID != original.SourceChatID ||
|
||||||
|
decoded.SourceThreadID != original.SourceThreadID {
|
||||||
|
t.Fatalf("source metadata was not preserved: got %+v", decoded)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(decoded.SearchedKnowledgeBaseIDs, original.SearchedKnowledgeBaseIDs) {
|
||||||
|
t.Fatalf("searched knowledge bases = %#v, want %#v", decoded.SearchedKnowledgeBaseIDs, original.SearchedKnowledgeBaseIDs)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(decoded.TokenUsage, original.TokenUsage) {
|
||||||
|
t.Fatalf("token usage = %#v, want %#v", decoded.TokenUsage, original.TokenUsage)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -4,22 +4,21 @@ package types
|
|||||||
// replacing the previous 14-parameter method signatures.
|
// replacing the previous 14-parameter method signatures.
|
||||||
// EventBus is passed separately to avoid circular dependency with the event package.
|
// EventBus is passed separately to avoid circular dependency with the event package.
|
||||||
type QARequest struct {
|
type QARequest struct {
|
||||||
Session *Session // The conversation session
|
Session *Session // The conversation session
|
||||||
Query string // User query text
|
Query string // User query text
|
||||||
AssistantMessageID string // Pre-created assistant message ID
|
AssistantMessageID string // Pre-created assistant message ID
|
||||||
SummaryModelID string // Optional model override; empty = use agent/KB default
|
SummaryModelID string // Optional model override; empty = use agent/KB default
|
||||||
CustomAgent *CustomAgent // Optional custom agent for config override
|
CustomAgent *CustomAgent // Optional custom agent for config override
|
||||||
KnowledgeBaseIDs []string // Knowledge base IDs to search (from request + @mentions)
|
KnowledgeBaseIDs []string // Knowledge base IDs to search (from request + @mentions)
|
||||||
KnowledgeIDs []string // Specific knowledge (file) IDs to search
|
KnowledgeIDs []string // Specific knowledge (file) IDs to search
|
||||||
TagScopes []TagScope // Tag-constrained KB scopes from @mentions
|
TagScopes []TagScope // Tag-constrained KB scopes from @mentions
|
||||||
MCPServiceIDs []string // Per-request MCP service IDs from @mentions
|
MCPServiceIDs []string // Per-request MCP service IDs from @mentions
|
||||||
SkillNames []string // Per-request preloaded skill names from @mentions
|
SkillNames []string // Per-request preloaded skill names from @mentions
|
||||||
ImageURLs []string // Image URLs for multimodal input
|
ImageURLs []string // Image URLs for multimodal input
|
||||||
ImageDescription string // VLM-generated image description (fallback for non-vision models)
|
ImageDescription string // VLM-generated image description (fallback for non-vision models)
|
||||||
|
UserMessageID string // Created user message ID
|
||||||
UserMessageID string // Created user message ID
|
WebSearchEnabled bool // Whether web search is enabled for this request
|
||||||
WebSearchEnabled bool // Whether web search is enabled for this request
|
EnableMemory bool // Whether memory feature is enabled
|
||||||
EnableMemory bool // Whether memory feature is enabled
|
QuotedContext string // Quoted message content from IM quote-reply (appended at LLM prompt stage, not used for retrieval)
|
||||||
QuotedContext string // Quoted message content from IM quote-reply (appended at LLM prompt stage, not used for retrieval)
|
Attachments MessageAttachments // File attachments (processed and ready for prompt injection)
|
||||||
Attachments MessageAttachments // File attachments (processed and ready for prompt injection)
|
|
||||||
}
|
}
|
||||||
|
|||||||
+14
-121
@@ -1,136 +1,29 @@
|
|||||||
package utils
|
package utils
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"regexp"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var imageDataURLPatternForLog = regexp.MustCompile(`data:image\/[a-zA-Z0-9.+-]+;base64,[A-Za-z0-9+/=]+`)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
defaultMaxLogChars = 12000
|
defaultMaxLogChars = 12000
|
||||||
dataURLRedaction = "[image-data-redacted]"
|
defaultMaxDataURLPreview = 96
|
||||||
)
|
)
|
||||||
|
|
||||||
// CompactImageDataURLForLog removes every data URI before log or dump output.
|
// CompactImageDataURLForLog shortens large image data URLs for log output.
|
||||||
// It deliberately handles non-canonical forms because debug paths may receive
|
|
||||||
// JSON-escaped or percent-encoded requests in addition to ordinary data: URLs.
|
|
||||||
func CompactImageDataURLForLog(raw string) string {
|
func CompactImageDataURLForLog(raw string) string {
|
||||||
masked := RedactDataURLsForLog(raw)
|
masked := imageDataURLPatternForLog.ReplaceAllStringFunc(raw, func(match string) string {
|
||||||
|
if len(match) <= defaultMaxDataURLPreview {
|
||||||
|
return match
|
||||||
|
}
|
||||||
|
hidden := len(match) - defaultMaxDataURLPreview
|
||||||
|
return match[:defaultMaxDataURLPreview] + "...<omitted " + strconv.Itoa(hidden) + " chars>"
|
||||||
|
})
|
||||||
|
|
||||||
if len(masked) <= defaultMaxLogChars {
|
if len(masked) <= defaultMaxLogChars {
|
||||||
return masked
|
return masked
|
||||||
}
|
}
|
||||||
return masked[:defaultMaxLogChars] + "... (truncated, total " + strconv.Itoa(len(masked)) + " chars)"
|
return masked[:defaultMaxLogChars] + "... (truncated, total " + strconv.Itoa(len(masked)) + " chars)"
|
||||||
}
|
}
|
||||||
|
|
||||||
// RedactDataURLsForLog removes data URIs without truncating surrounding text.
|
|
||||||
func RedactDataURLsForLog(raw string) string {
|
|
||||||
return redactDataURLs(raw)
|
|
||||||
}
|
|
||||||
|
|
||||||
func redactDataURLs(raw string) string {
|
|
||||||
var out strings.Builder
|
|
||||||
out.Grow(len(raw))
|
|
||||||
for i := 0; i < len(raw); {
|
|
||||||
prefixLen := dataURLPrefixLen(raw[i:])
|
|
||||||
if prefixLen == 0 {
|
|
||||||
out.WriteByte(raw[i])
|
|
||||||
i++
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
out.WriteString(dataURLRedaction)
|
|
||||||
i += prefixLen
|
|
||||||
for i < len(raw) && !isDataURLTerminator(raw, i) {
|
|
||||||
i++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
// dataURLPrefixLen recognizes "data:" when each ASCII character is either
|
|
||||||
// literal, percent encoded, or represented as one-or-more JSON escape layers.
|
|
||||||
func dataURLPrefixLen(value string) int {
|
|
||||||
pos := 0
|
|
||||||
for i := 0; i < len("data:"); i++ {
|
|
||||||
consumed := encodedASCIILen(value[pos:], "data:"[i])
|
|
||||||
if consumed == 0 {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
pos += consumed
|
|
||||||
}
|
|
||||||
return pos
|
|
||||||
}
|
|
||||||
|
|
||||||
func encodedASCIILen(value string, want byte) int {
|
|
||||||
if len(value) > 0 && strings.EqualFold(value[:1], string(want)) {
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
if len(value) >= 3 && value[0] == '%' {
|
|
||||||
pos := 0
|
|
||||||
for len(value) >= pos+3 && strings.EqualFold(value[pos:pos+3], "%25") {
|
|
||||||
pos += 3
|
|
||||||
}
|
|
||||||
if pos == 0 {
|
|
||||||
if decoded, ok := decodeHexByte(value[1], value[2]); ok && decoded == want {
|
|
||||||
return 3
|
|
||||||
}
|
|
||||||
} else if len(value) >= pos+2 {
|
|
||||||
if decoded, ok := decodeHexByte(value[pos], value[pos+1]); ok && decoded == want {
|
|
||||||
return pos + 2
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
backslashes := 0
|
|
||||||
for backslashes < len(value) && value[backslashes] == '\\' {
|
|
||||||
backslashes++
|
|
||||||
}
|
|
||||||
if backslashes > 0 && len(value) >= backslashes+5 && (value[backslashes] == 'u' || value[backslashes] == 'U') {
|
|
||||||
hex := value[backslashes+1 : backslashes+5]
|
|
||||||
if strings.EqualFold(hex[:2], "00") {
|
|
||||||
if decoded, ok := decodeHexByte(hex[2], hex[3]); ok && decoded == want {
|
|
||||||
return backslashes + 5
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
func decodeHexByte(high, low byte) (byte, bool) {
|
|
||||||
hex := func(ch byte) (byte, bool) {
|
|
||||||
switch {
|
|
||||||
case ch >= '0' && ch <= '9':
|
|
||||||
return ch - '0', true
|
|
||||||
case ch >= 'a' && ch <= 'f':
|
|
||||||
return ch - 'a' + 10, true
|
|
||||||
case ch >= 'A' && ch <= 'F':
|
|
||||||
return ch - 'A' + 10, true
|
|
||||||
default:
|
|
||||||
return 0, false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
hi, ok := hex(high)
|
|
||||||
if !ok {
|
|
||||||
return 0, false
|
|
||||||
}
|
|
||||||
lo, ok := hex(low)
|
|
||||||
if !ok {
|
|
||||||
return 0, false
|
|
||||||
}
|
|
||||||
return hi<<4 | lo, true
|
|
||||||
}
|
|
||||||
|
|
||||||
// Data characters may legally include punctuation such as parentheses and
|
|
||||||
// backslashes. Stop only at whitespace or an unescaped serialized-string quote.
|
|
||||||
func isDataURLTerminator(raw string, pos int) bool {
|
|
||||||
switch raw[pos] {
|
|
||||||
case ' ', '\t', '\r', '\n':
|
|
||||||
return true
|
|
||||||
case '"':
|
|
||||||
backslashes := 0
|
|
||||||
for i := pos - 1; i >= 0 && raw[i] == '\\'; i-- {
|
|
||||||
backslashes++
|
|
||||||
}
|
|
||||||
return backslashes%2 == 0
|
|
||||||
default:
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,29 +0,0 @@
|
|||||||
package utils
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestCompactImageDataURLForLogRedactsEscapedAndLegalPayloadForms(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
raw string
|
|
||||||
secret string
|
|
||||||
}{
|
|
||||||
{"encoded leading d", `\u0064ata:image/png;base64,ESCAPEDLEADING`, "ESCAPEDLEADING"},
|
|
||||||
{"fully percent encoded prefix", `%64%61%74%61%3Aimage/png;base64,PERCENTPREFIX`, "PERCENTPREFIX"},
|
|
||||||
{"escaped payload sequence", `data:image/png;base64,BEFORE\u002bAFTER`, "AFTER"},
|
|
||||||
{"parenthesis is legal payload", `data:text/plain,BEFORE(AFTER)SUFFIX next`, "SUFFIX"},
|
|
||||||
{"apostrophe is legal payload", `data:image/png;name=foo'bar;base64,APOSTROPHESECRET next`, "APOSTROPHESECRET"},
|
|
||||||
{"double percent encoded prefix", `%2564%2561%2574%2561%253Aimage/png;base64,DOUBLEPERCENT`, "DOUBLEPERCENT"},
|
|
||||||
}
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
got := CompactImageDataURLForLog(tt.raw)
|
|
||||||
if strings.Contains(got, tt.secret) || !strings.Contains(got, dataURLRedaction) {
|
|
||||||
t.Fatalf("CompactImageDataURLForLog(%q) = %q", tt.raw, got)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
package utils
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestCompactImageDataURLForLogFullyRedactsImages(t *testing.T) {
|
|
||||||
cases := []string{
|
|
||||||
"data:image/png;base64,YWJj",
|
|
||||||
"DATA:image/png;BASE64," + strings.Repeat("QUJD", 100),
|
|
||||||
`{"image":"data:image\\/png;base64,TOPSECRET"}`,
|
|
||||||
"data:image/png,%89PNG%0D%0A",
|
|
||||||
"data:text/plain,NONIMAGESECRET",
|
|
||||||
"data%3Aimage%2Fpng%3Bbase64%2CENCODEDSECRET",
|
|
||||||
`data\u003aimage/png;base64,ESCAPEDSECRET`,
|
|
||||||
}
|
|
||||||
for _, raw := range cases {
|
|
||||||
got := CompactImageDataURLForLog(raw)
|
|
||||||
if !strings.Contains(got, "[image-data-redacted]") {
|
|
||||||
t.Fatalf("CompactImageDataURLForLog(%q) = %q", raw, got)
|
|
||||||
}
|
|
||||||
for _, secret := range []string{"YWJj", "QUJD", "TOPSECRET", "%89PNG", "NONIMAGESECRET", "ENCODEDSECRET", "ESCAPEDSECRET"} {
|
|
||||||
if strings.Contains(got, secret) {
|
|
||||||
t.Fatalf("sanitized output retained %q: %q", secret, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+13
-37
@@ -226,14 +226,6 @@ var restrictedHostSuffixes = []string{
|
|||||||
var restrictedIPv4Ranges = []*net.IPNet{
|
var restrictedIPv4Ranges = []*net.IPNet{
|
||||||
// 100.64.0.0/10 - Carrier-grade NAT (RFC 6598)
|
// 100.64.0.0/10 - Carrier-grade NAT (RFC 6598)
|
||||||
mustParseCIDR("100.64.0.0/10"),
|
mustParseCIDR("100.64.0.0/10"),
|
||||||
// 192.31.196.0/24 - AS112-v4 special-purpose service
|
|
||||||
mustParseCIDR("192.31.196.0/24"),
|
|
||||||
// 192.52.193.0/24 - AMT special-purpose service
|
|
||||||
mustParseCIDR("192.52.193.0/24"),
|
|
||||||
// 192.88.99.0/24 - Deprecated 6to4 relay anycast
|
|
||||||
mustParseCIDR("192.88.99.0/24"),
|
|
||||||
// 192.175.48.0/24 - Direct Delegation AS112 service
|
|
||||||
mustParseCIDR("192.175.48.0/24"),
|
|
||||||
// 198.18.0.0/15 - Network device benchmark testing (RFC 2544)
|
// 198.18.0.0/15 - Network device benchmark testing (RFC 2544)
|
||||||
mustParseCIDR("198.18.0.0/15"),
|
mustParseCIDR("198.18.0.0/15"),
|
||||||
// 198.51.100.0/24 - TEST-NET-2 for documentation (RFC 5737)
|
// 198.51.100.0/24 - TEST-NET-2 for documentation (RFC 5737)
|
||||||
@@ -258,22 +250,6 @@ var restrictedIPv4Ranges = []*net.IPNet{
|
|||||||
mustParseCIDR("172.20.0.0/16"),
|
mustParseCIDR("172.20.0.0/16"),
|
||||||
}
|
}
|
||||||
|
|
||||||
// restrictedIPv6Ranges contains special-purpose address space that must not be
|
|
||||||
// used as an outbound media-fetch destination. These ranges are not ordinary
|
|
||||||
// public unicast destinations and some can translate or encapsulate IPv4.
|
|
||||||
var restrictedIPv6Ranges = []*net.IPNet{
|
|
||||||
mustParseCIDR("64:ff9b::/96"),
|
|
||||||
mustParseCIDR("64:ff9b:1::/48"),
|
|
||||||
mustParseCIDR("100::/64"),
|
|
||||||
mustParseCIDR("100:0:0:1::/64"),
|
|
||||||
mustParseCIDR("2001::/23"),
|
|
||||||
mustParseCIDR("2001:db8::/32"),
|
|
||||||
mustParseCIDR("2002::/16"),
|
|
||||||
mustParseCIDR("2620:4f:8000::/48"),
|
|
||||||
mustParseCIDR("3fff::/20"),
|
|
||||||
mustParseCIDR("5f00::/16"),
|
|
||||||
}
|
|
||||||
|
|
||||||
// mustParseCIDR parses a CIDR string and panics on error
|
// mustParseCIDR parses a CIDR string and panics on error
|
||||||
func mustParseCIDR(s string) *net.IPNet {
|
func mustParseCIDR(s string) *net.IPNet {
|
||||||
_, ipNet, err := net.ParseCIDR(s)
|
_, ipNet, err := net.ParseCIDR(s)
|
||||||
@@ -313,19 +289,6 @@ func isRestrictedIP(ip net.IP) (bool, string) {
|
|||||||
|
|
||||||
// Check IPv6-specific restrictions
|
// Check IPv6-specific restrictions
|
||||||
if ip.To4() == nil && len(ip) == 16 {
|
if ip.To4() == nil && len(ip) == 16 {
|
||||||
// Preserve explicit diagnostics for tunneling prefixes before the broader
|
|
||||||
// IANA special-purpose range table.
|
|
||||||
if ip[0] == 0x20 && ip[1] == 0x01 && ip[2] == 0x00 && ip[3] == 0x00 {
|
|
||||||
return true, "Teredo tunneling address"
|
|
||||||
}
|
|
||||||
if ip[0] == 0x20 && ip[1] == 0x02 {
|
|
||||||
return true, "6to4 special-purpose address"
|
|
||||||
}
|
|
||||||
for _, cidr := range restrictedIPv6Ranges {
|
|
||||||
if cidr.Contains(ip) {
|
|
||||||
return true, fmt.Sprintf("restricted range %s", cidr.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Site-local (deprecated but still blocked): fec0::/10
|
// Site-local (deprecated but still blocked): fec0::/10
|
||||||
if ip[0] == 0xfe && (ip[1]&0xc0) == 0xc0 {
|
if ip[0] == 0xfe && (ip[1]&0xc0) == 0xc0 {
|
||||||
return true, "site-local IPv6 address"
|
return true, "site-local IPv6 address"
|
||||||
@@ -341,6 +304,19 @@ func isRestrictedIP(ip net.IP) (bool, string) {
|
|||||||
return true, fmt.Sprintf("IPv4-mapped %s", reason)
|
return true, fmt.Sprintf("IPv4-mapped %s", reason)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// Teredo tunneling addresses: 2001:0000::/32
|
||||||
|
// Embed arbitrary IPv4 in the payload; can reach internal hosts via relay.
|
||||||
|
if ip[0] == 0x20 && ip[1] == 0x01 && ip[2] == 0x00 && ip[3] == 0x00 {
|
||||||
|
return true, "Teredo tunneling address"
|
||||||
|
}
|
||||||
|
// 6to4 addresses: 2002::/16
|
||||||
|
// Bits 16-47 carry an IPv4 address; block when embedded IPv4 is restricted.
|
||||||
|
if ip[0] == 0x20 && ip[1] == 0x02 {
|
||||||
|
embeddedIP := net.IP(ip[2:6])
|
||||||
|
if restricted, reason := isRestrictedIP(embeddedIP); restricted {
|
||||||
|
return true, fmt.Sprintf("6to4 embedded %s", reason)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return false, ""
|
return false, ""
|
||||||
|
|||||||
@@ -1,25 +0,0 @@
|
|||||||
package utils
|
|
||||||
|
|
||||||
import "testing"
|
|
||||||
|
|
||||||
func TestIsPublicIPRejectsSpecialPurposeIPv6(t *testing.T) {
|
|
||||||
for _, raw := range []string{
|
|
||||||
"192.31.196.1",
|
|
||||||
"192.52.193.1",
|
|
||||||
"192.88.99.1",
|
|
||||||
"192.175.48.1",
|
|
||||||
"100::1",
|
|
||||||
"100:0:0:1::1",
|
|
||||||
"2001:1::1",
|
|
||||||
"2001:2::1",
|
|
||||||
"2001:10::1",
|
|
||||||
"2001:20::1",
|
|
||||||
"64:ff9b:1::1",
|
|
||||||
"2002:0808:0808::1",
|
|
||||||
"2620:4f:8000::1",
|
|
||||||
} {
|
|
||||||
if IsPublicIP(parseIPForTest(t, raw)) {
|
|
||||||
t.Errorf("IsPublicIP(%s) = true; want false for special-purpose IPv6", raw)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -266,7 +266,7 @@ func TestIsRestrictedIP_IPv6(t *testing.T) {
|
|||||||
{"site-local", "fec0::1", true, "site-local"},
|
{"site-local", "fec0::1", true, "site-local"},
|
||||||
{"Teredo", "2001:0000:4136:e378:8000:63bf:3fff:fdd2", true, "Teredo"},
|
{"Teredo", "2001:0000:4136:e378:8000:63bf:3fff:fdd2", true, "Teredo"},
|
||||||
{"6to4 private", "2002:c0a8:0101::1", true, "6to4"},
|
{"6to4 private", "2002:c0a8:0101::1", true, "6to4"},
|
||||||
{"6to4 public endpoint still special-purpose", "2002:0808:0808::1", true, "6to4"},
|
{"6to4 public", "2002:0808:0808::1", false, ""},
|
||||||
{"public IPv6", "2001:4860:4860::8888", false, ""},
|
{"public IPv6", "2001:4860:4860::8888", false, ""},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -168,6 +168,15 @@ CREATE TABLE IF NOT EXISTS messages (
|
|||||||
is_completed BOOLEAN NOT NULL DEFAULT 0,
|
is_completed BOOLEAN NOT NULL DEFAULT 0,
|
||||||
is_fallback BOOLEAN NOT NULL DEFAULT 0,
|
is_fallback BOOLEAN NOT NULL DEFAULT 0,
|
||||||
channel VARCHAR(50) NOT NULL DEFAULT '',
|
channel VARCHAR(50) NOT NULL DEFAULT '',
|
||||||
|
source_platform VARCHAR(20) NOT NULL DEFAULT '',
|
||||||
|
source_channel_id VARCHAR(36) NOT NULL DEFAULT '',
|
||||||
|
source_message_id VARCHAR(128) NOT NULL DEFAULT '',
|
||||||
|
source_delivery_id VARCHAR(128) NOT NULL DEFAULT '',
|
||||||
|
source_user_id VARCHAR(128) NOT NULL DEFAULT '',
|
||||||
|
source_chat_id VARCHAR(128) NOT NULL DEFAULT '',
|
||||||
|
source_thread_id VARCHAR(128) NOT NULL DEFAULT '',
|
||||||
|
searched_knowledge_base_ids TEXT NOT NULL DEFAULT '[]',
|
||||||
|
token_usage TEXT,
|
||||||
agent_duration_ms INTEGER DEFAULT 0,
|
agent_duration_ms INTEGER DEFAULT 0,
|
||||||
knowledge_id VARCHAR(36),
|
knowledge_id VARCHAR(36),
|
||||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||||
@@ -177,6 +186,11 @@ CREATE TABLE IF NOT EXISTS messages (
|
|||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_messages_session_id ON messages(session_id);
|
CREATE INDEX IF NOT EXISTS idx_messages_session_id ON messages(session_id);
|
||||||
CREATE INDEX IF NOT EXISTS idx_messages_knowledge_id ON messages(knowledge_id);
|
CREATE INDEX IF NOT EXISTS idx_messages_knowledge_id ON messages(knowledge_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_messages_request_role ON messages(request_id, role);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_messages_source_channel ON messages(source_platform, source_channel_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_messages_source_message ON messages(source_platform, source_message_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_messages_source_delivery ON messages(source_platform, source_delivery_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_messages_source_user ON messages(source_platform, source_user_id);
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS chunks (
|
CREATE TABLE IF NOT EXISTS chunks (
|
||||||
id VARCHAR(36) PRIMARY KEY,
|
id VARCHAR(36) PRIMARY KEY,
|
||||||
|
|||||||
@@ -0,0 +1,20 @@
|
|||||||
|
-- Migration: 000064_message_audit (rollback)
|
||||||
|
DO $$ BEGIN RAISE NOTICE '[Migration 000064 rollback] Removing message audit metadata'; END $$;
|
||||||
|
|
||||||
|
DROP INDEX IF EXISTS idx_messages_source_user;
|
||||||
|
DROP INDEX IF EXISTS idx_messages_source_delivery;
|
||||||
|
DROP INDEX IF EXISTS idx_messages_source_message;
|
||||||
|
DROP INDEX IF EXISTS idx_messages_source_channel;
|
||||||
|
DROP INDEX IF EXISTS idx_messages_request_role;
|
||||||
|
|
||||||
|
ALTER TABLE messages DROP COLUMN IF EXISTS token_usage;
|
||||||
|
ALTER TABLE messages DROP COLUMN IF EXISTS searched_knowledge_base_ids;
|
||||||
|
ALTER TABLE messages DROP COLUMN IF EXISTS source_thread_id;
|
||||||
|
ALTER TABLE messages DROP COLUMN IF EXISTS source_chat_id;
|
||||||
|
ALTER TABLE messages DROP COLUMN IF EXISTS source_user_id;
|
||||||
|
ALTER TABLE messages DROP COLUMN IF EXISTS source_delivery_id;
|
||||||
|
ALTER TABLE messages DROP COLUMN IF EXISTS source_message_id;
|
||||||
|
ALTER TABLE messages DROP COLUMN IF EXISTS source_channel_id;
|
||||||
|
ALTER TABLE messages DROP COLUMN IF EXISTS source_platform;
|
||||||
|
|
||||||
|
DO $$ BEGIN RAISE NOTICE '[Migration 000064 rollback] Message audit metadata removed'; END $$;
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
-- Migration: 000064_message_audit
|
||||||
|
-- Description: Persist per-message source metadata, delivery correlation, retrieval scope, and model token usage.
|
||||||
|
DO $$ BEGIN RAISE NOTICE '[Migration 000064] Adding message audit metadata'; END $$;
|
||||||
|
|
||||||
|
ALTER TABLE messages ADD COLUMN IF NOT EXISTS source_platform VARCHAR(20) NOT NULL DEFAULT '';
|
||||||
|
ALTER TABLE messages ADD COLUMN IF NOT EXISTS source_channel_id VARCHAR(36) NOT NULL DEFAULT '';
|
||||||
|
ALTER TABLE messages ADD COLUMN IF NOT EXISTS source_message_id VARCHAR(128) NOT NULL DEFAULT '';
|
||||||
|
ALTER TABLE messages ADD COLUMN IF NOT EXISTS source_delivery_id VARCHAR(128) NOT NULL DEFAULT '';
|
||||||
|
ALTER TABLE messages ADD COLUMN IF NOT EXISTS source_user_id VARCHAR(128) NOT NULL DEFAULT '';
|
||||||
|
ALTER TABLE messages ADD COLUMN IF NOT EXISTS source_chat_id VARCHAR(128) NOT NULL DEFAULT '';
|
||||||
|
ALTER TABLE messages ADD COLUMN IF NOT EXISTS source_thread_id VARCHAR(128) NOT NULL DEFAULT '';
|
||||||
|
ALTER TABLE messages ADD COLUMN IF NOT EXISTS searched_knowledge_base_ids JSONB NOT NULL DEFAULT '[]'::jsonb;
|
||||||
|
ALTER TABLE messages ADD COLUMN IF NOT EXISTS token_usage JSONB;
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_messages_request_role
|
||||||
|
ON messages (request_id, role);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_messages_source_channel
|
||||||
|
ON messages (source_platform, source_channel_id)
|
||||||
|
WHERE source_platform != '';
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_messages_source_message
|
||||||
|
ON messages (source_platform, source_message_id)
|
||||||
|
WHERE source_message_id != '';
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_messages_source_delivery
|
||||||
|
ON messages (source_platform, source_delivery_id)
|
||||||
|
WHERE source_delivery_id != '';
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_messages_source_user
|
||||||
|
ON messages (source_platform, source_user_id)
|
||||||
|
WHERE source_user_id != '';
|
||||||
|
|
||||||
|
COMMENT ON COLUMN messages.source_platform IS 'Concrete source platform such as dingtalk; channel remains im for compatibility';
|
||||||
|
COMMENT ON COLUMN messages.source_channel_id IS 'WeKnora im_channels.id that received the message';
|
||||||
|
COMMENT ON COLUMN messages.source_message_id IS 'External platform message identifier';
|
||||||
|
COMMENT ON COLUMN messages.source_delivery_id IS 'External delivery or card identifier used to correlate callbacks';
|
||||||
|
COMMENT ON COLUMN messages.source_user_id IS 'External platform user identifier for this individual message';
|
||||||
|
COMMENT ON COLUMN messages.source_chat_id IS 'External platform conversation or group identifier';
|
||||||
|
COMMENT ON COLUMN messages.source_thread_id IS 'External platform thread identifier when available';
|
||||||
|
COMMENT ON COLUMN messages.searched_knowledge_base_ids IS 'Knowledge-base search scope, retained even when retrieval has zero hits';
|
||||||
|
COMMENT ON COLUMN messages.token_usage IS 'Answer-generation token usage: prompt, completion, total, and cached tokens';
|
||||||
|
|
||||||
|
DO $$ BEGIN RAISE NOTICE '[Migration 000064] Message audit metadata ready'; END $$;
|
||||||
Reference in New Issue
Block a user