wizardchen
578 lines
16 KiB
Go
578 lines
16 KiB
Go
package tools
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"regexp"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/PuerkitoBio/goquery"
|
|
"github.com/chromedp/chromedp"
|
|
|
|
"github.com/Tencent/WeKnora/internal/logger"
|
|
"github.com/Tencent/WeKnora/internal/models/chat"
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
)
|
|
|
|
const (
|
|
webFetchTimeout = 60 * time.Second
|
|
webFetchMaxChars = 100000
|
|
)
|
|
|
|
type webFetchParams struct {
|
|
URL string
|
|
Prompt string
|
|
}
|
|
|
|
type webFetchItemResult struct {
|
|
output string
|
|
data map[string]interface{}
|
|
err error
|
|
}
|
|
|
|
// WebFetchTool 拉取网页内容并根据提示执行总结
|
|
type WebFetchTool struct {
|
|
BaseTool
|
|
client *http.Client
|
|
chatModel chat.Chat
|
|
}
|
|
|
|
// NewWebFetchTool 创建 web_fetch 工具实例
|
|
func NewWebFetchTool(chatModel chat.Chat) *WebFetchTool {
|
|
description := `Fetch detailed web content from previously discovered URLs and analyze it with an LLM.
|
|
|
|
## Usage
|
|
- Receive one or more {url, prompt} combinations
|
|
- Fetch web page content and convert to Markdown text
|
|
- Use prompt to call small model for analysis and summary (if model is available)
|
|
- Return summary result and original content fragment
|
|
|
|
## When to Use
|
|
- **MANDATORY**: After web_search returns results, if content is truncated or incomplete, use web_fetch to get full page content
|
|
- When web_search snippet is insufficient for answering the question
|
|
`
|
|
|
|
return &WebFetchTool{
|
|
BaseTool: NewBaseTool("web_fetch", description),
|
|
client: &http.Client{
|
|
Timeout: webFetchTimeout,
|
|
},
|
|
chatModel: chatModel,
|
|
}
|
|
}
|
|
|
|
// Parameters 返回工具参数的 JSON Schema
|
|
func (t *WebFetchTool) Parameters() map[string]interface{} {
|
|
return map[string]interface{}{
|
|
"type": "object",
|
|
"properties": map[string]interface{}{
|
|
"items": map[string]interface{}{
|
|
"type": "array",
|
|
"description": "批量抓取任务，每项包含 url 与 prompt",
|
|
"items": map[string]interface{}{
|
|
"type": "object",
|
|
"properties": map[string]interface{}{
|
|
"url": map[string]interface{}{
|
|
"type": "string",
|
|
"description": "待抓取的网页 URL，需来自 web_search 结果",
|
|
},
|
|
"prompt": map[string]interface{}{
|
|
"type": "string",
|
|
"description": "分析该网页内容时使用的提示词",
|
|
},
|
|
},
|
|
"required": []string{"url", "prompt"},
|
|
},
|
|
"minItems": 1,
|
|
},
|
|
},
|
|
"required": []string{"items"},
|
|
}
|
|
}
|
|
|
|
// Execute 执行 web_fetch 工具
|
|
func (t *WebFetchTool) Execute(ctx context.Context, args map[string]interface{}) (*types.ToolResult, error) {
|
|
logger.Infof(ctx, "[Tool][WebFetch] Execute started")
|
|
|
|
rawItems, ok := args["items"]
|
|
if !ok {
|
|
logger.Errorf(ctx, "[Tool][WebFetch] 参数缺失: items")
|
|
return &types.ToolResult{
|
|
Success: false,
|
|
Error: "missing required parameter: items",
|
|
}, nil
|
|
}
|
|
|
|
itemsSlice, ok := rawItems.([]interface{})
|
|
if !ok || len(itemsSlice) == 0 {
|
|
logger.Errorf(ctx, "[Tool][WebFetch] items 解析失败或为空: %#v", rawItems)
|
|
return &types.ToolResult{
|
|
Success: false,
|
|
Error: "'items' must be a non-empty array",
|
|
}, nil
|
|
}
|
|
|
|
results := make([]*webFetchItemResult, len(itemsSlice))
|
|
|
|
var wg sync.WaitGroup
|
|
wg.Add(len(itemsSlice))
|
|
|
|
for idx := range itemsSlice {
|
|
i := idx
|
|
itemAny := itemsSlice[i]
|
|
|
|
params := t.parseParams(itemAny)
|
|
|
|
go func(index int, p webFetchParams) {
|
|
defer wg.Done()
|
|
|
|
if err := t.validateParams(p); err != nil {
|
|
results[index] = &webFetchItemResult{
|
|
err: err,
|
|
data: map[string]interface{}{
|
|
"url": p.URL,
|
|
"prompt": p.Prompt,
|
|
"error": err.Error(),
|
|
},
|
|
output: fmt.Sprintf("URL: %s\n错误: %v\n\n", p.URL, err),
|
|
}
|
|
return
|
|
}
|
|
|
|
output, data, err := t.executeFetch(ctx, p)
|
|
results[index] = &webFetchItemResult{
|
|
output: output,
|
|
data: data,
|
|
err: err,
|
|
}
|
|
}(i, params)
|
|
}
|
|
|
|
wg.Wait()
|
|
|
|
var builder strings.Builder
|
|
builder.WriteString("=== Web Fetch Results ===\n\n")
|
|
|
|
aggregated := make([]map[string]interface{}, 0, len(results))
|
|
success := true
|
|
var firstErr error
|
|
|
|
for idx, res := range results {
|
|
if res == nil {
|
|
success = false
|
|
if firstErr == nil {
|
|
firstErr = fmt.Errorf("fetch item %d returned nil", idx)
|
|
}
|
|
builder.WriteString(fmt.Sprintf("#%d: 无结果（内部错误）\n\n", idx+1))
|
|
continue
|
|
}
|
|
|
|
builder.WriteString(fmt.Sprintf("#%d:\n%s", idx+1, res.output))
|
|
if !strings.HasSuffix(res.output, "\n") {
|
|
builder.WriteString("\n")
|
|
}
|
|
builder.WriteString("\n")
|
|
|
|
if res.data != nil {
|
|
aggregated = append(aggregated, res.data)
|
|
}
|
|
|
|
if res.err != nil {
|
|
success = false
|
|
if firstErr == nil {
|
|
firstErr = res.err
|
|
}
|
|
}
|
|
}
|
|
|
|
// Add guidance for next steps
|
|
builder.WriteString("\n=== Next Steps ===\n")
|
|
if len(aggregated) > 0 {
|
|
builder.WriteString("- ✅ Full page content has been fetched and analyzed.\n")
|
|
builder.WriteString("- Evaluate if the content is sufficient to answer the question completely.\n")
|
|
builder.WriteString("- Synthesize information from all fetched pages for comprehensive answers.\n")
|
|
if !success {
|
|
builder.WriteString("- ⚠️ Some URLs failed to fetch. Use available content or try alternative sources.\n")
|
|
}
|
|
} else {
|
|
builder.WriteString("- ❌ No content was successfully fetched. Consider:\n")
|
|
builder.WriteString(" - Verify URLs are accessible\n")
|
|
builder.WriteString(" - Try alternative sources from web_search results\n")
|
|
builder.WriteString(" - Check if information can be found in knowledge base instead\n")
|
|
}
|
|
|
|
data := map[string]interface{}{
|
|
"results": aggregated,
|
|
"count": len(aggregated),
|
|
"display_type": "web_fetch_results",
|
|
}
|
|
|
|
logger.Infof(ctx, "[Tool][WebFetch] Completed with success=%v, items=%d", success, len(aggregated))
|
|
|
|
return &types.ToolResult{
|
|
Success: success,
|
|
Output: builder.String(),
|
|
Data: data,
|
|
Error: func() string {
|
|
if firstErr != nil {
|
|
return firstErr.Error()
|
|
}
|
|
return ""
|
|
}(),
|
|
}, nil
|
|
}
|
|
|
|
func (t *WebFetchTool) parseParams(item interface{}) webFetchParams {
|
|
params := webFetchParams{}
|
|
if m, ok := item.(map[string]interface{}); ok {
|
|
if v, ok := m["url"].(string); ok {
|
|
params.URL = strings.TrimSpace(v)
|
|
}
|
|
if v, ok := m["prompt"].(string); ok {
|
|
params.Prompt = strings.TrimSpace(v)
|
|
}
|
|
}
|
|
return params
|
|
}
|
|
|
|
func (t *WebFetchTool) validateParams(p webFetchParams) error {
|
|
if p.URL == "" {
|
|
return fmt.Errorf("url is required")
|
|
}
|
|
if p.Prompt == "" {
|
|
return fmt.Errorf("prompt is required")
|
|
}
|
|
if !strings.HasPrefix(p.URL, "http://") && !strings.HasPrefix(p.URL, "https://") {
|
|
return fmt.Errorf("invalid URL format")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (t *WebFetchTool) executeFetch(ctx context.Context, params webFetchParams) (string, map[string]interface{}, error) {
|
|
logger.Infof(ctx, "[Tool][WebFetch] Fetching URL: %s", params.URL)
|
|
|
|
finalURL := t.normalizeGitHubURL(params.URL)
|
|
|
|
htmlContent, method, err := t.fetchHTMLContent(ctx, finalURL)
|
|
if err != nil {
|
|
logger.Errorf(ctx, "[Tool][WebFetch] 获取页面失败 url=%s err=%v", finalURL, err)
|
|
return fmt.Sprintf("URL: %s\n错误: %v\n", params.URL, err),
|
|
map[string]interface{}{
|
|
"url": params.URL,
|
|
"prompt": params.Prompt,
|
|
"error": err.Error(),
|
|
}, err
|
|
}
|
|
|
|
textContent := t.convertHTMLToText(htmlContent)
|
|
|
|
resultData := map[string]interface{}{
|
|
"url": params.URL,
|
|
"prompt": params.Prompt,
|
|
"raw_content": textContent,
|
|
"content_length": len(textContent),
|
|
"method": method,
|
|
}
|
|
var summary string
|
|
var summaryErr error
|
|
summary, summaryErr = t.processWithLLM(ctx, params, textContent)
|
|
if summaryErr != nil {
|
|
logger.Warnf(ctx, "[Tool][WebFetch] LLM 处理失败 url=%s err=%v", params.URL, summaryErr)
|
|
} else if summary != "" {
|
|
resultData["summary"] = summary
|
|
}
|
|
|
|
output := t.buildOutputText(params, textContent, summary, summaryErr)
|
|
|
|
return output, resultData, summaryErr
|
|
}
|
|
|
|
func (t *WebFetchTool) normalizeGitHubURL(source string) string {
|
|
if strings.Contains(source, "github.com") && strings.Contains(source, "/blob/") {
|
|
source = strings.Replace(source, "github.com", "raw.githubusercontent.com", 1)
|
|
source = strings.Replace(source, "/blob/", "/", 1)
|
|
}
|
|
return source
|
|
}
|
|
|
|
func (t *WebFetchTool) processWithLLM(ctx context.Context, params webFetchParams, content string) (string, error) {
|
|
if t.chatModel == nil {
|
|
return "", fmt.Errorf("chat model not available for web_fetch")
|
|
}
|
|
|
|
systemMessage := "你是一名擅长阅读网页内容的智能助手，请根据提供的网页文本回答用户需求，严禁编造未在文本中出现的信息。"
|
|
userTemplate := `用户请求:
|
|
%s
|
|
|
|
网页内容:
|
|
%s`
|
|
|
|
messages := []chat.Message{
|
|
{
|
|
Role: "system",
|
|
Content: systemMessage,
|
|
},
|
|
{
|
|
Role: "user",
|
|
Content: fmt.Sprintf(userTemplate, params.Prompt, content),
|
|
},
|
|
}
|
|
|
|
response, err := t.chatModel.Chat(ctx, messages, &chat.ChatOptions{
|
|
Temperature: 0.3,
|
|
MaxTokens: 1024,
|
|
})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
return strings.TrimSpace(response.Content), nil
|
|
}
|
|
|
|
func (t *WebFetchTool) buildOutputText(params webFetchParams, content string, summary string, summaryErr error) string {
|
|
var builder strings.Builder
|
|
builder.WriteString(fmt.Sprintf("URL: %s\n", params.URL))
|
|
builder.WriteString(fmt.Sprintf("Prompt: %s\n", params.Prompt))
|
|
|
|
if summaryErr == nil && summary != "" {
|
|
builder.WriteString("Summary:\n")
|
|
builder.WriteString(summary)
|
|
builder.WriteString("\n")
|
|
} else {
|
|
builder.WriteString("Content Preview:\n")
|
|
builder.WriteString(content)
|
|
builder.WriteString("\n")
|
|
}
|
|
|
|
return builder.String()
|
|
}
|
|
|
|
func (t *WebFetchTool) fetchHTMLContent(ctx context.Context, targetURL string) (string, string, error) {
|
|
html, err := t.fetchWithChromedp(ctx, targetURL)
|
|
if err == nil && strings.TrimSpace(html) != "" {
|
|
return html, "chromedp", nil
|
|
}
|
|
|
|
if err != nil {
|
|
logger.Debugf(ctx, "[Tool][WebFetch] Chromedp 抓取失败 url=%s err=%v，尝试直接请求", targetURL, err)
|
|
}
|
|
|
|
html, httpErr := t.fetchWithHTTP(ctx, targetURL)
|
|
if httpErr != nil {
|
|
if err != nil {
|
|
return "", "", fmt.Errorf("chromedp error: %v; http error: %w", err, httpErr)
|
|
}
|
|
return "", "", httpErr
|
|
}
|
|
|
|
return html, "http", nil
|
|
}
|
|
|
|
func (t *WebFetchTool) fetchWithChromedp(ctx context.Context, targetURL string) (string, error) {
|
|
logger.Debugf(ctx, "[Tool][WebFetch] Chromedp 抓取开始 url=%s", targetURL)
|
|
|
|
opts := append(chromedp.DefaultExecAllocatorOptions[:],
|
|
chromedp.Flag("headless", true),
|
|
chromedp.Flag("disable-setuid-sandbox", true),
|
|
chromedp.Flag("disable-dev-shm-usage", true),
|
|
chromedp.Flag("disable-gpu", true),
|
|
chromedp.Flag("disable-blink-features", "AutomationControlled"),
|
|
chromedp.Flag("disable-features", "VizDisplayCompositor"),
|
|
chromedp.UserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"),
|
|
)
|
|
|
|
allocCtx, cancel := chromedp.NewExecAllocator(ctx, opts...)
|
|
defer cancel()
|
|
|
|
ctx, cancel = chromedp.NewContext(allocCtx)
|
|
defer cancel()
|
|
|
|
ctx, cancel = context.WithTimeout(ctx, webFetchTimeout)
|
|
defer cancel()
|
|
|
|
var html string
|
|
err := chromedp.Run(ctx,
|
|
chromedp.Navigate(targetURL),
|
|
chromedp.WaitReady("body", chromedp.ByQuery),
|
|
chromedp.OuterHTML("html", &html),
|
|
)
|
|
if err != nil {
|
|
return "", fmt.Errorf("chromedp run failed: %w", err)
|
|
}
|
|
|
|
logger.Debugf(ctx, "[Tool][WebFetch] Chromedp 抓取成功 url=%s", targetURL)
|
|
return html, nil
|
|
}
|
|
|
|
func (t *WebFetchTool) fetchWithHTTP(ctx context.Context, targetURL string) (string, error) {
|
|
resp, err := t.fetchWithTimeout(ctx, targetURL)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
return "", fmt.Errorf("request failed with status %d %s", resp.StatusCode, resp.Status)
|
|
}
|
|
|
|
limitedReader := io.LimitReader(resp.Body, webFetchMaxChars*2)
|
|
htmlBytes, err := io.ReadAll(limitedReader)
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to read response body: %w", err)
|
|
}
|
|
|
|
return string(htmlBytes), nil
|
|
}
|
|
|
|
func (t *WebFetchTool) fetchWithTimeout(ctx context.Context, targetURL string) (*http.Response, error) {
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create request: %w", err)
|
|
}
|
|
|
|
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; WebFetchTool/1.0)")
|
|
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8")
|
|
req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8")
|
|
req.Header.Set("Cache-Control", "no-cache")
|
|
|
|
return t.client.Do(req)
|
|
}
|
|
|
|
func (t *WebFetchTool) convertHTMLToText(html string) string {
|
|
doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
|
|
if err != nil {
|
|
return t.basicTextExtraction(html)
|
|
}
|
|
|
|
doc.Find("script, style, nav, footer, header").Remove()
|
|
|
|
var markdown strings.Builder
|
|
doc.Find("body").Each(func(i int, body *goquery.Selection) {
|
|
t.processNode(body, &markdown)
|
|
})
|
|
|
|
result := markdown.String()
|
|
result = regexp.MustCompile(`\n{3,}`).ReplaceAllString(result, "\n\n")
|
|
return strings.TrimSpace(result)
|
|
}
|
|
|
|
func (t *WebFetchTool) processNode(s *goquery.Selection, markdown *strings.Builder) {
|
|
s.Contents().Each(func(i int, node *goquery.Selection) {
|
|
nodeName := goquery.NodeName(node)
|
|
|
|
switch nodeName {
|
|
case "h1", "h2", "h3", "h4", "h5", "h6":
|
|
headerLevel := int(nodeName[1] - '0')
|
|
markdown.WriteString("\n")
|
|
markdown.WriteString(strings.Repeat("#", headerLevel))
|
|
markdown.WriteString(" ")
|
|
markdown.WriteString(strings.TrimSpace(node.Text()))
|
|
markdown.WriteString("\n\n")
|
|
case "p":
|
|
t.processNode(node, markdown)
|
|
markdown.WriteString("\n\n")
|
|
case "a":
|
|
href, exists := node.Attr("href")
|
|
text := strings.TrimSpace(node.Text())
|
|
if exists && text != "" {
|
|
markdown.WriteString("[")
|
|
markdown.WriteString(text)
|
|
markdown.WriteString("](")
|
|
markdown.WriteString(href)
|
|
markdown.WriteString(")")
|
|
} else if text != "" {
|
|
markdown.WriteString(text)
|
|
}
|
|
case "img":
|
|
src, _ := node.Attr("src")
|
|
alt, _ := node.Attr("alt")
|
|
if src != "" {
|
|
markdown.WriteString("![")
|
|
markdown.WriteString(alt)
|
|
markdown.WriteString("](")
|
|
markdown.WriteString(src)
|
|
markdown.WriteString(")\n\n")
|
|
}
|
|
case "ul", "ol":
|
|
markdown.WriteString("\n")
|
|
isOrdered := nodeName == "ol"
|
|
node.Find("li").Each(func(idx int, li *goquery.Selection) {
|
|
if isOrdered {
|
|
fmt.Fprintf(markdown, "%d. ", idx+1)
|
|
} else {
|
|
markdown.WriteString("- ")
|
|
}
|
|
markdown.WriteString(strings.TrimSpace(li.Text()))
|
|
markdown.WriteString("\n")
|
|
})
|
|
markdown.WriteString("\n")
|
|
case "br":
|
|
markdown.WriteString("\n")
|
|
case "code":
|
|
parent := node.Parent()
|
|
if goquery.NodeName(parent) == "pre" {
|
|
markdown.WriteString("\n```\n")
|
|
markdown.WriteString(node.Text())
|
|
markdown.WriteString("\n```\n\n")
|
|
} else {
|
|
markdown.WriteString("`")
|
|
markdown.WriteString(node.Text())
|
|
markdown.WriteString("`")
|
|
}
|
|
case "blockquote":
|
|
lines := strings.Split(strings.TrimSpace(node.Text()), "\n")
|
|
for _, line := range lines {
|
|
markdown.WriteString("> ")
|
|
markdown.WriteString(strings.TrimSpace(line))
|
|
markdown.WriteString("\n")
|
|
}
|
|
markdown.WriteString("\n")
|
|
case "strong", "b":
|
|
markdown.WriteString("**")
|
|
markdown.WriteString(strings.TrimSpace(node.Text()))
|
|
markdown.WriteString("**")
|
|
case "em", "i":
|
|
markdown.WriteString("*")
|
|
markdown.WriteString(strings.TrimSpace(node.Text()))
|
|
markdown.WriteString("*")
|
|
case "hr":
|
|
markdown.WriteString("\n---\n\n")
|
|
case "table":
|
|
markdown.WriteString("\n")
|
|
node.Find("tr").Each(func(idx int, tr *goquery.Selection) {
|
|
tr.Find("th, td").Each(func(i int, cell *goquery.Selection) {
|
|
markdown.WriteString("| ")
|
|
markdown.WriteString(strings.TrimSpace(cell.Text()))
|
|
markdown.WriteString(" ")
|
|
})
|
|
markdown.WriteString("|\n")
|
|
if idx == 0 {
|
|
tr.Find("th").Each(func(i int, _ *goquery.Selection) {
|
|
markdown.WriteString("|---")
|
|
})
|
|
markdown.WriteString("|\n")
|
|
}
|
|
})
|
|
markdown.WriteString("\n")
|
|
case "#text":
|
|
text := node.Text()
|
|
if strings.TrimSpace(text) != "" {
|
|
markdown.WriteString(text)
|
|
}
|
|
default:
|
|
t.processNode(node, markdown)
|
|
}
|
|
})
|
|
}
|
|
|
|
func (t *WebFetchTool) basicTextExtraction(html string) string {
|
|
re := regexp.MustCompile(`<[^>]*>`)
|
|
text := re.ReplaceAllString(html, " ")
|
|
text = regexp.MustCompile(`\s+`).ReplaceAllString(text, " ")
|
|
return strings.TrimSpace(text)
|
|
}
|