diff --git a/.env.example b/.env.example index 35bd5322..20ccc0a7 100644 --- a/.env.example +++ b/.env.example @@ -55,7 +55,7 @@ REDIS_DB=0 REDIS_PREFIX=stream: # 当使用本地存储时,文件保存的基础目录路径 -LOCAL_STORAGE_BASE_DIR=./data/files +LOCAL_STORAGE_BASE_DIR=/data/files # 是否自动恢复脏数据 AUTO_RECOVER_DIRTY=true diff --git a/docker/Dockerfile.app b/docker/Dockerfile.app index 93fac820..7abdc3d9 100644 --- a/docker/Dockerfile.app +++ b/docker/Dockerfile.app @@ -1,5 +1,5 @@ # Build stage -FROM golang:1.24-alpine AS builder +FROM golang:1.24-bookworm AS builder WORKDIR /app @@ -16,9 +16,10 @@ ENV GOSUMDB=${GOSUMDB_ARG} # Install dependencies RUN if [ -n "$APK_MIRROR_ARG" ]; then \ - sed -i "s@dl-cdn.alpinelinux.org@${APK_MIRROR_ARG}@g" /etc/apk/repositories; \ + sed -i "s@deb.debian.org@${APK_MIRROR_ARG}@g" /etc/apt/sources.list.d/debian.sources; \ fi && \ - apk add --no-cache git build-base + apt-get update && \ + apt-get install -y git build-essential # Install migrate tool RUN go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@latest @@ -45,28 +46,31 @@ RUN --mount=type=cache,target=/go/pkg/mod make build-prod RUN --mount=type=cache,target=/go/pkg/mod cp -r /go/pkg/mod/github.com/yanyiwu/ /app/yanyiwu/ # Final stage -FROM alpine:3.23 +FROM debian:12.12-slim WORKDIR /app ARG APK_MIRROR_ARG="mirrors.tencent.com" # Create a non-root user first -RUN id -u appuser >/dev/null 2>&1 || adduser -D -g '' appuser +RUN useradd -m -s /bin/bash appuser RUN if [ -n "$APK_MIRROR_ARG" ]; then \ - sed -i "s@dl-cdn.alpinelinux.org@${APK_MIRROR_ARG}@g" /etc/apk/repositories; \ + sed -i "s@deb.debian.org@${APK_MIRROR_ARG}@g" /etc/apt/sources.list.d/debian.sources; \ fi && \ - apk update && apk upgrade && \ - apk add --no-cache build-base postgresql-client mysql-client ca-certificates tzdata sed curl bash vim wget \ - python3 py3-pip python3-dev libffi-dev openssl-dev && \ + apt-get update && apt-get upgrade -y && \ + apt-get install -y --no-install-recommends \ + build-essential postgresql-client default-mysql-client ca-certificates tzdata sed curl bash vim wget \ + python3 python3-pip python3-dev libffi-dev libssl-dev \ + nodejs npm && \ python3 -m pip install --break-system-packages --upgrade pip setuptools wheel && \ - apk add --no-cache nodejs-current npm && \ mkdir -p /home/appuser/.local/bin && \ curl -LsSf https://astral.sh/uv/install.sh | CARGO_HOME=/home/appuser/.cargo UV_INSTALL_DIR=/home/appuser/.local/bin sh && \ chown -R appuser:appuser /home/appuser && \ ln -sf /home/appuser/.local/bin/uvx /usr/local/bin/uvx && \ - chmod +x /usr/local/bin/uvx + chmod +x /usr/local/bin/uvx && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* # Create data directories and set permissions RUN mkdir -p /data/files && \ diff --git a/docreader/parser/web_parser.py b/docreader/parser/web_parser.py index a7906902..48e8f24c 100644 --- a/docreader/parser/web_parser.py +++ b/docreader/parser/web_parser.py @@ -16,7 +16,7 @@ logger = logging.getLogger(__name__) class StdWebParser(BaseParser): """Standard web page parser using Playwright and Trafilatura. - + This parser scrapes web pages using Playwright's WebKit browser and extracts clean content using Trafilatura library. It supports proxy configuration and converts HTML content to markdown format. @@ -24,7 +24,7 @@ class StdWebParser(BaseParser): def __init__(self, title: str, **kwargs): """Initialize the web parser. - + Args: title: Title of the web page to be used as file name **kwargs: Additional arguments passed to BaseParser @@ -37,10 +37,10 @@ class StdWebParser(BaseParser): async def scrape(self, url: str) -> str: """Scrape web page content using Playwright. - + Args: url: The URL of the web page to scrape - + Returns: HTML content of the web page as string, empty string on error """ @@ -74,8 +74,7 @@ class StdWebParser(BaseParser): logger.info("Browser closed") # Return raw HTML content for further processing - logger.info("Parsing HTML with BeautifulSoup") - logger.info("Successfully parsed HTML content") + logger.info("Successfully retrieved HTML content") return content except Exception as e: @@ -107,7 +106,6 @@ class StdWebParser(BaseParser): include_images=True, include_tables=True, include_links=True, - deduplicate=True, # Remove duplicate content ) if not md_text: logger.error("Failed to parse web page") @@ -117,11 +115,12 @@ class StdWebParser(BaseParser): class WebParser(PipelineParser): """Web parser using pipeline pattern. - + This parser chains StdWebParser (for web scraping and HTML to markdown conversion) with MarkdownParser (for markdown processing). The pipeline processes content sequentially through both parsers. """ + # Parser classes to be executed in sequence _parser_cls = (StdWebParser, MarkdownParser) diff --git a/docs/QA.md b/docs/QA.md index fd24773d..ccfaa78f 100644 --- a/docs/QA.md +++ b/docs/QA.md @@ -96,5 +96,26 @@ COS_PATH_PREFIX=your_cos_path_prefix 3. 查看文档解析模块日志,查看OCR和Caption是否正确解析和打印 +## 5. 如何使用数据分析功能? + +在使用数据分析功能前,请确保智能体已配置相关工具: + +1. **智能推理**:需在工具配置中勾选以下两个工具: + - 查看数据元信息 + - 数据分析 + +2. **快速问答智能体**:无需手动选择工具,即可直接进行简单的数据查询操作。 + +### 注意事项与使用规范 + +1. **支持的文件格式** + - 目前仅支持 **CSV** (`.csv`) 和 **Excel** (`.xlsx`, `.xls`) 格式的文件。 + - 对于复杂的 Excel 文件,如果读取失败,建议将其转换为标准的 CSV 格式后重新上传。 + +2. **查询限制** + - 仅支持 **只读查询**,包括 `SELECT`, `SHOW`, `DESCRIBE`, `EXPLAIN`, `PRAGMA` 等语句。 + - 禁止执行任何修改数据的操作,如 `INSERT`, `UPDATE`, `DELETE`, `CREATE`, `DROP` 等。 + + ## P.S. 如果以上方式未解决问题,请在issue中描述您的问题,并提供必要的日志信息辅助我们进行问题排查 \ No newline at end of file diff --git a/frontend/src/views/agent/AgentEditorModal.vue b/frontend/src/views/agent/AgentEditorModal.vue index 7fd22a64..98e3ff88 100644 --- a/frontend/src/views/agent/AgentEditorModal.vue +++ b/frontend/src/views/agent/AgentEditorModal.vue @@ -993,6 +993,8 @@ const allTools = [ { value: 'query_knowledge_graph', label: '查询知识图谱', description: '从知识图谱中查询关系', requiresKB: true }, { value: 'get_document_info', label: '获取文档信息', description: '查看文档元数据', requiresKB: true }, { value: 'database_query', label: '查询数据库', description: '查询数据库中的信息', requiresKB: true }, + { value: 'data_analysis', label: '数据分析', description: '理解数据文件并进行数据分析', requiresKB: true }, + { value: 'data_schema', label: '查看数据元信息', description: '获取表格文件的元信息', requiresKB: true }, ]; // 根据知识库配置动态计算是否有知识库能力 diff --git a/go.mod b/go.mod index 3c042e81..49694a64 100644 --- a/go.mod +++ b/go.mod @@ -7,13 +7,15 @@ toolchain go1.24.2 require ( github.com/PuerkitoBio/goquery v1.10.3 github.com/chromedp/chromedp v0.14.2 + github.com/duckdb/duckdb-go/v2 v2.5.4 github.com/elastic/go-elasticsearch/v7 v7.17.10 github.com/elastic/go-elasticsearch/v8 v8.18.0 github.com/gin-contrib/cors v1.7.5 github.com/gin-gonic/gin v1.11.0 - github.com/go-viper/mapstructure/v2 v2.2.1 + github.com/go-viper/mapstructure/v2 v2.4.0 github.com/golang-jwt/jwt/v5 v5.3.0 github.com/golang-migrate/migrate/v4 v4.19.0 + github.com/google/jsonschema-go v0.4.2 github.com/google/uuid v1.6.0 github.com/hibiken/asynq v0.25.1 github.com/mark3labs/mcp-go v0.43.0 @@ -42,10 +44,11 @@ require ( go.opentelemetry.io/otel/sdk v1.37.0 go.opentelemetry.io/otel/trace v1.38.0 go.uber.org/dig v1.18.1 - golang.org/x/crypto v0.44.0 - golang.org/x/sync v0.18.0 + golang.org/x/crypto v0.46.0 + golang.org/x/sync v0.19.0 google.golang.org/grpc v1.76.0 google.golang.org/protobuf v1.36.10 + gopkg.in/yaml.v3 v3.0.1 gorm.io/driver/postgres v1.5.11 gorm.io/gorm v1.25.12 ) @@ -54,8 +57,9 @@ require ( github.com/KyleBanks/depth v1.2.1 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect - github.com/andybalholm/brotli v1.1.0 // indirect + github.com/andybalholm/brotli v1.2.0 // indirect github.com/andybalholm/cascadia v1.3.3 // indirect + github.com/apache/arrow-go/v18 v18.4.1 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/buger/jsonparser v1.1.1 // indirect github.com/bytedance/sonic v1.14.0 // indirect @@ -66,8 +70,16 @@ require ( github.com/chromedp/sysutil v1.1.0 // indirect github.com/clbanning/mxj v1.8.4 // indirect github.com/cloudwego/base64x v0.1.6 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/duckdb/duckdb-go-bindings v0.1.24 // indirect + github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.24 // indirect + github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.24 // indirect + github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.24 // indirect + github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.24 // indirect + github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.24 // indirect + github.com/duckdb/duckdb-go/arrowmapping v0.0.27 // indirect + github.com/duckdb/duckdb-go/mapping v0.0.27 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/elastic/elastic-transport-go/v8 v8.7.0 // indirect github.com/fsnotify/fsnotify v1.8.0 // indirect @@ -89,6 +101,7 @@ require ( github.com/gobwas/ws v1.4.0 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/goccy/go-yaml v1.18.0 // indirect + github.com/google/flatbuffers v25.9.23+incompatible // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect @@ -102,13 +115,13 @@ require ( github.com/jinzhu/now v1.1.5 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.18.1 // indirect + github.com/klauspost/compress v1.18.2 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/lib/pq v1.10.9 // indirect github.com/mailru/easyjson v0.9.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/mattn/go-runewidth v0.0.16 // indirect github.com/minio/crc64nvme v1.0.1 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/mitchellh/mapstructure v1.4.3 // indirect @@ -117,8 +130,8 @@ require ( github.com/mozillazg/go-httpheader v0.2.1 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect - github.com/pierrec/lz4/v4 v4.1.21 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pierrec/lz4/v4 v4.1.22 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/quic-go/qpack v0.5.1 // indirect github.com/quic-go/quic-go v0.54.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect @@ -134,20 +147,23 @@ require ( github.com/ugorji/go/codec v1.3.0 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect + github.com/zeebo/xxh3 v1.0.2 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/otel/metric v1.38.0 // indirect go.opentelemetry.io/proto/otlp v1.7.0 // indirect go.uber.org/mock v0.5.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.20.0 // indirect - golang.org/x/mod v0.29.0 // indirect - golang.org/x/net v0.47.0 // indirect - golang.org/x/sys v0.38.0 // indirect - golang.org/x/text v0.31.0 // indirect + golang.org/x/exp v0.0.0-20251209150349-8475f28825e9 // indirect + golang.org/x/mod v0.31.0 // indirect + golang.org/x/net v0.48.0 // indirect + golang.org/x/sys v0.39.0 // indirect + golang.org/x/telemetry v0.0.0-20251208220230-2638a1023523 // indirect + golang.org/x/text v0.32.0 // indirect golang.org/x/time v0.14.0 // indirect - golang.org/x/tools v0.38.0 // indirect + golang.org/x/tools v0.40.0 // indirect + golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250804133106-a7a43d27e69b // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index e8cd3620..5f2e4b61 100644 --- a/go.sum +++ b/go.sum @@ -13,10 +13,14 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= -github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= -github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= +github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= +github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= +github.com/apache/arrow-go/v18 v18.4.1 h1:q/jVkBWCJOB9reDgaIZIdruLQUb1kbkvOnOFezVH1C4= +github.com/apache/arrow-go/v18 v18.4.1/go.mod h1:tLyFubsAl17bvFdUAy24bsSvA/6ww95Iqi67fTpGu3E= +github.com/apache/thrift v0.22.0 h1:r7mTJdj51TMDe6RtcmNdQxgn9XcyfGDOzegMDRg47uc= +github.com/apache/thrift v0.22.0/go.mod h1:1e7J/O1Ae6ZQMTYdy9xa3w9k+XHWPfRvdPyJeynQ+/g= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= @@ -49,8 +53,9 @@ github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151X github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4= @@ -63,6 +68,24 @@ github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pM github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/duckdb/duckdb-go-bindings v0.1.24 h1:p1v3GruGHGcZD69cWauH6QrOX32oooqdUAxrWK3Fo6o= +github.com/duckdb/duckdb-go-bindings v0.1.24/go.mod h1:WA7U/o+b37MK2kiOPPueVZ+FIxt5AZFCjszi8hHeH18= +github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.24 h1:XhqMj+bvpTIm+hMeps1Kk94r2eclAswk2ISFs4jMm+g= +github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.24/go.mod h1:jfbOHwGZqNCpMAxV4g4g5jmWr0gKdMvh2fGusPubxC4= +github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.24 h1:OyHr5PykY5FG81jchpRoESMDQX1HK66PdNsfxoHxbwM= +github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.24/go.mod h1:zLVtv1a7TBuTPvuAi32AIbnuw7jjaX5JElZ+urv1ydc= +github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.24 h1:6Y4VarmcT7Oe8stwta4dOLlUX8aG4ciG9VhFKnp91a4= +github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.24/go.mod h1:GCaBoYnuLZEva7BXzdXehTbqh9VSvpLB80xcmxGBGs8= +github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.24 h1:NCAGH7o1RsJv631EQGOqs94ABtmYZO6JjMHkv7GIgG8= +github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.24/go.mod h1:kpQSpJmDSSZQ3ikbZR1/8UqecqMeUkWFjFX2xZxlCuI= +github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.24 h1:JOupXaHMMu8zLgq7v9uxPjl1CXSJHlISCxopMiqtkzU= +github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.24/go.mod h1:wa+egSGXTPS16NPADFCK1yFyt3VSXxUS6Pt2fLnvRPM= +github.com/duckdb/duckdb-go/arrowmapping v0.0.27 h1:w0XKX+EJpAN4XOQlKxSxSKZq/tCVbRfTRBp98jA0q8M= +github.com/duckdb/duckdb-go/arrowmapping v0.0.27/go.mod h1:VkFx49Icor1bbxOPxAU8jRzwL0nTXICOthxVq4KqOqQ= +github.com/duckdb/duckdb-go/mapping v0.0.27 h1:QEta+qPEKmfhd89U8vnm4MVslj1UscmkyJwu8x+OtME= +github.com/duckdb/duckdb-go/mapping v0.0.27/go.mod h1:7C4QWJWG6UOV9b0iWanfF5ML1ivJPX45Kz+VmlvRlTA= +github.com/duckdb/duckdb-go/v2 v2.5.4 h1:+ip+wPCwf7Eu/dXxp19aLCxwpLUaeOy2UV/peBphXK0= +github.com/duckdb/duckdb-go/v2 v2.5.4/go.mod h1:CeobOFmWpf7MTDb+MW08/zIWP8TQ2jbPbMgGo5761tY= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/elastic/elastic-transport-go/v8 v8.7.0 h1:OgTneVuXP2uip4BA658Xi6Hfw+PeIOod2rY3GVMGoVE= @@ -118,8 +141,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.27.0 h1:w8+XrWVMhGkxOaaowyKH35gFydVHOvC0/uWoy2Fzwn4= github.com/go-playground/validator/v10 v10.27.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= -github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= -github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= @@ -138,6 +161,10 @@ github.com/golang-migrate/migrate/v4 v4.19.0/go.mod h1:9dyEcu+hO+G9hPSw8AIg50yg6 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v25.9.23+incompatible h1:rGZKv+wOb6QPzIdkM2KxhBZCDrA0DeN6DNmRDrqIsQU= +github.com/google/flatbuffers v25.9.23+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -147,6 +174,8 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/jsonschema-go v0.4.2 h1:tmrUohrwoLZZS/P3x7ex0WAVknEkBZM46iALbcqoRA8= +github.com/google/jsonschema-go v0.4.2/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -181,8 +210,10 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co= -github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0= +github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= +github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= @@ -209,8 +240,12 @@ github.com/mark3labs/mcp-go v0.43.0/go.mod h1:YnJfOL382MIWDx1kMY+2zsRHU/q78dBg9a github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= -github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= github.com/minio/crc64nvme v1.0.1 h1:DHQPrYPdqK7jQG/Ls5CTBZWeex/2FMS3G5XGkycuFrY= github.com/minio/crc64nvme v1.0.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= @@ -255,12 +290,13 @@ github.com/pganalyze/pg_query_go/v6 v6.1.0 h1:jG5ZLhcVgL1FAw4C/0VNQaVmX1SUJx71wB github.com/pganalyze/pg_query_go/v6 v6.1.0/go.mod h1:nvTHIuoud6e1SfrUaFwHqT0i4b5Nr+1rPWVds3B5+50= github.com/pgvector/pgvector-go v0.3.0 h1:Ij+Yt78R//uYqs3Zk35evZFvr+G0blW0OUN+Q2D1RWc= github.com/pgvector/pgvector-go v0.3.0/go.mod h1:duFy+PXWfW7QQd5ibqutBO4GxLsUZ9RVXhFZGIBsWSA= -github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= -github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= +github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/qdrant/go-client v1.16.1 h1:Jr47kz0k8I+U2sUm2UUO2eq2kL0fTcgjLPIz6a0RKuQ= github.com/qdrant/go-client v1.16.1/go.mod h1:I+EL3h4HRoRTeHtbfOd/4kDXwCukZfkd41j/9wryGkw= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= @@ -341,11 +377,17 @@ github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/ github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yanyiwu/gojieba v1.4.5 h1:VyZogGtdFSnJbACHvDRvDreXPPVPCg8axKFUdblU/JI= github.com/yanyiwu/gojieba v1.4.5/go.mod h1:JUq4DddFVGdHXJHxxepxRmhrKlDpaBxR8O28v6fKYLY= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18= @@ -384,15 +426,17 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.44.0 h1:A97SsFvM3AIwEEmTBiaxPPTYpDC47w720rdiiUvgoAU= -golang.org/x/crypto v0.44.0/go.mod h1:013i+Nw79BMiQiMsOPcVCB5ZIJbYkerPrGnOa00tvmc= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= +golang.org/x/exp v0.0.0-20251209150349-8475f28825e9 h1:MDfG8Cvcqlt9XXrmEiD4epKn7VJHZO84hejP9Jmp0MM= +golang.org/x/exp v0.0.0-20251209150349-8475f28825e9/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= -golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= +golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI= +golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= @@ -404,8 +448,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -413,8 +457,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= -golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -429,9 +473,11 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= +golang.org/x/telemetry v0.0.0-20251208220230-2638a1023523 h1:H52Mhyrc44wBgLTGzq6+0cmuVuF3LURCSXsLMOqfFos= +golang.org/x/telemetry v0.0.0-20251208220230-2638a1023523/go.mod h1:ArQvPJS723nJQietgilmZA+shuB3CZxH1n2iXq9VSfs= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -440,8 +486,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= -golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= -golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= +golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= +golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -452,8 +498,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -462,10 +508,12 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= -golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= +golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= +golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/genproto/googleapis/api v0.0.0-20250804133106-a7a43d27e69b h1:ULiyYQ0FdsJhwwZUwbaXpZF5yUE3h+RA+gxvBu37ucc= diff --git a/internal/agent/engine.go b/internal/agent/engine.go index a13eec39..46739685 100644 --- a/internal/agent/engine.go +++ b/internal/agent/engine.go @@ -80,6 +80,9 @@ func (e *AgentEngine) Execute( llmContext []chat.Message, ) (*types.AgentState, error) { logger.Infof(ctx, "========== Agent Execution Started ==========") + // Ensure tools are cleaned up after execution + defer e.toolRegistry.Cleanup(ctx) + logger.Infof(ctx, "[Agent] SessionID: %s, MessageID: %s", sessionID, messageID) logger.Infof(ctx, "[Agent] User Query: %s", query) logger.Infof(ctx, "[Agent] LLM Context Messages: %d", len(llmContext)) @@ -304,7 +307,7 @@ func (e *AgentEngine) executeLoop( "tool_call_id": tc.ID, "tool_index": fmt.Sprintf("%d/%d", i+1, len(response.ToolCalls)), }) - result, err := e.toolRegistry.ExecuteTool(ctx, tc.Function.Name, args) + result, err := e.toolRegistry.ExecuteTool(ctx, tc.Function.Name, json.RawMessage(tc.Function.Arguments)) duration := time.Since(toolCallStartTime).Milliseconds() logger.Infof(ctx, "[Agent][Round-%d][Tool-%d/%d] Tool execution completed in %dms", state.CurrentRound+1, i+1, len(response.ToolCalls), duration) diff --git a/internal/agent/tools/data_analysis.go b/internal/agent/tools/data_analysis.go new file mode 100644 index 00000000..864bffad --- /dev/null +++ b/internal/agent/tools/data_analysis.go @@ -0,0 +1,460 @@ +package tools + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "strings" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/Tencent/WeKnora/internal/utils" + _ "github.com/duckdb/duckdb-go/v2" +) + +var dataAnalysisTool = BaseTool{ + name: ToolDataAnalysis, + description: "Use this tool when the knowledge is CSV or Excel files. It loads the data into memory and executes SQL for data analysis. If the user's question requires data statistics, convert the question into SQL and execute it.", + schema: utils.GenerateSchema[DataAnalysisInput](), +} + +type DataAnalysisInput struct { + KnowledgeID string `json:"knowledge_id" jsonschema:"id of the knowledge to query"` + Sql string `json:"sql" jsonschema:"SQL to be executed on knowledge"` +} + +type DataAnalysisTool struct { + BaseTool + knowledgeService interfaces.KnowledgeService + db *sql.DB + sessionID string + createdTables []string // Track tables created in this session +} + +func NewDataAnalysisTool(knowledgeService interfaces.KnowledgeService, db *sql.DB, sessionID string) *DataAnalysisTool { + return &DataAnalysisTool{ + BaseTool: dataAnalysisTool, + knowledgeService: knowledgeService, + db: db, + sessionID: sessionID, + } +} + +// recordCreatedTable records a table name for cleanup, ensuring uniqueness +// Returns true if the table was newly recorded, false if it already existed +func (t *DataAnalysisTool) recordCreatedTable(tableName string) bool { + for _, name := range t.createdTables { + if name == tableName { + return false + } + } + t.createdTables = append(t.createdTables, tableName) + return true +} + +// Cleanup cleans up the session-specific schema +func (t *DataAnalysisTool) Cleanup(ctx context.Context) { + if len(t.createdTables) == 0 { + logger.Infof(ctx, "[Tool][DataAnalysis] No tables to clean up for session: %s", t.sessionID) + return + } + + logger.Infof(ctx, "[Tool][DataAnalysis] Cleaning up %d tables for session: %s", len(t.createdTables), t.sessionID) + + for _, tableName := range t.createdTables { + dropSQL := fmt.Sprintf("DROP TABLE IF EXISTS \"%s\"", tableName) + if _, err := t.db.ExecContext(ctx, dropSQL); err != nil { + logger.Errorf(ctx, "[Tool][DataAnalysis] Failed to drop table '%s': %v", tableName, err) + // Continue to drop other tables even if one fails + continue + } + logger.Infof(ctx, "[Tool][DataAnalysis] Successfully dropped table '%s'", tableName) + } + + // Clear the list after cleanup + t.createdTables = nil +} + +// Execute executes the SQL query on DuckDB (only read-only queries are allowed) +func (t *DataAnalysisTool) Execute(ctx context.Context, args json.RawMessage) (*types.ToolResult, error) { + logger.Infof(ctx, "[Tool][DataAnalysis] Execute started for session: %s", t.sessionID) + var input DataAnalysisInput + if err := json.Unmarshal(args, &input); err != nil { + logger.Errorf(ctx, "[Tool][DataAnalysis] Failed to parse input args: %v", err) + return &types.ToolResult{ + Success: false, + Error: fmt.Sprintf("Failed to parse input args: %v", err), + }, err + } + + schema, err := t.LoadFromKnowledgeID(ctx, input.KnowledgeID) + if err != nil { + logger.Errorf(ctx, "[Tool][DataAnalysis] Failed to load knowledge ID '%s': %v", input.KnowledgeID, err) + return &types.ToolResult{ + Success: false, + Error: fmt.Sprintf("Failed to load knowledge ID '%s': %v", input.KnowledgeID, err), + }, err + } + + // Replace knowledge ID with table name + input.Sql = strings.ReplaceAll(input.Sql, input.KnowledgeID, schema.TableName) + + // Check if this is a read-only query + normalizedSQL := strings.TrimSpace(strings.ToLower(input.Sql)) + isReadOnly := strings.HasPrefix(normalizedSQL, "select") || + strings.HasPrefix(normalizedSQL, "show") || + strings.HasPrefix(normalizedSQL, "describe") || + strings.HasPrefix(normalizedSQL, "explain") || + strings.HasPrefix(normalizedSQL, "pragma") + + if !isReadOnly { + // Reject modification queries + logger.Warnf(ctx, "[Tool][DataAnalysis] Modification query rejected for session %s: %s", t.sessionID, input.Sql) + return &types.ToolResult{ + Success: false, + Error: "DuckDB tool only supports read-only queries (SELECT, SHOW, DESCRIBE, EXPLAIN, PRAGMA). Modification operations (INSERT, UPDATE, DELETE, CREATE, DROP, etc.) are not allowed.", + }, fmt.Errorf("modification queries are not allowed") + } + + logger.Infof(ctx, "[Tool][DataAnalysis] Received SQL query for session %s: %s", t.sessionID, input.Sql) + // Execute single query and get results + results, err := t.executeSingleQuery(ctx, input.Sql) + if err != nil { + return &types.ToolResult{ + Success: false, + Error: fmt.Sprintf("Query execution failed: %v", err), + }, err + } + + queryOutput := t.formatQueryResults(results, input.Sql) + logger.Infof(ctx, "[Tool][DataAnalysis] Completed execution query, total %d rows for session %s", len(results), t.sessionID) + return &types.ToolResult{ + Success: true, + Output: queryOutput, + Data: map[string]interface{}{ + "rows": results, + "row_count": len(results), + "query": input.Sql, + "display_type": ToolDataAnalysis, + "session_id": t.sessionID, + }, + }, nil +} + +// executeSingleQuery executes a single SQL query and returns columns and results +// Parameters: +// - ctx: context for cancellation and timeout +// - sqlQuery: the SQL query to execute +// - existingColumns: existing column names to merge with (can be nil or empty) +// +// Returns: +// - []string: merged column names (existing + new columns, deduplicated) +// - []map[string]string: query results +// - error: any error that occurred during execution +func (t *DataAnalysisTool) executeSingleQuery(ctx context.Context, sqlQuery string) ([]map[string]string, error) { + rows, err := t.db.QueryContext(ctx, sqlQuery) + if err != nil { + logger.Errorf(ctx, "[Tool][DataAnalysis] Query execution failed: %v", err) + return nil, fmt.Errorf("query execution failed: %w", err) + } + defer rows.Close() + + // Get column names + columns, err := rows.Columns() + if err != nil { + logger.Errorf(ctx, "[Tool][DataAnalysis] Failed to get columns: %v", err) + return nil, fmt.Errorf("failed to get columns: %w", err) + } + + // Process results + results := make([]map[string]string, 0) + for rows.Next() { + columnValues := make([]interface{}, len(columns)) + columnPointers := make([]interface{}, len(columns)) + for i := range columnValues { + columnPointers[i] = &columnValues[i] + } + + if err := rows.Scan(columnPointers...); err != nil { + logger.Errorf(ctx, "[Tool][DataAnalysis] Failed to scan row: %v", err) + return nil, fmt.Errorf("failed to scan row: %w", err) + } + + rowMap := make(map[string]string) + for i, colName := range columns { + val := columnValues[i] + // Convert []byte to string for better readability + if b, ok := val.([]byte); ok { + rowMap[colName] = string(b) + } else { + rowMap[colName] = fmt.Sprintf("%v", val) + } + } + results = append(results, rowMap) + } + + if err := rows.Err(); err != nil { + logger.Errorf(ctx, "[Tool][DataAnalysis] Error iterating rows: %v", err) + return nil, fmt.Errorf("error iterating rows: %w", err) + } + + return results, nil +} + +// formatQueryResults formats query results into JSONL format (one JSON object per line) +func (t *DataAnalysisTool) formatQueryResults(results []map[string]string, query string) string { + var output strings.Builder + + output.WriteString("=== DuckDB 查询结果 ===\n\n") + output.WriteString(fmt.Sprintf("执行的SQL: %s\n\n", query)) + output.WriteString(fmt.Sprintf("返回 %d 行数据\n\n", len(results))) + + if len(results) == 0 { + output.WriteString("未找到匹配的记录。\n") + return output.String() + } + + output.WriteString("=== 数据详情 ===\n\n") + if len(results) > 10 { + output.WriteString(fmt.Sprintf("显示了所有 %d 条记录。建议使用 LIMIT 子句限制结果数量以提高性能。\n\n", len(results))) + } + + // Write each record as a separate JSON line + for i, record := range results { + recordBytes, _ := json.Marshal(record) + + // Remove the trailing newline added by Encode + recordStr := strings.Trim(string(recordBytes), "\n") + output.WriteString(fmt.Sprintf("record %d: %s\n", i+1, recordStr)) + } + + return output.String() +} + +// TableSchema represents the schema information of a table +type TableSchema struct { + TableName string `json:"table_name"` + Columns []ColumnInfo `json:"columns"` + RowCount int64 `json:"row_count"` + Metadata map[string]interface{} `json:"metadata,omitempty"` +} + +// ColumnInfo represents information about a single column +type ColumnInfo struct { + Name string `json:"name"` + Type string `json:"type"` + Nullable string `json:"nullable"` +} + +// LoadFromCSV loads data from a CSV file into a DuckDB table and returns the table schema +// Parameters: +// - ctx: context for cancellation and timeout +// - filename: path to the CSV file +// - tableName: name of the table to create +// +// Returns: +// - *TableSchema: schema information of the created table +// - error: any error that occurred during the operation +func (t *DataAnalysisTool) LoadFromCSV(ctx context.Context, filename string, tableName string) (*TableSchema, error) { + logger.Infof(ctx, "[Tool][DataAnalysis] Loading CSV file '%s' into table '%s' for session %s", filename, tableName, t.sessionID) + + // Record the created table for cleanup. If already exists, skip creation + if t.recordCreatedTable(tableName) { + // Create table from CSV using DuckDB's read_csv_auto function + // Table will be created in the session schema + createTableSQL := fmt.Sprintf("CREATE TABLE \"%s\" AS SELECT * FROM read_csv_auto('%s')", tableName, filename) + + _, err := t.db.ExecContext(ctx, createTableSQL) + if err != nil { + logger.Errorf(ctx, "[Tool][DataAnalysis] Failed to create table from CSV: %v", err) + return nil, fmt.Errorf("failed to create table from CSV: %w", err) + } + + logger.Infof(ctx, "[Tool][DataAnalysis] Successfully created table '%s' from CSV file in session %s", tableName, t.sessionID) + } + + // Get and return the table schema + return t.LoadFromTable(ctx, tableName) +} + +// LoadFromExcel loads data from an Excel file into a DuckDB table and returns the table schema +// Parameters: +// - ctx: context for cancellation and timeout +// - filename: path to the Excel file +// - tableName: name of the table to create +// +// Returns: +// - *TableSchema: schema information of the created table +// - error: any error that occurred during the operation +// +// Note: This function requires the spatial extension to be installed in DuckDB +func (t *DataAnalysisTool) LoadFromExcel(ctx context.Context, filename string, tableName string) (*TableSchema, error) { + logger.Infof(ctx, "[Tool][DataAnalysis] Loading Excel file '%s' into table '%s' for session %s", filename, tableName, t.sessionID) + + // Record the created table for cleanup. If already exists, skip creation + if t.recordCreatedTable(tableName) { + // Try to read Excel file using st_read (from spatial extension) + // If spatial extension doesn't support Excel, we'll need to convert to CSV first + createTableSQL := fmt.Sprintf("CREATE TABLE \"%s\" AS SELECT * FROM st_read('%s')", tableName, filename) + + _, err := t.db.ExecContext(ctx, createTableSQL) + if err != nil { + logger.Errorf(ctx, "[Tool][DataAnalysis] Failed to create table from Excel: %v", err) + return nil, fmt.Errorf("failed to create table from Excel file. Consider converting to CSV first: %w", err) + } + + logger.Infof(ctx, "[Tool][DataAnalysis] Successfully created table '%s' from Excel file in session %s", tableName, t.sessionID) + } + + // Get and return the table schema + return t.LoadFromTable(ctx, tableName) +} + +// LoadFromKnowledge loads data from a Knowledge entity into a DuckDB table and returns the table schema +// It automatically determines the file type and calls the appropriate loading method +// Parameters: +// - ctx: context for cancellation and timeout +// - knowledge: the Knowledge entity containing file information +// +// Returns: +// - *TableSchema: schema information of the created table +// - error: any error that occurred during the operation +func (t *DataAnalysisTool) LoadFromKnowledge(ctx context.Context, knowledge *types.Knowledge) (*TableSchema, error) { + if knowledge == nil { + return nil, fmt.Errorf("knowledge cannot be nil") + } + tableName := t.TableName(knowledge) + + // Normalize file type to lowercase for comparison + fileType := strings.ToLower(knowledge.FileType) + + logger.Infof(ctx, "[Tool][DataAnalysis] Loading knowledge '%s' (type: %s) into table '%s' for session %s", + knowledge.ID, fileType, tableName, t.sessionID) + + switch fileType { + case "csv": + return t.LoadFromCSV(ctx, knowledge.FilePath, tableName) + case "xlsx", "xls": + return t.LoadFromExcel(ctx, knowledge.FilePath, tableName) + default: + logger.Warnf(ctx, "[Tool][DataAnalysis] Unsupported file type '%s' for knowledge '%s' in session %s", + fileType, knowledge.ID, t.sessionID) + return nil, fmt.Errorf("unsupported file type: %s (supported types: csv, xlsx, xls)", fileType) + } +} + +// LoadFromKnowledgeID loads data from a Knowledge ID into a DuckDB table and returns the table schema +// Parameters: +// - ctx: context for cancellation and timeout +// - knowledgeID: the ID of the Knowledge entity +// +// Returns: +// - string: the name of the created table +// - *TableSchema: schema information of the created table +// - error: any error that occurred during the operation +func (t *DataAnalysisTool) LoadFromKnowledgeID(ctx context.Context, knowledgeID string) (*TableSchema, error) { + knowledge, err := t.knowledgeService.GetKnowledgeByID(ctx, knowledgeID) + if err != nil { + logger.Errorf(ctx, "[Tool][DataAnalysis] Failed to get knowledge by ID '%s': %v", knowledgeID, err) + return nil, fmt.Errorf("failed to get knowledge by ID: %w", err) + } + + return t.LoadFromKnowledge(ctx, knowledge) +} + +// LoadFromTable retrieves the schema information of an existing table +// Parameters: +// - ctx: context for cancellation and timeout +// - tableName: name of the table to query +// +// Returns: +// - *TableSchema: schema information of the table +// - error: any error that occurred during the operation +// +// Note: This function does NOT create the table, it only retrieves schema information +func (t *DataAnalysisTool) LoadFromTable(ctx context.Context, tableName string) (*TableSchema, error) { + logger.Infof(ctx, "[Tool][DataAnalysis] Getting schema for table '%s' in session %s", tableName, t.sessionID) + + // Query to get column information using PRAGMA table_info or DESCRIBE + schemaSQL := fmt.Sprintf("DESCRIBE \"%s\"", tableName) + + rows, err := t.db.QueryContext(ctx, schemaSQL) + if err != nil { + logger.Errorf(ctx, "[Tool][DataAnalysis] Failed to get table schema: %v", err) + return nil, fmt.Errorf("failed to get table schema: %w", err) + } + defer rows.Close() + + // Parse column information + columns := make([]ColumnInfo, 0) + for rows.Next() { + var colName, colType, nullable string + var extra1, extra2, extra3 interface{} // DuckDB DESCRIBE may return additional columns + + // Try to scan with different column counts + err := rows.Scan(&colName, &colType, &nullable, &extra1, &extra2, &extra3) + if err != nil { + // Try with fewer columns + err = rows.Scan(&colName, &colType, &nullable) + if err != nil { + logger.Errorf(ctx, "[Tool][DataAnalysis] Failed to scan column info: %v", err) + return nil, fmt.Errorf("failed to scan column info: %w", err) + } + } + + columns = append(columns, ColumnInfo{ + Name: colName, + Type: colType, + Nullable: nullable, + }) + } + + if err := rows.Err(); err != nil { + logger.Errorf(ctx, "[Tool][DataAnalysis] Error iterating schema rows: %v", err) + return nil, fmt.Errorf("error iterating schema rows: %w", err) + } + + // Get row count + countSQL := fmt.Sprintf("SELECT COUNT(*) FROM \"%s\"", tableName) + var rowCount int64 + if err := t.db.QueryRowContext(ctx, countSQL).Scan(&rowCount); err != nil { + logger.Errorf(ctx, "[Tool][DataAnalysis] Failed to get row count: %v", err) + return nil, fmt.Errorf("failed to get row count: %w", err) + } + + schema := &TableSchema{ + TableName: tableName, + Columns: columns, + RowCount: rowCount, + Metadata: map[string]interface{}{ + "column_count": len(columns), + "session_id": t.sessionID, + }, + } + + logger.Infof(ctx, "[Tool][DataAnalysis] Retrieved schema for table '%s' in session %s: %d columns, %d rows", + tableName, t.sessionID, len(columns), rowCount) + + return schema, nil +} + +func (t *DataAnalysisTool) TableName(knowledge *types.Knowledge) string { + return "k_" + strings.ReplaceAll(knowledge.ID, "-", "_") +} + +// buildSchemaDescription builds a formatted schema description +func (t *TableSchema) Description() string { + var builder strings.Builder + builder.WriteString(fmt.Sprintf("表名: %s\n", t.TableName)) + builder.WriteString(fmt.Sprintf("列数: %d\n", len(t.Columns))) + builder.WriteString(fmt.Sprintf("行数: %d\n\n", t.RowCount)) + builder.WriteString("列信息:\n") + + for _, col := range t.Columns { + builder.WriteString(fmt.Sprintf("- %s (%s)\n", col.Name, col.Type)) + } + + return builder.String() +} diff --git a/internal/agent/tools/data_schema.go b/internal/agent/tools/data_schema.go new file mode 100644 index 00000000..a0777486 --- /dev/null +++ b/internal/agent/tools/data_schema.go @@ -0,0 +1,114 @@ +package tools + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/Tencent/WeKnora/internal/utils" +) + +var dataSchemaTool = BaseTool{ + name: ToolDataSchema, + description: "Use this tool to get the schema information of a CSV or Excel file loaded into DuckDB. It returns the table name, columns, and row count.", + schema: utils.GenerateSchema[DataSchemaInput](), +} + +type DataSchemaInput struct { + KnowledgeID string `json:"knowledge_id" jsonschema:"id of the knowledge to query"` +} + +type DataSchemaTool struct { + BaseTool + knowledgeService interfaces.KnowledgeService + chunkRepo interfaces.ChunkRepository + targetChunkTypes []types.ChunkType +} + +func NewDataSchemaTool(knowledgeService interfaces.KnowledgeService, chunkRepo interfaces.ChunkRepository, targetChunkTypes ...types.ChunkType) *DataSchemaTool { + if len(targetChunkTypes) == 0 { + targetChunkTypes = []types.ChunkType{types.ChunkTypeTableSummary, types.ChunkTypeTableColumn} + } + return &DataSchemaTool{ + BaseTool: dataSchemaTool, + knowledgeService: knowledgeService, + chunkRepo: chunkRepo, + targetChunkTypes: targetChunkTypes, + } +} + +// Execute executes the tool logic +func (t *DataSchemaTool) Execute(ctx context.Context, args json.RawMessage) (*types.ToolResult, error) { + var input DataSchemaInput + if err := json.Unmarshal(args, &input); err != nil { + return &types.ToolResult{ + Success: false, + Error: fmt.Sprintf("Failed to parse input args: %v", err), + }, err + } + + // Get knowledge to get TenantID + knowledge, err := t.knowledgeService.GetKnowledgeByID(ctx, input.KnowledgeID) + if err != nil { + return &types.ToolResult{ + Success: false, + Error: fmt.Sprintf("Failed to get knowledge '%s': %v", input.KnowledgeID, err), + }, err + } + + // Get chunks for the knowledge ID using ChunkRepository + // We only need table summary and column chunks + chunkTypes := t.targetChunkTypes + page := &types.Pagination{ + Page: 1, + PageSize: 100, // Should be enough for schema chunks + } + + chunks, _, err := t.chunkRepo.ListPagedChunksByKnowledgeID( + ctx, + knowledge.TenantID, + input.KnowledgeID, + page, + chunkTypes, + "", // tagID + "", // keyword + "", // searchField + "", // sortOrder + "", // knowledgeType + ) + if err != nil { + return &types.ToolResult{ + Success: false, + Error: fmt.Sprintf("Failed to list chunks for knowledge ID '%s': %v", input.KnowledgeID, err), + }, err + } + + var summaryContent, columnContent string + for _, chunk := range chunks { + if chunk.ChunkType == types.ChunkTypeTableSummary { + summaryContent = chunk.Content + } else if chunk.ChunkType == types.ChunkTypeTableColumn { + columnContent = chunk.Content + } + } + + if summaryContent == "" || columnContent == "" { + return &types.ToolResult{ + Success: false, + Error: fmt.Sprintf("No table schema information found for knowledge ID '%s'", input.KnowledgeID), + }, fmt.Errorf("no schema info found") + } + + output := fmt.Sprintf("%s\n\n%s", summaryContent, columnContent) + + return &types.ToolResult{ + Success: true, + Output: output, + Data: map[string]interface{}{ + "summary": summaryContent, + "columns": columnContent, + }, + }, nil +} diff --git a/internal/agent/tools/database_query.go b/internal/agent/tools/database_query.go index b84d8d72..251950ea 100644 --- a/internal/agent/tools/database_query.go +++ b/internal/agent/tools/database_query.go @@ -9,92 +9,14 @@ import ( "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/utils" pg_query "github.com/pganalyze/pg_query_go/v6" "gorm.io/gorm" ) -// SQLSecurityValidator provides comprehensive SQL injection protection using PostgreSQL's official parser -type SQLSecurityValidator struct { - allowedTables map[string]bool - allowedFunctions map[string]bool - tenantID uint64 -} - -// NewSQLSecurityValidator creates a new SQL security validator -func NewSQLSecurityValidator(tenantID uint64) *SQLSecurityValidator { - return &SQLSecurityValidator{ - allowedTables: map[string]bool{ - "tenants": true, - "knowledge_bases": true, - "knowledges": true, - "sessions": true, - "messages": true, - "chunks": true, - "embeddings": true, - "models": true, - }, - // Whitelist of allowed SQL functions (aggregates and safe functions only) - allowedFunctions: map[string]bool{ - // Aggregate functions - "count": true, - "sum": true, - "avg": true, - "min": true, - "max": true, - "array_agg": true, - "string_agg": true, - "bool_and": true, - "bool_or": true, - "json_agg": true, - "jsonb_agg": true, - "json_object_agg": true, - "jsonb_object_agg": true, - // Safe scalar functions - "coalesce": true, - "nullif": true, - "greatest": true, - "least": true, - "abs": true, - "ceil": true, - "floor": true, - "round": true, - "length": true, - "lower": true, - "upper": true, - "trim": true, - "ltrim": true, - "rtrim": true, - "substring": true, - "concat": true, - "concat_ws": true, - "replace": true, - "left": true, - "right": true, - "now": true, - "current_date": true, - "current_timestamp": true, - "date_trunc": true, - "extract": true, - "to_char": true, - "to_date": true, - "to_timestamp": true, - "date_part": true, - "age": true, - }, - tenantID: tenantID, - } -} - -// DatabaseQueryTool allows AI to query the database with auto-injected tenant_id for security -type DatabaseQueryTool struct { - BaseTool - db *gorm.DB - tenantID uint64 -} - -// NewDatabaseQueryTool creates a new database query tool -func NewDatabaseQueryTool(db *gorm.DB, tenantID uint64) *DatabaseQueryTool { - description := `Execute SQL queries to retrieve information from the database. +var databaseQueryTool = BaseTool{ + name: ToolDatabaseQuery, + description: `Execute SQL queries to retrieve information from the database. ## Security Features - Automatic tenant_id injection: All queries are automatically filtered by the logged-in user's tenant_id @@ -171,36 +93,123 @@ Join knowledge bases and documents: - Only SELECT queries are allowed - Limit results with LIMIT clause for better performance - Use appropriate JOINs when querying across tables -- All timestamps are in UTC with time zone` +- All timestamps are in UTC with time zone`, + schema: utils.GenerateSchema[DatabaseQueryInput](), +} - return &DatabaseQueryTool{ - BaseTool: NewBaseTool("database_query", description), - db: db, +type DatabaseQueryInput struct { + SQL string `json:"sql" jsonschema:"The SELECT SQL query to execute. DO NOT include tenant_id condition - it will be automatically added for security."` +} + +// SQLSecurityValidator provides comprehensive SQL injection protection using PostgreSQL's official parser +type SQLSecurityValidator struct { + allowedTables map[string]bool + allowedFunctions map[string]bool + tenantID uint64 +} + +// NewSQLSecurityValidator creates a new SQL security validator +func NewSQLSecurityValidator(tenantID uint64) *SQLSecurityValidator { + return &SQLSecurityValidator{ + allowedTables: map[string]bool{ + "tenants": true, + "knowledge_bases": true, + "knowledges": true, + "sessions": true, + "messages": true, + "chunks": true, + "embeddings": true, + "models": true, + }, + // Whitelist of allowed SQL functions (aggregates and safe functions only) + allowedFunctions: map[string]bool{ + // Aggregate functions + "count": true, + "sum": true, + "avg": true, + "min": true, + "max": true, + "array_agg": true, + "string_agg": true, + "bool_and": true, + "bool_or": true, + "json_agg": true, + "jsonb_agg": true, + "json_object_agg": true, + "jsonb_object_agg": true, + // Safe scalar functions + "coalesce": true, + "nullif": true, + "greatest": true, + "least": true, + "abs": true, + "ceil": true, + "floor": true, + "round": true, + "length": true, + "lower": true, + "upper": true, + "trim": true, + "ltrim": true, + "rtrim": true, + "substring": true, + "concat": true, + "concat_ws": true, + "replace": true, + "left": true, + "right": true, + "now": true, + "current_date": true, + "current_timestamp": true, + "date_trunc": true, + "extract": true, + "to_char": true, + "to_date": true, + "to_timestamp": true, + "date_part": true, + "age": true, + }, tenantID: tenantID, } } -// Parameters returns the JSON schema for the tool's parameters -func (t *DatabaseQueryTool) Parameters() map[string]interface{} { - return map[string]interface{}{ - "type": "object", - "properties": map[string]interface{}{ - "sql": map[string]interface{}{ - "type": "string", - "description": "The SELECT SQL query to execute. DO NOT include tenant_id condition - it will be automatically added for security.", - }, - }, - "required": []string{"sql"}, +// DatabaseQueryInput defines the input parameters for database query tool + +// DatabaseQueryTool allows AI to query the database with auto-injected tenant_id for security +type DatabaseQueryTool struct { + BaseTool + db *gorm.DB +} + +// NewDatabaseQueryTool creates a new database query tool +func NewDatabaseQueryTool(db *gorm.DB) *DatabaseQueryTool { + return &DatabaseQueryTool{ + BaseTool: databaseQueryTool, + db: db, } } // Execute executes the database query tool -func (t *DatabaseQueryTool) Execute(ctx context.Context, args map[string]interface{}) (*types.ToolResult, error) { +func (t *DatabaseQueryTool) Execute(ctx context.Context, args json.RawMessage) (*types.ToolResult, error) { logger.Infof(ctx, "[Tool][DatabaseQuery] Execute started") - // Extract SQL from args - sqlQuery, ok := args["sql"].(string) - if !ok || sqlQuery == "" { + tenantID := uint64(0) + if tid, ok := ctx.Value(types.TenantIDContextKey).(uint64); ok { + tenantID = tid + } + + // Parse args from json.RawMessage + var input DatabaseQueryInput + if err := json.Unmarshal(args, &input); err != nil { + logger.Errorf(ctx, "[Tool][DatabaseQuery] Failed to parse args: %v", err) + return &types.ToolResult{ + Success: false, + Error: fmt.Sprintf("Failed to parse args: %v", err), + }, err + } + + // Extract SQL from input + if input.SQL == "" { logger.Errorf(ctx, "[Tool][DatabaseQuery] Missing or invalid SQL parameter") return &types.ToolResult{ Success: false, @@ -208,12 +217,12 @@ func (t *DatabaseQueryTool) Execute(ctx context.Context, args map[string]interfa }, fmt.Errorf("missing sql parameter") } - logger.Infof(ctx, "[Tool][DatabaseQuery] Original SQL query:\n%s", sqlQuery) - logger.Infof(ctx, "[Tool][DatabaseQuery] Tenant ID: %d", t.tenantID) + logger.Infof(ctx, "[Tool][DatabaseQuery] Original SQL query:\n%s", input.SQL) + logger.Infof(ctx, "[Tool][DatabaseQuery] Tenant ID: %d", tenantID) // Validate and secure the SQL query logger.Debugf(ctx, "[Tool][DatabaseQuery] Validating and securing SQL...") - securedSQL, err := t.validateAndSecureSQL(sqlQuery) + securedSQL, err := t.validateAndSecureSQL(input.SQL, tenantID) if err != nil { logger.Errorf(ctx, "[Tool][DatabaseQuery] SQL validation failed: %v", err) return &types.ToolResult{ @@ -224,7 +233,7 @@ func (t *DatabaseQueryTool) Execute(ctx context.Context, args map[string]interfa logger.Infof(ctx, "[Tool][DatabaseQuery] Secured SQL query:\n%s", securedSQL) logger.Infof(ctx, "Executing secured SQL query - original: %s, secured: %s, tenant_id: %d", - sqlQuery, securedSQL, t.tenantID) + input.SQL, securedSQL, tenantID) // Execute the query logger.Infof(ctx, "[Tool][DatabaseQuery] Executing query against database...") @@ -312,15 +321,15 @@ func (t *DatabaseQueryTool) Execute(ctx context.Context, args map[string]interfa "rows": results, "row_count": len(results), "query": securedSQL, - "tenant_id": t.tenantID, + "tenant_id": tenantID, "display_type": "database_query", }, }, nil } // validateAndSecureSQL validates the SQL query and injects tenant_id conditions -func (t *DatabaseQueryTool) validateAndSecureSQL(sqlQuery string) (string, error) { - validator := NewSQLSecurityValidator(t.tenantID) +func (t *DatabaseQueryTool) validateAndSecureSQL(sqlQuery string, tenantID uint64) (string, error) { + validator := NewSQLSecurityValidator(tenantID) return validator.ValidateAndSecure(sqlQuery) } diff --git a/internal/agent/tools/definitions.go b/internal/agent/tools/definitions.go index dabac9b5..f96f0a64 100644 --- a/internal/agent/tools/definitions.go +++ b/internal/agent/tools/definitions.go @@ -1,5 +1,21 @@ package tools +// Tool names constants +const ( + ToolThinking = "thinking" + ToolTodoWrite = "todo_write" + ToolGrepChunks = "grep_chunks" + ToolKnowledgeSearch = "knowledge_search" + ToolListKnowledgeChunks = "list_knowledge_chunks" + ToolQueryKnowledgeGraph = "query_knowledge_graph" + ToolGetDocumentInfo = "get_document_info" + ToolDatabaseQuery = "database_query" + ToolDataAnalysis = "data_analysis" + ToolDataSchema = "data_schema" + ToolWebSearch = "web_search" + ToolWebFetch = "web_fetch" +) + // AvailableTool defines a simple tool metadata used by settings APIs. type AvailableTool struct { Name string `json:"name"` @@ -11,27 +27,31 @@ type AvailableTool struct { // Keep this in sync with registered tools in this package. func AvailableToolDefinitions() []AvailableTool { return []AvailableTool{ - {Name: "thinking", Label: "思考", Description: "动态和反思性的问题解决思考工具"}, - {Name: "todo_write", Label: "制定计划", Description: "创建结构化的研究计划"}, - {Name: "grep_chunks", Label: "关键词搜索", Description: "快速定位包含特定关键词的文档和分块"}, - {Name: "knowledge_search", Label: "语义搜索", Description: "理解问题并查找语义相关内容"}, - {Name: "list_knowledge_chunks", Label: "查看文档分块", Description: "获取文档完整分块内容"}, - {Name: "query_knowledge_graph", Label: "查询知识图谱", Description: "从知识图谱中查询关系"}, - {Name: "get_document_info", Label: "获取文档信息", Description: "查看文档元数据"}, - {Name: "database_query", Label: "查询数据库", Description: "查询数据库中的信息"}, + {Name: ToolThinking, Label: "思考", Description: "动态和反思性的问题解决思考工具"}, + {Name: ToolTodoWrite, Label: "制定计划", Description: "创建结构化的研究计划"}, + {Name: ToolGrepChunks, Label: "关键词搜索", Description: "快速定位包含特定关键词的文档和分块"}, + {Name: ToolKnowledgeSearch, Label: "语义搜索", Description: "理解问题并查找语义相关内容"}, + {Name: ToolListKnowledgeChunks, Label: "查看文档分块", Description: "获取文档完整分块内容"}, + {Name: ToolQueryKnowledgeGraph, Label: "查询知识图谱", Description: "从知识图谱中查询关系"}, + {Name: ToolGetDocumentInfo, Label: "获取文档信息", Description: "查看文档元数据"}, + {Name: ToolDatabaseQuery, Label: "查询数据库", Description: "查询数据库中的信息"}, + {Name: ToolDataAnalysis, Label: "数据分析", Description: "理解数据文件并进行数据分析"}, + {Name: ToolDataSchema, Label: "查看数据元信息", Description: "获取表格文件的元信息"}, } } // DefaultAllowedTools returns the default allowed tools list. func DefaultAllowedTools() []string { return []string{ - "thinking", - "todo_write", - "knowledge_search", - "grep_chunks", - "list_knowledge_chunks", - "query_knowledge_graph", - "get_document_info", - "database_query", + ToolThinking, + ToolTodoWrite, + ToolKnowledgeSearch, + ToolGrepChunks, + ToolListKnowledgeChunks, + ToolQueryKnowledgeGraph, + ToolGetDocumentInfo, + ToolDatabaseQuery, + ToolDataAnalysis, + ToolDataSchema, } } diff --git a/internal/agent/tools/get_document_info.go b/internal/agent/tools/get_document_info.go index c4ddebd6..2937490c 100644 --- a/internal/agent/tools/get_document_info.go +++ b/internal/agent/tools/get_document_info.go @@ -2,28 +2,18 @@ package tools import ( "context" + "encoding/json" "fmt" "sync" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/Tencent/WeKnora/internal/utils" ) -// GetDocumentInfoTool retrieves detailed information about a document/knowledge -type GetDocumentInfoTool struct { - BaseTool - tenantID uint64 - knowledgeService interfaces.KnowledgeService - chunkService interfaces.ChunkService -} - -// NewGetDocumentInfoTool creates a new get document info tool -func NewGetDocumentInfoTool( - tenantID uint64, - knowledgeService interfaces.KnowledgeService, - chunkService interfaces.ChunkService, -) *GetDocumentInfoTool { - description := `Retrieve detailed metadata information about documents. +var getDocumentInfoTool = BaseTool{ + name: ToolGetDocumentInfo, + description: `Retrieve detailed metadata information about documents. ## When to Use @@ -50,56 +40,61 @@ Do not use when: - Concurrent query for multiple documents provides better performance - Returns complete document metadata, not just title -- Can check document processing status (parse_status)` +- Can check document processing status (parse_status)`, + schema: utils.GenerateSchema[GetDocumentInfoInput](), +} +// GetDocumentInfoInput defines the input parameters for get document info tool +type GetDocumentInfoInput struct { + KnowledgeIDs []string `json:"knowledge_ids" jsonschema:"Array of document/knowledge IDs, obtained from knowledge_id field in search results, supports concurrent batch queries"` +} + +// GetDocumentInfoTool retrieves detailed information about a document/knowledge +type GetDocumentInfoTool struct { + BaseTool + knowledgeService interfaces.KnowledgeService + chunkService interfaces.ChunkService +} + +// NewGetDocumentInfoTool creates a new get document info tool +func NewGetDocumentInfoTool( + knowledgeService interfaces.KnowledgeService, + chunkService interfaces.ChunkService, +) *GetDocumentInfoTool { return &GetDocumentInfoTool{ - BaseTool: NewBaseTool("get_document_info", description), - tenantID: tenantID, + BaseTool: getDocumentInfoTool, knowledgeService: knowledgeService, chunkService: chunkService, } } -// Parameters returns the JSON schema for the tool's parameters -func (t *GetDocumentInfoTool) Parameters() map[string]interface{} { - return map[string]interface{}{ - "type": "object", - "properties": map[string]interface{}{ - "knowledge_ids": map[string]interface{}{ - "type": "array", - "description": "Array of document/knowledge IDs, obtained from knowledge_id field in search results, supports concurrent batch queries", - "items": map[string]interface{}{ - "type": "string", - }, - "minItems": 1, - "maxItems": 10, - }, - }, - "required": []string{"knowledge_ids"}, - } -} - // Execute retrieves document information with concurrent processing -func (t *GetDocumentInfoTool) Execute(ctx context.Context, args map[string]interface{}) (*types.ToolResult, error) { +func (t *GetDocumentInfoTool) Execute(ctx context.Context, args json.RawMessage) (*types.ToolResult, error) { + tenantID := uint64(0) + if tid, ok := ctx.Value(types.TenantIDContextKey).(uint64); ok { + tenantID = tid + } + + // Parse args from json.RawMessage + var input GetDocumentInfoInput + if err := json.Unmarshal(args, &input); err != nil { + return &types.ToolResult{ + Success: false, + Error: fmt.Sprintf("Failed to parse args: %v", err), + }, err + } + // Extract knowledge_ids array - knowledgeIDsRaw, ok := args["knowledge_ids"].([]interface{}) - if !ok || len(knowledgeIDsRaw) == 0 { + knowledgeIDs := input.KnowledgeIDs + if len(knowledgeIDs) == 0 { return &types.ToolResult{ Success: false, Error: "knowledge_ids is required and must be a non-empty array", }, fmt.Errorf("knowledge_ids is required") } - // Convert to string slice - var knowledgeIDs []string - for _, id := range knowledgeIDsRaw { - if idStr, ok := id.(string); ok && idStr != "" { - knowledgeIDs = append(knowledgeIDs, idStr) - } - } - - // Check if knowledge IDs are valid - if len(knowledgeIDs) == 0 { + // Validate max 10 documents + if len(knowledgeIDs) > 10 { return &types.ToolResult{ Success: false, Error: "knowledge_ids must contain at least one valid knowledge ID", @@ -124,7 +119,7 @@ func (t *GetDocumentInfoTool) Execute(ctx context.Context, args map[string]inter defer wg.Done() // Get knowledge metadata - knowledge, err := t.knowledgeService.GetRepository().GetKnowledgeByID(ctx, t.tenantID, id) + knowledge, err := t.knowledgeService.GetRepository().GetKnowledgeByID(ctx, tenantID, id) if err != nil { mu.Lock() results[id] = &docInfo{ @@ -136,7 +131,7 @@ func (t *GetDocumentInfoTool) Execute(ctx context.Context, args map[string]inter // Get chunk count _, total, err := t.chunkService.GetRepository(). - ListPagedChunksByKnowledgeID(ctx, t.tenantID, id, &types.Pagination{ + ListPagedChunksByKnowledgeID(ctx, tenantID, id, &types.Pagination{ Page: 1, PageSize: 1000, }, []types.ChunkType{"text"}, "", "", "", "", "") diff --git a/internal/agent/tools/grep_chunks.go b/internal/agent/tools/grep_chunks.go index af5abe3c..c9f74167 100644 --- a/internal/agent/tools/grep_chunks.go +++ b/internal/agent/tools/grep_chunks.go @@ -2,6 +2,7 @@ package tools import ( "context" + "encoding/json" "fmt" "math" "sort" @@ -13,19 +14,9 @@ import ( "gorm.io/gorm" ) -// GrepChunksTool performs text pattern matching in knowledge base chunks -// Similar to grep command in Unix-like systems, but operates on knowledge base content -type GrepChunksTool struct { - BaseTool - db *gorm.DB - tenantID uint64 - knowledgeBaseIDs []string - knowledgeIDs []string // Specific knowledge/document IDs to search within -} - -// NewGrepChunksTool creates a new grep chunks tool -func NewGrepChunksTool(db *gorm.DB, tenantID uint64, knowledgeBaseIDs []string, knowledgeIDs []string) *GrepChunksTool { - description := `Unix-style text pattern matching tool for knowledge base chunks. +var grepChunksTool = BaseTool{ + name: ToolGrepChunks, + description: `Unix-style text pattern matching tool for knowledge base chunks. Searches for text patterns in chunk content using strict literal text matching (fixed-string search). This tool performs exact keyword lookup, not semantic search. @@ -61,76 +52,81 @@ grep_chunks scans enabled chunks across the specified knowledge bases and return - Exact keyword presence checks - Fast preliminary filtering before semantic search - Situations requiring deterministic text search +`, + schema: json.RawMessage(`{ + "type": "object", + "properties": { + "pattern": { + "type": "array", + "description": "REQUIRED: Text patterns to search for. Can be a single pattern or multiple patterns. Treated as literal text (fixed string matching). Results match any of the patterns (OR logic).", + "items": { + "type": "string" + }, + "minItems": 1 + }, + "knowledge_base_ids": { + "type": "array", + "description": "Filter by knowledge base IDs. If empty, searches all allowed KBs.", + "items": { + "type": "string" + } + }, + "max_results": { + "type": "integer", + "description": "Maximum number of matching chunks to return (default: 50, max: 200)", + "default": 50, + "minimum": 1, + "maximum": 200 + } + }, + "required": ["pattern"] +}`), +} -` +// GrepChunksInput defines the input parameters for grep chunks tool +type GrepChunksInput struct { + Pattern []string `json:"pattern" ` + KnowledgeBaseIDs []string `json:"knowledge_base_ids,omitempty"` + MaxResults int `json:"max_results,omitempty"` +} +// GrepChunksTool performs text pattern matching in knowledge base chunks +// Similar to grep command in Unix-like systems, but operates on knowledge base content +type GrepChunksTool struct { + BaseTool + db *gorm.DB + knowledgeBaseIDs []string + knowledgeIDs []string // Specific knowledge/document IDs to search within +} + +// NewGrepChunksTool creates a new grep chunks tool +func NewGrepChunksTool(db *gorm.DB, knowledgeBaseIDs []string, knowledgeIDs []string) *GrepChunksTool { return &GrepChunksTool{ - BaseTool: NewBaseTool("grep_chunks", description), + BaseTool: grepChunksTool, db: db, - tenantID: tenantID, knowledgeBaseIDs: knowledgeBaseIDs, knowledgeIDs: knowledgeIDs, } } -// Parameters returns the JSON schema for the tool's parameters -func (t *GrepChunksTool) Parameters() map[string]interface{} { - return map[string]interface{}{ - "type": "object", - "properties": map[string]interface{}{ - "pattern": map[string]interface{}{ - "type": "array", - "description": "REQUIRED: Text patterns to search for. Can be a single pattern or multiple patterns. Treated as literal text (fixed string matching). Results match any of the patterns (OR logic).", - "items": map[string]interface{}{ - "type": "string", - }, - "minItems": 1, - }, - "knowledge_base_ids": map[string]interface{}{ - "type": "array", - "description": "Filter by knowledge base IDs. If empty, searches all allowed KBs.", - "items": map[string]interface{}{ - "type": "string", - }, - }, - // "knowledge_ids": map[string]interface{}{ - // "type": "array", - // "description": "Filter by document/knowledge IDs. If empty, searches all documents.", - // "items": map[string]interface{}{ - // "type": "string", - // }, - // }, - "max_results": map[string]interface{}{ - "type": "integer", - "description": "Maximum number of matching chunks to return (default: 50, max: 200)", - "default": 50, - "minimum": 1, - "maximum": 200, - }, - }, - "required": []string{"pattern"}, - } -} - // Execute executes the grep chunks tool -func (t *GrepChunksTool) Execute(ctx context.Context, args map[string]interface{}) (*types.ToolResult, error) { +func (t *GrepChunksTool) Execute(ctx context.Context, args json.RawMessage) (*types.ToolResult, error) { logger.Infof(ctx, "[Tool][GrepChunks] Execute started") + // Parse args from json.RawMessage + var input GrepChunksInput + if err := json.Unmarshal(args, &input); err != nil { + logger.Errorf(ctx, "[Tool][GrepChunks] Failed to parse args: %v", err) + return &types.ToolResult{ + Success: false, + Error: fmt.Sprintf("Failed to parse args: %v", err), + }, err + } + // Parse pattern parameter (required) - support multiple patterns - var patterns []string - if patternsRaw, ok := args["pattern"].([]interface{}); ok && len(patternsRaw) > 0 { - for _, p := range patternsRaw { - if pStr, ok := p.(string); ok && strings.TrimSpace(pStr) != "" { - patterns = append(patterns, strings.TrimSpace(pStr)) - } - } - } - // Also support single string for backward compatibility - if len(patterns) == 0 { - if patternStr, ok := args["pattern"].(string); ok && strings.TrimSpace(patternStr) != "" { - patterns = append(patterns, strings.TrimSpace(patternStr)) - } - } + patterns := input.Pattern + + // Validate patterns if len(patterns) == 0 { logger.Errorf(ctx, "[Tool][GrepChunks] Missing or invalid pattern parameter") return &types.ToolResult{ @@ -143,8 +139,8 @@ func (t *GrepChunksTool) Execute(ctx context.Context, args map[string]interface{ countOnly := false // default: show results maxResults := 50 - if mr, ok := args["max_results"].(float64); ok { - maxResults = int(mr) + if input.MaxResults > 0 { + maxResults = input.MaxResults if maxResults < 1 { maxResults = 1 } else if maxResults > 200 { @@ -153,14 +149,7 @@ func (t *GrepChunksTool) Execute(ctx context.Context, args map[string]interface{ } // Parse knowledge_base_ids filter - var kbIDs []string - if kbIDsRaw, ok := args["knowledge_base_ids"].([]interface{}); ok { - for _, id := range kbIDsRaw { - if idStr, ok := id.(string); ok && idStr != "" { - kbIDs = append(kbIDs, idStr) - } - } - } + kbIDs := input.KnowledgeBaseIDs if len(kbIDs) == 0 { kbIDs = t.knowledgeBaseIDs } @@ -273,11 +262,15 @@ func (t *GrepChunksTool) searchChunks( kbIDs []string, knowledgeIDs []string, ) ([]chunkWithTitle, int64, error) { + tenantID := uint64(0) + if tid, ok := ctx.Value(types.TenantIDContextKey).(uint64); ok { + tenantID = tid + } // Build base query query := t.db.Debug().WithContext(ctx).Table("chunks"). Select("chunks.id, chunks.content, chunks.chunk_index, chunks.knowledge_id, chunks.knowledge_base_id, chunks.chunk_type, chunks.created_at, knowledges.title as knowledge_title, COUNT(*) OVER (PARTITION BY chunks.knowledge_id) AS total_chunk_count"). Joins("LEFT JOIN knowledges ON chunks.knowledge_id = knowledges.id"). - Where("chunks.tenant_id = ?", t.tenantID). + Where("chunks.tenant_id = ?", tenantID). Where("chunks.is_enabled = ?", true). Where("chunks.deleted_at IS NULL"). Where("knowledges.deleted_at IS NULL") diff --git a/internal/agent/tools/knowledge_search.go b/internal/agent/tools/knowledge_search.go index dbdcd8cc..3d69184b 100644 --- a/internal/agent/tools/knowledge_search.go +++ b/internal/agent/tools/knowledge_search.go @@ -19,40 +19,9 @@ import ( "github.com/Tencent/WeKnora/internal/types/interfaces" ) -// searchResultWithMeta wraps search result with metadata about which query matched it -type searchResultWithMeta struct { - *types.SearchResult - SourceQuery string - QueryType string // "vector" or "keyword" - KnowledgeBaseID string // ID of the knowledge base this result came from - KnowledgeBaseType string // Type of the knowledge base (document, faq, etc.) -} - -// KnowledgeSearchTool searches knowledge bases with flexible query modes -type KnowledgeSearchTool struct { - BaseTool - knowledgeBaseService interfaces.KnowledgeBaseService - knowledgeService interfaces.KnowledgeService - chunkService interfaces.ChunkService - tenantID uint64 - searchTargets types.SearchTargets // Pre-computed unified search targets - rerankModel rerank.Reranker - chatModel chat.Chat // Optional chat model for LLM-based reranking - config *config.Config // Global config for fallback values -} - -// NewKnowledgeSearchTool creates a new knowledge search tool -func NewKnowledgeSearchTool( - knowledgeBaseService interfaces.KnowledgeBaseService, - knowledgeService interfaces.KnowledgeService, - chunkService interfaces.ChunkService, - tenantID uint64, - searchTargets types.SearchTargets, - rerankModel rerank.Reranker, - chatModel chat.Chat, - cfg *config.Config, -) *KnowledgeSearchTool { - description := `Semantic/vector search tool for retrieving knowledge by meaning, intent, and conceptual relevance. +var knowledgeSearchTool = BaseTool{ + name: ToolKnowledgeSearch, + description: `Semantic/vector search tool for retrieving knowledge by meaning, intent, and conceptual relevance. This tool uses embeddings to understand the user's query and find semantically similar content across knowledge base chunks. @@ -106,15 +75,75 @@ Avoid: ## Output Returns chunks ranked by semantic similarity, reranked when applicable. -Results represent conceptual relevance, not literal keyword overlap. -` +Results represent conceptual relevance, not literal keyword overlap.`, + schema: json.RawMessage(`{ + "type": "object", + "properties": { + "queries": { + "type": "array", + "description": "REQUIRED: 1-5 semantic questions/topics (e.g., ['What is RAG?', 'RAG benefits'])", + "items": { + "type": "string" + }, + "minItems": 1, + "maxItems": 5 + }, + "knowledge_base_ids": { + "type": "array", + "description": "Optional: KB IDs to search", + "items": { + "type": "string" + }, + "minItems": 0, + "maxItems": 10 + } + }, + "required": ["queries"] +}`), +} +// KnowledgeSearchInput defines the input parameters for knowledge search tool +type KnowledgeSearchInput struct { + Queries []string `json:"queries"` + KnowledgeBaseIDs []string `json:"knowledge_base_ids,omitempty"` +} + +// searchResultWithMeta wraps search result with metadata about which query matched it +type searchResultWithMeta struct { + *types.SearchResult + SourceQuery string + QueryType string // "vector" or "keyword" + KnowledgeBaseID string // ID of the knowledge base this result came from + KnowledgeBaseType string // Type of the knowledge base (document, faq, etc.) +} + +// KnowledgeSearchTool searches knowledge bases with flexible query modes +type KnowledgeSearchTool struct { + BaseTool + knowledgeBaseService interfaces.KnowledgeBaseService + knowledgeService interfaces.KnowledgeService + chunkService interfaces.ChunkService + searchTargets types.SearchTargets // Pre-computed unified search targets + rerankModel rerank.Reranker + chatModel chat.Chat // Optional chat model for LLM-based reranking + config *config.Config // Global config for fallback values +} + +// NewKnowledgeSearchTool creates a new knowledge search tool +func NewKnowledgeSearchTool( + knowledgeBaseService interfaces.KnowledgeBaseService, + knowledgeService interfaces.KnowledgeService, + chunkService interfaces.ChunkService, + searchTargets types.SearchTargets, + rerankModel rerank.Reranker, + chatModel chat.Chat, + cfg *config.Config, +) *KnowledgeSearchTool { return &KnowledgeSearchTool{ - BaseTool: NewBaseTool("knowledge_search", description), + BaseTool: knowledgeSearchTool, knowledgeBaseService: knowledgeBaseService, knowledgeService: knowledgeService, chunkService: chunkService, - tenantID: tenantID, searchTargets: searchTargets, rerankModel: rerankModel, chatModel: chatModel, @@ -122,50 +151,28 @@ Results represent conceptual relevance, not literal keyword overlap. } } -// Parameters returns the JSON schema for the tool's parameters -func (t *KnowledgeSearchTool) Parameters() map[string]interface{} { - return map[string]interface{}{ - "type": "object", - "properties": map[string]interface{}{ - "queries": map[string]interface{}{ - "type": "array", - "description": "REQUIRED: 1-5 semantic questions/topics (e.g., ['What is RAG?', 'RAG benefits'])", - "items": map[string]interface{}{ - "type": "string", - }, - "minItems": 1, - "maxItems": 5, - }, - "knowledge_base_ids": map[string]interface{}{ - "type": "array", - "description": "Optional: KB IDs to search", - "items": map[string]interface{}{ - "type": "string", - }, - "minItems": 0, - "maxItems": 10, - }, - }, - "required": []string{"queries"}, - } -} - // Execute executes the knowledge search tool -func (t *KnowledgeSearchTool) Execute(ctx context.Context, args map[string]interface{}) (*types.ToolResult, error) { +func (t *KnowledgeSearchTool) Execute(ctx context.Context, args json.RawMessage) (*types.ToolResult, error) { logger.Infof(ctx, "[Tool][KnowledgeSearch] Execute started") + // Parse args from json.RawMessage + var input KnowledgeSearchInput + if err := json.Unmarshal(args, &input); err != nil { + logger.Errorf(ctx, "[Tool][KnowledgeSearch] Failed to parse args: %v", err) + return &types.ToolResult{ + Success: false, + Error: fmt.Sprintf("Failed to parse args: %v", err), + }, err + } + // Log input arguments - argsJSON, _ := json.MarshalIndent(args, "", " ") + argsJSON, _ := json.MarshalIndent(input, "", " ") logger.Debugf(ctx, "[Tool][KnowledgeSearch] Input args:\n%s", string(argsJSON)) // Determine which KBs to search - user can optionally filter to specific KBs var userSpecifiedKBs []string - if kbIDsRaw, ok := args["knowledge_base_ids"].([]interface{}); ok && len(kbIDsRaw) > 0 { - for _, id := range kbIDsRaw { - if idStr, ok := id.(string); ok && idStr != "" { - userSpecifiedKBs = append(userSpecifiedKBs, idStr) - } - } + if len(input.KnowledgeBaseIDs) > 0 { + userSpecifiedKBs = input.KnowledgeBaseIDs logger.Infof(ctx, "[Tool][KnowledgeSearch] User specified %d knowledge bases: %v", len(userSpecifiedKBs), userSpecifiedKBs) } @@ -199,14 +206,7 @@ func (t *KnowledgeSearchTool) Execute(ctx context.Context, args map[string]inter logger.Infof(ctx, "[Tool][KnowledgeSearch] Using %d search targets across %d KBs", len(searchTargets), len(kbIDs)) // Parse query parameter - var queries []string - if queriesRaw, ok := args["queries"].([]interface{}); ok && len(queriesRaw) > 0 { - for _, q := range queriesRaw { - if qStr, ok := q.(string); ok && qStr != "" { - queries = append(queries, qStr) - } - } - } + queries := input.Queries // Validate: query must be provided if len(queries) == 0 { @@ -965,6 +965,10 @@ func (t *KnowledgeSearchTool) formatOutput( kbsToSearch []string, queries []string, ) (*types.ToolResult, error) { + tenantID := uint64(0) + if tid, ok := ctx.Value(types.TenantIDContextKey).(uint64); ok { + tenantID = tid + } if len(results) == 0 { data := map[string]interface{}{ "knowledge_base_ids": kbsToSearch, @@ -1050,7 +1054,7 @@ func (t *KnowledgeSearchTool) formatOutput( // Get total chunk count for this knowledge (cache it) if _, exists := knowledgeTotalMap[result.KnowledgeID]; !exists { _, total, err := t.chunkService.GetRepository().ListPagedChunksByKnowledgeID(ctx, - t.tenantID, result.KnowledgeID, + tenantID, result.KnowledgeID, &types.Pagination{Page: 1, PageSize: 1}, []types.ChunkType{types.ChunkTypeText}, "", "", "", "", "", ) diff --git a/internal/agent/tools/list_knowledge_chunks.go b/internal/agent/tools/list_knowledge_chunks.go index 80998e9c..042b0153 100644 --- a/internal/agent/tools/list_knowledge_chunks.go +++ b/internal/agent/tools/list_knowledge_chunks.go @@ -10,21 +10,9 @@ import ( "github.com/Tencent/WeKnora/internal/types/interfaces" ) -// ListKnowledgeChunksTool retrieves chunk snapshots for a specific knowledge document. -type ListKnowledgeChunksTool struct { - BaseTool - tenantID uint64 - chunkService interfaces.ChunkService - knowledgeService interfaces.KnowledgeService -} - -// NewListKnowledgeChunksTool creates a new tool instance. -func NewListKnowledgeChunksTool( - tenantID uint64, - knowledgeService interfaces.KnowledgeService, - chunkService interfaces.ChunkService, -) *ListKnowledgeChunksTool { - description := `Retrieve full chunk content for a document by knowledge_id. +var listKnowledgeChunksTool = BaseTool{ + name: ToolListKnowledgeChunks, + description: `Retrieve full chunk content for a document by knowledge_id. ## Use After grep_chunks or knowledge_search: 1. grep_chunks(["keyword", "变体"]) → get knowledge_id @@ -41,46 +29,76 @@ func NewListKnowledgeChunksTool( - offset (optional): Start position (default 0) ## Output: -Full chunk content with chunk_id, chunk_index, and content text.` +Full chunk content with chunk_id, chunk_index, and content text.`, + schema: json.RawMessage(`{ + "type": "object", + "properties": { + "knowledge_id": { + "type": "string", + "description": "Document ID to retrieve chunks from" + }, + "limit": { + "type": "integer", + "description": "Chunks per page (default 20, max 100)", + "default": 20, + "minimum": 1, + "maximum": 100 + }, + "offset": { + "type": "integer", + "description": "Start position (default 0)", + "default": 0, + "minimum": 0 + } + }, + "required": ["knowledge_id", "limit", "offset"] +}`), +} +// ListKnowledgeChunksInput defines the input parameters for list knowledge chunks tool +type ListKnowledgeChunksInput struct { + KnowledgeID string `json:"knowledge_id"` + Limit int `json:"limit"` + Offset int `json:"offset"` +} + +// ListKnowledgeChunksTool retrieves chunk snapshots for a specific knowledge document. +type ListKnowledgeChunksTool struct { + BaseTool + chunkService interfaces.ChunkService + knowledgeService interfaces.KnowledgeService +} + +// NewListKnowledgeChunksTool creates a new tool instance. +func NewListKnowledgeChunksTool( + knowledgeService interfaces.KnowledgeService, + chunkService interfaces.ChunkService, +) *ListKnowledgeChunksTool { return &ListKnowledgeChunksTool{ - BaseTool: NewBaseTool("list_knowledge_chunks", description), - tenantID: tenantID, + BaseTool: listKnowledgeChunksTool, chunkService: chunkService, knowledgeService: knowledgeService, } } -// Parameters returns the JSON schema describing accepted arguments. -func (t *ListKnowledgeChunksTool) Parameters() map[string]interface{} { - return map[string]interface{}{ - "type": "object", - "properties": map[string]interface{}{ - "knowledge_id": map[string]interface{}{ - "type": "string", - "description": "Document ID to retrieve chunks from", - }, - "limit": map[string]interface{}{ - "type": "integer", - "description": "Chunks per page (default 20, max 100)", - "default": 20, - "minimum": 1, - "maximum": 100, - }, - "offset": map[string]interface{}{ - "type": "integer", - "description": "Start position (default 0)", - "default": 0, - "minimum": 0, - }, - }, - "required": []string{"knowledge_id", "limit", "offset"}, - } -} - // Execute performs the chunk fetch against the chunk service. -func (t *ListKnowledgeChunksTool) Execute(ctx context.Context, args map[string]interface{}) (*types.ToolResult, error) { - knowledgeID, ok := args["knowledge_id"].(string) +func (t *ListKnowledgeChunksTool) Execute(ctx context.Context, args json.RawMessage) (*types.ToolResult, error) { + tenantID := uint64(0) + if tid, ok := ctx.Value(types.TenantIDContextKey).(uint64); ok { + tenantID = tid + } + + // Parse args from json.RawMessage + var input ListKnowledgeChunksInput + if err := json.Unmarshal(args, &input); err != nil { + return &types.ToolResult{ + Success: false, + Error: fmt.Sprintf("Failed to parse args: %v", err), + }, err + } + + knowledgeID := input.KnowledgeID + ok := knowledgeID != "" if !ok || strings.TrimSpace(knowledgeID) == "" { return &types.ToolResult{ Success: false, @@ -90,22 +108,12 @@ func (t *ListKnowledgeChunksTool) Execute(ctx context.Context, args map[string]i knowledgeID = strings.TrimSpace(knowledgeID) chunkLimit := 20 - offset := 0 - if rawLimit, exists := args["limit"]; exists { - switch v := rawLimit.(type) { - case float64: - chunkLimit = int(v) - case int: - chunkLimit = v - } + if input.Limit > 0 { + chunkLimit = input.Limit } - if rawOffset, exists := args["offset"]; exists { - switch v := rawOffset.(type) { - case float64: - offset = int(v) - case int: - offset = v - } + offset := 0 + if input.Offset > 0 { + offset = input.Offset } if offset < 0 { offset = 0 @@ -117,7 +125,7 @@ func (t *ListKnowledgeChunksTool) Execute(ctx context.Context, args map[string]i } chunks, total, err := t.chunkService.GetRepository().ListPagedChunksByKnowledgeID(ctx, - t.tenantID, knowledgeID, pagination, []types.ChunkType{types.ChunkTypeText, types.ChunkTypeFAQ}, "", "", "", "", "") + tenantID, knowledgeID, pagination, []types.ChunkType{types.ChunkTypeText, types.ChunkTypeFAQ}, "", "", "", "", "") if err != nil { return &types.ToolResult{ Success: false, diff --git a/internal/agent/tools/mcp_tool.go b/internal/agent/tools/mcp_tool.go index d757d624..a2faab9d 100644 --- a/internal/agent/tools/mcp_tool.go +++ b/internal/agent/tools/mcp_tool.go @@ -12,6 +12,8 @@ import ( "github.com/Tencent/WeKnora/internal/types" ) +type MCPInput = map[string]any + // MCPTool wraps an MCP service tool to implement the Tool interface type MCPTool struct { service *types.MCPService @@ -47,22 +49,31 @@ func (t *MCPTool) Description() string { } // Parameters returns the JSON Schema for tool parameters -func (t *MCPTool) Parameters() map[string]interface{} { - if t.mcpTool.InputSchema != nil { +func (t *MCPTool) Parameters() json.RawMessage { + if len(t.mcpTool.InputSchema) > 0 { return t.mcpTool.InputSchema } - // Return a default schema if none provided - return map[string]interface{}{ - "type": "object", - "properties": map[string]interface{}{}, - } + return json.RawMessage(`{ + "type": "object", + "properties": {} + }`) } // Execute executes the MCP tool -func (t *MCPTool) Execute(ctx context.Context, args map[string]interface{}) (*types.ToolResult, error) { +func (t *MCPTool) Execute(ctx context.Context, args json.RawMessage) (*types.ToolResult, error) { logger.GetLogger(ctx).Infof("Executing MCP tool: %s from service: %s", t.mcpTool.Name, t.service.Name) + // Parse args from json.RawMessage + var input MCPInput + if err := json.Unmarshal(args, &input); err != nil { + logger.Errorf(ctx, "[Tool][DatabaseQuery] Failed to parse args: %v", err) + return &types.ToolResult{ + Success: false, + Error: fmt.Sprintf("Failed to parse args: %v", err), + }, err + } + // Get or create MCP client client, err := t.mcpManager.GetOrCreateClient(t.service) if err != nil { @@ -86,7 +97,7 @@ func (t *MCPTool) Execute(ctx context.Context, args map[string]interface{}) (*ty } // Call the tool via MCP - result, err := client.CallTool(ctx, t.mcpTool.Name, args) + result, err := client.CallTool(ctx, t.mcpTool.Name, input) if err != nil { logger.GetLogger(ctx).Errorf("MCP tool call failed: %v", err) return &types.ToolResult{ diff --git a/internal/agent/tools/query_knowledge_graph.go b/internal/agent/tools/query_knowledge_graph.go index 2a58a0b9..9324573f 100644 --- a/internal/agent/tools/query_knowledge_graph.go +++ b/internal/agent/tools/query_knowledge_graph.go @@ -2,23 +2,19 @@ package tools import ( "context" + "encoding/json" "fmt" "sort" "sync" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/Tencent/WeKnora/internal/utils" ) -// QueryKnowledgeGraphTool queries the knowledge graph for entities and relationships -type QueryKnowledgeGraphTool struct { - BaseTool - knowledgeService interfaces.KnowledgeBaseService -} - -// NewQueryKnowledgeGraphTool creates a new query knowledge graph tool -func NewQueryKnowledgeGraphTool(knowledgeService interfaces.KnowledgeBaseService) *QueryKnowledgeGraphTool { - description := `Query knowledge graph to explore entity relationships and knowledge networks. +var queryKnowledgeGraphTool = BaseTool{ + name: ToolQueryKnowledgeGraph, + description: `Query knowledge graph to explore entity relationships and knowledge networks. ## Core Function Explores relationships between entities in knowledge bases that have graph extraction configured. @@ -54,65 +50,59 @@ If KB is not configured with graph, tool will return regular search results. ## Notes - Results indicate graph configuration status - Cross-KB results are automatically deduplicated -- Results are sorted by relevance` +- Results are sorted by relevance`, + schema: utils.GenerateSchema[QueryKnowledgeGraphInput](), +} +// QueryKnowledgeGraphInput defines the input parameters for query knowledge graph tool +type QueryKnowledgeGraphInput struct { + KnowledgeBaseIDs []string `json:"knowledge_base_ids" jsonschema:"Array of knowledge base IDs to query"` + Query string `json:"query" jsonschema:"查询内容(实体名称或查询文本)"` +} + +// QueryKnowledgeGraphTool queries the knowledge graph for entities and relationships +type QueryKnowledgeGraphTool struct { + BaseTool + knowledgeService interfaces.KnowledgeBaseService +} + +// NewQueryKnowledgeGraphTool creates a new query knowledge graph tool +func NewQueryKnowledgeGraphTool(knowledgeService interfaces.KnowledgeBaseService) *QueryKnowledgeGraphTool { return &QueryKnowledgeGraphTool{ - BaseTool: NewBaseTool("query_knowledge_graph", description), + BaseTool: queryKnowledgeGraphTool, knowledgeService: knowledgeService, } } -// Parameters returns the JSON schema for the tool's parameters -func (t *QueryKnowledgeGraphTool) Parameters() map[string]interface{} { - return map[string]interface{}{ - "type": "object", - "properties": map[string]interface{}{ - "knowledge_base_ids": map[string]interface{}{ - "type": "array", - "description": "Array of knowledge base IDs to query", - "items": map[string]interface{}{ - "type": "string", - }, - "minItems": 1, - "maxItems": 10, - }, - "query": map[string]interface{}{ - "type": "string", - "description": "查询内容(实体名称或查询文本)", - }, - }, - "required": []string{"knowledge_base_ids", "query"}, - } -} - // Execute performs the knowledge graph query with concurrent KB processing -func (t *QueryKnowledgeGraphTool) Execute(ctx context.Context, args map[string]interface{}) (*types.ToolResult, error) { +func (t *QueryKnowledgeGraphTool) Execute(ctx context.Context, args json.RawMessage) (*types.ToolResult, error) { + // Parse args from json.RawMessage + var input QueryKnowledgeGraphInput + if err := json.Unmarshal(args, &input); err != nil { + return &types.ToolResult{ + Success: false, + Error: fmt.Sprintf("Failed to parse args: %v", err), + }, err + } + // Extract knowledge_base_ids array - kbIDsRaw, ok := args["knowledge_base_ids"].([]interface{}) - if !ok || len(kbIDsRaw) == 0 { + if len(input.KnowledgeBaseIDs) == 0 { return &types.ToolResult{ Success: false, Error: "knowledge_base_ids is required and must be a non-empty array", }, fmt.Errorf("knowledge_base_ids is required") } - // Convert to string slice - var kbIDs []string - for _, id := range kbIDsRaw { - if idStr, ok := id.(string); ok && idStr != "" { - kbIDs = append(kbIDs, idStr) - } - } - - if len(kbIDs) == 0 { + // Validate max 10 KBs + if len(input.KnowledgeBaseIDs) > 10 { return &types.ToolResult{ Success: false, - Error: "knowledge_base_ids must contain at least one valid KB ID", - }, fmt.Errorf("no valid KB IDs provided") + Error: "knowledge_base_ids must contain at most 10 KB IDs", + }, fmt.Errorf("too many KB IDs") } - query, ok := args["query"].(string) - if !ok || query == "" { + query := input.Query + if query == "" { return &types.ToolResult{ Success: false, Error: "query is required", @@ -136,7 +126,7 @@ func (t *QueryKnowledgeGraphTool) Execute(ctx context.Context, args map[string]i MatchCount: 10, } - for _, kbID := range kbIDs { + for _, kbID := range input.KnowledgeBaseIDs { wg.Add(1) go func(id string) { defer wg.Done() @@ -181,7 +171,7 @@ func (t *QueryKnowledgeGraphTool) Execute(ctx context.Context, args map[string]i graphConfigs := make(map[string]map[string]interface{}) kbCounts := make(map[string]int) - for _, kbID := range kbIDs { + for _, kbID := range input.KnowledgeBaseIDs { result := kbResults[kbID] if result.err != nil { errors = append(errors, fmt.Sprintf("KB %s: %v", kbID, result.err)) @@ -218,7 +208,7 @@ func (t *QueryKnowledgeGraphTool) Execute(ctx context.Context, args map[string]i Success: true, Output: "未找到相关的图谱信息。", Data: map[string]interface{}{ - "knowledge_base_ids": kbIDs, + "knowledge_base_ids": input.KnowledgeBaseIDs, "query": query, "results": []interface{}{}, "graph_configs": graphConfigs, @@ -230,7 +220,7 @@ func (t *QueryKnowledgeGraphTool) Execute(ctx context.Context, args map[string]i // Format output with enhanced graph information output := "=== 知识图谱查询 ===\n\n" output += fmt.Sprintf("📊 查询: %s\n", query) - output += fmt.Sprintf("🎯 目标知识库: %v\n", kbIDs) + output += fmt.Sprintf("🎯 目标知识库: %v\n", input.KnowledgeBaseIDs) output += fmt.Sprintf("✓ 找到 %d 条相关结果(已去重)\n\n", len(allResults)) if len(errors) > 0 { @@ -354,7 +344,7 @@ func (t *QueryKnowledgeGraphTool) Execute(ctx context.Context, args map[string]i Success: true, Output: output, Data: map[string]interface{}{ - "knowledge_base_ids": kbIDs, + "knowledge_base_ids": input.KnowledgeBaseIDs, "query": query, "results": formattedResults, "count": len(allResults), diff --git a/internal/agent/tools/registry.go b/internal/agent/tools/registry.go index 53bce94c..4f661eda 100644 --- a/internal/agent/tools/registry.go +++ b/internal/agent/tools/registry.go @@ -2,33 +2,22 @@ package tools import ( "context" + "encoding/json" "fmt" "github.com/Tencent/WeKnora/internal/common" "github.com/Tencent/WeKnora/internal/types" - "github.com/Tencent/WeKnora/internal/types/interfaces" - "gorm.io/gorm" ) // ToolRegistry manages the registration and retrieval of tools type ToolRegistry struct { - tools map[string]types.Tool - knowledgeService interfaces.KnowledgeService - chunkService interfaces.ChunkService - db *gorm.DB + tools map[string]types.Tool } // NewToolRegistry creates a new tool registry -func NewToolRegistry( - knowledgeService interfaces.KnowledgeService, - chunkService interfaces.ChunkService, - db *gorm.DB, // gorm.DB for database operations -) *ToolRegistry { +func NewToolRegistry() *ToolRegistry { return &ToolRegistry{ - tools: make(map[string]types.Tool), - knowledgeService: knowledgeService, - chunkService: chunkService, - db: db, + tools: make(map[string]types.Tool), } } @@ -72,7 +61,7 @@ func (r *ToolRegistry) GetFunctionDefinitions() []types.FunctionDefinition { func (r *ToolRegistry) ExecuteTool( ctx context.Context, name string, - args map[string]interface{}, + args json.RawMessage, ) (*types.ToolResult, error) { common.PipelineInfo(ctx, "AgentTool", "execute_start", map[string]interface{}{ "tool": name, @@ -112,3 +101,13 @@ func (r *ToolRegistry) ExecuteTool( return result, execErr } + +// Cleanup cleans up all registered tools that implement the Cleanup method +func (r *ToolRegistry) Cleanup(ctx context.Context) { + // Check specifically for DataAnalysisTool + if tool, exists := r.tools[ToolDataAnalysis]; exists { + if dataAnalysisTool, ok := tool.(*DataAnalysisTool); ok { + dataAnalysisTool.Cleanup(ctx) + } + } +} diff --git a/internal/agent/tools/sequentialthinking.go b/internal/agent/tools/sequentialthinking.go index c0a8241d..da7b7385 100644 --- a/internal/agent/tools/sequentialthinking.go +++ b/internal/agent/tools/sequentialthinking.go @@ -2,36 +2,16 @@ package tools import ( "context" + "encoding/json" "fmt" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" ) -// SequentialThinkingTool is a dynamic and reflective problem-solving tool -// This tool helps analyze problems through a flexible thinking process that can adapt and evolve -type SequentialThinkingTool struct { - BaseTool - thoughtHistory []ThoughtData - branches map[string][]ThoughtData -} - -// ThoughtData represents a single thought in the sequential thinking process -type ThoughtData struct { - Thought string `json:"thought"` - ThoughtNumber int `json:"thought_number"` - TotalThoughts int `json:"total_thoughts"` - IsRevision bool `json:"is_revision,omitempty"` - RevisesThought *int `json:"revises_thought,omitempty"` - BranchFromThought *int `json:"branch_from_thought,omitempty"` - BranchID string `json:"branch_id,omitempty"` - NeedsMoreThoughts bool `json:"needs_more_thoughts,omitempty"` - NextThoughtNeeded bool `json:"next_thought_needed"` -} - -// NewSequentialThinkingTool creates a new sequential thinking tool instance -func NewSequentialThinkingTool() *SequentialThinkingTool { - description := `A detailed tool for dynamic and reflective problem-solving through thoughts. +var sequentialThinkingTool = BaseTool{ + name: ToolThinking, + description: `A detailed tool for dynamic and reflective problem-solving through thoughts. This tool helps analyze problems through a flexible thinking process that can adapt and evolve. @@ -99,72 +79,101 @@ Each thought can build on, question, or revise previous insights as understandin 8. Verify the hypothesis based on the Chain of Thought steps 9. Repeat the process until satisfied with the solution 10. Provide a single, ideally correct answer as the final output -11. Only set next_thought_needed to false when truly done and a satisfactory answer is reached` - - return &SequentialThinkingTool{ - BaseTool: NewBaseTool("thinking", description), - thoughtHistory: make([]ThoughtData, 0), - branches: make(map[string][]ThoughtData), - } +11. Only set next_thought_needed to false when truly done and a satisfactory answer is reached`, + schema: json.RawMessage(`{ + "type": "object", + "properties": { + "thought": { + "type": "string", + "description": "Your current thinking step. Write in natural, user-friendly language. NEVER mention tool names (like \"grep_chunks\", \"knowledge_search\", \"web_search\", etc.). Instead, describe actions in plain language (e.g., \"I'll search for key terms\" instead of \"I'll use grep_chunks\"). Focus on WHAT you're trying to find and WHY, not HOW (which tools you'll use)." + }, + "nextThoughtNeeded": { + "type": "boolean", + "description": "Whether another thought step is needed" + }, + "thoughtNumber": { + "type": "integer", + "description": "Current thought number (numeric value, e.g., 1, 2, 3)", + "minimum": 1 + }, + "totalThoughts": { + "type": "integer", + "description": "Estimated total thoughts needed (numeric value, e.g., 5, 10)", + "minimum": 5 + }, + "isRevision": { + "type": "boolean", + "description": "Whether this revises previous thinking" + }, + "revisesThought": { + "type": "integer", + "description": "Which thought is being reconsidered", + "minimum": 1 + }, + "branchFromThought": { + "type": "integer", + "description": "Branching point thought number", + "minimum": 1 + }, + "branchId": { + "type": "string", + "description": "Branch identifier" + }, + "needsMoreThoughts": { + "type": "boolean", + "description": "If more thoughts are needed" + } + }, + "required": ["thought", "nextThoughtNeeded", "thoughtNumber", "totalThoughts"] +}`), } -// Parameters returns the JSON schema for the tool's parameters -func (t *SequentialThinkingTool) Parameters() map[string]interface{} { - return map[string]interface{}{ - "type": "object", - "properties": map[string]interface{}{ - "thought": map[string]interface{}{ - "type": "string", - "description": `Your current thinking step. Write in natural, user-friendly language. NEVER mention tool names (like "grep_chunks", "knowledge_search", "web_search", etc.). Instead, describe actions in plain language (e.g., "I'll search for key terms" instead of "I'll use grep_chunks"). Focus on WHAT you're trying to find and WHY, not HOW (which tools you'll use).`, - }, - "nextThoughtNeeded": map[string]interface{}{ - "type": "boolean", - "description": "Whether another thought step is needed", - }, - "thoughtNumber": map[string]interface{}{ - "type": "integer", - "description": "Current thought number (numeric value, e.g., 1, 2, 3)", - "minimum": 1, - }, - "totalThoughts": map[string]interface{}{ - "type": "integer", - "description": "Estimated total thoughts needed (numeric value, e.g., 5, 10)", - "minimum": 5, - }, - "isRevision": map[string]interface{}{ - "type": "boolean", - "description": "Whether this revises previous thinking", - }, - "revisesThought": map[string]interface{}{ - "type": "integer", - "description": "Which thought is being reconsidered", - "minimum": 1, - }, - "branchFromThought": map[string]interface{}{ - "type": "integer", - "description": "Branching point thought number", - "minimum": 1, - }, - "branchId": map[string]interface{}{ - "type": "string", - "description": "Branch identifier", - }, - "needsMoreThoughts": map[string]interface{}{ - "type": "boolean", - "description": "If more thoughts are needed", - }, - }, - "required": []string{"thought", "nextThoughtNeeded", "thoughtNumber", "totalThoughts"}, +// SequentialThinkingTool is a dynamic and reflective problem-solving tool +// This tool helps analyze problems through a flexible thinking process that can adapt and evolve +type SequentialThinkingTool struct { + BaseTool + thoughtHistory []SequentialThinkingInput + branches map[string][]SequentialThinkingInput +} + +// SequentialThinkingInput defines the input parameters for sequential thinking tool +type SequentialThinkingInput struct { + Thought string `json:"thought"` + ThoughtNumber int `json:"thought_number"` + TotalThoughts int `json:"total_thoughts"` + IsRevision bool `json:"is_revision,omitempty"` + RevisesThought *int `json:"revises_thought,omitempty"` + BranchFromThought *int `json:"branch_from_thought,omitempty"` + BranchID string `json:"branch_id,omitempty"` + NeedsMoreThoughts bool `json:"needs_more_thoughts,omitempty"` + NextThoughtNeeded bool `json:"next_thought_needed"` +} + +// NewSequentialThinkingTool creates a new sequential thinking tool instance +func NewSequentialThinkingTool() *SequentialThinkingTool { + return &SequentialThinkingTool{ + BaseTool: sequentialThinkingTool, + thoughtHistory: make([]SequentialThinkingInput, 0), + branches: make(map[string][]SequentialThinkingInput), } } // Execute executes the sequential thinking tool -func (t *SequentialThinkingTool) Execute(ctx context.Context, args map[string]interface{}) (*types.ToolResult, error) { +func (t *SequentialThinkingTool) Execute(ctx context.Context, args json.RawMessage) (*types.ToolResult, error) { logger.Infof(ctx, "[Tool][SequentialThinking] Execute started") + // Parse args from json.RawMessage + var input SequentialThinkingInput + if err := json.Unmarshal(args, &input); err != nil { + logger.Errorf(ctx, "[Tool][SequentialThinking] Failed to parse args: %v", err) + return &types.ToolResult{ + Success: false, + Error: fmt.Sprintf("Failed to parse args: %v", err), + }, err + } + // Validate and parse input - thoughtData, err := t.validateThoughtData(args) - if err != nil { + if err := t.validate(input); err != nil { logger.Errorf(ctx, "[Tool][SequentialThinking] Validation failed: %v", err) return &types.ToolResult{ Success: false, @@ -173,22 +182,22 @@ func (t *SequentialThinkingTool) Execute(ctx context.Context, args map[string]in } // Adjust totalThoughts if thoughtNumber exceeds it - if thoughtData.ThoughtNumber > thoughtData.TotalThoughts { - thoughtData.TotalThoughts = thoughtData.ThoughtNumber + if input.ThoughtNumber > input.TotalThoughts { + input.TotalThoughts = input.ThoughtNumber } // Add to thought history - t.thoughtHistory = append(t.thoughtHistory, thoughtData) + t.thoughtHistory = append(t.thoughtHistory, input) // Handle branching - if thoughtData.BranchFromThought != nil && thoughtData.BranchID != "" { - if t.branches[thoughtData.BranchID] == nil { - t.branches[thoughtData.BranchID] = make([]ThoughtData, 0) + if input.BranchFromThought != nil && input.BranchID != "" { + if t.branches[input.BranchID] == nil { + t.branches[input.BranchID] = make([]SequentialThinkingInput, 0) } - t.branches[thoughtData.BranchID] = append(t.branches[thoughtData.BranchID], thoughtData) + t.branches[input.BranchID] = append(t.branches[input.BranchID], input) } - logger.Debugf(ctx, "[Tool][SequentialThinking] %s", thoughtData.Thought) + logger.Debugf(ctx, "[Tool][SequentialThinking] %s", input.Thought) // Prepare response data branchKeys := make([]string, 0, len(t.branches)) @@ -196,25 +205,25 @@ func (t *SequentialThinkingTool) Execute(ctx context.Context, args map[string]in branchKeys = append(branchKeys, k) } - incomplete := thoughtData.NextThoughtNeeded || thoughtData.NeedsMoreThoughts || - thoughtData.ThoughtNumber < thoughtData.TotalThoughts + incomplete := input.NextThoughtNeeded || input.NeedsMoreThoughts || + input.ThoughtNumber < input.TotalThoughts responseData := map[string]interface{}{ - "thought_number": thoughtData.ThoughtNumber, - "total_thoughts": thoughtData.TotalThoughts, - "next_thought_needed": thoughtData.NextThoughtNeeded, + "thought_number": input.ThoughtNumber, + "total_thoughts": input.TotalThoughts, + "next_thought_needed": input.NextThoughtNeeded, "branches": branchKeys, "thought_history_length": len(t.thoughtHistory), "display_type": "thinking", - "thought": thoughtData.Thought, + "thought": input.Thought, "incomplete_steps": incomplete, } logger.Infof( ctx, "[Tool][SequentialThinking] Execute completed - Thought %d/%d", - thoughtData.ThoughtNumber, - thoughtData.TotalThoughts, + input.ThoughtNumber, + input.TotalThoughts, ) outputMsg := "Thought process recorded" @@ -229,82 +238,22 @@ func (t *SequentialThinkingTool) Execute(ctx context.Context, args map[string]in }, nil } -// validateThoughtData validates and parses the input arguments -func (t *SequentialThinkingTool) validateThoughtData(input map[string]interface{}) (ThoughtData, error) { - var data ThoughtData - +// validate validates the input thought data +func (t *SequentialThinkingTool) validate(data SequentialThinkingInput) error { // Validate thought (required) - thought, ok := input["thought"].(string) - if !ok || thought == "" { - return data, fmt.Errorf("invalid thought: must be a non-empty string") + if data.Thought == "" { + return fmt.Errorf("invalid thought: must be a non-empty string") } - data.Thought = thought // Validate thoughtNumber (required) - thoughtNumber, ok := input["thoughtNumber"].(float64) - if !ok { - // Try int type - if num, ok := input["thoughtNumber"].(int); ok { - data.ThoughtNumber = num - } else { - return data, fmt.Errorf("invalid thoughtNumber: must be a number") - } - } else { - data.ThoughtNumber = int(thoughtNumber) - } if data.ThoughtNumber < 1 { - return data, fmt.Errorf("invalid thoughtNumber: must be >= 1") + return fmt.Errorf("invalid thoughtNumber: must be >= 1") } // Validate totalThoughts (required) - totalThoughts, ok := input["totalThoughts"].(float64) - if !ok { - // Try int type - if num, ok := input["totalThoughts"].(int); ok { - data.TotalThoughts = num - } else { - return data, fmt.Errorf("invalid totalThoughts: must be a number") - } - } else { - data.TotalThoughts = int(totalThoughts) - } if data.TotalThoughts < 1 { - return data, fmt.Errorf("invalid totalThoughts: must be >= 1") + return fmt.Errorf("invalid totalThoughts: must be >= 1") } - // Validate nextThoughtNeeded (required) - nextThoughtNeeded, ok := input["nextThoughtNeeded"].(bool) - if !ok { - return data, fmt.Errorf("invalid nextThoughtNeeded: must be a boolean") - } - data.NextThoughtNeeded = nextThoughtNeeded - - // Optional fields - if isRevision, ok := input["isRevision"].(bool); ok { - data.IsRevision = isRevision - } - - if revisesThought, ok := input["revisesThought"].(float64); ok { - num := int(revisesThought) - data.RevisesThought = &num - } else if revisesThought, ok := input["revisesThought"].(int); ok { - data.RevisesThought = &revisesThought - } - - if branchFromThought, ok := input["branchFromThought"].(float64); ok { - num := int(branchFromThought) - data.BranchFromThought = &num - } else if branchFromThought, ok := input["branchFromThought"].(int); ok { - data.BranchFromThought = &branchFromThought - } - - if branchID, ok := input["branchId"].(string); ok { - data.BranchID = branchID - } - - if needsMoreThoughts, ok := input["needsMoreThoughts"].(bool); ok { - data.NeedsMoreThoughts = needsMoreThoughts - } - - return data, nil + return nil } diff --git a/internal/agent/tools/todo_write.go b/internal/agent/tools/todo_write.go index 8cf50e15..1f2116de 100644 --- a/internal/agent/tools/todo_write.go +++ b/internal/agent/tools/todo_write.go @@ -6,25 +6,12 @@ import ( "fmt" "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/utils" ) -// TodoWriteTool implements a planning tool for complex tasks -// This is an optional tool that helps organize multi-step research -type TodoWriteTool struct { - BaseTool -} - -// PlanStep represents a single step in the research plan -type PlanStep struct { - ID string `json:"id"` - Description string `json:"description"` - ToolsToUse []string `json:"tools_to_use"` - Status string `json:"status"` // pending, in_progress, completed, skipped -} - -// NewTodoWriteTool creates a new todo_write tool instance -func NewTodoWriteTool() *TodoWriteTool { - description := `Use this tool to create and manage a structured task list for retrieval and research tasks. This helps you track progress, organize complex retrieval operations, and demonstrate thoroughness to the user. +var todoWriteTool = BaseTool{ + name: ToolTodoWrite, + description: `Use this tool to create and manage a structured task list for retrieval and research tasks. This helps you track progress, organize complex retrieval operations, and demonstrate thoroughness to the user. **CRITICAL - Focus on Retrieval Tasks Only**: - This tool is for tracking RETRIEVAL and RESEARCH tasks (e.g., searching knowledge bases, retrieving documents, gathering information) @@ -144,82 +131,56 @@ The assistant did not use the todo list because this is an informational request **Important**: After completing all retrieval tasks in todo_write, use the thinking tool to synthesize findings and generate the final answer. The todo_write tool tracks WHAT to retrieve, while thinking tool handles HOW to synthesize and present the information. -When in doubt, use this tool. Being proactive with task management demonstrates attentiveness and ensures you complete all retrieval requirements successfully.` - - return &TodoWriteTool{ - BaseTool: NewBaseTool("todo_write", description), - } +When in doubt, use this tool. Being proactive with task management demonstrates attentiveness and ensures you complete all retrieval requirements successfully.`, + schema: utils.GenerateSchema[TodoWriteInput](), } -// Parameters returns the JSON schema for the tool's parameters -func (t *TodoWriteTool) Parameters() map[string]interface{} { - return map[string]interface{}{ - "type": "object", - "properties": map[string]interface{}{ - "task": map[string]interface{}{ - "type": "string", - "description": "The complex task or question you need to create a plan for", - }, - "steps": map[string]interface{}{ - "type": "array", - "description": "Array of research plan steps with status tracking", - "items": map[string]interface{}{ - "type": "object", - "properties": map[string]interface{}{ - "id": map[string]interface{}{ - "type": "string", - "description": "Unique identifier for this step (e.g., 'step1', 'step2')", - }, - "description": map[string]interface{}{ - "type": "string", - "description": "Clear description of what to investigate or accomplish in this step", - }, - // "tools_to_use": map[string]interface{}{ - // "type": "array", - // "description": "Suggested tools for this step (e.g., ['knowledge_search', 'list_knowledge_chunks'])", - // "items": map[string]interface{}{ - // "type": "string", - // }, - // }, - "status": map[string]interface{}{ - "type": "string", - "enum": []string{"pending", "in_progress", "completed"}, - "description": "Current status: pending (not started), in_progress (executing), completed (finished)", - }, - }, - "required": []string{"id", "description", "status"}, - }, - }, - }, - "required": []string{"task", "steps"}, +// TodoWriteTool implements a planning tool for complex tasks +// This is an optional tool that helps organize multi-step research +type TodoWriteTool struct { + BaseTool +} + +// TodoWriteInput defines the input parameters for todo_write tool +type TodoWriteInput struct { + Task string `json:"task" jsonschema:"The complex task or question you need to create a plan for"` + Steps []PlanStep `json:"steps" jsonschema:"Array of research plan steps with status tracking"` +} + +// PlanStep represents a single step in the research plan +type PlanStep struct { + ID string `json:"id" jsonschema:"Unique identifier for this step (e.g., 'step1', 'step2')"` + Description string `json:"description" jsonschema:"Clear description of what to investigate or accomplish in this step"` + Status string `json:"status" jsonschema:"Current status: pending (not started), in_progress (executing), completed (finished)"` +} + +// NewTodoWriteTool creates a new todo_write tool instance +func NewTodoWriteTool() *TodoWriteTool { + return &TodoWriteTool{ + BaseTool: todoWriteTool, } } // Execute executes the todo_write tool -func (t *TodoWriteTool) Execute(ctx context.Context, args map[string]interface{}) (*types.ToolResult, error) { - task, ok := args["task"].(string) - if !ok { - task = "未提供任务描述" +func (t *TodoWriteTool) Execute(ctx context.Context, args json.RawMessage) (*types.ToolResult, error) { + // Parse args from json.RawMessage + var input TodoWriteInput + if err := json.Unmarshal(args, &input); err != nil { + return &types.ToolResult{ + Success: false, + Error: fmt.Sprintf("Failed to parse args: %v", err), + }, err + } + + if input.Task == "" { + input.Task = "未提供任务描述" } // Parse plan steps - var planSteps []PlanStep - if stepsData, ok := args["steps"].([]interface{}); ok { - for _, stepData := range stepsData { - if stepMap, ok := stepData.(map[string]interface{}); ok { - step := PlanStep{ - ID: getStringField(stepMap, "id"), - Description: getStringField(stepMap, "description"), - ToolsToUse: getStringArrayField(stepMap, "tools_to_use"), - Status: getStringField(stepMap, "status"), - } - planSteps = append(planSteps, step) - } - } - } + planSteps := input.Steps // Generate formatted output - output := generatePlanOutput(task, planSteps) + output := generatePlanOutput(input.Task, planSteps) // Prepare structured data for response stepsJSON, _ := json.Marshal(planSteps) @@ -228,7 +189,7 @@ func (t *TodoWriteTool) Execute(ctx context.Context, args map[string]interface{} Success: true, Output: output, Data: map[string]interface{}{ - "task": task, + "task": input.Task, "steps": planSteps, "steps_json": string(stepsJSON), "total_steps": len(planSteps), diff --git a/internal/agent/tools/tool.go b/internal/agent/tools/tool.go index 9768e198..2ba3f619 100644 --- a/internal/agent/tools/tool.go +++ b/internal/agent/tools/tool.go @@ -1,6 +1,7 @@ package tools import ( + "encoding/json" "fmt" "github.com/Tencent/WeKnora/internal/types" @@ -10,13 +11,15 @@ import ( type BaseTool struct { name string description string + schema json.RawMessage } // NewBaseTool creates a new base tool -func NewBaseTool(name, description string) BaseTool { +func NewBaseTool(name, description string, schema json.RawMessage) BaseTool { return BaseTool{ name: name, description: description, + schema: schema, } } @@ -30,6 +33,11 @@ func (t *BaseTool) Description() string { return t.description } +// Parameters returns the tool parameters schema +func (t *BaseTool) Parameters() json.RawMessage { + return t.schema +} + // ToolExecutor is a helper interface for executing tools type ToolExecutor interface { types.Tool diff --git a/internal/agent/tools/web_fetch.go b/internal/agent/tools/web_fetch.go index 449645df..ba81d785 100644 --- a/internal/agent/tools/web_fetch.go +++ b/internal/agent/tools/web_fetch.go @@ -2,6 +2,7 @@ package tools import ( "context" + "encoding/json" "fmt" "io" "net/http" @@ -16,6 +17,7 @@ import ( "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/models/chat" "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/utils" ) const ( @@ -23,6 +25,33 @@ const ( webFetchMaxChars = 100000 // maximum number of characters to fetch ) +var webFetchTool = BaseTool{ + name: ToolWebFetch, + description: `Fetch detailed web content from previously discovered URLs and analyze it with an LLM. + +## Usage +- Receive one or more {url, prompt} combinations +- Fetch web page content and convert to Markdown text +- Use prompt to call small model for analysis and summary (if model is available) +- Return summary result and original content fragment + +## When to Use +- **MANDATORY**: After web_search returns results, if content is truncated or incomplete, use web_fetch to get full page content +- When web_search snippet is insufficient for answering the question`, + schema: utils.GenerateSchema[WebFetchInput](), +} + +// WebFetchInput defines the input parameters for web fetch tool +type WebFetchInput struct { + Items []WebFetchItem `json:"items" jsonschema:"批量抓取任务,每项包含 url 与 prompt"` +} + +// WebFetchItem represents a single web fetch task +type WebFetchItem struct { + URL string `json:"url" jsonschema:"待抓取的网页 URL,需来自 web_search 结果"` + Prompt string `json:"prompt" jsonschema:"分析该网页内容时使用的提示词"` +} + // webFetchParams is the parameters for the web fetch tool type webFetchParams struct { URL string @@ -45,21 +74,8 @@ type WebFetchTool struct { // NewWebFetchTool creates a new web_fetch tool instance func NewWebFetchTool(chatModel chat.Chat) *WebFetchTool { - description := `Fetch detailed web content from previously discovered URLs and analyze it with an LLM. - -## Usage -- Receive one or more {url, prompt} combinations -- Fetch web page content and convert to Markdown text -- Use prompt to call small model for analysis and summary (if model is available) -- Return summary result and original content fragment - -## When to Use -- **MANDATORY**: After web_search returns results, if content is truncated or incomplete, use web_fetch to get full page content -- When web_search snippet is insufficient for answering the question -` - return &WebFetchTool{ - BaseTool: NewBaseTool("web_fetch", description), + BaseTool: webFetchTool, client: &http.Client{ Timeout: webFetchTimeout, }, @@ -67,41 +83,21 @@ func NewWebFetchTool(chatModel chat.Chat) *WebFetchTool { } } -// Parameters 返回工具参数的 JSON Schema -func (t *WebFetchTool) Parameters() map[string]interface{} { - return map[string]interface{}{ - "type": "object", - "properties": map[string]interface{}{ - "items": map[string]interface{}{ - "type": "array", - "description": "批量抓取任务,每项包含 url 与 prompt", - "items": map[string]interface{}{ - "type": "object", - "properties": map[string]interface{}{ - "url": map[string]interface{}{ - "type": "string", - "description": "待抓取的网页 URL,需来自 web_search 结果", - }, - "prompt": map[string]interface{}{ - "type": "string", - "description": "分析该网页内容时使用的提示词", - }, - }, - "required": []string{"url", "prompt"}, - }, - "minItems": 1, - }, - }, - "required": []string{"items"}, - } -} - // Execute 执行 web_fetch 工具 -func (t *WebFetchTool) Execute(ctx context.Context, args map[string]interface{}) (*types.ToolResult, error) { +func (t *WebFetchTool) Execute(ctx context.Context, args json.RawMessage) (*types.ToolResult, error) { logger.Infof(ctx, "[Tool][WebFetch] Execute started") - rawItems, ok := args["items"] - if !ok { + // Parse args from json.RawMessage + var input WebFetchInput + if err := json.Unmarshal(args, &input); err != nil { + logger.Errorf(ctx, "[Tool][WebFetch] Failed to parse args: %v", err) + return &types.ToolResult{ + Success: false, + Error: fmt.Sprintf("Failed to parse args: %v", err), + }, err + } + + if len(input.Items) == 0 { logger.Errorf(ctx, "[Tool][WebFetch] 参数缺失: items") return &types.ToolResult{ Success: false, @@ -109,25 +105,19 @@ func (t *WebFetchTool) Execute(ctx context.Context, args map[string]interface{}) }, nil } - itemsSlice, ok := rawItems.([]interface{}) - if !ok || len(itemsSlice) == 0 { - logger.Errorf(ctx, "[Tool][WebFetch] items 解析失败或为空: %#v", rawItems) - return &types.ToolResult{ - Success: false, - Error: "'items' must be a non-empty array", - }, nil - } - - results := make([]*webFetchItemResult, len(itemsSlice)) + results := make([]*webFetchItemResult, len(input.Items)) var wg sync.WaitGroup - wg.Add(len(itemsSlice)) + wg.Add(len(input.Items)) - for idx := range itemsSlice { + for idx := range input.Items { i := idx - itemAny := itemsSlice[i] + item := input.Items[i] - params := t.parseParams(itemAny) + params := webFetchParams{ + URL: item.URL, + Prompt: item.Prompt, + } go func(index int, p webFetchParams) { defer wg.Done() diff --git a/internal/agent/tools/web_search.go b/internal/agent/tools/web_search.go index b5ec41ff..c8abfceb 100644 --- a/internal/agent/tools/web_search.go +++ b/internal/agent/tools/web_search.go @@ -2,6 +2,7 @@ package tools import ( "context" + "encoding/json" "fmt" "strings" "time" @@ -9,29 +10,12 @@ import ( "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/Tencent/WeKnora/internal/utils" ) -// WebSearchTool performs web searches and returns results -type WebSearchTool struct { - BaseTool - webSearchService interfaces.WebSearchService - knowledgeBaseService interfaces.KnowledgeBaseService - knowledgeService interfaces.KnowledgeService - sessionService interfaces.SessionService - sessionID string - maxResults int -} - -// NewWebSearchTool creates a new web search tool -func NewWebSearchTool( - webSearchService interfaces.WebSearchService, - knowledgeBaseService interfaces.KnowledgeBaseService, - knowledgeService interfaces.KnowledgeService, - sessionService interfaces.SessionService, - sessionID string, - maxResults int, -) *WebSearchTool { - description := `Search the web for current information and news. This tool searches the internet to find up-to-date information that may not be in the knowledge base. +var webSearchTool = BaseTool{ + name: ToolWebSearch, + description: `Search the web for current information and news. This tool searches the internet to find up-to-date information that may not be in the knowledge base. ## CRITICAL - KB First Rule **ABSOLUTE RULE**: You MUST complete KB retrieval (grep_chunks AND knowledge_search) FIRST before using this tool. @@ -57,10 +41,7 @@ func NewWebSearchTool( **Parameters**: - query (required): Search query string -**Returns**: Web search results with title, URL, snippet, and content (up to ` + fmt.Sprintf( - "%d", - maxResults, - ) + ` results) +**Returns**: Web search results with title, URL, snippet, and content (up to %d results) ## Examples @@ -83,42 +64,66 @@ func NewWebSearchTool( - Use this tool when knowledge bases don't have the information you need - Results include URL, title, snippet, and content snippet (may be truncated) - **CRITICAL**: If content is truncated or you need full details, use **web_fetch** to fetch complete page content -- Maximum ` + fmt.Sprintf( - "%d", - maxResults, - ) + ` results will be returned per search` - - return &WebSearchTool{ - BaseTool: NewBaseTool("web_search", description), - webSearchService: webSearchService, - knowledgeBaseService: knowledgeBaseService, - knowledgeService: knowledgeService, - sessionService: sessionService, - sessionID: sessionID, - maxResults: maxResults, - } +- Maximum %d results will be returned per search`, + schema: utils.GenerateSchema[WebSearchInput](), } -// Parameters returns the JSON schema for the tool's parameters -func (t *WebSearchTool) Parameters() map[string]interface{} { - return map[string]interface{}{ - "type": "object", - "properties": map[string]interface{}{ - "query": map[string]interface{}{ - "type": "string", - "description": "Search query string", - }, - }, - "required": []string{"query"}, +// WebSearchInput defines the input parameters for web search tool +type WebSearchInput struct { + Query string `json:"query" jsonschema:"Search query string"` +} + +// WebSearchTool performs web searches and returns results +type WebSearchTool struct { + BaseTool + webSearchService interfaces.WebSearchService + knowledgeBaseService interfaces.KnowledgeBaseService + knowledgeService interfaces.KnowledgeService + webSearchStateService interfaces.WebSearchStateService + sessionID string + maxResults int +} + +// NewWebSearchTool creates a new web search tool +func NewWebSearchTool( + webSearchService interfaces.WebSearchService, + knowledgeBaseService interfaces.KnowledgeBaseService, + knowledgeService interfaces.KnowledgeService, + webSearchStateService interfaces.WebSearchStateService, + sessionID string, + maxResults int, +) *WebSearchTool { + tool := webSearchTool + tool.description = fmt.Sprintf(tool.description, maxResults, maxResults) + + return &WebSearchTool{ + BaseTool: tool, + webSearchService: webSearchService, + knowledgeBaseService: knowledgeBaseService, + knowledgeService: knowledgeService, + webSearchStateService: webSearchStateService, + sessionID: sessionID, + maxResults: maxResults, } } // Execute executes the web search tool -func (t *WebSearchTool) Execute(ctx context.Context, args map[string]interface{}) (*types.ToolResult, error) { +func (t *WebSearchTool) Execute(ctx context.Context, args json.RawMessage) (*types.ToolResult, error) { logger.Infof(ctx, "[Tool][WebSearch] Execute started") + // Parse args from json.RawMessage + var input WebSearchInput + if err := json.Unmarshal(args, &input); err != nil { + logger.Errorf(ctx, "[Tool][WebSearch] Failed to parse args: %v", err) + return &types.ToolResult{ + Success: false, + Error: fmt.Sprintf("Failed to parse args: %v", err), + }, err + } + // Parse query - query, ok := args["query"].(string) + query := input.Query + ok := query != "" if !ok || query == "" { logger.Errorf(ctx, "[Tool][WebSearch] Query is required") return &types.ToolResult{ @@ -178,8 +183,8 @@ func (t *WebSearchTool) Execute(ctx context.Context, args map[string]interface{} // Apply RAG compression if configured if len(webResults) > 0 && tenant.WebSearchConfig.CompressionMethod != "none" && tenant.WebSearchConfig.CompressionMethod != "" { - // Load session-scoped temp KB state from Redis using SessionService - tempKBID, seen, ids := t.sessionService.GetWebSearchTempKBState(ctx, t.sessionID) + // Load session-scoped temp KB state from Redis using WebSearchStateRepository + tempKBID, seen, ids := t.webSearchStateService.GetWebSearchTempKBState(ctx, t.sessionID) // Build questions for RAG compression questions := []string{strings.TrimSpace(query)} @@ -193,8 +198,8 @@ func (t *WebSearchTool) Execute(ctx context.Context, args map[string]interface{} logger.Warnf(ctx, "[Tool][WebSearch] RAG compression failed, using raw results: %v", err) } else { webResults = compressed - // Persist temp KB state back into Redis using SessionService - t.sessionService.SaveWebSearchTempKBState(ctx, t.sessionID, kbID, newSeen, newIDs) + // Persist temp KB state back into Redis using WebSearchStateRepository + t.webSearchStateService.SaveWebSearchTempKBState(ctx, t.sessionID, kbID, newSeen, newIDs) logger.Infof(ctx, "[Tool][WebSearch] RAG compression completed, %d results", len(webResults)) } } diff --git a/internal/application/service/agent_service.go b/internal/application/service/agent_service.go index c953118f..aa97352e 100644 --- a/internal/application/service/agent_service.go +++ b/internal/application/service/agent_service.go @@ -2,6 +2,7 @@ package service import ( "context" + "database/sql" "fmt" "github.com/Tencent/WeKnora/internal/agent" @@ -22,16 +23,18 @@ const MAX_ITERATIONS = 100 // Max iterations for agent execution // agentService implements agent-related business logic type agentService struct { - cfg *config.Config - modelService interfaces.ModelService - mcpServiceService interfaces.MCPServiceService - mcpManager *mcp.MCPManager - eventBus *event.EventBus - db *gorm.DB - webSearchService interfaces.WebSearchService - knowledgeBaseService interfaces.KnowledgeBaseService - knowledgeService interfaces.KnowledgeService - chunkService interfaces.ChunkService + cfg *config.Config + modelService interfaces.ModelService + mcpServiceService interfaces.MCPServiceService + mcpManager *mcp.MCPManager + eventBus *event.EventBus + db *gorm.DB + webSearchService interfaces.WebSearchService + knowledgeBaseService interfaces.KnowledgeBaseService + knowledgeService interfaces.KnowledgeService + chunkService interfaces.ChunkService + duckdb *sql.DB + webSearchStateService interfaces.WebSearchStateService } // NewAgentService creates a new agent service @@ -46,18 +49,22 @@ func NewAgentService( eventBus *event.EventBus, db *gorm.DB, webSearchService interfaces.WebSearchService, + duckdb *sql.DB, + webSearchStateService interfaces.WebSearchStateService, ) interfaces.AgentService { return &agentService{ - cfg: cfg, - modelService: modelService, - knowledgeBaseService: knowledgeBaseService, - knowledgeService: knowledgeService, - chunkService: chunkService, - mcpServiceService: mcpServiceService, - mcpManager: mcpManager, - eventBus: eventBus, - db: db, - webSearchService: webSearchService, + cfg: cfg, + modelService: modelService, + knowledgeBaseService: knowledgeBaseService, + knowledgeService: knowledgeService, + chunkService: chunkService, + mcpServiceService: mcpServiceService, + mcpManager: mcpManager, + eventBus: eventBus, + db: db, + webSearchService: webSearchService, + duckdb: duckdb, + webSearchStateService: webSearchStateService, } } @@ -70,7 +77,6 @@ func (s *agentService) CreateAgentEngine( eventBus *event.EventBus, contextManager interfaces.ContextManager, sessionID string, - sessionService interfaces.SessionService, ) (interfaces.AgentEngine, error) { logger.Infof(ctx, "Creating agent engine with custom EventBus") @@ -87,10 +93,10 @@ func (s *agentService) CreateAgentEngine( // The registerTools function will filter out knowledge-related tools in this case // Create tool registry - toolRegistry := tools.NewToolRegistry(s.knowledgeService, s.chunkService, s.db) + toolRegistry := tools.NewToolRegistry() // Register tools - if err := s.registerTools(ctx, toolRegistry, config, rerankModel, chatModel, sessionID, sessionService); err != nil { + if err := s.registerTools(ctx, toolRegistry, config, rerankModel, chatModel, sessionID); err != nil { return nil, fmt.Errorf("failed to register tools: %w", err) } @@ -202,7 +208,6 @@ func (s *agentService) registerTools( rerankModel rerank.Reranker, chatModel chat.Chat, sessionID string, - sessionService interfaces.SessionService, ) error { // Use config's allowed tools if specified, otherwise use defaults var allowedTools []string @@ -220,17 +225,19 @@ func (s *agentService) registerTools( if !hasKnowledge { filteredTools := make([]string, 0) kbTools := map[string]bool{ - "knowledge_search": true, - "grep_chunks": true, - "list_knowledge_chunks": true, - "query_knowledge_graph": true, - "get_document_info": true, - "database_query": true, + tools.ToolKnowledgeSearch: true, + tools.ToolGrepChunks: true, + tools.ToolListKnowledgeChunks: true, + tools.ToolQueryKnowledgeGraph: true, + tools.ToolGetDocumentInfo: true, + tools.ToolDatabaseQuery: true, + tools.ToolDataAnalysis: true, + tools.ToolDataSchema: true, } // If no knowledge and no web search, also disable todo_write (not useful for simple chat) if !config.WebSearchEnabled { - kbTools["todo_write"] = true + kbTools[tools.ToolTodoWrite] = true } for _, toolName := range allowedTools { @@ -244,76 +251,74 @@ func (s *agentService) registerTools( // If web search is enabled, add web_search to allowedTools if config.WebSearchEnabled { - allowedTools = append(allowedTools, "web_search") - allowedTools = append(allowedTools, "web_fetch") + allowedTools = append(allowedTools, tools.ToolWebSearch) + allowedTools = append(allowedTools, tools.ToolWebFetch) } - - // Get tenant ID from context - tenantID := uint64(0) - if tid, ok := ctx.Value(types.TenantIDContextKey).(uint64); ok { - tenantID = tid - } - logger.Infof( - ctx, - "Registering tools: %v, tenant ID: %d, webSearchEnabled: %v", - allowedTools, - tenantID, - config.WebSearchEnabled, - ) + logger.Infof(ctx, "Registering tools: %v, webSearchEnabled: %v", allowedTools, config.WebSearchEnabled) // Register each allowed tool for _, toolName := range allowedTools { + var toolToRegister types.Tool + switch toolName { - case "thinking": - registry.RegisterTool(tools.NewSequentialThinkingTool()) - case "todo_write": - registry.RegisterTool(tools.NewTodoWriteTool()) - case "knowledge_search": - registry.RegisterTool( - tools.NewKnowledgeSearchTool( - s.knowledgeBaseService, - s.knowledgeService, - s.chunkService, - tenantID, - config.SearchTargets, - rerankModel, - chatModel, - s.cfg, - )) - case "grep_chunks": - registry.RegisterTool(tools.NewGrepChunksTool(s.db, tenantID, config.KnowledgeBases, config.KnowledgeIDs)) - logger.Infof(ctx, "Registered grep_chunks tool for tenant: %d, KBs: %d, KnowledgeIDs: %d", tenantID, len(config.KnowledgeBases), len(config.KnowledgeIDs)) - case "list_knowledge_chunks": - registry.RegisterTool(tools.NewListKnowledgeChunksTool(tenantID, s.knowledgeService, s.chunkService)) - case "query_knowledge_graph": - registry.RegisterTool(tools.NewQueryKnowledgeGraphTool(s.knowledgeBaseService)) - case "get_document_info": - registry.RegisterTool(tools.NewGetDocumentInfoTool(tenantID, s.knowledgeService, s.chunkService)) - case "database_query": - registry.RegisterTool(tools.NewDatabaseQueryTool(s.db, tenantID)) - case "web_search": - registry.RegisterTool(tools.NewWebSearchTool( + case tools.ToolThinking: + toolToRegister = tools.NewSequentialThinkingTool() + case tools.ToolTodoWrite: + toolToRegister = tools.NewTodoWriteTool() + case tools.ToolKnowledgeSearch: + toolToRegister = tools.NewKnowledgeSearchTool( + s.knowledgeBaseService, + s.knowledgeService, + s.chunkService, + config.SearchTargets, + rerankModel, + chatModel, + s.cfg, + ) + case tools.ToolGrepChunks: + toolToRegister = tools.NewGrepChunksTool(s.db, config.KnowledgeBases, config.KnowledgeIDs) + logger.Infof(ctx, "Registered grep_chunks tool, KBs: %d, KnowledgeIDs: %d", len(config.KnowledgeBases), len(config.KnowledgeIDs)) + case tools.ToolListKnowledgeChunks: + toolToRegister = tools.NewListKnowledgeChunksTool(s.knowledgeService, s.chunkService) + case tools.ToolQueryKnowledgeGraph: + toolToRegister = tools.NewQueryKnowledgeGraphTool(s.knowledgeBaseService) + case tools.ToolGetDocumentInfo: + toolToRegister = tools.NewGetDocumentInfoTool(s.knowledgeService, s.chunkService) + case tools.ToolDatabaseQuery: + toolToRegister = tools.NewDatabaseQueryTool(s.db) + case tools.ToolWebSearch: + toolToRegister = tools.NewWebSearchTool( s.webSearchService, s.knowledgeBaseService, s.knowledgeService, - sessionService, - sessionID, - config.WebSearchMaxResults, - )) - logger.Infof( - ctx, - "Registered web_search tool for session: %s, maxResults: %d", + s.webSearchStateService, sessionID, config.WebSearchMaxResults, ) + logger.Infof(ctx, "Registered web_search tool for session: %s, maxResults: %d", sessionID, config.WebSearchMaxResults) - case "web_fetch": - registry.RegisterTool(tools.NewWebFetchTool(chatModel)) + case tools.ToolWebFetch: + toolToRegister = tools.NewWebFetchTool(chatModel) logger.Infof(ctx, "Registered web_fetch tool for session: %s", sessionID) + case tools.ToolDataAnalysis: + toolToRegister = tools.NewDataAnalysisTool(s.knowledgeService, s.duckdb, sessionID) + logger.Infof(ctx, "Registered data_analysis tool for session: %s", sessionID) + + case tools.ToolDataSchema: + toolToRegister = tools.NewDataSchemaTool(s.knowledgeService, s.chunkService.GetRepository()) + logger.Infof(ctx, "Registered data_schema tool") + default: logger.Warnf(ctx, "Unknown tool: %s", toolName) } + + if toolToRegister != nil { + if toolToRegister.Name() != toolName { + logger.Warnf(ctx, "Tool name mismatch: expected %s, got %s", toolName, toolToRegister.Name()) + } + registry.RegisterTool(toolToRegister) + } } logger.Infof(ctx, "Registered %d tools", len(registry.ListTools())) diff --git a/internal/application/service/chat_pipline/data_analysis.go b/internal/application/service/chat_pipline/data_analysis.go new file mode 100644 index 00000000..2807e04d --- /dev/null +++ b/internal/application/service/chat_pipline/data_analysis.go @@ -0,0 +1,163 @@ +package chatpipline + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "slices" + "strings" + + "github.com/Tencent/WeKnora/internal/agent/tools" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/models/chat" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/Tencent/WeKnora/internal/utils" +) + +type PluginDataAnalysis struct { + modelService interfaces.ModelService + knowledgeService interfaces.KnowledgeService + chunkRepo interfaces.ChunkRepository + db *sql.DB +} + +func NewPluginDataAnalysis( + eventManager *EventManager, + modelService interfaces.ModelService, + knowledgeService interfaces.KnowledgeService, + chunkRepo interfaces.ChunkRepository, + db *sql.DB, +) *PluginDataAnalysis { + p := &PluginDataAnalysis{ + modelService: modelService, + knowledgeService: knowledgeService, + chunkRepo: chunkRepo, + db: db, + } + eventManager.Register(p) + return p +} + +func (p *PluginDataAnalysis) ActivationEvents() []types.EventType { + return []types.EventType{types.DATA_ANALYSIS} +} + +func (p *PluginDataAnalysis) OnEvent( + ctx context.Context, + eventType types.EventType, + chatManage *types.ChatManage, + next func() *PluginError, +) *PluginError { + // 1. Check if there are any CSV/Excel files in MergeResult + var dataFiles []*types.SearchResult + for _, result := range chatManage.MergeResult { + if isDataFile(result.KnowledgeFilename) { + dataFiles = append(dataFiles, result) + } + } + + // Filter out table column and table summary chunks from MergeResult + chatManage.MergeResult = filterOutTableChunks(chatManage.MergeResult) + + if len(dataFiles) == 0 { + return next() + } + + // 2. Ask LLM if data analysis is needed + // We only process the first data file for now to avoid complexity + targetFile := dataFiles[0] + + // Get Knowledge details to get file path + knowledge, err := p.knowledgeService.GetKnowledgeByID(ctx, targetFile.KnowledgeID) + if err != nil { + logger.Errorf(ctx, "Failed to get knowledge %s: %v", targetFile.KnowledgeID, err) + return next() + } + + // Initialize DataAnalysisTool + tool := tools.NewDataAnalysisTool(p.knowledgeService, p.db, chatManage.SessionID) + defer tool.Cleanup(ctx) + + // Load data into DuckDB + schema, err := tool.LoadFromKnowledge(ctx, knowledge) + if err != nil { + logger.Errorf(ctx, "Failed to get data schema: %v", err) + return next() + } + + // Ask LLM to generate SQL for data analysis + chatModel, err := p.modelService.GetChatModel(ctx, chatManage.ChatModelID) + if err != nil { + return ErrGetChatModel.WithError(err) + } + + // Use utils.GenerateSchema to generate format schema for DataAnalysisInput + formatSchema := utils.GenerateSchema[tools.DataAnalysisInput]() + + analysisPrompt := fmt.Sprintf(` +User Question: %s +Knowledge ID: %s +Table Schema: %s + +Determine if the user's question requires data analysis (e.g., statistics, aggregation, filtering) on this table. +If YES, generate a DuckDB SQL query to answer the user's question and fill in the knowledge_id and sql fields. +If NO, leave the sql field empty. + +Return your response in the specified JSON format.`, chatManage.Query, knowledge.ID, schema.Description()) + + response, err := chatModel.Chat(ctx, []chat.Message{ + {Role: "user", Content: analysisPrompt}, + }, &chat.ChatOptions{ + Temperature: 0.1, + Format: formatSchema, + }) + if err != nil { + logger.Errorf(ctx, "Failed to generate analysis response: %v", err) + return next() + } + // logger.Debugf(ctx, "Data analysis LLM response: %s", response.Content) + + // Execute SQL using the tool + // Initialize DataAnalysisTool + toolResult, err := tool.Execute(ctx, json.RawMessage(response.Content)) + if err != nil { + logger.Errorf(ctx, "Failed to execute SQL: %v", err) + return next() + } + + // 5. Store result + // Create a new SearchResult for the analysis output + analysisResult := &types.SearchResult{ + ID: "analysis_" + knowledge.ID, + Content: toolResult.Output, + Score: 1.0, + MatchType: types.MatchTypeDataAnalysis, + KnowledgeID: knowledge.ID, + KnowledgeTitle: knowledge.Title, + KnowledgeFilename: knowledge.FileName, + } + + chatManage.MergeResult = append(chatManage.MergeResult, analysisResult) + + return next() +} + +func isDataFile(filename string) bool { + lower := strings.ToLower(filename) + return strings.HasSuffix(lower, ".csv") || strings.HasSuffix(lower, ".xlsx") || strings.HasSuffix(lower, ".xls") +} + +// filterOutTableChunks filters out table column and table summary chunks from search results +func filterOutTableChunks(results []*types.SearchResult) []*types.SearchResult { + filtered := make([]*types.SearchResult, 0, len(results)) + filterList := []string{string(types.ChunkTypeTableColumn), string(types.ChunkTypeTableSummary)} + for _, result := range results { + if slices.Contains(filterList, result.ChunkType) { + continue + } + filtered = append(filtered, result) + } + return filtered +} diff --git a/internal/application/service/chat_pipline/merge.go b/internal/application/service/chat_pipline/merge.go index 868f37c0..0ed92be4 100644 --- a/internal/application/service/chat_pipline/merge.go +++ b/internal/application/service/chat_pipline/merge.go @@ -61,9 +61,12 @@ func (p *PluginMerge) OnEvent(ctx context.Context, } // Group chunks by their knowledge source ID - knowledgeGroup := make(map[string][]*types.SearchResult) + knowledgeGroup := make(map[string]map[string][]*types.SearchResult) for _, chunk := range searchResult { - knowledgeGroup[chunk.KnowledgeID] = append(knowledgeGroup[chunk.KnowledgeID], chunk) + if _, ok := knowledgeGroup[chunk.KnowledgeID]; !ok { + knowledgeGroup[chunk.KnowledgeID] = make(map[string][]*types.SearchResult) + } + knowledgeGroup[chunk.KnowledgeID][chunk.ChunkType] = append(knowledgeGroup[chunk.KnowledgeID][chunk.ChunkType], chunk) } pipelineInfo(ctx, "Merge", "group_summary", map[string]interface{}{ @@ -72,71 +75,64 @@ func (p *PluginMerge) OnEvent(ctx context.Context, mergedChunks := []*types.SearchResult{} // Process each knowledge source separately - for knowledgeID, chunks := range knowledgeGroup { - pipelineInfo(ctx, "Merge", "group_process", map[string]interface{}{ - "knowledge_id": knowledgeID, - "chunk_cnt": len(chunks), - }) + for knowledgeID, chunkGroup := range knowledgeGroup { + for _, chunks := range chunkGroup { + pipelineInfo(ctx, "Merge", "group_process", map[string]interface{}{ + "knowledge_id": knowledgeID, + "chunk_cnt": len(chunks), + }) - // Sort chunks by their start position in the original document - sort.Slice(chunks, func(i, j int) bool { - if chunks[i].StartAt == chunks[j].StartAt { - return chunks[i].EndAt < chunks[j].EndAt - } - return chunks[i].StartAt < chunks[j].StartAt - }) + // Sort chunks by their start position in the original document + sort.Slice(chunks, func(i, j int) bool { + if chunks[i].StartAt == chunks[j].StartAt { + return chunks[i].EndAt < chunks[j].EndAt + } + return chunks[i].StartAt < chunks[j].StartAt + }) - // Merge overlapping or adjacent chunks - knowledgeMergedChunks := []*types.SearchResult{} - if chunks[0].ChunkType == types.ChunkTypeSummary { - knowledgeMergedChunks = append(knowledgeMergedChunks, chunks[0]) - // skip the first chunk if it is summary chunk - // This is to avoid merging the summary chunk with the first content chunk - chunks = chunks[1:] - } - if len(chunks) > 0 { - knowledgeMergedChunks = append(knowledgeMergedChunks, chunks[0]) - } - for i := 1; i < len(chunks); i++ { - lastChunk := knowledgeMergedChunks[len(knowledgeMergedChunks)-1] - // If the current chunk starts after the last chunk ends, add it to the merged chunks - if chunks[i].StartAt > lastChunk.EndAt { - knowledgeMergedChunks = append(knowledgeMergedChunks, chunks[i]) - continue - } - // Merge overlapping chunks - if chunks[i].EndAt > lastChunk.EndAt { - lastChunk.Content = lastChunk.Content + - string([]rune(chunks[i].Content)[lastChunk.EndAt-chunks[i].StartAt:]) - lastChunk.EndAt = chunks[i].EndAt - lastChunk.SubChunkID = append(lastChunk.SubChunkID, chunks[i].ID) + // Merge overlapping or adjacent chunks + knowledgeMergedChunks := []*types.SearchResult{chunks[0]} + for i := 1; i < len(chunks); i++ { + lastChunk := knowledgeMergedChunks[len(knowledgeMergedChunks)-1] + // If the current chunk starts after the last chunk ends, add it to the merged chunks + if chunks[i].StartAt > lastChunk.EndAt { + knowledgeMergedChunks = append(knowledgeMergedChunks, chunks[i]) + continue + } + // Merge overlapping chunks + if chunks[i].EndAt > lastChunk.EndAt { + lastChunk.Content = lastChunk.Content + + string([]rune(chunks[i].Content)[lastChunk.EndAt-chunks[i].StartAt:]) + lastChunk.EndAt = chunks[i].EndAt + lastChunk.SubChunkID = append(lastChunk.SubChunkID, chunks[i].ID) - // 合并 ImageInfo - if err := mergeImageInfo(ctx, lastChunk, chunks[i]); err != nil { - pipelineWarn(ctx, "Merge", "image_merge", map[string]interface{}{ - "knowledge_id": knowledgeID, - "error": err.Error(), - }) + // 合并 ImageInfo + if err := mergeImageInfo(ctx, lastChunk, chunks[i]); err != nil { + pipelineWarn(ctx, "Merge", "image_merge", map[string]interface{}{ + "knowledge_id": knowledgeID, + "error": err.Error(), + }) + } + } + if chunks[i].Score > lastChunk.Score { + lastChunk.Score = chunks[i].Score } } - if chunks[i].Score > lastChunk.Score { - lastChunk.Score = chunks[i].Score - } + + pipelineInfo(ctx, "Merge", "group_output", map[string]interface{}{ + "knowledge_id": knowledgeID, + "merged_chunks": len(knowledgeMergedChunks), + }) + + // Sort merged chunks by their score (highest first) + sort.Slice(knowledgeMergedChunks, func(i, j int) bool { + return knowledgeMergedChunks[i].Score > knowledgeMergedChunks[j].Score + }) + + mergedChunks = append(mergedChunks, knowledgeMergedChunks...) } - - pipelineInfo(ctx, "Merge", "group_output", map[string]interface{}{ - "knowledge_id": knowledgeID, - "merged_chunks": len(knowledgeMergedChunks), - }) - - mergedChunks = append(mergedChunks, knowledgeMergedChunks...) } - // Sort merged chunks by their score (highest first) - sort.Slice(mergedChunks, func(i, j int) bool { - return mergedChunks[i].Score > mergedChunks[j].Score - }) - pipelineInfo(ctx, "Merge", "output", map[string]interface{}{ "merged_total": len(mergedChunks), }) diff --git a/internal/application/service/chat_pipline/search.go b/internal/application/service/chat_pipline/search.go index 60e24e60..a49f859a 100644 --- a/internal/application/service/chat_pipline/search.go +++ b/internal/application/service/chat_pipline/search.go @@ -17,13 +17,14 @@ import ( // PluginSearch implements search functionality for chat pipeline type PluginSearch struct { - knowledgeBaseService interfaces.KnowledgeBaseService - knowledgeService interfaces.KnowledgeService - chunkService interfaces.ChunkService - config *config.Config - webSearchService interfaces.WebSearchService - tenantService interfaces.TenantService - sessionService interfaces.SessionService + knowledgeBaseService interfaces.KnowledgeBaseService + knowledgeService interfaces.KnowledgeService + chunkService interfaces.ChunkService + config *config.Config + webSearchService interfaces.WebSearchService + tenantService interfaces.TenantService + sessionService interfaces.SessionService + webSearchStateService interfaces.WebSearchStateService } func NewPluginSearch(eventManager *EventManager, @@ -34,15 +35,17 @@ func NewPluginSearch(eventManager *EventManager, webSearchService interfaces.WebSearchService, tenantService interfaces.TenantService, sessionService interfaces.SessionService, + webSearchStateService interfaces.WebSearchStateService, ) *PluginSearch { res := &PluginSearch{ - knowledgeBaseService: knowledgeBaseService, - knowledgeService: knowledgeService, - chunkService: chunkService, - config: config, - webSearchService: webSearchService, - tenantService: tenantService, - sessionService: sessionService, + knowledgeBaseService: knowledgeBaseService, + knowledgeService: knowledgeService, + chunkService: chunkService, + config: config, + webSearchService: webSearchService, + tenantService: tenantService, + sessionService: sessionService, + webSearchStateService: webSearchStateService, } eventManager.Register(res) return res @@ -333,7 +336,7 @@ func (p *PluginSearch) searchByTargets( // Try direct loading for specific knowledge targets if t.Type == types.SearchTargetTypeKnowledge { directResults, skippedIDs := p.tryDirectChunkLoading(ctx, chatManage.TenantID, t.KnowledgeIDs) - + if len(directResults) > 0 { pipelineInfo(ctx, "Search", "direct_load", map[string]interface{}{ "kb_id": t.KnowledgeBaseID, @@ -513,8 +516,8 @@ func (p *PluginSearch) searchWebIfEnabled(ctx context.Context, chatManage *types } // Build questions using RewriteQuery only questions := []string{strings.TrimSpace(chatManage.RewriteQuery)} - // Load session-scoped temp KB state from Redis using SessionService - tempKBID, seen, ids := p.sessionService.GetWebSearchTempKBState(ctx, chatManage.SessionID) + // Load session-scoped temp KB state from Redis using WebSearchStateRepository + tempKBID, seen, ids := p.webSearchStateService.GetWebSearchTempKBState(ctx, chatManage.SessionID) compressed, kbID, newSeen, newIDs, err := p.webSearchService.CompressWithRAG( ctx, chatManage.SessionID, tempKBID, questions, webResults, tenant.WebSearchConfig, p.knowledgeBaseService, p.knowledgeService, seen, ids, @@ -525,8 +528,8 @@ func (p *PluginSearch) searchWebIfEnabled(ctx context.Context, chatManage *types }) } else { webResults = compressed - // Persist temp KB state back into Redis using SessionService - p.sessionService.SaveWebSearchTempKBState(ctx, chatManage.SessionID, kbID, newSeen, newIDs) + // Persist temp KB state back into Redis using WebSearchStateRepository + p.webSearchStateService.SaveWebSearchTempKBState(ctx, chatManage.SessionID, kbID, newSeen, newIDs) } res := searchutil.ConvertWebSearchResults(webResults) pipelineInfo(ctx, "Search", "web_hits", map[string]interface{}{ diff --git a/internal/application/service/extract.go b/internal/application/service/extract.go index a42c4a26..68e20d21 100644 --- a/internal/application/service/extract.go +++ b/internal/application/service/extract.go @@ -2,20 +2,80 @@ package service import ( "context" + "database/sql" "encoding/json" "fmt" "os" "strings" + "github.com/Tencent/WeKnora/internal/agent/tools" chatpipline "github.com/Tencent/WeKnora/internal/application/service/chat_pipline" + "github.com/Tencent/WeKnora/internal/application/service/retriever" "github.com/Tencent/WeKnora/internal/config" "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/models/chat" + "github.com/Tencent/WeKnora/internal/models/embedding" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" "github.com/google/uuid" "github.com/hibiken/asynq" ) +const ( + // tableDescriptionPromptTemplate 表格描述生成的 prompt 模板 + tableDescriptionPromptTemplate = `你是一个数据分析专家。请根据以下表格的结构信息和数据样本,生成一段简洁的表格元数据描述(200-300字)。 + +表名: %s + +%s + +%s + +请从以下维度描述这个表格: +1. **数据主题**:这个表格记录的是什么类型的数据?(如:用户信息、销售记录、日志数据等) +2. **核心字段**:列出3-5个最重要的字段及其含义 +3. **数据规模**:总行数和列数 +4. **业务场景**:这个表格可能用于什么业务分析或应用场景? +5. **关键特征**:数据有什么显著特点?(如:包含地理位置、有分类标签、存在层级关系等) + +**重要提示**: +- 不要输出具体的数据值或样本内容 +- 使用概括性的描述,让用户能快速判断这个表格是否包含他们需要的信息 +- 用简洁专业的语言,便于检索和理解` + + // columnDescriptionsPromptTemplate 列描述生成的 prompt 模板 + columnDescriptionsPromptTemplate = `你是一个数据分析专家。请根据以下表格的结构信息和数据样本,为每一列生成结构化的描述信息。 + +表名: %s + +%s + +%s + +请为每一列生成详细的描述,包含以下信息: +1. **字段含义**:这一列存储的是什么信息?(如:用户ID、订单金额、创建时间等) +2. **数据类型**:数据的类型和格式(如:整数、字符串、日期时间、布尔值等) +3. **业务用途**:这个字段在业务中的作用(如:用于用户识别、金额计算、时间排序等) +4. **数据特征**:数据的显著特点(如:唯一标识、可为空、有枚举值、有单位等) + +请按以下格式输出(每列一个段落): + +**列名1** (数据类型) +- 字段含义:xxx +- 业务用途:xxx +- 数据特征:xxx + +**列名2** (数据类型) +- 字段含义:xxx +- 业务用途:xxx +- 数据特征:xxx + +**重要提示**: +- 不要输出具体的数据值,只描述字段的元信息 +- 使用清晰的业务术语,便于用户理解和搜索 +- 如果从样本数据中能推断出枚举值范围,可以概括说明(如:状态字段包含待处理/进行中/已完成等状态)` +) + // NewChunkExtractTask creates a new chunk extract task func NewChunkExtractTask( ctx context.Context, @@ -46,6 +106,35 @@ func NewChunkExtractTask( return nil } +// NewTableExtractTask creates a new table extract task +func NewDataTableSummaryTask( + ctx context.Context, + client *asynq.Client, + tenantID uint64, + knowledgeID string, + summaryModel string, + embeddingModel string, +) error { + payload, err := json.Marshal(DataTableSummaryPayload{ + TenantID: tenantID, + KnowledgeID: knowledgeID, + SummaryModel: summaryModel, + EmbeddingModel: embeddingModel, + }) + if err != nil { + return err + } + task := asynq.NewTask(types.TypeDataTableSummary, payload, asynq.MaxRetry(3)) + info, err := client.Enqueue(task) + if err != nil { + logger.Errorf(ctx, "failed to enqueue data table summary task: %v", err) + return fmt.Errorf("failed to enqueue data table summary task: %v", err) + } + logger.Infof(ctx, "enqueued data table summary task: id=%s queue=%s knowledge=%s", + info.ID, info.Queue, knowledgeID) + return nil +} + // ChunkExtractService is a service for extracting chunks type ChunkExtractService struct { template *types.PromptTemplateStructured @@ -62,7 +151,7 @@ func NewChunkExtractService( knowledgeBaseRepo interfaces.KnowledgeBaseRepository, chunkRepo interfaces.ChunkRepository, graphEngine interfaces.RetrieveGraphRepository, -) interfaces.Extracter { +) interfaces.TaskHandler { // generator := chatpipline.NewQAPromptGenerator(chatpipline.NewFormater(), config.ExtractManager.ExtractGraph) // ctx := context.Background() // logger.Debugf(ctx, "chunk extract system prompt: %s", generator.System(ctx)) @@ -76,8 +165,8 @@ func NewChunkExtractService( } } -// Extract extracts a chunk -func (s *ChunkExtractService) Extract(ctx context.Context, t *asynq.Task) error { +// Handle handles the chunk extraction task +func (s *ChunkExtractService) Handle(ctx context.Context, t *asynq.Task) error { var p types.ExtractChunkPayload if err := json.Unmarshal(t.Payload(), &p); err != nil { logger.Errorf(ctx, "failed to unmarshal task payload: %v", err) @@ -143,3 +232,392 @@ func (s *ChunkExtractService) Extract(ctx context.Context, t *asynq.Task) error } return nil } + +// DataTableExtractPayload represents the table extract task payload +type DataTableSummaryPayload struct { + TenantID uint64 `json:"tenant_id"` + KnowledgeID string `json:"knowledge_id"` + SummaryModel string `json:"summary_model"` + EmbeddingModel string `json:"embedding_model"` +} + +// DataTableSummaryService is a service for extracting tables +type DataTableSummaryService struct { + modelService interfaces.ModelService + knowledgeService interfaces.KnowledgeService + chunkService interfaces.ChunkService + tenantService interfaces.TenantService + retrieveEngine interfaces.RetrieveEngineRegistry + sqlDB *sql.DB +} + +// NewDataTableSummaryService creates a new DataTableSummaryService +func NewDataTableSummaryService( + modelService interfaces.ModelService, + knowledgeService interfaces.KnowledgeService, + chunkService interfaces.ChunkService, + tenantService interfaces.TenantService, + retrieveEngine interfaces.RetrieveEngineRegistry, + sqlDB *sql.DB, +) interfaces.TaskHandler { + return &DataTableSummaryService{ + modelService: modelService, + knowledgeService: knowledgeService, + chunkService: chunkService, + tenantService: tenantService, + retrieveEngine: retrieveEngine, + sqlDB: sqlDB, + } +} + +// Handle implements the TaskHandler interface for table extraction +// 整体流程:初始化 -> 准备资源 -> 加载数据 -> 生成摘要 -> 创建索引 +func (s *DataTableSummaryService) Handle(ctx context.Context, t *asynq.Task) error { + // 1. 解析任务并初始化上下文 + var payload DataTableSummaryPayload + if err := json.Unmarshal(t.Payload(), &payload); err != nil { + logger.Errorf(ctx, "failed to unmarshal table extract task payload: %v", err) + return err + } + + ctx = logger.WithRequestID(ctx, uuid.New().String()) + ctx = logger.WithField(ctx, "knowledge", payload.KnowledgeID) + ctx = context.WithValue(ctx, types.TenantIDContextKey, payload.TenantID) + + logger.Infof(ctx, "Processing table extraction for knowledge: %s", payload.KnowledgeID) + + // 2. 准备所有必需的资源(知识、模型、引擎等) + resources, err := s.prepareResources(ctx, payload) + if err != nil { + return err + } + + // 3. 加载表格数据并生成摘要 + chunks, err := s.processTableData(ctx, resources) + if err != nil { + return err + } + + // 4. 索引到向量数据库 + if err := s.indexToVectorDB(ctx, chunks, resources.retrieveEngine, resources.embeddingModel); err != nil { + s.cleanupOnFailure(ctx, resources, chunks, err) + return err + } + + logger.Infof(ctx, "Table extraction completed for knowledge: %s", payload.KnowledgeID) + return nil +} + +// extractionResources 封装提取过程所需的所有资源 +type extractionResources struct { + knowledge *types.Knowledge + chatModel chat.Chat + embeddingModel embedding.Embedder + retrieveEngine *retriever.CompositeRetrieveEngine +} + +// prepareResources 准备提取所需的所有资源 +// 思路:集中加载所有依赖,统一错误处理,避免分散的资源获取逻辑 +func (s *DataTableSummaryService) prepareResources(ctx context.Context, payload DataTableSummaryPayload) (*extractionResources, error) { + // 获取并验证知识文件 + knowledge, err := s.knowledgeService.GetKnowledgeByID(ctx, payload.KnowledgeID) + if err != nil { + logger.Errorf(ctx, "failed to get knowledge: %v", err) + return nil, err + } + + // 验证文件类型 + fileType := strings.ToLower(knowledge.FileType) + if fileType != "csv" && fileType != "xlsx" && fileType != "xls" { + logger.Warnf(ctx, "knowledge %s is not a CSV or Excel file, skipping table summary", payload.KnowledgeID) + return nil, fmt.Errorf("unsupported file type: %s", fileType) + } + + // 获取租户信息 + tenantInfo, err := s.tenantService.GetTenantByID(ctx, payload.TenantID) + if err != nil { + logger.Errorf(ctx, "failed to get tenant: %v", err) + return nil, err + } + + // 获取聊天模型(用于生成摘要) + chatModel, err := s.modelService.GetChatModel(ctx, payload.SummaryModel) + if err != nil { + logger.Errorf(ctx, "failed to get chat model: %v", err) + return nil, err + } + + // 获取嵌入模型(用于向量化) + embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, payload.EmbeddingModel) + if err != nil { + logger.Errorf(ctx, "failed to get embedding model: %v", err) + return nil, err + } + + // 获取检索引擎 + retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines()) + if err != nil { + logger.Errorf(ctx, "failed to get retrieve engine: %v", err) + return nil, err + } + + return &extractionResources{ + knowledge: knowledge, + chatModel: chatModel, + embeddingModel: embeddingModel, + retrieveEngine: retrieveEngine, + }, nil +} + +// processTableData 处理表格数据:加载 -> 分析 -> 生成摘要 -> 创建chunks +// 思路:将数据处理的核心流程集中在一起,保持逻辑连贯性 +func (s *DataTableSummaryService) processTableData(ctx context.Context, resources *extractionResources) ([]*types.Chunk, error) { + // 创建DuckDB会话并加载数据 + sessionID := fmt.Sprintf("table_summary_%s", resources.knowledge.ID) + duckdbTool := tools.NewDataAnalysisTool(s.knowledgeService, s.sqlDB, sessionID) + defer duckdbTool.Cleanup(ctx) + + // 使用knowledge.ID作为表名,根据文件类型自动加载数据 + tableSchema, err := duckdbTool.LoadFromKnowledge(ctx, resources.knowledge) + if err != nil { + logger.Errorf(ctx, "failed to load data into DuckDB: %v", err) + return nil, err + } + + logger.Infof(ctx, "Loaded table %s with %d columns and %d rows", tableSchema.TableName, len(tableSchema.Columns), tableSchema.RowCount) + + // 获取样本数据用于生成摘要 + input := tools.DataAnalysisInput{ + KnowledgeID: resources.knowledge.ID, + Sql: fmt.Sprintf("SELECT * FROM \"%s\" LIMIT 10", tableSchema.TableName), + } + jsonData, err := json.Marshal(input) + if err != nil { + logger.Errorf(ctx, "failed to marshal input: %v", err) + return nil, err + } + sampleResult, err := duckdbTool.Execute(ctx, jsonData) + if err != nil { + logger.Errorf(ctx, "failed to get sample data: %v", err) + return nil, err + } + + // 构建共用的schema和样本数据描述 + schemaDesc := tableSchema.Description() + sampleDesc := s.buildSampleDataDescription(sampleResult, 10) + + // 使用AI生成表格摘要和列描述 + tableDescription, err := s.generateTableDescription(ctx, resources.chatModel, tableSchema.TableName, schemaDesc, sampleDesc) + if err != nil { + logger.Errorf(ctx, "failed to generate table description: %v", err) + return nil, err + } + logger.Debugf(ctx, "table describe of knowledge %s: %s", resources.knowledge.ID, tableDescription) + + columnDescription, err := s.generateColumnDescriptions(ctx, resources.chatModel, tableSchema.TableName, schemaDesc, sampleDesc) + if err != nil { + logger.Errorf(ctx, "failed to generate column descriptions: %v", err) + return nil, err + } + logger.Debugf(ctx, "column describe of knowledge %s: %s", resources.knowledge.ID, columnDescription) + + // 构建chunks:一个表格摘要chunk + 多个列描述chunks + chunks := s.buildChunks(resources, tableDescription, columnDescription) + return chunks, nil +} + +// buildChunks 构建chunk对象 +// tableDescription和columnDescriptions分别生成一个chunk +func (s *DataTableSummaryService) buildChunks(resources *extractionResources, tableDescription string, columnDescription string) []*types.Chunk { + chunks := make([]*types.Chunk, 0, 2) + + // 表格摘要chunk + summaryChunk := &types.Chunk{ + ID: uuid.New().String(), + TenantID: resources.knowledge.TenantID, + KnowledgeID: resources.knowledge.ID, + KnowledgeBaseID: resources.knowledge.KnowledgeBaseID, + Content: tableDescription, + ChunkIndex: 0, + IsEnabled: true, + ChunkType: types.ChunkTypeTableSummary, + Status: int(types.ChunkStatusStored), + } + chunks = append(chunks, summaryChunk) + + // 列描述chunk(所有列的描述合并为一个chunk) + columnChunk := &types.Chunk{ + ID: uuid.New().String(), + TenantID: resources.knowledge.TenantID, + KnowledgeID: resources.knowledge.ID, + KnowledgeBaseID: resources.knowledge.KnowledgeBaseID, + Content: columnDescription, + ChunkIndex: 1, + IsEnabled: true, + ChunkType: types.ChunkTypeTableColumn, + ParentChunkID: summaryChunk.ID, + Status: int(types.ChunkStatusStored), + } + chunks = append(chunks, columnChunk) + + summaryChunk.NextChunkID = columnChunk.ID + columnChunk.PreChunkID = summaryChunk.ID + + return chunks +} + +// indexToVectorDB 将chunks索引到向量数据库 +// 思路:批量构建索引信息,统一索引,更新状态 +func (s *DataTableSummaryService) indexToVectorDB( + ctx context.Context, + chunks []*types.Chunk, + engine *retriever.CompositeRetrieveEngine, + embedder embedding.Embedder, +) error { + // 构建索引信息列表 + indexInfoList := make([]*types.IndexInfo, 0, len(chunks)) + for _, chunk := range chunks { + indexInfoList = append(indexInfoList, &types.IndexInfo{ + Content: chunk.Content, + SourceID: chunk.ID, + SourceType: types.ChunkSourceType, + ChunkID: chunk.ID, + KnowledgeID: chunk.KnowledgeID, + KnowledgeBaseID: chunk.KnowledgeBaseID, + }) + } + + // 保存到数据库 + if err := s.chunkService.CreateChunks(ctx, chunks); err != nil { + logger.Errorf(ctx, "failed to create chunks: %v", err) + return err + } + logger.Infof(ctx, "Created %d chunks for data table", len(chunks)) + + // 批量索引 + if err := engine.BatchIndex(ctx, embedder, indexInfoList); err != nil { + logger.Errorf(ctx, "failed to index chunks: %v", err) + return err + } + + // 更新chunk状态为已索引 + for _, chunk := range chunks { + chunk.Status = int(types.ChunkStatusIndexed) + } + if err := s.chunkService.UpdateChunks(ctx, chunks); err != nil { + logger.Errorf(ctx, "failed to update chunk status: %v", err) + return err + } + + return nil +} + +// cleanupOnFailure 索引失败时的清理工作 +// 思路:删除已创建的chunk和对应的向量索引,避免脏数据残留 +func (s *DataTableSummaryService) cleanupOnFailure(ctx context.Context, resources *extractionResources, chunks []*types.Chunk, indexErr error) { + logger.Warnf(ctx, "Starting cleanup due to failure: %v", indexErr) + + // 1. 更新知识状态为失败 + resources.knowledge.ParseStatus = types.ParseStatusFailed + resources.knowledge.ErrorMessage = indexErr.Error() + if err := s.knowledgeService.UpdateKnowledge(ctx, resources.knowledge); err != nil { + logger.Errorf(ctx, "Failed to update knowledge status: %v", err) + } else { + logger.Infof(ctx, "Updated knowledge %s status to failed", resources.knowledge.ID) + } + + // 提取chunk IDs + chunkIDs := make([]string, 0, len(chunks)) + for _, chunk := range chunks { + chunkIDs = append(chunkIDs, chunk.ID) + } + + // 删除已创建的chunks + if len(chunkIDs) > 0 { + if err := s.chunkService.DeleteChunks(ctx, chunkIDs); err != nil { + logger.Errorf(ctx, "Failed to delete chunks: %v", err) + } else { + logger.Infof(ctx, "Deleted %d chunks", len(chunkIDs)) + } + } + + // 删除对应的向量索引 + if len(chunkIDs) > 0 { + if err := resources.retrieveEngine.DeleteBySourceIDList( + ctx, chunkIDs, resources.embeddingModel.GetDimensions(), types.KnowledgeBaseTypeDocument, + ); err != nil { + logger.Errorf(ctx, "Failed to delete vector index: %v", err) + } else { + logger.Infof(ctx, "Deleted vector index for %d chunks", len(chunkIDs)) + } + } + + logger.Infof(ctx, "Cleanup completed") +} + +// generateTableDescription generates a summary description for the entire table +func (s *DataTableSummaryService) generateTableDescription(ctx context.Context, chatModel chat.Chat, tableName, schemaDesc, sampleDesc string) (string, error) { + prompt := fmt.Sprintf(tableDescriptionPromptTemplate, tableName, schemaDesc, sampleDesc) + // logger.Debugf(ctx, "generateTableDescription prompt: %s", prompt) + + thinking := false + response, err := chatModel.Chat(ctx, []chat.Message{ + {Role: "user", Content: prompt}, + }, &chat.ChatOptions{ + Temperature: 0.3, + MaxTokens: 512, + Thinking: &thinking, + }) + if err != nil { + return "", fmt.Errorf("failed to generate table description: %w", err) + } + + return fmt.Sprintf("# 表格摘要\n\n表名: %s\n\n%s", tableName, response.Content), nil +} + +// generateColumnDescriptions generates descriptions for each column in batch +func (s *DataTableSummaryService) generateColumnDescriptions(ctx context.Context, chatModel chat.Chat, tableName, schemaDesc, sampleDesc string) (string, error) { + // Build batch prompt for all columns + prompt := fmt.Sprintf(columnDescriptionsPromptTemplate, tableName, schemaDesc, sampleDesc) + // logger.Debugf(ctx, "generateColumnDescriptions prompt: %s", prompt) + + // Call LLM once for all columns + thinking := false + response, err := chatModel.Chat(ctx, []chat.Message{ + {Role: "user", Content: prompt}, + }, &chat.ChatOptions{ + Temperature: 0.3, + MaxTokens: 2048, + Thinking: &thinking, + }) + if err != nil { + return "", fmt.Errorf("failed to generate column descriptions: %w", err) + } + + return fmt.Sprintf("# 表格列信息\n\n表名: %s\n\n%s", tableName, response.Content), nil +} + +// buildSampleDataDescription builds a formatted sample data description +func (s *DataTableSummaryService) buildSampleDataDescription(sampleData *types.ToolResult, maxRows int) string { + var builder strings.Builder + builder.WriteString(fmt.Sprintf("前%d行数据示例:\n", maxRows)) + + rows, ok := sampleData.Data["rows"].([]map[string]interface{}) + if !ok { + return builder.String() + } + + for i, row := range rows { + if i >= maxRows { + break + } + jsonBytes, err := json.Marshal(row) + if err != nil { + continue + } + builder.WriteString(string(jsonBytes)) + builder.WriteString("\n") + } + + return builder.String() +} diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index a9594104..e2fb53e4 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -348,6 +348,10 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context, knowledge.ID, ) + if slices.Contains([]string{"csv", "xlsx", "xls"}, getFileType(safeFilename)) { + NewDataTableSummaryTask(ctx, s.task, tenantID, knowledge.ID, kb.SummaryModelID, kb.EmbeddingModelID) + } + logger.Infof(ctx, "Knowledge from file created successfully, ID: %s", knowledge.ID) return knowledge, nil } diff --git a/internal/application/service/knowledgebase.go b/internal/application/service/knowledgebase.go index 6c8fee1d..efdea99a 100644 --- a/internal/application/service/knowledgebase.go +++ b/internal/application/service/knowledgebase.go @@ -1030,7 +1030,7 @@ func (s *knowledgeBaseService) processSearchResults(ctx context.Context, } // Add nearby chunks (prev and next) - if chunk.ChunkType == types.ChunkTypeText { + if slices.Contains([]string{types.ChunkTypeText}, chunk.ChunkType) { if chunk.NextChunkID != "" && !processedChunkIDs[chunk.NextChunkID] { additionalChunkIDs = append(additionalChunkIDs, chunk.NextChunkID) processedChunkIDs[chunk.NextChunkID] = true @@ -1163,6 +1163,7 @@ func (s *knowledgeBaseService) buildSearchResult(chunk *types.Chunk, func (s *knowledgeBaseService) isValidTextChunk(chunk *types.Chunk) bool { return slices.Contains([]types.ChunkType{ types.ChunkTypeText, types.ChunkTypeSummary, + types.ChunkTypeTableColumn, types.ChunkTypeTableSummary, types.ChunkTypeFAQ, }, chunk.ChunkType) } diff --git a/internal/application/service/session.go b/internal/application/service/session.go index 962537b2..af36273f 100644 --- a/internal/application/service/session.go +++ b/internal/application/service/session.go @@ -19,7 +19,6 @@ import ( "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" "github.com/google/uuid" - "github.com/redis/go-redis/v9" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" ) @@ -31,18 +30,18 @@ func generateEventID(suffix string) string { // sessionService implements the SessionService interface for managing conversation sessions type sessionService struct { - cfg *config.Config // Application configuration - sessionRepo interfaces.SessionRepository // Repository for session data - messageRepo interfaces.MessageRepository // Repository for message data - knowledgeBaseService interfaces.KnowledgeBaseService // Service for knowledge base operations - modelService interfaces.ModelService // Service for model operations - tenantService interfaces.TenantService // Service for tenant operations - eventManager *chatpipline.EventManager // Event manager for chat pipeline - agentService interfaces.AgentService // Service for agent operations - sessionStorage llmcontext.ContextStorage // Session storage - knowledgeService interfaces.KnowledgeService // Service for knowledge operations - chunkService interfaces.ChunkService // Service for chunk operations - redisClient *redis.Client // Redis client for temp KB state + cfg *config.Config // Application configuration + sessionRepo interfaces.SessionRepository // Repository for session data + messageRepo interfaces.MessageRepository // Repository for message data + knowledgeBaseService interfaces.KnowledgeBaseService // Service for knowledge base operations + modelService interfaces.ModelService // Service for model operations + tenantService interfaces.TenantService // Service for tenant operations + eventManager *chatpipline.EventManager // Event manager for chat pipeline + agentService interfaces.AgentService // Service for agent operations + sessionStorage llmcontext.ContextStorage // Session storage + knowledgeService interfaces.KnowledgeService // Service for knowledge operations + chunkService interfaces.ChunkService // Service for chunk operations + webSearchStateRepo interfaces.WebSearchStateService // Service for web search state } // NewSessionService creates a new session service instance with all required dependencies @@ -57,7 +56,7 @@ func NewSessionService(cfg *config.Config, eventManager *chatpipline.EventManager, agentService interfaces.AgentService, sessionStorage llmcontext.ContextStorage, - redisClient *redis.Client, + webSearchStateRepo interfaces.WebSearchStateService, ) interfaces.SessionService { return &sessionService{ cfg: cfg, @@ -71,7 +70,7 @@ func NewSessionService(cfg *config.Config, eventManager: eventManager, agentService: agentService, sessionStorage: sessionStorage, - redisClient: redisClient, + webSearchStateRepo: webSearchStateRepo, } } @@ -200,7 +199,7 @@ func (s *sessionService) DeleteSession(ctx context.Context, id string) error { tenantID := ctx.Value(types.TenantIDContextKey).(uint64) // Cleanup temporary KB stored in Redis for this session - if err := s.DeleteWebSearchTempKBState(ctx, id); err != nil { + if err := s.webSearchStateRepo.DeleteWebSearchTempKBState(ctx, id); err != nil { logger.Warnf(ctx, "Failed to cleanup temporary KB for session %s: %v", id, err) } @@ -705,7 +704,6 @@ func (s *sessionService) selectChatModelID( knowledgeBaseIDs []string, knowledgeIDs []string, ) (string, error) { - // If no knowledge base IDs but have knowledge IDs, derive KB IDs from knowledge IDs if len(knowledgeBaseIDs) == 0 && len(knowledgeIDs) > 0 { tenantID := ctx.Value(types.TenantIDContextKey).(uint64) @@ -1238,7 +1236,6 @@ func (s *sessionService) AgentQA( eventBus, contextManager, session.ID, - s, ) if err != nil { logger.Errorf(ctx, "Failed to create agent engine: %v", err) @@ -1321,110 +1318,6 @@ func (s *sessionService) ClearContext(ctx context.Context, sessionID string) err return s.sessionStorage.Delete(ctx, sessionID) } -// GetWebSearchTempKBState retrieves the temporary KB state for web search from Redis -func (s *sessionService) GetWebSearchTempKBState( - ctx context.Context, - sessionID string, -) (tempKBID string, seenURLs map[string]bool, knowledgeIDs []string) { - stateKey := fmt.Sprintf("tempkb:%s", sessionID) - if raw, getErr := s.redisClient.Get(ctx, stateKey).Bytes(); getErr == nil && len(raw) > 0 { - var state struct { - KBID string `json:"kbID"` - KnowledgeIDs []string `json:"knowledgeIDs"` - SeenURLs map[string]bool `json:"seenURLs"` - } - if err := json.Unmarshal(raw, &state); err == nil { - tempKBID = state.KBID - ids := state.KnowledgeIDs - if state.SeenURLs != nil { - seenURLs = state.SeenURLs - } else { - seenURLs = make(map[string]bool) - } - return tempKBID, seenURLs, ids - } - } - return "", make(map[string]bool), []string{} -} - -// SaveWebSearchTempKBState saves the temporary KB state for web search to Redis -func (s *sessionService) SaveWebSearchTempKBState( - ctx context.Context, - sessionID string, - tempKBID string, - seenURLs map[string]bool, - knowledgeIDs []string, -) { - stateKey := fmt.Sprintf("tempkb:%s", sessionID) - state := struct { - KBID string `json:"kbID"` - KnowledgeIDs []string `json:"knowledgeIDs"` - SeenURLs map[string]bool `json:"seenURLs"` - }{ - KBID: tempKBID, - KnowledgeIDs: knowledgeIDs, - SeenURLs: seenURLs, - } - if b, err := json.Marshal(state); err == nil { - _ = s.redisClient.Set(ctx, stateKey, b, 0).Err() - } -} - -// DeleteWebSearchTempKBState deletes the temporary KB state for web search from Redis -// and cleans up associated knowledge base and knowledge items. -func (s *sessionService) DeleteWebSearchTempKBState(ctx context.Context, sessionID string) error { - if s.redisClient == nil { - return nil - } - - stateKey := fmt.Sprintf("tempkb:%s", sessionID) - raw, getErr := s.redisClient.Get(ctx, stateKey).Bytes() - if getErr != nil || len(raw) == 0 { - // No state found, nothing to clean up - return nil - } - - var state struct { - KBID string `json:"kbID"` - KnowledgeIDs []string `json:"knowledgeIDs"` - SeenURLs map[string]bool `json:"seenURLs"` - } - if err := json.Unmarshal(raw, &state); err != nil { - // Invalid state, just delete the key - _ = s.redisClient.Del(ctx, stateKey).Err() - return nil - } - - // If KBID is empty, just delete the Redis key - if strings.TrimSpace(state.KBID) == "" { - _ = s.redisClient.Del(ctx, stateKey).Err() - return nil - } - - logger.Infof(ctx, "Cleaning temporary KB for session %s: %s", sessionID, state.KBID) - - // Delete all knowledge items - for _, kid := range state.KnowledgeIDs { - if delErr := s.knowledgeService.DeleteKnowledge(ctx, kid); delErr != nil { - logger.Warnf(ctx, "Failed to delete temp knowledge %s: %v", kid, delErr) - } - } - - // Delete the knowledge base - if delErr := s.knowledgeBaseService.DeleteKnowledgeBase(ctx, state.KBID); delErr != nil { - logger.Warnf(ctx, "Failed to delete temp knowledge base %s: %v", state.KBID, delErr) - } - - // Delete the Redis key - if delErr := s.redisClient.Del(ctx, stateKey).Err(); delErr != nil { - logger.Warnf(ctx, "Failed to delete Redis key %s: %v", stateKey, delErr) - return fmt.Errorf("failed to delete Redis key: %w", delErr) - } - - logger.Infof(ctx, "Successfully cleaned up temporary KB for session %s", sessionID) - return nil -} - // handleFallbackResponse handles fallback response based on strategy func (s *sessionService) handleFallbackResponse(ctx context.Context, chatManage *types.ChatManage) { if chatManage.FallbackStrategy == types.FallbackStrategyModel { diff --git a/internal/application/service/web_search_state.go b/internal/application/service/web_search_state.go new file mode 100644 index 00000000..430e133c --- /dev/null +++ b/internal/application/service/web_search_state.go @@ -0,0 +1,136 @@ +package service + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/redis/go-redis/v9" +) + +// webSearchStateService implements the WebSearchStateService interface +type webSearchStateService struct { + redisClient *redis.Client + knowledgeService interfaces.KnowledgeService + knowledgeBaseService interfaces.KnowledgeBaseService +} + +// NewWebSearchStateService creates a new web search state service instance +func NewWebSearchStateService( + redisClient *redis.Client, + knowledgeService interfaces.KnowledgeService, + knowledgeBaseService interfaces.KnowledgeBaseService, +) interfaces.WebSearchStateService { + return &webSearchStateService{ + redisClient: redisClient, + knowledgeService: knowledgeService, + knowledgeBaseService: knowledgeBaseService, + } +} + +// GetWebSearchTempKBState retrieves the temporary KB state for web search from Redis +func (s *webSearchStateService) GetWebSearchTempKBState( + ctx context.Context, + sessionID string, +) (tempKBID string, seenURLs map[string]bool, knowledgeIDs []string) { + stateKey := fmt.Sprintf("tempkb:%s", sessionID) + if raw, getErr := s.redisClient.Get(ctx, stateKey).Bytes(); getErr == nil && len(raw) > 0 { + var state struct { + KBID string `json:"kbID"` + KnowledgeIDs []string `json:"knowledgeIDs"` + SeenURLs map[string]bool `json:"seenURLs"` + } + if err := json.Unmarshal(raw, &state); err == nil { + tempKBID = state.KBID + ids := state.KnowledgeIDs + if state.SeenURLs != nil { + seenURLs = state.SeenURLs + } else { + seenURLs = make(map[string]bool) + } + return tempKBID, seenURLs, ids + } + } + return "", make(map[string]bool), []string{} +} + +// SaveWebSearchTempKBState saves the temporary KB state for web search to Redis +func (s *webSearchStateService) SaveWebSearchTempKBState( + ctx context.Context, + sessionID string, + tempKBID string, + seenURLs map[string]bool, + knowledgeIDs []string, +) { + stateKey := fmt.Sprintf("tempkb:%s", sessionID) + state := struct { + KBID string `json:"kbID"` + KnowledgeIDs []string `json:"knowledgeIDs"` + SeenURLs map[string]bool `json:"seenURLs"` + }{ + KBID: tempKBID, + KnowledgeIDs: knowledgeIDs, + SeenURLs: seenURLs, + } + if b, err := json.Marshal(state); err == nil { + _ = s.redisClient.Set(ctx, stateKey, b, 0).Err() + } +} + +// DeleteWebSearchTempKBState deletes the temporary KB state for web search from Redis +// and cleans up associated knowledge base and knowledge items. +func (s *webSearchStateService) DeleteWebSearchTempKBState(ctx context.Context, sessionID string) error { + if s.redisClient == nil { + return nil + } + + stateKey := fmt.Sprintf("tempkb:%s", sessionID) + raw, getErr := s.redisClient.Get(ctx, stateKey).Bytes() + if getErr != nil || len(raw) == 0 { + // No state found, nothing to clean up + return nil + } + + var state struct { + KBID string `json:"kbID"` + KnowledgeIDs []string `json:"knowledgeIDs"` + SeenURLs map[string]bool `json:"seenURLs"` + } + if err := json.Unmarshal(raw, &state); err != nil { + // Invalid state, just delete the key + _ = s.redisClient.Del(ctx, stateKey).Err() + return nil + } + + // If KBID is empty, just delete the Redis key + if strings.TrimSpace(state.KBID) == "" { + _ = s.redisClient.Del(ctx, stateKey).Err() + return nil + } + + logger.Infof(ctx, "Cleaning temporary KB for session %s: %s", sessionID, state.KBID) + + // Delete all knowledge items + for _, kid := range state.KnowledgeIDs { + if delErr := s.knowledgeService.DeleteKnowledge(ctx, kid); delErr != nil { + logger.Warnf(ctx, "Failed to delete temp knowledge %s: %v", kid, delErr) + } + } + + // Delete the knowledge base + if delErr := s.knowledgeBaseService.DeleteKnowledgeBase(ctx, state.KBID); delErr != nil { + logger.Warnf(ctx, "Failed to delete temp knowledge base %s: %v", state.KBID, delErr) + } + + // Delete the Redis key + if delErr := s.redisClient.Del(ctx, stateKey).Err(); delErr != nil { + logger.Warnf(ctx, "Failed to delete Redis key %s: %v", stateKey, delErr) + return fmt.Errorf("failed to delete Redis key: %w", delErr) + } + + logger.Infof(ctx, "Successfully cleaned up temporary KB for session %s", sessionID) + return nil +} diff --git a/internal/container/container.go b/internal/container/container.go index d9920432..dd594768 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -5,6 +5,7 @@ package container import ( "context" + "database/sql" "fmt" "net/url" "os" @@ -83,6 +84,7 @@ func BuildContainer(container *dig.Container) *dig.Container { must(container.Provide(initOllamaService)) must(container.Provide(initNeo4jClient)) must(container.Provide(stream.NewStreamManager)) + must(container.Provide(NewDuckDB)) // Data repositories layer must(container.Provide(repository.NewTenantRepository)) @@ -98,6 +100,7 @@ func BuildContainer(container *dig.Container) *dig.Container { must(container.Provide(neo4jRepo.NewNeo4jRepository)) must(container.Provide(repository.NewMCPServiceRepository)) must(container.Provide(repository.NewCustomAgentRepository)) + must(container.Provide(service.NewWebSearchStateService)) // MCP manager for managing MCP client connections must(container.Provide(mcp.NewMCPManager)) @@ -113,7 +116,11 @@ func BuildContainer(container *dig.Container) *dig.Container { must(container.Provide(service.NewDatasetService)) must(container.Provide(service.NewEvaluationService)) must(container.Provide(service.NewUserService)) - must(container.Provide(service.NewChunkExtractService)) + + // Extract services - register individual extracters with names + must(container.Provide(service.NewChunkExtractService, dig.Name("chunkExtracter"))) + must(container.Provide(service.NewDataTableSummaryService, dig.Name("dataTableSummary"))) + must(container.Provide(service.NewMessageService)) must(container.Provide(service.NewMCPServiceService)) must(container.Provide(service.NewCustomAgentService)) @@ -139,6 +146,7 @@ func BuildContainer(container *dig.Container) *dig.Container { must(container.Invoke(chatpipline.NewPluginSearch)) must(container.Invoke(chatpipline.NewPluginRerank)) must(container.Invoke(chatpipline.NewPluginMerge)) + must(container.Invoke(chatpipline.NewPluginDataAnalysis)) must(container.Invoke(chatpipline.NewPluginIntoChatMessage)) must(container.Invoke(chatpipline.NewPluginChatCompletion)) must(container.Invoke(chatpipline.NewPluginChatCompletionStream)) @@ -600,3 +608,24 @@ func initNeo4jClient() (neo4j.Driver, error) { return nil, fmt.Errorf("failed to connect to Neo4j after %d attempts: %w", maxRetries, err) } + +func NewDuckDB() (*sql.DB, error) { + sqlDB, err := sql.Open("duckdb", ":memory:") + if err != nil { + return nil, fmt.Errorf("failed to open duckdb: %w", err) + } + ctx := context.Background() + // Install and load the spatial extension which includes Excel support + // Note: DuckDB doesn't have native Excel support, we need to use a workaround + // Option 1: Install spatial extension (if available) + installSQL := "INSTALL spatial;" + if _, err := sqlDB.ExecContext(ctx, installSQL); err != nil { + logger.Warnf(ctx, "[DuckDB] Failed to install spatial extension (may already be installed): %v", err) + } + + loadSQL := "LOAD spatial;" + if _, err := sqlDB.ExecContext(ctx, loadSQL); err != nil { + logger.Warnf(ctx, "[DuckDB] Failed to load spatial extension: %v", err) + } + return sqlDB, nil +} diff --git a/internal/mcp/client.go b/internal/mcp/client.go index a9d4ec4d..70315c7f 100644 --- a/internal/mcp/client.go +++ b/internal/mcp/client.go @@ -245,21 +245,11 @@ func (c *mcpGoClient) ListTools(ctx context.Context) ([]*types.MCPTool, error) { // Convert to our types tools := make([]*types.MCPTool, len(result.Tools)) for i, tool := range result.Tools { - // Convert InputSchema - var inputSchema map[string]interface{} - // Convert InputSchema to map - data, err := json.Marshal(tool.InputSchema) - if err == nil { - var schemaMap map[string]interface{} - if err := json.Unmarshal(data, &schemaMap); err == nil { - inputSchema = schemaMap - } - } - + data, _ := json.Marshal(tool.InputSchema) tools[i] = &types.MCPTool{ Name: tool.Name, Description: tool.Description, - InputSchema: inputSchema, + InputSchema: data, } } diff --git a/internal/models/chat/chat.go b/internal/models/chat/chat.go index 9d907497..635edaac 100644 --- a/internal/models/chat/chat.go +++ b/internal/models/chat/chat.go @@ -2,6 +2,7 @@ package chat import ( "context" + "encoding/json" "fmt" "strings" @@ -18,23 +19,24 @@ type Tool struct { // FunctionDef represents a function definition type FunctionDef struct { - Name string `json:"name"` - Description string `json:"description"` - Parameters map[string]interface{} `json:"parameters"` + Name string `json:"name"` + Description string `json:"description"` + Parameters json.RawMessage `json:"parameters"` } // ChatOptions 聊天选项 type ChatOptions struct { - Temperature float64 `json:"temperature"` // 温度参数 - TopP float64 `json:"top_p"` // Top P 参数 - Seed int `json:"seed"` // 随机种子 - MaxTokens int `json:"max_tokens"` // 最大 token 数 - MaxCompletionTokens int `json:"max_completion_tokens"` // 最大完成 token 数 - FrequencyPenalty float64 `json:"frequency_penalty"` // 频率惩罚 - PresencePenalty float64 `json:"presence_penalty"` // 存在惩罚 - Thinking *bool `json:"thinking"` // 是否启用思考 - Tools []Tool `json:"tools,omitempty"` // 可用工具列表 - ToolChoice string `json:"tool_choice,omitempty"` // "auto", "required", "none", or specific tool + Temperature float64 `json:"temperature"` // 温度参数 + TopP float64 `json:"top_p"` // Top P 参数 + Seed int `json:"seed"` // 随机种子 + MaxTokens int `json:"max_tokens"` // 最大 token 数 + MaxCompletionTokens int `json:"max_completion_tokens"` // 最大完成 token 数 + FrequencyPenalty float64 `json:"frequency_penalty"` // 频率惩罚 + PresencePenalty float64 `json:"presence_penalty"` // 存在惩罚 + Thinking *bool `json:"thinking"` // 是否启用思考 + Tools []Tool `json:"tools,omitempty"` // 可用工具列表 + ToolChoice string `json:"tool_choice,omitempty"` // "auto", "required", "none", or specific tool + Format json.RawMessage `json:"format,omitempty"` // 响应格式定义 } // Message 表示聊天消息 diff --git a/internal/models/chat/ollama.go b/internal/models/chat/ollama.go index 4371f292..647ef002 100644 --- a/internal/models/chat/ollama.go +++ b/internal/models/chat/ollama.go @@ -2,7 +2,9 @@ package chat import ( "context" + "encoding/json" "fmt" + "strconv" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/models/utils/ollama" @@ -28,12 +30,17 @@ func NewOllamaChat(config *ChatConfig, ollamaService *ollama.OllamaService) (*Ol // convertMessages 转换消息格式为Ollama API格式 func (c *OllamaChat) convertMessages(messages []Message) []ollamaapi.Message { - ollamaMessages := make([]ollamaapi.Message, len(messages)) - for i, msg := range messages { - ollamaMessages[i] = ollamaapi.Message{ - Role: msg.Role, - Content: msg.Content, + ollamaMessages := make([]ollamaapi.Message, 0, len(messages)) + for _, msg := range messages { + msgOllama := ollamaapi.Message{ + Role: msg.Role, + Content: msg.Content, + ToolCalls: c.toolCallFrom(msg.ToolCalls), } + if msg.Role == "tool" { + msgOllama.ToolName = msg.Name + } + ollamaMessages = append(ollamaMessages, msgOllama) } return ollamaMessages } @@ -67,6 +74,12 @@ func (c *OllamaChat) buildChatRequest(messages []Message, opts *ChatOptions, isS Value: *opts.Thinking, } } + if len(opts.Format) > 0 { + chatReq.Format = opts.Format + } + if len(opts.Tools) > 0 { + chatReq.Tools = c.toolFrom(opts.Tools) + } } return chatReq @@ -86,11 +99,13 @@ func (c *OllamaChat) Chat(ctx context.Context, messages []Message, opts *ChatOpt logger.GetLogger(ctx).Infof("发送聊天请求到模型 %s", c.modelName) var responseContent string + var toolCalls []types.LLMToolCall var promptTokens, completionTokens int // 使用 Ollama 客户端发送请求 err := c.ollamaService.Chat(ctx, chatReq, func(resp ollamaapi.ChatResponse) error { responseContent = resp.Message.Content + toolCalls = c.toolCallTo(resp.Message.ToolCalls) // 获取token计数 if resp.EvalCount > 0 { @@ -106,7 +121,8 @@ func (c *OllamaChat) Chat(ctx context.Context, messages []Message, opts *ChatOpt // 构建响应 return &types.ChatResponse{ - Content: responseContent, + Content: responseContent, + ToolCalls: toolCalls, Usage: struct { PromptTokens int `json:"prompt_tokens"` CompletionTokens int `json:"completion_tokens"` @@ -152,6 +168,14 @@ func (c *OllamaChat) ChatStream( } } + if len(resp.Message.ToolCalls) > 0 { + streamChan <- types.StreamResponse{ + ResponseType: types.ResponseTypeToolCall, + ToolCalls: c.toolCallTo(resp.Message.ToolCalls), + Done: false, + } + } + if resp.Done { streamChan <- types.StreamResponse{ ResponseType: types.ResponseTypeAnswer, @@ -189,3 +213,97 @@ func (c *OllamaChat) GetModelName() string { func (c *OllamaChat) GetModelID() string { return c.modelID } + +// toolFrom 将本模块的 Tool 转换为 Ollama 的 Tool +func (c *OllamaChat) toolFrom(tools []Tool) ollamaapi.Tools { + if len(tools) == 0 { + return nil + } + ollamaTools := make(ollamaapi.Tools, 0, len(tools)) + for _, tool := range tools { + function := ollamaapi.ToolFunction{ + Name: tool.Function.Name, + Description: tool.Function.Description, + } + if len(tool.Function.Parameters) > 0 { + _ = json.Unmarshal(tool.Function.Parameters, &function.Parameters) + } + + ollamaTools = append(ollamaTools, ollamaapi.Tool{ + Type: tool.Type, + Function: function, + }) + } + return ollamaTools +} + +// toolTo 将 Ollama 的 Tool 转换为本模块的 Tool +func (c *OllamaChat) toolTo(ollamaTools ollamaapi.Tools) []Tool { + if len(ollamaTools) == 0 { + return nil + } + tools := make([]Tool, 0, len(ollamaTools)) + for _, tool := range ollamaTools { + paramsBytes, _ := json.Marshal(tool.Function.Parameters) + tools = append(tools, Tool{ + Type: tool.Type, + Function: FunctionDef{ + Name: tool.Function.Name, + Description: tool.Function.Description, + Parameters: paramsBytes, + }, + }) + } + return tools +} + +// toolCallFrom 将本模块的 ToolCall 转换为 Ollama 的 ToolCall +func (c *OllamaChat) toolCallFrom(toolCalls []ToolCall) []ollamaapi.ToolCall { + if len(toolCalls) == 0 { + return nil + } + ollamaToolCalls := make([]ollamaapi.ToolCall, 0, len(toolCalls)) + for _, tc := range toolCalls { + var args map[string]interface{} + if tc.Function.Arguments != "" { + _ = json.Unmarshal([]byte(tc.Function.Arguments), &args) + } + ollamaToolCalls = append(ollamaToolCalls, ollamaapi.ToolCall{ + Function: ollamaapi.ToolCallFunction{ + Index: tools2i(tc.ID), + Name: tc.Function.Name, + Arguments: args, + }, + }) + } + return ollamaToolCalls +} + +// toolCallTo 将 Ollama 的 ToolCall 转换为本模块的 ToolCall +func (c *OllamaChat) toolCallTo(ollamaToolCalls []ollamaapi.ToolCall) []types.LLMToolCall { + if len(ollamaToolCalls) == 0 { + return nil + } + toolCalls := make([]types.LLMToolCall, 0, len(ollamaToolCalls)) + for _, tc := range ollamaToolCalls { + argsBytes, _ := json.Marshal(tc.Function.Arguments) + toolCalls = append(toolCalls, types.LLMToolCall{ + ID: tooli2s(tc.Function.Index), + Type: "function", + Function: types.FunctionCall{ + Name: tc.Function.Name, + Arguments: string(argsBytes), + }, + }) + } + return toolCalls +} + +func tooli2s(i int) string { + return strconv.Itoa(i) +} + +func tools2i(s string) int { + i, _ := strconv.Atoi(s) + return i +} diff --git a/internal/models/chat/remote_api.go b/internal/models/chat/remote_api.go index 1833dc02..99ad7068 100644 --- a/internal/models/chat/remote_api.go +++ b/internal/models/chat/remote_api.go @@ -202,6 +202,13 @@ func (c *RemoteAPIChat) buildChatCompletionRequest(messages []Message, } } } + + if len(opts.Format) > 0 { + req.ResponseFormat = &openai.ChatCompletionResponseFormat{ + Type: openai.ChatCompletionResponseFormatTypeJSONObject, + } + req.Messages[len(req.Messages)-1].Content += fmt.Sprintf("\nUse this JSON schema: %s", opts.Format) + } } // ChatTemplateKwargs is only supported by custom backends like vLLM. diff --git a/internal/router/task.go b/internal/router/task.go index a3b0fa9d..2836aad3 100644 --- a/internal/router/task.go +++ b/internal/router/task.go @@ -15,10 +15,11 @@ type AsynqTaskParams struct { dig.In Server *asynq.Server - Extracter interfaces.Extracter KnowledgeService interfaces.KnowledgeService KnowledgeBaseService interfaces.KnowledgeBaseService TagService interfaces.KnowledgeTagService + ChunkExtracter interfaces.TaskHandler `name:"chunkExtracter"` + DataTableSummary interfaces.TaskHandler `name:"dataTableSummary"` } func getAsynqRedisClientOpt() *asynq.RedisClientOpt { @@ -61,7 +62,9 @@ func RunAsynqServer(params AsynqTaskParams) *asynq.ServeMux { // Create a new mux and register all handlers mux := asynq.NewServeMux() - mux.HandleFunc(types.TypeChunkExtract, params.Extracter.Extract) + // Register extract handlers - router will dispatch to appropriate handler + mux.HandleFunc(types.TypeChunkExtract, params.ChunkExtracter.Handle) + mux.HandleFunc(types.TypeDataTableSummary, params.DataTableSummary.Handle) // Register document processing handler mux.HandleFunc(types.TypeDocumentProcess, params.KnowledgeService.ProcessDocument) diff --git a/internal/types/agent.go b/internal/types/agent.go index 04dc5331..203559f8 100644 --- a/internal/types/agent.go +++ b/internal/types/agent.go @@ -10,22 +10,22 @@ import ( // AgentConfig represents the full agent configuration (used at tenant level and runtime) // This includes all configuration parameters for agent execution type AgentConfig struct { - MaxIterations int `json:"max_iterations"` // Maximum number of ReAct iterations - ReflectionEnabled bool `json:"reflection_enabled"` // Whether to enable reflection - AllowedTools []string `json:"allowed_tools"` // List of allowed tool names - Temperature float64 `json:"temperature"` // LLM temperature for agent - KnowledgeBases []string `json:"knowledge_bases"` // Accessible knowledge base IDs - KnowledgeIDs []string `json:"knowledge_ids"` // Accessible knowledge IDs (individual documents) - SystemPrompt string `json:"system_prompt,omitempty"` // Unified system prompt (uses {{web_search_status}} placeholder for dynamic behavior) + MaxIterations int `json:"max_iterations"` // Maximum number of ReAct iterations + ReflectionEnabled bool `json:"reflection_enabled"` // Whether to enable reflection + AllowedTools []string `json:"allowed_tools"` // List of allowed tool names + Temperature float64 `json:"temperature"` // LLM temperature for agent + KnowledgeBases []string `json:"knowledge_bases"` // Accessible knowledge base IDs + KnowledgeIDs []string `json:"knowledge_ids"` // Accessible knowledge IDs (individual documents) + SystemPrompt string `json:"system_prompt,omitempty"` // Unified system prompt (uses {{web_search_status}} placeholder for dynamic behavior) // Deprecated: Use SystemPrompt instead. Kept for backward compatibility during migration. - SystemPromptWebEnabled string `json:"system_prompt_web_enabled,omitempty"` // Deprecated: Custom prompt when web search is enabled - SystemPromptWebDisabled string `json:"system_prompt_web_disabled,omitempty"` // Deprecated: Custom prompt when web search is disabled - UseCustomSystemPrompt bool `json:"use_custom_system_prompt"` // Whether to use custom system prompt instead of default - WebSearchEnabled bool `json:"web_search_enabled"` // Whether web search tool is enabled - WebSearchMaxResults int `json:"web_search_max_results"` // Maximum number of web search results (default: 5) - MultiTurnEnabled bool `json:"multi_turn_enabled"` // Whether multi-turn conversation is enabled - HistoryTurns int `json:"history_turns"` // Number of history turns to keep in context - SearchTargets SearchTargets `json:"-"` // Pre-computed unified search targets (runtime only) + SystemPromptWebEnabled string `json:"system_prompt_web_enabled,omitempty"` // Deprecated: Custom prompt when web search is enabled + SystemPromptWebDisabled string `json:"system_prompt_web_disabled,omitempty"` // Deprecated: Custom prompt when web search is disabled + UseCustomSystemPrompt bool `json:"use_custom_system_prompt"` // Whether to use custom system prompt instead of default + WebSearchEnabled bool `json:"web_search_enabled"` // Whether web search tool is enabled + WebSearchMaxResults int `json:"web_search_max_results"` // Maximum number of web search results (default: 5) + MultiTurnEnabled bool `json:"multi_turn_enabled"` // Whether multi-turn conversation is enabled + HistoryTurns int `json:"history_turns"` // Number of history turns to keep in context + SearchTargets SearchTargets `json:"-"` // Pre-computed unified search targets (runtime only) // MCP service selection MCPSelectionMode string `json:"mcp_selection_mode"` // MCP selection mode: "all", "selected", "none" MCPServices []string `json:"mcp_services"` // Selected MCP service IDs (when mode is "selected") @@ -109,10 +109,10 @@ type Tool interface { Description() string // Parameters returns the JSON Schema for the tool's parameters - Parameters() map[string]interface{} + Parameters() json.RawMessage // Execute runs the tool with the given arguments - Execute(ctx context.Context, args map[string]interface{}) (*ToolResult, error) + Execute(ctx context.Context, args json.RawMessage) (*ToolResult, error) } // ToolResult represents the result of a tool execution @@ -167,7 +167,7 @@ type AgentState struct { // FunctionDefinition represents a function definition for LLM function calling type FunctionDefinition struct { - Name string `json:"name"` - Description string `json:"description"` - Parameters map[string]interface{} `json:"parameters"` + Name string `json:"name"` + Description string `json:"description"` + Parameters json.RawMessage `json:"parameters"` } diff --git a/internal/types/chat_manage.go b/internal/types/chat_manage.go index 40457b86..2843f81e 100644 --- a/internal/types/chat_manage.go +++ b/internal/types/chat_manage.go @@ -140,6 +140,7 @@ const ( ENTITY_SEARCH EventType = "entity_search" // Search for relevant entities CHUNK_RERANK EventType = "chunk_rerank" // Rerank search results CHUNK_MERGE EventType = "chunk_merge" // Merge similar chunks + DATA_ANALYSIS EventType = "data_analysis" // Data analysis for CSV/Excel files INTO_CHAT_MESSAGE EventType = "into_chat_message" // Convert chunks into chat messages CHAT_COMPLETION EventType = "chat_completion" // Generate chat completion CHAT_COMPLETION_STREAM EventType = "chat_completion_stream" // Stream chat completion @@ -174,6 +175,7 @@ var Pipline = map[string][]EventType{ CHUNK_RERANK, CHUNK_MERGE, FILTER_TOP_K, + DATA_ANALYSIS, INTO_CHAT_MESSAGE, CHAT_COMPLETION_STREAM, STREAM_FILTER, diff --git a/internal/types/chunk.go b/internal/types/chunk.go index 5242bb1c..0c79314f 100644 --- a/internal/types/chunk.go +++ b/internal/types/chunk.go @@ -9,7 +9,7 @@ import ( ) // ChunkType 定义了不同类型的 Chunk -type ChunkType string +type ChunkType = string const ( // ChunkTypeText 表示普通的文本 Chunk @@ -28,6 +28,10 @@ const ( ChunkTypeFAQ ChunkType = "faq" // ChunkTypeWebSearch 表示 Web 搜索结果的 Chunk ChunkTypeWebSearch ChunkType = "web_search" + // ChunkTypeTableSummary 表示数据表摘要的 Chunk + ChunkTypeTableSummary ChunkType = "table_summary" + // ChunkTypeTableColumn 表示数据表列描述的 Chunk + ChunkTypeTableColumn ChunkType = "table_column" ) // ChunkStatus 定义了不同状态的 Chunk diff --git a/internal/types/embedding.go b/internal/types/embedding.go index 6fd36b4a..897b1def 100644 --- a/internal/types/embedding.go +++ b/internal/types/embedding.go @@ -20,8 +20,9 @@ const ( MatchTypeParentChunk // 父Chunk匹配类型 MatchTypeRelationChunk // 关系Chunk匹配类型 MatchTypeGraph - MatchTypeWebSearch // 网络搜索匹配类型 - MatchTypeDirectLoad // 直接加载匹配类型 + MatchTypeWebSearch // 网络搜索匹配类型 + MatchTypeDirectLoad // 直接加载匹配类型 + MatchTypeDataAnalysis // 数据分析匹配类型 ) // IndexInfo contains information about indexed content diff --git a/internal/types/extract_graph.go b/internal/types/extract_graph.go index 91ccd2de..aac06184 100644 --- a/internal/types/extract_graph.go +++ b/internal/types/extract_graph.go @@ -1,14 +1,15 @@ package types const ( - TypeChunkExtract = "chunk:extract" - TypeDocumentProcess = "document:process" // 文档处理任务 - TypeFAQImport = "faq:import" // FAQ导入任务 - TypeQuestionGeneration = "question:generation" // 问题生成任务 - TypeSummaryGeneration = "summary:generation" // 摘要生成任务 - TypeKBClone = "kb:clone" // 知识库复制任务 - TypeIndexDelete = "index:delete" // 索引删除任务 - TypeKBDelete = "kb:delete" // 知识库删除任务 + TypeChunkExtract = "chunk:extract" + TypeDocumentProcess = "document:process" // 文档处理任务 + TypeFAQImport = "faq:import" // FAQ导入任务 + TypeQuestionGeneration = "question:generation" // 问题生成任务 + TypeSummaryGeneration = "summary:generation" // 摘要生成任务 + TypeKBClone = "kb:clone" // 知识库复制任务 + TypeIndexDelete = "index:delete" // 索引删除任务 + TypeKBDelete = "kb:delete" // 知识库删除任务 + TypeDataTableSummary = "datatable:summary" // 表格摘要任务 ) // ExtractChunkPayload represents the extract chunk task payload @@ -24,11 +25,11 @@ type DocumentProcessPayload struct { TenantID uint64 `json:"tenant_id"` KnowledgeID string `json:"knowledge_id"` KnowledgeBaseID string `json:"knowledge_base_id"` - FilePath string `json:"file_path,omitempty"` // 文件路径(文件导入时使用) - FileName string `json:"file_name,omitempty"` // 文件名(文件导入时使用) - FileType string `json:"file_type,omitempty"` // 文件类型(文件导入时使用) - URL string `json:"url,omitempty"` // URL(URL导入时使用) - Passages []string `json:"passages,omitempty"` // 文本段落(文本导入时使用) + FilePath string `json:"file_path,omitempty"` // 文件路径(文件导入时使用) + FileName string `json:"file_name,omitempty"` // 文件名(文件导入时使用) + FileType string `json:"file_type,omitempty"` // 文件类型(文件导入时使用) + URL string `json:"url,omitempty"` // URL(URL导入时使用) + Passages []string `json:"passages,omitempty"` // 文本段落(文本导入时使用) EnableMultimodel bool `json:"enable_multimodel"` EnableQuestionGeneration bool `json:"enable_question_generation"` // 是否启用问题生成 QuestionCount int `json:"question_count,omitempty"` // 每个chunk生成的问题数量 @@ -111,10 +112,10 @@ type KBCloneProgress struct { // ChunkContext represents chunk content with surrounding context type ChunkContext struct { - ChunkID string `json:"chunk_id"` - Content string `json:"content"` - PrevContent string `json:"prev_content,omitempty"` // Previous chunk content for context - NextContent string `json:"next_content,omitempty"` // Next chunk content for context + ChunkID string `json:"chunk_id"` + Content string `json:"content"` + PrevContent string `json:"prev_content,omitempty"` // Previous chunk content for context + NextContent string `json:"next_content,omitempty"` // Next chunk content for context } // PromptTemplateStructured represents the prompt template structured diff --git a/internal/types/interfaces/agent.go b/internal/types/interfaces/agent.go index a063dcbb..8c100caa 100644 --- a/internal/types/interfaces/agent.go +++ b/internal/types/interfaces/agent.go @@ -39,7 +39,6 @@ type AgentService interface { eventBus *event.EventBus, contextManager ContextManager, sessionID string, - sessionService SessionService, ) (AgentEngine, error) // ValidateConfig validates an agent configuration diff --git a/internal/types/interfaces/extracter.go b/internal/types/interfaces/extracter.go deleted file mode 100644 index 7dc81e1a..00000000 --- a/internal/types/interfaces/extracter.go +++ /dev/null @@ -1,13 +0,0 @@ -package interfaces - -import ( - "context" - - "github.com/hibiken/asynq" -) - -// Extracter is a interface for extracting entities -type Extracter interface { - // Extract extracts entities from a task - Extract(ctx context.Context, t *asynq.Task) error -} diff --git a/internal/types/interfaces/session.go b/internal/types/interfaces/session.go index 423f8ecd..1c326f3a 100644 --- a/internal/types/interfaces/session.go +++ b/internal/types/interfaces/session.go @@ -61,22 +61,6 @@ type SessionService interface { ) error // ClearContext clears the LLM context for a session ClearContext(ctx context.Context, sessionID string) error - // GetWebSearchTempKBState retrieves the temporary KB state for web search from Redis - GetWebSearchTempKBState( - ctx context.Context, - sessionID string, - ) (tempKBID string, seenURLs map[string]bool, knowledgeIDs []string) - // SaveWebSearchTempKBState saves the temporary KB state for web search to Redis - SaveWebSearchTempKBState( - ctx context.Context, - sessionID string, - tempKBID string, - seenURLs map[string]bool, - knowledgeIDs []string, - ) - // DeleteWebSearchTempKBState deletes the temporary KB state for web search from Redis - // and cleans up associated knowledge base and knowledge items - DeleteWebSearchTempKBState(ctx context.Context, sessionID string) error } // SessionRepository defines the session repository interface diff --git a/internal/types/interfaces/task_handler.go b/internal/types/interfaces/task_handler.go new file mode 100644 index 00000000..fc93bd28 --- /dev/null +++ b/internal/types/interfaces/task_handler.go @@ -0,0 +1,13 @@ +package interfaces + +import ( + "context" + + "github.com/hibiken/asynq" +) + +// TaskHandler is a interface for handling asynchronous tasks +type TaskHandler interface { + // Handle handles the task + Handle(ctx context.Context, t *asynq.Task) error +} diff --git a/internal/types/interfaces/web_search_state.go b/internal/types/interfaces/web_search_state.go new file mode 100644 index 00000000..40cca831 --- /dev/null +++ b/internal/types/interfaces/web_search_state.go @@ -0,0 +1,26 @@ +package interfaces + +import ( + "context" +) + +// WebSearchStateService defines the service interface for managing web search temporary KB state +type WebSearchStateService interface { + // GetWebSearchTempKBState retrieves the temporary KB state for web search from Redis + GetWebSearchTempKBState( + ctx context.Context, + sessionID string, + ) (tempKBID string, seenURLs map[string]bool, knowledgeIDs []string) + + // SaveWebSearchTempKBState saves the temporary KB state for web search to Redis + SaveWebSearchTempKBState( + ctx context.Context, + sessionID string, + tempKBID string, + seenURLs map[string]bool, + knowledgeIDs []string, + ) + + // DeleteWebSearchTempKBState deletes the temporary KB state for web search from Redis + DeleteWebSearchTempKBState(ctx context.Context, sessionID string) error +} diff --git a/internal/types/mcp.go b/internal/types/mcp.go index 7d1fee36..393ee22b 100644 --- a/internal/types/mcp.go +++ b/internal/types/mcp.go @@ -65,9 +65,9 @@ type MCPEnvVars map[string]string // MCPTool represents a tool exposed by an MCP service type MCPTool struct { - Name string `json:"name"` - Description string `json:"description"` - InputSchema map[string]interface{} `json:"inputSchema"` // JSON Schema for tool parameters + Name string `json:"name"` + Description string `json:"description"` + InputSchema json.RawMessage `json:"inputSchema"` // JSON Schema for tool parameters } // MCPResource represents a resource exposed by an MCP service diff --git a/internal/utils/json.go b/internal/utils/json.go index b7fe5315..2f94385b 100644 --- a/internal/utils/json.go +++ b/internal/utils/json.go @@ -1,6 +1,11 @@ package utils -import "encoding/json" +import ( + "encoding/json" + "fmt" + + jsonschema "github.com/google/jsonschema-go/jsonschema" +) // ToJSON converts a value to a JSON string func ToJSON(v interface{}) string { @@ -10,3 +15,21 @@ func ToJSON(v interface{}) string { } return string(json) } + +// GenerateSchema generates JSON schema for type T and returns it as a map +// This is optimized to avoid unnecessary serialization/deserialization +func GenerateSchema[T any]() json.RawMessage { + schema, err := jsonschema.For[T](nil) + if err != nil { + panic(fmt.Sprintf("failed to generate schema: %v", err)) + } + + // Convert schema to map directly through JSON marshaling + // This is necessary because the schema object doesn't expose its internal structure + schemaBytes, err := json.Marshal(schema) + if err != nil { + panic(fmt.Sprintf("failed to marshal schema: %v", err)) + } + + return schemaBytes +}