From daf7dee4d2aa0bf52863662e09e87860f4885ec0 Mon Sep 17 00:00:00 2001 From: begoniezhao Date: Thu, 26 Feb 2026 15:26:03 +0800 Subject: [PATCH] refactor: Unify /url route and file URL handling --- client/knowledge.go | 61 +++----------- internal/application/service/knowledge.go | 32 ++++++-- internal/handler/knowledge.go | 98 ++++------------------- internal/router/router.go | 4 +- internal/types/interfaces/knowledge.go | 13 +-- 5 files changed, 56 insertions(+), 152 deletions(-) diff --git a/client/knowledge.go b/client/knowledge.go index 0ca34d44..b2fa7934 100644 --- a/client/knowledge.go +++ b/client/knowledge.go @@ -188,51 +188,15 @@ func (c *Client) CreateKnowledgeFromFile(ctx context.Context, return &response.Data, nil } -// CreateKnowledgeFromURL creates a knowledge entry from a web URL -func (c *Client) CreateKnowledgeFromURL( - ctx context.Context, - knowledgeBaseID string, - url string, - enableMultimodel *bool, - title string, -) (*Knowledge, error) { - path := fmt.Sprintf("/api/v1/knowledge-bases/%s/knowledge/url", knowledgeBaseID) - - reqBody := struct { - URL string `json:"url"` - EnableMultimodel *bool `json:"enable_multimodel"` - Title string `json:"title"` - }{ - URL: url, - EnableMultimodel: enableMultimodel, - Title: title, - } - - resp, err := c.doRequest(ctx, http.MethodPost, path, reqBody, nil) - if err != nil { - return nil, err - } - - var response KnowledgeResponse - if resp.StatusCode == http.StatusConflict { - if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { - return nil, fmt.Errorf("failed to parse response: %w", err) - } - return &response.Data, ErrDuplicateURL - } else if err := parseResponse(resp, &response); err != nil { - return nil, err - } - - return &response.Data, nil -} - -// CreateKnowledgeFromFileURLRequest contains the parameters for creating a knowledge entry from a remote file URL -type CreateKnowledgeFromFileURLRequest struct { - // URL is the remote file URL to download (must be http/https, required) +// CreateKnowledgeFromURLRequest contains the parameters for creating a knowledge entry from a URL. +// When FileName or FileType is provided (or the URL path has a known file extension such as .pdf/.docx/.doc/.txt/.md), +// the server automatically switches to file-download mode instead of web-page crawling. +type CreateKnowledgeFromURLRequest struct { + // URL is the target URL (required) URL string `json:"url"` - // FileName is the optional file name; if empty, the server resolves it from Content-Disposition or URL path + // FileName is the optional file name; used to hint file-download mode when URL has no extension FileName string `json:"file_name,omitempty"` - // FileType is the optional file type (e.g. "pdf"); if empty, the server resolves it from the file name + // FileType is the optional file type (e.g. "pdf"); used to hint file-download mode FileType string `json:"file_type,omitempty"` // EnableMultimodel is the optional flag to enable multimodal processing EnableMultimodel *bool `json:"enable_multimodel,omitempty"` @@ -242,14 +206,15 @@ type CreateKnowledgeFromFileURLRequest struct { TagID string `json:"tag_id,omitempty"` } -// CreateKnowledgeFromFileURL creates a knowledge entry from a remote file URL. -// The server downloads the file (.txt/.md/.pdf/.docx/.doc) and parses it asynchronously. -func (c *Client) CreateKnowledgeFromFileURL( +// CreateKnowledgeFromURL creates a knowledge entry from a URL. +// When req.FileName or req.FileType is provided (or the URL path has a known file extension), +// the server automatically switches to file-download mode instead of web-page crawling. +func (c *Client) CreateKnowledgeFromURL( ctx context.Context, knowledgeBaseID string, - req CreateKnowledgeFromFileURLRequest, + req CreateKnowledgeFromURLRequest, ) (*Knowledge, error) { - path := fmt.Sprintf("/api/v1/knowledge-bases/%s/knowledge/file-url", knowledgeBaseID) + path := fmt.Sprintf("/api/v1/knowledge-bases/%s/knowledge/url", knowledgeBaseID) reqBody := req diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index 66ed88ae..b116f8ad 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -367,11 +367,32 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context, // CreateKnowledgeFromURL creates a knowledge entry from a URL source // tagID is optional - when provided, the knowledge will be assigned to the specified tag/category. +// isFileURL reports whether the given URL should be treated as a direct file download. +// Priority: URL path has a known file extension first, then fall back to user-provided fileName/fileType hints. +func isFileURL(rawURL, fileName, fileType string) bool { + u, err := url.Parse(rawURL) + if err == nil { + ext := strings.ToLower(strings.TrimPrefix(path.Ext(u.Path), ".")) + if ext != "" && allowedFileURLExtensions[ext] { + return true + } + } + // Fall back to user-provided hints + return fileName != "" || fileType != "" +} + func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context, - kbID string, url string, enableMultimodel *bool, title string, tagID string, + kbID string, rawURL string, fileName string, fileType string, enableMultimodel *bool, title string, tagID string, ) (*types.Knowledge, error) { logger.Info(ctx, "Start creating knowledge from URL") - logger.Infof(ctx, "Knowledge base ID: %s, URL: %s", kbID, url) + logger.Infof(ctx, "Knowledge base ID: %s, URL: %s", kbID, rawURL) + + // Route to file_url logic when the URL points to a downloadable file + if isFileURL(rawURL, fileName, fileType) { + return s.createKnowledgeFromFileURL(ctx, kbID, rawURL, fileName, fileType, enableMultimodel, title, tagID) + } + + url := rawURL // Get knowledge base configuration logger.Info(ctx, "Getting knowledge base configuration") @@ -540,10 +561,9 @@ func extractFileNameFromContentDisposition(header string) string { return "" } -// CreateKnowledgeFromFileURL creates a knowledge entry by downloading a file from a remote URL. -// Sync phase: URL format + SSRF validation only (no HEAD request). -// Async phase: download file to temp dir, pass binary to docreader. -func (s *knowledgeService) CreateKnowledgeFromFileURL( +// createKnowledgeFromFileURL is the internal implementation for file URL knowledge creation. +// Called by CreateKnowledgeFromURL when the URL is detected as a direct file download. +func (s *knowledgeService) createKnowledgeFromFileURL( ctx context.Context, kbID string, fileURL string, diff --git a/internal/handler/knowledge.go b/internal/handler/knowledge.go index c6269e52..d60cff1d 100644 --- a/internal/handler/knowledge.go +++ b/internal/handler/knowledge.go @@ -366,12 +366,12 @@ func (h *KnowledgeHandler) CreateKnowledgeFromFile(c *gin.Context) { // CreateKnowledgeFromURL godoc // @Summary 从URL创建知识 -// @Description 从指定URL抓取内容并创建知识条目 +// @Description 从指定URL抓取内容并创建知识条目。当提供 file_name/file_type 或 URL 路径含已知文件扩展名时，自动切换为文件下载模式 // @Tags 知识管理 // @Accept json // @Produce json // @Param id path string true "知识库ID" -// @Param request body object{url=string,enable_multimodel=bool,title=string,tag_id=string} true "URL请求" +// @Param request body object{url=string,file_name=string,file_type=string,enable_multimodel=bool,title=string,tag_id=string} true "URL请求" // @Success 201 {object} map[string]interface{} "创建的知识" // @Failure 400 {object} errors.AppError "请求参数错误" // @Failure 409 {object} map[string]interface{} "URL重复" @@ -399,6 +399,8 @@ func (h *KnowledgeHandler) CreateKnowledgeFromURL(c *gin.Context) { // Parse URL from request body var req struct { URL string `json:"url" binding:"required"` + FileName string `json:"file_name"` + FileType string `json:"file_type"` EnableMultimodel *bool `json:"enable_multimodel"` Title string `json:"title"` TagID string `json:"tag_id"` @@ -409,98 +411,24 @@ func (h *KnowledgeHandler) CreateKnowledgeFromURL(c *gin.Context) { return } - logger.Infof(ctx, "Received URL request: %s", secutils.SanitizeForLog(req.URL)) - logger.Infof( - ctx, + logger.Infof(ctx, "Received URL request: %s, file_name: %s, file_type: %s", + secutils.SanitizeForLog(req.URL), + secutils.SanitizeForLog(req.FileName), + secutils.SanitizeForLog(req.FileType), + ) + logger.Infof(ctx, "Creating knowledge from URL, knowledge base ID: %s, URL: %s", secutils.SanitizeForLog(kbID), secutils.SanitizeForLog(req.URL), ) // Create knowledge entry from the URL - knowledge, err := h.kgService.CreateKnowledgeFromURL(ctx, kbID, req.URL, req.EnableMultimodel, req.Title, req.TagID) + knowledge, err := h.kgService.CreateKnowledgeFromURL(ctx, kbID, req.URL, req.FileName, req.FileType, req.EnableMultimodel, req.Title, req.TagID) // Check for duplicate knowledge error if err != nil { if h.handleDuplicateKnowledgeError(c, err, knowledge, "url") { return } - logger.ErrorWithFields(ctx, err, nil) - c.Error(errors.NewInternalServerError(err.Error())) - return - } - - logger.Infof( - ctx, - "Knowledge created successfully from URL, ID: %s, title: %s", - secutils.SanitizeForLog(knowledge.ID), - secutils.SanitizeForLog(knowledge.Title), - ) - c.JSON(http.StatusCreated, gin.H{ - "success": true, - "data": knowledge, - }) -} - -// CreateKnowledgeFromFileURL godoc -// @Summary 从文件资源链接创建知识 -// @Description 提供远程文件链接（.txt/.md/.pdf/.docx/.doc），后台下载并解析创建知识条目 -// @Tags 知识管理 -// @Accept json -// @Produce json -// @Param id path string true "知识库ID" -// @Param request body object{url=string,file_name=string,file_type=string,enable_multimodel=bool,title=string,tag_id=string} true "文件URL请求" -// @Success 201 {object} map[string]interface{} "创建的知识" -// @Failure 400 {object} errors.AppError "请求参数错误" -// @Failure 409 {object} map[string]interface{} "文件重复" -// @Security Bearer -// @Security ApiKeyAuth -// @Router /knowledge-bases/{id}/knowledge/file-url [post] -func (h *KnowledgeHandler) CreateKnowledgeFromFileURL(c *gin.Context) { - ctx := c.Request.Context() - logger.Info(ctx, "Start creating knowledge from file URL") - - // Validate access to the knowledge base (only owner or admin/editor can create) - _, kbID, effectiveTenantID, permission, err := h.validateKnowledgeBaseAccess(c) - if err != nil { - c.Error(err) - return - } - ctx = context.WithValue(ctx, types.TenantIDContextKey, effectiveTenantID) - - // Check write permission - if permission != types.OrgRoleAdmin && permission != types.OrgRoleEditor { - c.Error(errors.NewForbiddenError("No permission to create knowledge")) - return - } - - // Parse request body - var req struct { - URL string `json:"url" binding:"required"` - FileName string `json:"file_name"` - FileType string `json:"file_type"` - EnableMultimodel *bool `json:"enable_multimodel"` - Title string `json:"title"` - TagID string `json:"tag_id"` - } - if err := c.ShouldBindJSON(&req); err != nil { - logger.Error(ctx, "Failed to parse file URL request", err) - c.Error(errors.NewBadRequestError(err.Error())) - return - } - - logger.Infof(ctx, "Received file URL request: %s, file_name: %s, file_type: %s", - secutils.SanitizeForLog(req.URL), - secutils.SanitizeForLog(req.FileName), - secutils.SanitizeForLog(req.FileType), - ) - - knowledge, err := h.kgService.CreateKnowledgeFromFileURL( - ctx, kbID, req.URL, req.FileName, req.FileType, req.EnableMultimodel, req.Title, req.TagID, - ) - if err != nil { - if h.handleDuplicateKnowledgeError(c, err, knowledge, "file_url") { - return - } if appErr, ok := errors.IsAppError(err); ok { c.Error(appErr) return @@ -510,7 +438,9 @@ func (h *KnowledgeHandler) CreateKnowledgeFromFileURL(c *gin.Context) { return } - logger.Infof(ctx, "Knowledge created successfully from file URL, ID: %s, title: %s", + logger.Infof( + ctx, + "Knowledge created successfully from URL, ID: %s, title: %s", secutils.SanitizeForLog(knowledge.ID), secutils.SanitizeForLog(knowledge.Title), ) diff --git a/internal/router/router.go b/internal/router/router.go index 5539a644..6e9d9ff0 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -148,10 +148,8 @@ func RegisterKnowledgeRoutes(r *gin.RouterGroup, handler *handler.KnowledgeHandl { // 从文件创建知识 kb.POST("/file", handler.CreateKnowledgeFromFile) - // 从URL创建知识 + // 从URL创建知识（支持网页URL和文件URL，传 file_name/file_type 或 URL 含已知扩展名时自动切换为文件下载模式） kb.POST("/url", handler.CreateKnowledgeFromURL) - // 从文件资源链接创建知识 - kb.POST("/file-url", handler.CreateKnowledgeFromFileURL) // 手工 Markdown 录入 kb.POST("/manual", handler.CreateManualKnowledge) // 获取知识库下的知识列表 diff --git a/internal/types/interfaces/knowledge.go b/internal/types/interfaces/knowledge.go index 0d7f602c..6a37461c 100644 --- a/internal/types/interfaces/knowledge.go +++ b/internal/types/interfaces/knowledge.go @@ -23,22 +23,13 @@ type KnowledgeService interface { tagID string, ) (*types.Knowledge, error) // CreateKnowledgeFromURL creates knowledge from a URL. + // When fileName or fileType is provided (or the URL path has a known file extension), + // the URL is treated as a direct file download instead of a web page crawl. // tagID is optional - when provided, the knowledge will be assigned to the specified tag/category. CreateKnowledgeFromURL( ctx context.Context, kbID string, url string, - enableMultimodel *bool, - title string, - tagID string, - ) (*types.Knowledge, error) - // CreateKnowledgeFromFileURL creates knowledge by downloading a file from a remote URL. - // fileName and fileType are optional - when not provided, they will be inferred from the URL or Content-Disposition header. - // tagID is optional - when provided, the knowledge will be assigned to the specified tag/category. - CreateKnowledgeFromFileURL( - ctx context.Context, - kbID string, - fileURL string, fileName string, fileType string, enableMultimodel *bool,