Weknora/Enterprise-Knowledge-Base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: Unify /url route and file URL handling

Browse Source
This commit is contained in:
begoniezhao
2026-02-26 15:26:03 +08:00
parent e24bfb05a2
commit daf7dee4d2
5 changed files with 56 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/knowledge.go
+13 -48
View File
@@ -188,51 +188,15 @@ func (c *Client) CreateKnowledgeFromFile(ctx context.Context,
return &response.Data, nil
}
// CreateKnowledgeFromURL creates a knowledge entry from a web URL
func (c *Client) CreateKnowledgeFromURL(
ctx context.Context,
knowledgeBaseID string,
url string,
enableMultimodel *bool,
title string,
) (*Knowledge, error) {
path := fmt.Sprintf("/api/v1/knowledge-bases/%s/knowledge/url", knowledgeBaseID)
reqBody := struct {
URL string `json:"url"`
EnableMultimodel *bool `json:"enable_multimodel"`
Title string `json:"title"`
}{
URL: url,
EnableMultimodel: enableMultimodel,
Title: title,
}
resp, err := c.doRequest(ctx, http.MethodPost, path, reqBody, nil)
if err != nil {
return nil, err
}
var response KnowledgeResponse
if resp.StatusCode == http.StatusConflict {
if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
return nil, fmt.Errorf("failed to parse response: %w", err)
}
return &response.Data, ErrDuplicateURL
} else if err := parseResponse(resp, &response); err != nil {
return nil, err
}
return &response.Data, nil
}
// CreateKnowledgeFromFileURLRequest contains the parameters for creating a knowledge entry from a remote file URL
type CreateKnowledgeFromFileURLRequest struct {
// URL is the remote file URL to download (must be http/https, required)
// CreateKnowledgeFromURLRequest contains the parameters for creating a knowledge entry from a URL.
// When FileName or FileType is provided (or the URL path has a known file extension such as .pdf/.docx/.doc/.txt/.md),
// the server automatically switches to file-download mode instead of web-page crawling.
type CreateKnowledgeFromURLRequest struct {
// URL is the target URL (required)
URL string `json:"url"`
// FileName is the optional file name; if empty, the server resolves it from Content-Disposition or URL path
// FileName is the optional file name; used to hint file-download mode when URL has no extension
FileName string `json:"file_name,omitempty"`
// FileType is the optional file type (e.g. "pdf"); if empty, the server resolves it from the file name
// FileType is the optional file type (e.g. "pdf"); used to hint file-download mode
FileType string `json:"file_type,omitempty"`
// EnableMultimodel is the optional flag to enable multimodal processing
EnableMultimodel *bool `json:"enable_multimodel,omitempty"`
@@ -242,14 +206,15 @@ type CreateKnowledgeFromFileURLRequest struct {
TagID string `json:"tag_id,omitempty"`
}
// CreateKnowledgeFromFileURL creates a knowledge entry from a remote file URL.
// The server downloads the file (.txt/.md/.pdf/.docx/.doc) and parses it asynchronously.
func (c *Client) CreateKnowledgeFromFileURL(
// CreateKnowledgeFromURL creates a knowledge entry from a URL.
// When req.FileName or req.FileType is provided (or the URL path has a known file extension),
// the server automatically switches to file-download mode instead of web-page crawling.
func (c *Client) CreateKnowledgeFromURL(
ctx context.Context,
knowledgeBaseID string,
req CreateKnowledgeFromFileURLRequest,
req CreateKnowledgeFromURLRequest,
) (*Knowledge, error) {
path := fmt.Sprintf("/api/v1/knowledge-bases/%s/knowledge/file-url", knowledgeBaseID)
path := fmt.Sprintf("/api/v1/knowledge-bases/%s/knowledge/url", knowledgeBaseID)
reqBody := req
internal/application/service/knowledge.go
+26 -6
View File
@@ -367,11 +367,32 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
// CreateKnowledgeFromURL creates a knowledge entry from a URL source
// tagID is optional - when provided, the knowledge will be assigned to the specified tag/category.
// isFileURL reports whether the given URL should be treated as a direct file download.
// Priority: URL path has a known file extension first, then fall back to user-provided fileName/fileType hints.
func isFileURL(rawURL, fileName, fileType string) bool {
u, err := url.Parse(rawURL)
if err == nil {
ext := strings.ToLower(strings.TrimPrefix(path.Ext(u.Path), "."))
if ext != "" && allowedFileURLExtensions[ext] {
return true
}
}
// Fall back to user-provided hints
return fileName != "" || fileType != ""
}
func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context,
kbID string, url string, enableMultimodel *bool, title string, tagID string,
kbID string, rawURL string, fileName string, fileType string, enableMultimodel *bool, title string, tagID string,
) (*types.Knowledge, error) {
logger.Info(ctx, "Start creating knowledge from URL")
logger.Infof(ctx, "Knowledge base ID: %s, URL: %s", kbID, url)
logger.Infof(ctx, "Knowledge base ID: %s, URL: %s", kbID, rawURL)
// Route to file_url logic when the URL points to a downloadable file
if isFileURL(rawURL, fileName, fileType) {
return s.createKnowledgeFromFileURL(ctx, kbID, rawURL, fileName, fileType, enableMultimodel, title, tagID)
}
url := rawURL
// Get knowledge base configuration
logger.Info(ctx, "Getting knowledge base configuration")
@@ -540,10 +561,9 @@ func extractFileNameFromContentDisposition(header string) string {
return ""
}
// CreateKnowledgeFromFileURL creates a knowledge entry by downloading a file from a remote URL.
// Sync phase: URL format + SSRF validation only (no HEAD request).
// Async phase: download file to temp dir, pass binary to docreader.
func (s *knowledgeService) CreateKnowledgeFromFileURL(
// createKnowledgeFromFileURL is the internal implementation for file URL knowledge creation.
// Called by CreateKnowledgeFromURL when the URL is detected as a direct file download.
func (s *knowledgeService) createKnowledgeFromFileURL(
ctx context.Context,
kbID string,
fileURL string,
internal/handler/knowledge.go
+14 -84
View File
@@ -366,12 +366,12 @@ func (h *KnowledgeHandler) CreateKnowledgeFromFile(c *gin.Context) {
// CreateKnowledgeFromURL godoc
// @Summary 从URL创建知识
// @Description 从指定URL抓取内容并创建知识条目
// @Description 从指定URL抓取内容并创建知识条目。当提供 file_name/file_type 或 URL 路径含已知文件扩展名时，自动切换为文件下载模式
// @Tags 知识管理
// @Accept json
// @Produce json
// @Param id path string true "知识库ID"
// @Param request body object{url=string,enable_multimodel=bool,title=string,tag_id=string} true "URL请求"
// @Param request body object{url=string,file_name=string,file_type=string,enable_multimodel=bool,title=string,tag_id=string} true "URL请求"
// @Success 201 {object} map[string]interface{} "创建的知识"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Failure 409 {object} map[string]interface{} "URL重复"
@@ -399,6 +399,8 @@ func (h *KnowledgeHandler) CreateKnowledgeFromURL(c *gin.Context) {
// Parse URL from request body
var req struct {
URL string `json:"url" binding:"required"`
FileName string `json:"file_name"`
FileType string `json:"file_type"`
EnableMultimodel *bool `json:"enable_multimodel"`
Title string `json:"title"`
TagID string `json:"tag_id"`
@@ -409,98 +411,24 @@ func (h *KnowledgeHandler) CreateKnowledgeFromURL(c *gin.Context) {
return
}
logger.Infof(ctx, "Received URL request: %s", secutils.SanitizeForLog(req.URL))
logger.Infof(
ctx,
logger.Infof(ctx, "Received URL request: %s, file_name: %s, file_type: %s",
secutils.SanitizeForLog(req.URL),
secutils.SanitizeForLog(req.FileName),
secutils.SanitizeForLog(req.FileType),
)
logger.Infof(ctx,
"Creating knowledge from URL, knowledge base ID: %s, URL: %s",
secutils.SanitizeForLog(kbID),
secutils.SanitizeForLog(req.URL),
)
// Create knowledge entry from the URL
knowledge, err := h.kgService.CreateKnowledgeFromURL(ctx, kbID, req.URL, req.EnableMultimodel, req.Title, req.TagID)
knowledge, err := h.kgService.CreateKnowledgeFromURL(ctx, kbID, req.URL, req.FileName, req.FileType, req.EnableMultimodel, req.Title, req.TagID)
// Check for duplicate knowledge error
if err != nil {
if h.handleDuplicateKnowledgeError(c, err, knowledge, "url") {
return
}
logger.ErrorWithFields(ctx, err, nil)
c.Error(errors.NewInternalServerError(err.Error()))
return
}
logger.Infof(
ctx,
"Knowledge created successfully from URL, ID: %s, title: %s",
secutils.SanitizeForLog(knowledge.ID),
secutils.SanitizeForLog(knowledge.Title),
)
c.JSON(http.StatusCreated, gin.H{
"success": true,
"data": knowledge,
})
}
// CreateKnowledgeFromFileURL godoc
// @Summary 从文件资源链接创建知识
// @Description 提供远程文件链接（.txt/.md/.pdf/.docx/.doc），后台下载并解析创建知识条目
// @Tags 知识管理
// @Accept json
// @Produce json
// @Param id path string true "知识库ID"
// @Param request body object{url=string,file_name=string,file_type=string,enable_multimodel=bool,title=string,tag_id=string} true "文件URL请求"
// @Success 201 {object} map[string]interface{} "创建的知识"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Failure 409 {object} map[string]interface{} "文件重复"
// @Security Bearer
// @Security ApiKeyAuth
// @Router /knowledge-bases/{id}/knowledge/file-url [post]
func (h *KnowledgeHandler) CreateKnowledgeFromFileURL(c *gin.Context) {
ctx := c.Request.Context()
logger.Info(ctx, "Start creating knowledge from file URL")
// Validate access to the knowledge base (only owner or admin/editor can create)
_, kbID, effectiveTenantID, permission, err := h.validateKnowledgeBaseAccess(c)
if err != nil {
c.Error(err)
return
}
ctx = context.WithValue(ctx, types.TenantIDContextKey, effectiveTenantID)
// Check write permission
if permission != types.OrgRoleAdmin && permission != types.OrgRoleEditor {
c.Error(errors.NewForbiddenError("No permission to create knowledge"))
return
}
// Parse request body
var req struct {
URL string `json:"url" binding:"required"`
FileName string `json:"file_name"`
FileType string `json:"file_type"`
EnableMultimodel *bool `json:"enable_multimodel"`
Title string `json:"title"`
TagID string `json:"tag_id"`
}
if err := c.ShouldBindJSON(&req); err != nil {
logger.Error(ctx, "Failed to parse file URL request", err)
c.Error(errors.NewBadRequestError(err.Error()))
return
}
logger.Infof(ctx, "Received file URL request: %s, file_name: %s, file_type: %s",
secutils.SanitizeForLog(req.URL),
secutils.SanitizeForLog(req.FileName),
secutils.SanitizeForLog(req.FileType),
)
knowledge, err := h.kgService.CreateKnowledgeFromFileURL(
ctx, kbID, req.URL, req.FileName, req.FileType, req.EnableMultimodel, req.Title, req.TagID,
)
if err != nil {
if h.handleDuplicateKnowledgeError(c, err, knowledge, "file_url") {
return
}
if appErr, ok := errors.IsAppError(err); ok {
c.Error(appErr)
return
@@ -510,7 +438,9 @@ func (h *KnowledgeHandler) CreateKnowledgeFromFileURL(c *gin.Context) {
return
}
logger.Infof(ctx, "Knowledge created successfully from file URL, ID: %s, title: %s",
logger.Infof(
ctx,
"Knowledge created successfully from URL, ID: %s, title: %s",
secutils.SanitizeForLog(knowledge.ID),
secutils.SanitizeForLog(knowledge.Title),
)
internal/router/router.go
+1 -3
View File
@@ -148,10 +148,8 @@ func RegisterKnowledgeRoutes(r *gin.RouterGroup, handler *handler.KnowledgeHandl
{
// 从文件创建知识
kb.POST("/file", handler.CreateKnowledgeFromFile)
// 从URL创建知识
// 从URL创建知识（支持网页URL和文件URL，传 file_name/file_type 或 URL 含已知扩展名时自动切换为文件下载模式）
kb.POST("/url", handler.CreateKnowledgeFromURL)
// 从文件资源链接创建知识
kb.POST("/file-url", handler.CreateKnowledgeFromFileURL)
// 手工 Markdown 录入
kb.POST("/manual", handler.CreateManualKnowledge)
// 获取知识库下的知识列表
internal/types/interfaces/knowledge.go
+2 -11
View File
@@ -23,22 +23,13 @@ type KnowledgeService interface {
tagID string,
) (*types.Knowledge, error)
// CreateKnowledgeFromURL creates knowledge from a URL.
// When fileName or fileType is provided (or the URL path has a known file extension),
// the URL is treated as a direct file download instead of a web page crawl.
// tagID is optional - when provided, the knowledge will be assigned to the specified tag/category.
CreateKnowledgeFromURL(
ctx context.Context,
kbID string,
url string,
enableMultimodel *bool,
title string,
tagID string,
) (*types.Knowledge, error)
// CreateKnowledgeFromFileURL creates knowledge by downloading a file from a remote URL.
// fileName and fileType are optional - when not provided, they will be inferred from the URL or Content-Disposition header.
// tagID is optional - when provided, the knowledge will be assigned to the specified tag/category.
CreateKnowledgeFromFileURL(
ctx context.Context,
kbID string,
fileURL string,
fileName string,
fileType string,
enableMultimodel *bool,