feat: Add knowledge creation support from remote file URLs

This commit is contained in:
begoniezhao
2026-02-26 14:28:14 +08:00
parent d3a0fbf218
commit e24bfb05a2
6 changed files with 543 additions and 1 deletions
+45
View File
@@ -226,6 +226,51 @@ func (c *Client) CreateKnowledgeFromURL(
return &response.Data, nil
}
// CreateKnowledgeFromFileURLRequest contains the parameters for creating a knowledge entry from a remote file URL
type CreateKnowledgeFromFileURLRequest struct {
// URL is the remote file URL to download (must be http/https, required)
URL string `json:"url"`
// FileName is the optional file name; if empty, the server resolves it from Content-Disposition or URL path
FileName string `json:"file_name,omitempty"`
// FileType is the optional file type (e.g. "pdf"); if empty, the server resolves it from the file name
FileType string `json:"file_type,omitempty"`
// EnableMultimodel is the optional flag to enable multimodal processing
EnableMultimodel *bool `json:"enable_multimodel,omitempty"`
// Title is the optional title for the knowledge entry
Title string `json:"title,omitempty"`
// TagID is the optional tag ID to associate with the knowledge entry
TagID string `json:"tag_id,omitempty"`
}
// CreateKnowledgeFromFileURL creates a knowledge entry from a remote file URL.
// The server downloads the file (.txt/.md/.pdf/.docx/.doc) and parses it asynchronously.
func (c *Client) CreateKnowledgeFromFileURL(
ctx context.Context,
knowledgeBaseID string,
req CreateKnowledgeFromFileURLRequest,
) (*Knowledge, error) {
path := fmt.Sprintf("/api/v1/knowledge-bases/%s/knowledge/file-url", knowledgeBaseID)
reqBody := req
resp, err := c.doRequest(ctx, http.MethodPost, path, reqBody, nil)
if err != nil {
return nil, err
}
var response KnowledgeResponse
if resp.StatusCode == http.StatusConflict {
if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
return nil, fmt.Errorf("failed to parse response: %w", err)
}
return &response.Data, ErrDuplicateURL
} else if err := parseResponse(resp, &response); err != nil {
return nil, err
}
return &response.Data, nil
}
// GetKnowledge retrieves a knowledge entry by its ID
func (c *Client) GetKnowledge(ctx context.Context, knowledgeID string) (*Knowledge, error) {
path := fmt.Sprintf("/api/v1/knowledge/%s", knowledgeID)
+403 -1
View File
@@ -9,6 +9,10 @@ import (
"fmt"
"io"
"mime/multipart"
"net/http"
"net/url"
"os"
"path"
"regexp"
"runtime"
"slices"
@@ -494,6 +498,216 @@ func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context,
return knowledge, nil
}
// allowedFileURLExtensions defines the supported file extensions for file URL import
var allowedFileURLExtensions = map[string]bool{
"txt": true,
"md": true,
"pdf": true,
"docx": true,
"doc": true,
}
// maxFileURLSize is the maximum allowed file size for file URL import (10MB)
const maxFileURLSize = 10 * 1024 * 1024
// extractFileNameFromURL extracts the filename from a URL path
func extractFileNameFromURL(rawURL string) string {
u, err := url.Parse(rawURL)
if err != nil {
return ""
}
base := path.Base(u.Path)
if base == "." || base == "/" {
return ""
}
return base
}
// extractFileNameFromContentDisposition extracts filename from Content-Disposition header
func extractFileNameFromContentDisposition(header string) string {
// e.g. attachment; filename="document.pdf" or filename*=UTF-8''document.pdf
for _, part := range strings.Split(header, ";") {
part = strings.TrimSpace(part)
if strings.HasPrefix(strings.ToLower(part), "filename=") {
name := strings.TrimPrefix(part, "filename=")
name = strings.TrimPrefix(part[len("filename="):], "")
name = strings.Trim(name, `"'`)
if name != "" {
return name
}
}
}
return ""
}
// CreateKnowledgeFromFileURL creates a knowledge entry by downloading a file from a remote URL.
// Sync phase: URL format + SSRF validation only (no HEAD request).
// Async phase: download file to temp dir, pass binary to docreader.
func (s *knowledgeService) CreateKnowledgeFromFileURL(
ctx context.Context,
kbID string,
fileURL string,
fileName string,
fileType string,
enableMultimodel *bool,
title string,
tagID string,
) (*types.Knowledge, error) {
logger.Info(ctx, "Start creating knowledge from file URL")
logger.Infof(ctx, "Knowledge base ID: %s, file URL: %s", kbID, fileURL)
// Get knowledge base configuration
kb, err := s.kbService.GetKnowledgeBaseByID(ctx, kbID)
if err != nil {
logger.Errorf(ctx, "Failed to get knowledge base: %v", err)
return nil, err
}
// Validate URL format and security (static check only, no HEAD request)
if !isValidURL(fileURL) || !secutils.IsValidURL(fileURL) {
logger.Error(ctx, "Invalid or unsafe file URL format")
return nil, ErrInvalidURL
}
if safe, reason := secutils.IsSSRFSafeURL(fileURL); !safe {
logger.Errorf(ctx, "File URL rejected for SSRF protection: %s, reason: %s", fileURL, reason)
return nil, ErrInvalidURL
}
// Resolve fileName: user-provided > extracted from URL path
if fileName == "" {
fileName = extractFileNameFromURL(fileURL)
}
// Resolve fileType: user-provided > inferred from fileName
if fileType == "" && fileName != "" {
fileType = getFileType(fileName)
}
// Validate file extension against whitelist (if we can determine it)
if fileType != "" {
if !allowedFileURLExtensions[strings.ToLower(fileType)] {
logger.Errorf(ctx, "Unsupported file type for file URL import: %s", fileType)
return nil, werrors.NewBadRequestError(fmt.Sprintf("不支持的文件类型: %s,仅支持 txt, md, pdf, docx, doc", fileType))
}
}
// Use title as display name if fileName is still empty
displayName := fileName
if displayName == "" {
displayName = title
}
if displayName == "" {
// Fallback: use last segment of URL
displayName = extractFileNameFromURL(fileURL)
}
if displayName == "" {
displayName = fileURL
}
// Check for duplicate (by URL hash)
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
fileHash := calculateStr(fileURL)
exists, existingKnowledge, err := s.repo.CheckKnowledgeExists(ctx, tenantID, kbID, &types.KnowledgeCheckParams{
Type: "file_url",
URL: fileURL,
FileHash: fileHash,
})
if err != nil {
logger.Errorf(ctx, "Failed to check knowledge existence: %v", err)
return nil, err
}
if exists {
logger.Infof(ctx, "File URL already exists: %s", fileURL)
existingKnowledge.CreatedAt = time.Now()
existingKnowledge.UpdatedAt = time.Now()
if err := s.repo.UpdateKnowledge(ctx, existingKnowledge); err != nil {
logger.Errorf(ctx, "Failed to update existing knowledge: %v", err)
return nil, err
}
return existingKnowledge, types.NewDuplicateURLError(existingKnowledge)
}
// Check storage quota
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
if tenantInfo.StorageQuota > 0 && tenantInfo.StorageUsed >= tenantInfo.StorageQuota {
logger.Error(ctx, "Storage quota exceeded")
return nil, types.NewStorageQuotaExceededError()
}
// Create knowledge record
knowledge := &types.Knowledge{
ID: uuid.New().String(),
TenantID: tenantID,
KnowledgeBaseID: kbID,
Type: "file_url",
Title: title,
FileName: displayName,
FileType: fileType,
Source: fileURL,
FileHash: fileHash,
ParseStatus: "pending",
EnableStatus: "disabled",
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
EmbeddingModelID: kb.EmbeddingModelID,
TagID: tagID,
}
if knowledge.Title == "" {
knowledge.Title = displayName
}
if err := s.repo.CreateKnowledge(ctx, knowledge); err != nil {
logger.Errorf(ctx, "Failed to create knowledge record: %v", err)
return nil, err
}
// Build async task payload
enableMultimodelValue := false
if enableMultimodel != nil {
enableMultimodelValue = *enableMultimodel
} else {
enableMultimodelValue = kb.IsMultimodalEnabled()
}
enableQuestionGeneration := false
questionCount := 3
if kb.QuestionGenerationConfig != nil && kb.QuestionGenerationConfig.Enabled {
enableQuestionGeneration = true
if kb.QuestionGenerationConfig.QuestionCount > 0 {
questionCount = kb.QuestionGenerationConfig.QuestionCount
}
}
taskPayload := types.DocumentProcessPayload{
TenantID: tenantID,
KnowledgeID: knowledge.ID,
KnowledgeBaseID: kbID,
FileURL: fileURL,
FileName: fileName,
FileType: fileType,
EnableMultimodel: enableMultimodelValue,
EnableQuestionGeneration: enableQuestionGeneration,
QuestionCount: questionCount,
}
payloadBytes, err := json.Marshal(taskPayload)
if err != nil {
logger.Errorf(ctx, "Failed to marshal file URL process task payload: %v", err)
return knowledge, nil
}
task := asynq.NewTask(types.TypeDocumentProcess, payloadBytes, asynq.Queue("default"))
info, err := s.task.Enqueue(task)
if err != nil {
logger.Errorf(ctx, "Failed to enqueue file URL process task: %v", err)
return knowledge, nil
}
logger.Infof(ctx, "Enqueued file URL process task: id=%s queue=%s knowledge_id=%s", info.ID, info.Queue, knowledge.ID)
logger.Infof(ctx, "Knowledge from file URL created successfully, ID: %s", knowledge.ID)
return knowledge, nil
}
// CreateKnowledgeFromPassage creates a knowledge entry from text passages
func (s *knowledgeService) CreateKnowledgeFromPassage(ctx context.Context,
kbID string, passage []string,
@@ -2304,6 +2518,51 @@ func (s *knowledgeService) ReparseKnowledge(ctx context.Context, knowledgeID str
return existing, nil
}
// For file-URL-based knowledge, enqueue document processing task with FileURL field
if existing.Type == "file_url" && existing.Source != "" {
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
enableMultimodel := kb.IsMultimodalEnabled()
// Check question generation config
enableQuestionGeneration := false
questionCount := 3
if kb.QuestionGenerationConfig != nil && kb.QuestionGenerationConfig.Enabled {
enableQuestionGeneration = true
if kb.QuestionGenerationConfig.QuestionCount > 0 {
questionCount = kb.QuestionGenerationConfig.QuestionCount
}
}
taskPayload := types.DocumentProcessPayload{
TenantID: tenantID,
KnowledgeID: existing.ID,
KnowledgeBaseID: existing.KnowledgeBaseID,
FileURL: existing.Source,
FileName: existing.FileName,
FileType: existing.FileType,
EnableMultimodel: enableMultimodel,
EnableQuestionGeneration: enableQuestionGeneration,
QuestionCount: questionCount,
}
payloadBytes, err := json.Marshal(taskPayload)
if err != nil {
logger.Errorf(ctx, "Failed to marshal file URL reparse task payload: %v", err)
return existing, nil
}
task := asynq.NewTask(types.TypeDocumentProcess, payloadBytes, asynq.Queue("default"))
info, err := s.task.Enqueue(task)
if err != nil {
logger.Errorf(ctx, "Failed to enqueue file URL reparse task: %v", err)
return existing, nil
}
logger.Infof(ctx, "Enqueued file URL reparse task: id=%s queue=%s knowledge_id=%s", info.ID, info.Queue, existing.ID)
return existing, nil
}
// For URL-based knowledge, enqueue URL processing task
if existing.Type == "url" && existing.Source != "" {
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
@@ -6512,6 +6771,70 @@ func IsImageType(fileType string) bool {
}
}
// downloadFileFromURL downloads a remote file to a temp file and returns its binary content.
// payloadFileName and payloadFileType are in/out pointers: if they point to an empty string,
// the function resolves the value from Content-Disposition / URL path and writes it back.
// It does NOT perform SSRF validation — callers are responsible for that.
func downloadFileFromURL(ctx context.Context, fileURL string, payloadFileName, payloadFileType *string) ([]byte, error) {
httpClient := &http.Client{Timeout: 60 * time.Second}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, fileURL, nil)
if err != nil {
return nil, fmt.Errorf("failed to create request for file URL: %w", err)
}
resp, err := httpClient.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to download file from URL: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("remote server returned status %d", resp.StatusCode)
}
// Reject oversized files early via Content-Length
if contentLength := resp.ContentLength; contentLength > maxFileURLSize {
return nil, fmt.Errorf("file size %d bytes exceeds limit of %d bytes (10MB)", contentLength, maxFileURLSize)
}
// Resolve fileName: payload > Content-Disposition > URL path
if *payloadFileName == "" {
if cd := resp.Header.Get("Content-Disposition"); cd != "" {
*payloadFileName = extractFileNameFromContentDisposition(cd)
}
}
if *payloadFileName == "" {
*payloadFileName = extractFileNameFromURL(fileURL)
}
if *payloadFileType == "" && *payloadFileName != "" {
*payloadFileType = getFileType(*payloadFileName)
}
// Stream response body into a temp file, capped at maxFileURLSize
tmpFile, err := os.CreateTemp("", "weknora-fileurl-*")
if err != nil {
return nil, fmt.Errorf("failed to create temp file: %w", err)
}
tmpPath := tmpFile.Name()
defer os.Remove(tmpPath)
limiter := &io.LimitedReader{R: resp.Body, N: maxFileURLSize + 1}
written, err := io.Copy(tmpFile, limiter)
tmpFile.Close()
if err != nil {
return nil, fmt.Errorf("failed to write temp file: %w", err)
}
if written > maxFileURLSize {
return nil, fmt.Errorf("file size exceeds limit of 10MB")
}
contentBytes, err := os.ReadFile(tmpPath)
if err != nil {
return nil, fmt.Errorf("failed to read temp file: %w", err)
}
return contentBytes, nil
}
// ProcessDocument handles Asynq document processing tasks
func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) error {
var payload types.DocumentProcessPayload
@@ -6626,7 +6949,86 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
// 处理不同类型的导入:文件、URL、文本段落
var chunks []*proto.Chunk
if payload.URL != "" {
if payload.FileURL != "" {
// file_url 导入:再次 SSRF 校验(防 DNS 重绑定),下载到临时文件,传二进制给 docreader
if safe, reason := secutils.IsSSRFSafeURL(payload.FileURL); !safe {
logger.Errorf(ctx, "File URL rejected for SSRF protection in ProcessDocument: %s, reason: %s", payload.FileURL, reason)
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = "File URL is not allowed for security reasons"
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
return nil
}
// Download the remote file (SSRF already validated above).
// payloadFileName/payloadFileType are in/out: resolved values are written back if empty.
resolvedFileName := payload.FileName
resolvedFileType := payload.FileType
contentBytes, err := downloadFileFromURL(ctx, payload.FileURL, &resolvedFileName, &resolvedFileType)
if err != nil {
logger.Errorf(ctx, "Failed to download file from URL: %s, error: %v", payload.FileURL, err)
if isLastRetry {
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = err.Error()
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
}
return fmt.Errorf("failed to download file from URL: %w", err)
}
// Validate resolved file type against whitelist
if resolvedFileType != "" && !allowedFileURLExtensions[strings.ToLower(resolvedFileType)] {
logger.Errorf(ctx, "Unsupported file type resolved from file URL: %s", resolvedFileType)
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = fmt.Sprintf("unsupported file type: %s", resolvedFileType)
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
return nil
}
// Persist resolved metadata back to the knowledge record
if resolvedFileName != "" && knowledge.FileName == "" {
knowledge.FileName = resolvedFileName
}
if resolvedFileType != "" && knowledge.FileType == "" {
knowledge.FileType = resolvedFileType
s.repo.UpdateKnowledge(ctx, knowledge)
}
fileResp, err := s.docReaderClient.ReadFromFile(ctx, &proto.ReadFromFileRequest{
FileContent: contentBytes,
FileName: resolvedFileName,
FileType: resolvedFileType,
ReadConfig: &proto.ReadConfig{
ChunkSize: int32(kb.ChunkingConfig.ChunkSize),
ChunkOverlap: int32(kb.ChunkingConfig.ChunkOverlap),
Separators: kb.ChunkingConfig.Separators,
EnableMultimodal: payload.EnableMultimodel,
StorageConfig: &proto.StorageConfig{
Provider: proto.StorageProvider(proto.StorageProvider_value[strings.ToUpper(kb.StorageConfig.Provider)]),
Region: kb.StorageConfig.Region,
BucketName: kb.StorageConfig.BucketName,
AccessKeyId: kb.StorageConfig.SecretID,
SecretAccessKey: kb.StorageConfig.SecretKey,
AppId: kb.StorageConfig.AppID,
PathPrefix: kb.StorageConfig.PathPrefix,
},
VlmConfig: vlmConfig,
},
RequestId: payload.RequestId,
})
if err != nil {
logger.Errorf(ctx, "Failed to read file from docreader (file_url): %v", err)
if isLastRetry {
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = err.Error()
knowledge.UpdatedAt = time.Now()
s.repo.UpdateKnowledge(ctx, knowledge)
}
return fmt.Errorf("failed to read file from docreader: %w", err)
}
chunks = fileResp.Chunks
} else if payload.URL != "" {
// URL导入 - 再次进行 SSRF 验证(防止 DNS 重绑定攻击)
if safe, reason := secutils.IsSSRFSafeURL(payload.URL); !safe {
logger.Errorf(ctx, "URL rejected for SSRF protection in ProcessDocument: %s, reason: %s", payload.URL, reason)
+79
View File
@@ -441,6 +441,85 @@ func (h *KnowledgeHandler) CreateKnowledgeFromURL(c *gin.Context) {
})
}
// CreateKnowledgeFromFileURL godoc
// @Summary 从文件资源链接创建知识
// @Description 提供远程文件链接(.txt/.md/.pdf/.docx/.doc),后台下载并解析创建知识条目
// @Tags 知识管理
// @Accept json
// @Produce json
// @Param id path string true "知识库ID"
// @Param request body object{url=string,file_name=string,file_type=string,enable_multimodel=bool,title=string,tag_id=string} true "文件URL请求"
// @Success 201 {object} map[string]interface{} "创建的知识"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Failure 409 {object} map[string]interface{} "文件重复"
// @Security Bearer
// @Security ApiKeyAuth
// @Router /knowledge-bases/{id}/knowledge/file-url [post]
func (h *KnowledgeHandler) CreateKnowledgeFromFileURL(c *gin.Context) {
ctx := c.Request.Context()
logger.Info(ctx, "Start creating knowledge from file URL")
// Validate access to the knowledge base (only owner or admin/editor can create)
_, kbID, effectiveTenantID, permission, err := h.validateKnowledgeBaseAccess(c)
if err != nil {
c.Error(err)
return
}
ctx = context.WithValue(ctx, types.TenantIDContextKey, effectiveTenantID)
// Check write permission
if permission != types.OrgRoleAdmin && permission != types.OrgRoleEditor {
c.Error(errors.NewForbiddenError("No permission to create knowledge"))
return
}
// Parse request body
var req struct {
URL string `json:"url" binding:"required"`
FileName string `json:"file_name"`
FileType string `json:"file_type"`
EnableMultimodel *bool `json:"enable_multimodel"`
Title string `json:"title"`
TagID string `json:"tag_id"`
}
if err := c.ShouldBindJSON(&req); err != nil {
logger.Error(ctx, "Failed to parse file URL request", err)
c.Error(errors.NewBadRequestError(err.Error()))
return
}
logger.Infof(ctx, "Received file URL request: %s, file_name: %s, file_type: %s",
secutils.SanitizeForLog(req.URL),
secutils.SanitizeForLog(req.FileName),
secutils.SanitizeForLog(req.FileType),
)
knowledge, err := h.kgService.CreateKnowledgeFromFileURL(
ctx, kbID, req.URL, req.FileName, req.FileType, req.EnableMultimodel, req.Title, req.TagID,
)
if err != nil {
if h.handleDuplicateKnowledgeError(c, err, knowledge, "file_url") {
return
}
if appErr, ok := errors.IsAppError(err); ok {
c.Error(appErr)
return
}
logger.ErrorWithFields(ctx, err, nil)
c.Error(errors.NewInternalServerError(err.Error()))
return
}
logger.Infof(ctx, "Knowledge created successfully from file URL, ID: %s, title: %s",
secutils.SanitizeForLog(knowledge.ID),
secutils.SanitizeForLog(knowledge.Title),
)
c.JSON(http.StatusCreated, gin.H{
"success": true,
"data": knowledge,
})
}
// CreateManualKnowledge godoc
// @Summary 手工创建知识
// @Description 手工录入Markdown格式的知识内容
+2
View File
@@ -150,6 +150,8 @@ func RegisterKnowledgeRoutes(r *gin.RouterGroup, handler *handler.KnowledgeHandl
kb.POST("/file", handler.CreateKnowledgeFromFile)
// 从URL创建知识
kb.POST("/url", handler.CreateKnowledgeFromURL)
// 从文件资源链接创建知识
kb.POST("/file-url", handler.CreateKnowledgeFromFileURL)
// 手工 Markdown 录入
kb.POST("/manual", handler.CreateManualKnowledge)
// 获取知识库下的知识列表
+1
View File
@@ -30,6 +30,7 @@ type DocumentProcessPayload struct {
FileName string `json:"file_name,omitempty"` // 文件名(文件导入时使用)
FileType string `json:"file_type,omitempty"` // 文件类型(文件导入时使用)
URL string `json:"url,omitempty"` // URL(URL导入时使用)
FileURL string `json:"file_url,omitempty"` // 文件资源链接(file_url导入时使用)
Passages []string `json:"passages,omitempty"` // 文本段落(文本导入时使用)
EnableMultimodel bool `json:"enable_multimodel"`
EnableQuestionGeneration bool `json:"enable_question_generation"` // 是否启用问题生成
+13
View File
@@ -32,6 +32,19 @@ type KnowledgeService interface {
title string,
tagID string,
) (*types.Knowledge, error)
// CreateKnowledgeFromFileURL creates knowledge by downloading a file from a remote URL.
// fileName and fileType are optional - when not provided, they will be inferred from the URL or Content-Disposition header.
// tagID is optional - when provided, the knowledge will be assigned to the specified tag/category.
CreateKnowledgeFromFileURL(
ctx context.Context,
kbID string,
fileURL string,
fileName string,
fileType string,
enableMultimodel *bool,
title string,
tagID string,
) (*types.Knowledge, error)
// CreateKnowledgeFromPassage creates knowledge from text passages.
CreateKnowledgeFromPassage(ctx context.Context, kbID string, passage []string) (*types.Knowledge, error)
// CreateKnowledgeFromPassageSync creates knowledge from text passages and waits until chunks are indexed.