feat: introduce WeKnora Lite edition with lightweight configuration and deployment

- Added a new `.env.lite.example` file for the Lite version, providing a minimal configuration template.
- Updated `.env.example` to remove deprecated variables and include new Docreader settings.
- Enhanced Docker configurations to support the Lite version, including a new Dockerfile for the Docreader service.
- Introduced a Makefile target for building and running the Lite version, along with packaging capabilities.
- Created GitHub workflows for building and releasing Lite binaries, including Homebrew formula support.
- Implemented a new service file for managing the Lite version as a system service.

This update enables a streamlined, single-binary deployment of WeKnora, reducing external dependencies and simplifying setup.
This commit is contained in:
wizardchen
2026-02-28 13:22:07 +08:00
committed by lyingbug
parent 5241dbc39e
commit 397689d2f3
140 changed files with 13365 additions and 5413 deletions
+36 -118
View File
@@ -13,8 +13,6 @@ import (
"sync"
"time"
"github.com/Tencent/WeKnora/docreader/client"
"github.com/Tencent/WeKnora/docreader/proto"
chatpipline "github.com/Tencent/WeKnora/internal/application/service/chat_pipline"
"github.com/Tencent/WeKnora/internal/config"
"github.com/Tencent/WeKnora/internal/errors"
@@ -57,7 +55,7 @@ type InitializationHandler struct {
kbRepository interfaces.KnowledgeBaseRepository
knowledgeService interfaces.KnowledgeService
ollamaService *ollama.OllamaService
docReaderClient *client.Client
documentReader interfaces.DocumentReader
pooler embedding.EmbedderPooler
}
@@ -70,7 +68,7 @@ func NewInitializationHandler(
kbRepository interfaces.KnowledgeBaseRepository,
knowledgeService interfaces.KnowledgeService,
ollamaService *ollama.OllamaService,
docReaderClient *client.Client,
documentReader interfaces.DocumentReader,
pooler embedding.EmbedderPooler,
) *InitializationHandler {
return &InitializationHandler{
@@ -81,7 +79,7 @@ func NewInitializationHandler(
kbRepository: kbRepository,
knowledgeService: knowledgeService,
ollamaService: ollamaService,
docReaderClient: docReaderClient,
documentReader: documentReader,
pooler: pooler,
}
}
@@ -94,30 +92,20 @@ type KBModelConfigRequest struct {
// 文档分块配置
DocumentSplitting struct {
ChunkSize int `json:"chunkSize"`
ChunkOverlap int `json:"chunkOverlap"`
Separators []string `json:"separators"`
ChunkSize int `json:"chunkSize"`
ChunkOverlap int `json:"chunkOverlap"`
Separators []string `json:"separators"`
ParserEngineRules []types.ParserEngineRule `json:"parserEngineRules,omitempty"`
} `json:"documentSplitting"`
// 多模态配置
// 多模态配置(仅模型相关;存储引擎在 storageProvider 中配置)
Multimodal struct {
Enabled bool `json:"enabled"`
StorageType string `json:"storageType"` // "cos" or "minio"
COS *struct {
SecretID string `json:"secretId"`
SecretKey string `json:"secretKey"`
Region string `json:"region"`
BucketName string `json:"bucketName"`
AppID string `json:"appId"`
PathPrefix string `json:"pathPrefix"`
} `json:"cos"`
Minio *struct {
BucketName string `json:"bucketName"`
UseSSL bool `json:"useSSL"`
PathPrefix string `json:"pathPrefix"`
} `json:"minio"`
Enabled bool `json:"enabled"`
} `json:"multimodal"`
// 存储引擎选择("local" | "minio" | "cos"),影响文档上传与文档内图片存储,参数从全局设置读取
StorageProvider string `json:"storageProvider"`
// 知识图谱配置
NodeExtract struct {
Enabled bool `json:"enabled"`
@@ -300,38 +288,22 @@ func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) {
if len(req.DocumentSplitting.Separators) > 0 {
kb.ChunkingConfig.Separators = req.DocumentSplitting.Separators
}
kb.ChunkingConfig.ParserEngineRules = req.DocumentSplitting.ParserEngineRules
// 更新多模态配置
if req.Multimodal.Enabled {
switch strings.ToLower(req.Multimodal.StorageType) {
case "cos":
if req.Multimodal.COS != nil {
kb.StorageConfig = types.StorageConfig{
SecretID: req.Multimodal.COS.SecretID,
SecretKey: req.Multimodal.COS.SecretKey,
Region: req.Multimodal.COS.Region,
BucketName: req.Multimodal.COS.BucketName,
AppID: req.Multimodal.COS.AppID,
PathPrefix: req.Multimodal.COS.PathPrefix,
Provider: "cos",
}
}
case "minio":
if req.Multimodal.Minio != nil {
kb.StorageConfig = types.StorageConfig{
BucketName: req.Multimodal.Minio.BucketName,
PathPrefix: req.Multimodal.Minio.PathPrefix,
Provider: "minio",
SecretID: os.Getenv("MINIO_ACCESS_KEY_ID"),
SecretKey: os.Getenv("MINIO_SECRET_ACCESS_KEY"),
}
}
}
// VLM model already set above
} else {
// 多模态未启用时,清空存储配置
kb.StorageConfig = types.StorageConfig{}
kb.VLMConfig.ModelID = ""
}
// 存储引擎:仅写入 provider,参数从租户全局 StorageEngineConfig 读取
provider := strings.ToLower(strings.TrimSpace(req.StorageProvider))
if provider == "" {
provider = "local"
}
kb.StorageConfig = types.StorageConfig{Provider: provider}
// 更新知识图谱配置
if req.NodeExtract.Enabled {
// 转换 Nodes 和 Relations 为指针类型
@@ -1910,97 +1882,43 @@ func (h *InitializationHandler) TestMultimodalFunction(c *gin.Context) {
})
}
// testMultimodalWithDocReader 调用docreader服务进行多模态处理
// testMultimodalWithDocReader uses DocumentReader.Read for document reading,
// then returns basic information about the result.
func (h *InitializationHandler) testMultimodalWithDocReader(
ctx context.Context,
imageContent []byte, filename string,
chunkSize, chunkOverlap int32, separators []string,
req *testMultimodalForm,
) (map[string]string, error) {
// 获取文件扩展名
fileExt := ""
if idx := strings.LastIndex(filename, "."); idx != -1 {
fileExt = strings.ToLower(filename[idx+1:])
}
// 检查docreader服务配置
if h.docReaderClient == nil {
if h.documentReader == nil {
return nil, fmt.Errorf("DocReader service not configured")
}
// 构造请求
request := &proto.ReadFromFileRequest{
requestID, _ := ctx.Value(types.RequestIDContextKey).(string)
readResult, err := h.documentReader.Read(ctx, &types.ReadRequest{
FileContent: imageContent,
FileName: filename,
FileType: fileExt,
ReadConfig: &proto.ReadConfig{
ChunkSize: chunkSize,
ChunkOverlap: chunkOverlap,
Separators: separators,
EnableMultimodal: true, // 启用多模态处理
VlmConfig: &proto.VLMConfig{
ModelName: req.VLMModel,
BaseUrl: req.VLMBaseURL,
ApiKey: req.VLMAPIKey,
InterfaceType: req.VLMInterfaceType,
},
},
RequestId: ctx.Value(types.RequestIDContextKey).(string),
}
// 设置对象存储配置(通用)
switch strings.ToLower(req.StorageType) {
case "cos":
request.ReadConfig.StorageConfig = &proto.StorageConfig{
Provider: proto.StorageProvider_COS,
Region: req.COSRegion,
BucketName: req.COSBucketName,
AccessKeyId: req.COSSecretID,
SecretAccessKey: req.COSSecretKey,
AppId: req.COSAppID,
PathPrefix: req.COSPathPrefix,
}
case "minio":
request.ReadConfig.StorageConfig = &proto.StorageConfig{
Provider: proto.StorageProvider_MINIO,
BucketName: req.MinioBucketName,
PathPrefix: req.MinioPathPrefix,
AccessKeyId: os.Getenv("MINIO_ACCESS_KEY_ID"),
SecretAccessKey: os.Getenv("MINIO_SECRET_ACCESS_KEY"),
}
}
// 调用docreader服务
response, err := h.docReaderClient.ReadFromFile(ctx, request)
RequestID: requestID,
})
if err != nil {
return nil, fmt.Errorf("调用DocReader服务失败: %v", err)
}
if response.Error != "" {
return nil, fmt.Errorf("DocReader服务返回错误: %s", response.Error)
if readResult.Error != "" {
return nil, fmt.Errorf("DocReader服务返回错误: %s", readResult.Error)
}
// 处理响应,提取Caption和OCR信息
result := make(map[string]string)
var allCaptions, allOCRTexts []string
for _, chunk := range response.Chunks {
if len(chunk.Images) > 0 {
for _, image := range chunk.Images {
if image.Caption != "" {
allCaptions = append(allCaptions, image.Caption)
}
if image.OcrText != "" {
allOCRTexts = append(allOCRTexts, image.OcrText)
}
}
}
result := map[string]string{
"markdown": readResult.MarkdownContent,
"caption": "",
"ocr": "",
}
// 合并所有Caption和OCR结果
result["caption"] = strings.Join(allCaptions, "; ")
result["ocr"] = strings.Join(allOCRTexts, "; ")
return result, nil
}
+96
View File
@@ -738,6 +738,102 @@ func (h *KnowledgeHandler) DownloadKnowledgeFile(c *gin.Context) {
})
}
// mimeTypeByExt returns the MIME type for a given file extension.
func mimeTypeByExt(filename string) string {
ext := strings.ToLower(filename)
if idx := strings.LastIndex(ext, "."); idx >= 0 {
ext = ext[idx:]
} else {
ext = ""
}
m := map[string]string{
".pdf": "application/pdf",
".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
".doc": "application/msword",
".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
".ppt": "application/vnd.ms-powerpoint",
".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
".xls": "application/vnd.ms-excel",
".csv": "text/csv",
".jpg": "image/jpeg",
".jpeg": "image/jpeg",
".png": "image/png",
".gif": "image/gif",
".bmp": "image/bmp",
".webp": "image/webp",
".svg": "image/svg+xml",
".tiff": "image/tiff",
".txt": "text/plain; charset=utf-8",
".md": "text/markdown; charset=utf-8",
".markdown": "text/markdown; charset=utf-8",
".json": "application/json; charset=utf-8",
".xml": "application/xml; charset=utf-8",
".html": "text/html; charset=utf-8",
".css": "text/css; charset=utf-8",
".js": "text/javascript; charset=utf-8",
".ts": "text/typescript; charset=utf-8",
".py": "text/x-python; charset=utf-8",
".go": "text/x-go; charset=utf-8",
".java": "text/x-java; charset=utf-8",
".yaml": "text/yaml; charset=utf-8",
".yml": "text/yaml; charset=utf-8",
".sh": "text/x-shellscript; charset=utf-8",
}
if ct, ok := m[ext]; ok {
return ct
}
return "application/octet-stream"
}
// PreviewKnowledgeFile godoc
// @Summary 预览知识文件
// @Description 返回知识条目关联的原始文件,Content-Type 根据文件类型设置,用于浏览器内嵌预览
// @Tags 知识管理
// @Accept json
// @Produce application/pdf,image/jpeg,image/png,text/plain
// @Param id path string true "知识ID"
// @Success 200 {file} file "文件内容"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Security Bearer
// @Security ApiKeyAuth
// @Router /knowledge/{id}/preview [get]
func (h *KnowledgeHandler) PreviewKnowledgeFile(c *gin.Context) {
ctx := c.Request.Context()
id := secutils.SanitizeForLog(c.Param("id"))
if id == "" {
c.Error(errors.NewBadRequestError("Knowledge ID cannot be empty"))
return
}
_, effCtx, err := h.resolveKnowledgeAndValidateKBAccess(c, id, types.OrgRoleViewer)
if err != nil {
c.Error(err)
return
}
file, filename, err := h.kgService.GetKnowledgeFile(effCtx, id)
if err != nil {
logger.ErrorWithFields(ctx, err, nil)
c.Error(errors.NewInternalServerError("Failed to retrieve file").WithDetails(err.Error()))
return
}
defer file.Close()
contentType := mimeTypeByExt(filename)
c.Header("Content-Type", contentType)
c.Header("Content-Disposition", fmt.Sprintf("inline; filename=%s", filename))
c.Header("Cache-Control", "private, max-age=3600")
c.Stream(func(w io.Writer) bool {
if _, err := io.Copy(w, file); err != nil {
logger.Errorf(ctx, "Failed to stream preview: %v", err)
return false
}
return false
})
}
// GetKnowledgeBatchRequest defines parameters for batch knowledge retrieval
type GetKnowledgeBatchRequest struct {
IDs []string `form:"ids" binding:"required"` // List of knowledge IDs
+2 -2
View File
@@ -26,7 +26,7 @@ type KnowledgeBaseHandler struct {
knowledgeService interfaces.KnowledgeService
kbShareService interfaces.KBShareService
agentShareService interfaces.AgentShareService
asynqClient *asynq.Client
asynqClient interfaces.TaskEnqueuer
}
// NewKnowledgeBaseHandler creates a new knowledge base handler instance
@@ -35,7 +35,7 @@ func NewKnowledgeBaseHandler(
knowledgeService interfaces.KnowledgeService,
kbShareService interfaces.KBShareService,
agentShareService interfaces.AgentShareService,
asynqClient *asynq.Client,
asynqClient interfaces.TaskEnqueuer,
) *KnowledgeBaseHandler {
return &KnowledgeBaseHandler{
service: service,
@@ -232,12 +232,13 @@ func (h *AgentStreamHandler) handleReferences(ctx context.Context, evt event.Eve
} else if refMap, ok := ref.(map[string]interface{}); ok {
// Parse from map if needed
searchResult := &types.SearchResult{
ID: getString(refMap, "id"),
Content: getString(refMap, "content"),
Score: getFloat64(refMap, "score"),
KnowledgeID: getString(refMap, "knowledge_id"),
KnowledgeTitle: getString(refMap, "knowledge_title"),
ChunkIndex: int(getFloat64(refMap, "chunk_index")),
ID: getString(refMap, "id"),
Content: getString(refMap, "content"),
Score: getFloat64(refMap, "score"),
KnowledgeID: getString(refMap, "knowledge_id"),
KnowledgeTitle: getString(refMap, "knowledge_title"),
ChunkIndex: int(getFloat64(refMap, "chunk_index")),
KnowledgeBaseID: getString(refMap, "knowledge_base_id"),
}
if meta, ok := refMap["metadata"].(map[string]interface{}); ok {
+1
View File
@@ -84,6 +84,7 @@ func buildStreamResponse(evt interfaces.StreamEvent, requestID string) *types.St
ImageInfo: getString(refMap, "image_info"),
KnowledgeFilename: getString(refMap, "knowledge_filename"),
KnowledgeSource: getString(refMap, "knowledge_source"),
KnowledgeBaseID: getString(refMap, "knowledge_base_id"),
}
searchResults = append(searchResults, sr)
}
+420 -20
View File
@@ -3,12 +3,18 @@ package handler
import (
"context"
"encoding/json"
"fmt"
"net"
"os"
"regexp"
"strings"
"github.com/Tencent/WeKnora/internal/application/service/file"
"github.com/Tencent/WeKnora/internal/config"
"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/gin-gonic/gin"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
@@ -17,21 +23,24 @@ import (
// SystemHandler handles system-related requests
type SystemHandler struct {
cfg *config.Config
neo4jDriver neo4j.Driver
cfg *config.Config
neo4jDriver neo4j.Driver
documentReader interfaces.DocumentReader
}
// NewSystemHandler creates a new system handler
func NewSystemHandler(cfg *config.Config, neo4jDriver neo4j.Driver) *SystemHandler {
func NewSystemHandler(cfg *config.Config, neo4jDriver neo4j.Driver, documentReader interfaces.DocumentReader) *SystemHandler {
return &SystemHandler{
cfg: cfg,
neo4jDriver: neo4jDriver,
cfg: cfg,
neo4jDriver: neo4jDriver,
documentReader: documentReader,
}
}
// GetSystemInfoResponse defines the response structure for system info
type GetSystemInfoResponse struct {
Version string `json:"version"`
Edition string `json:"edition"`
CommitID string `json:"commit_id,omitempty"`
BuildTime string `json:"build_time,omitempty"`
GoVersion string `json:"go_version,omitempty"`
@@ -44,6 +53,7 @@ type GetSystemInfoResponse struct {
// 编译时注入的版本信息
var (
Version = "unknown"
Edition = "standard"
CommitID = "unknown"
BuildTime = "unknown"
GoVersion = "unknown"
@@ -70,10 +80,11 @@ func (h *SystemHandler) GetSystemInfo(c *gin.Context) {
graphDatabaseEngine := h.getGraphDatabaseEngine()
// Get MinIO enabled status
minioEnabled := h.isMinioEnabled()
minioEnabled := h.isMinioConfigured(c)
response := GetSystemInfoResponse{
Version: Version,
Edition: Edition,
CommitID: CommitID,
BuildTime: BuildTime,
GoVersion: GoVersion,
@@ -91,6 +102,125 @@ func (h *SystemHandler) GetSystemInfo(c *gin.Context) {
})
}
func (h *SystemHandler) getDocReaderConnInfo() (addr, transport string) {
addr = strings.TrimSpace(os.Getenv("DOCREADER_ADDR"))
transport = strings.TrimSpace(os.Getenv("DOCREADER_TRANSPORT"))
if transport == "" {
transport = "grpc"
}
transport = strings.ToLower(transport)
return addr, transport
}
// ListParserEngines returns available document parser engines.
// Merges Go-native static engines with engines discovered from the remote
// docreader service, so newly added Python engines are auto-discovered.
// @Summary 列出可用的文档解析引擎
// @Tags 系统
// @Produce json
// @Success 200 {object} map[string]interface{} "解析引擎列表"
// @Router /system/parser-engines [get]
func (h *SystemHandler) ListParserEngines(c *gin.Context) {
docreaderAddr, docreaderTransport := h.getDocReaderConnInfo()
connected := h.documentReader != nil && h.documentReader.IsConnected()
var overrides map[string]string
if v, exists := c.Get(types.TenantInfoContextKey.String()); exists {
if tenant, ok := v.(*types.Tenant); ok && tenant != nil && tenant.ParserEngineConfig != nil {
overrides = tenant.ParserEngineConfig.ToOverridesMap()
}
}
remoteEngines := h.fetchRemoteEngines(c.Request.Context(), overrides)
engines := docparser.ListAllEngines(connected, overrides, remoteEngines)
c.JSON(200, gin.H{"code": 0, "msg": "success", "data": engines, "docreader_addr": docreaderAddr, "docreader_transport": docreaderTransport, "connected": connected})
}
// ReconnectDocReader reconnects the document converter to a new (or same) DocReader address.
// @Summary 重连文档解析服务
// @Tags 系统
// @Accept json
// @Produce json
// @Param request body object{addr string} true "DocReader 地址"
// @Success 200
// @Router /system/docreader/reconnect [post]
func (h *SystemHandler) ReconnectDocReader(c *gin.Context) {
var req struct {
Addr string `json:"addr" binding:"required"`
}
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(400, gin.H{"code": 1, "msg": "请提供 addr 参数"})
return
}
addr := strings.TrimSpace(req.Addr)
if addr == "" {
c.JSON(400, gin.H{"code": 1, "msg": "addr 不能为空"})
return
}
if h.documentReader == nil {
c.JSON(500, gin.H{"code": 1, "msg": "document converter not initialized"})
return
}
if err := h.documentReader.Reconnect(addr); err != nil {
logger.Errorf(c.Request.Context(), "Failed to reconnect docreader to %s: %v", addr, err)
c.JSON(200, gin.H{"code": 1, "msg": fmt.Sprintf("连接失败: %v", err)})
return
}
var overrides map[string]string
if v, exists := c.Get(types.TenantInfoContextKey.String()); exists {
if tenant, ok := v.(*types.Tenant); ok && tenant != nil && tenant.ParserEngineConfig != nil {
overrides = tenant.ParserEngineConfig.ToOverridesMap()
}
}
remoteEngines := h.fetchRemoteEngines(c.Request.Context(), overrides)
engines := docparser.ListAllEngines(true, overrides, remoteEngines)
_, docreaderTransport := h.getDocReaderConnInfo()
c.JSON(200, gin.H{"code": 0, "msg": "连接成功", "data": engines, "docreader_addr": addr, "docreader_transport": docreaderTransport, "connected": true})
}
// CheckParserEngines runs availability check with the given config overrides (e.g. current form values).
// Used to test engine availability without saving; body shape matches ParserEngineConfig.
// @Summary 使用当前参数检测解析引擎可用性
// @Tags 系统
// @Accept json
// @Produce json
// @Param body body object true "解析引擎配置(与保存接口同结构)"
// @Success 200
// @Router /system/parser-engines/check [post]
func (h *SystemHandler) CheckParserEngines(c *gin.Context) {
docreaderAddr, docreaderTransport := h.getDocReaderConnInfo()
connected := h.documentReader != nil && h.documentReader.IsConnected()
var body types.ParserEngineConfig
if err := c.ShouldBindJSON(&body); err != nil {
c.JSON(400, gin.H{"code": 1, "msg": "请求体格式错误"})
return
}
overrides := body.ToOverridesMap()
remoteEngines := h.fetchRemoteEngines(c.Request.Context(), overrides)
engines := docparser.ListAllEngines(connected, overrides, remoteEngines)
c.JSON(200, gin.H{"code": 0, "msg": "success", "data": engines, "docreader_addr": docreaderAddr, "docreader_transport": docreaderTransport, "connected": connected})
}
// fetchRemoteEngines queries the remote docreader for its engine list.
// Returns nil on any error (e.g. not connected), letting the caller
// fall back to Go's static registry only.
func (h *SystemHandler) fetchRemoteEngines(ctx context.Context, overrides map[string]string) []types.ParserEngineInfo {
if h.documentReader == nil || !h.documentReader.IsConnected() {
return nil
}
engines, err := h.documentReader.ListEngines(ctx, overrides)
if err != nil {
logger.Warnf(ctx, "Failed to fetch remote engines from docreader: %v", err)
return nil
}
return engines
}
// getKeywordIndexEngine returns the keyword index engine name
func (h *SystemHandler) getKeywordIndexEngine() string {
retrieveDriver := os.Getenv("RETRIEVE_DRIVER")
@@ -172,16 +302,46 @@ func (h *SystemHandler) supportsRetrieverType(driver string, retrieverType types
return false
}
// isMinioEnabled checks if MinIO is enabled
func (h *SystemHandler) isMinioEnabled() bool {
// Check if all required MinIO environment variables are set
endpoint := os.Getenv("MINIO_ENDPOINT")
accessKeyID := os.Getenv("MINIO_ACCESS_KEY_ID")
secretAccessKey := os.Getenv("MINIO_SECRET_ACCESS_KEY")
// getMinioConfig resolves MinIO connection parameters from tenant config (if mode=remote) or env vars (mode=docker/default).
func (h *SystemHandler) getMinioConfig(c *gin.Context) (endpoint, accessKeyID, secretAccessKey string) {
if v, exists := c.Get(types.TenantInfoContextKey.String()); exists {
if tenant, ok := v.(*types.Tenant); ok && tenant != nil && tenant.StorageEngineConfig != nil && tenant.StorageEngineConfig.MinIO != nil {
m := tenant.StorageEngineConfig.MinIO
if m.Mode == "remote" {
return m.Endpoint, m.AccessKeyID, m.SecretAccessKey
}
}
}
endpoint = os.Getenv("MINIO_ENDPOINT")
accessKeyID = os.Getenv("MINIO_ACCESS_KEY_ID")
secretAccessKey = os.Getenv("MINIO_SECRET_ACCESS_KEY")
return
}
// isMinioConfigured checks whether MinIO connection info is available (from tenant config or env).
func (h *SystemHandler) isMinioConfigured(c *gin.Context) bool {
endpoint, accessKeyID, secretAccessKey := h.getMinioConfig(c)
return endpoint != "" && accessKeyID != "" && secretAccessKey != ""
}
// isMinioEnvAvailable checks whether MinIO env vars (MINIO_ENDPOINT etc.) are set.
func (h *SystemHandler) isMinioEnvAvailable() bool {
return os.Getenv("MINIO_ENDPOINT") != "" &&
os.Getenv("MINIO_ACCESS_KEY_ID") != "" &&
os.Getenv("MINIO_SECRET_ACCESS_KEY") != ""
}
// isCOSConfigured checks whether COS connection info is available from tenant config.
func (h *SystemHandler) isCOSConfigured(c *gin.Context) bool {
if v, exists := c.Get(types.TenantInfoContextKey.String()); exists {
if tenant, ok := v.(*types.Tenant); ok && tenant != nil && tenant.StorageEngineConfig != nil && tenant.StorageEngineConfig.COS != nil {
cosConf := tenant.StorageEngineConfig.COS
return cosConf.SecretID != "" && cosConf.SecretKey != "" && cosConf.Region != "" && cosConf.BucketName != ""
}
}
return false
}
// MinioBucketInfo represents bucket information with access policy
type MinioBucketInfo struct {
Name string `json:"name"`
@@ -194,6 +354,42 @@ type ListMinioBucketsResponse struct {
Buckets []MinioBucketInfo `json:"buckets"`
}
// StorageEngineStatusItem describes one storage engine's availability and description.
type StorageEngineStatusItem struct {
Name string `json:"name"` // "local", "minio", "cos"
Available bool `json:"available"` // whether the engine can be used
Description string `json:"description"` // short description for UI
}
// GetStorageEngineStatusResponse is the response for GET /system/storage-engine-status.
type GetStorageEngineStatusResponse struct {
Engines []StorageEngineStatusItem `json:"engines"`
MinioEnvAvailable bool `json:"minio_env_available"`
}
// GetStorageEngineStatus godoc
// @Summary 获取存储引擎状态
// @Description 返回 Local、MinIO、COS 各存储引擎的可用状态及说明,供全局设置与知识库选择使用
// @Tags 系统
// @Produce json
// @Success 200 {object} GetStorageEngineStatusResponse
// @Router /system/storage-engine-status [get]
func (h *SystemHandler) GetStorageEngineStatus(c *gin.Context) {
minioConfigured := h.isMinioConfigured(c)
minioEnvAvailable := h.isMinioEnvAvailable()
cosConfigured := h.isCOSConfigured(c)
engines := []StorageEngineStatusItem{
{Name: "local", Available: true, Description: "本地文件系统存储,仅适合单机部署"},
{Name: "minio", Available: minioConfigured || minioEnvAvailable, Description: "S3 兼容的自托管对象存储,适合内网和私有云部署"},
{Name: "cos", Available: cosConfigured, Description: "腾讯云对象存储服务,适合公有云部署,支持 CDN 加速"},
}
c.JSON(200, gin.H{
"code": 0,
"msg": "success",
"data": GetStorageEngineStatusResponse{Engines: engines, MinioEnvAvailable: minioEnvAvailable},
})
}
// ListMinioBuckets godoc
// @Summary 列出 MinIO 存储桶
// @Description 获取所有 MinIO 存储桶及其访问权限
@@ -207,22 +403,23 @@ type ListMinioBucketsResponse struct {
func (h *SystemHandler) ListMinioBuckets(c *gin.Context) {
ctx := logger.CloneContext(c.Request.Context())
// Check if MinIO is enabled
if !h.isMinioEnabled() {
logger.Warn(ctx, "MinIO is not enabled")
endpoint, accessKeyID, secretAccessKey := h.getMinioConfig(c)
if endpoint == "" || accessKeyID == "" || secretAccessKey == "" {
logger.Warn(ctx, "MinIO is not configured")
c.JSON(400, gin.H{
"code": 400,
"msg": "MinIO is not enabled",
"msg": "MinIO is not configured",
"success": false,
})
return
}
// Get MinIO configuration from environment
endpoint := os.Getenv("MINIO_ENDPOINT")
accessKeyID := os.Getenv("MINIO_ACCESS_KEY_ID")
secretAccessKey := os.Getenv("MINIO_SECRET_ACCESS_KEY")
useSSL := os.Getenv("MINIO_USE_SSL") == "true"
if v, exists := c.Get(types.TenantInfoContextKey.String()); exists {
if tenant, ok := v.(*types.Tenant); ok && tenant != nil && tenant.StorageEngineConfig != nil && tenant.StorageEngineConfig.MinIO != nil {
useSSL = tenant.StorageEngineConfig.MinIO.UseSSL
}
}
// Create MinIO client
minioClient, err := minio.New(endpoint, &minio.Options{
@@ -372,3 +569,206 @@ func hasGetObjectAction(action interface{}) bool {
}
return false
}
// --- Storage engine helpers ---
// cosFieldPattern validates COS region and bucket name format to prevent URL injection.
var cosFieldPattern = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]{0,62}$`)
// sanitizeStorageCheckError converts a raw storage connectivity error into a safe
// user-facing message that does not leak internal network details (hostnames, IPs, ports).
func sanitizeStorageCheckError(err error) string {
msg := err.Error()
switch {
case strings.Contains(msg, "no such host"):
return "DNS 解析失败,请检查地址是否正确"
case strings.Contains(msg, "connection refused"):
return "连接被拒绝,请确认服务已启动且端口正确"
case strings.Contains(msg, "no route to host"):
return "无法路由到目标地址,请检查网络配置"
case strings.Contains(msg, "i/o timeout") || strings.Contains(msg, "deadline exceeded") || strings.Contains(msg, "context deadline"):
return "连接超时,请检查网络或服务状态"
case strings.Contains(msg, "403") || strings.Contains(msg, "AccessDenied") || strings.Contains(msg, "access denied"):
return "认证失败,请检查访问凭证是否正确"
case strings.Contains(msg, "certificate") || strings.Contains(msg, "tls") || strings.Contains(msg, "x509"):
return "TLS/SSL 证书错误,请检查 SSL 配置"
case strings.Contains(msg, "404") || strings.Contains(msg, "NoSuchBucket"):
return "Bucket 不存在,请检查名称和 Region"
default:
return "连接失败,请检查配置参数是否正确"
}
}
// isBlockedStorageEndpoint checks whether a storage endpoint resolves to a dangerous
// address (cloud metadata, loopback, link-local). Unlike the stricter IsSSRFSafeURL,
// this allows private IPs since MinIO is commonly deployed on internal networks.
func isBlockedStorageEndpoint(endpoint string) (bool, string) {
host, _, err := net.SplitHostPort(endpoint)
if err != nil {
host = endpoint
}
hostLower := strings.ToLower(host)
blockedHosts := []string{
"metadata.google.internal",
"metadata.tencentyun.com",
"metadata.aws.internal",
"169.254.169.254",
}
for _, bh := range blockedHosts {
if hostLower == bh {
return true, "该地址不允许访问"
}
}
checkIP := func(ip net.IP) (bool, string) {
if ip.IsLoopback() {
return true, "不允许访问本地回环地址"
}
if ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() {
return true, "不允许访问链路本地地址"
}
if ip.IsUnspecified() {
return true, "无效的地址"
}
return false, ""
}
if ip := net.ParseIP(host); ip != nil {
return checkIP(ip)
}
ips, err := net.LookupIP(host)
if err != nil {
return false, ""
}
for _, ip := range ips {
if blocked, reason := checkIP(ip); blocked {
return blocked, reason
}
}
return false, ""
}
// --- Storage engine connectivity check ---
// StorageCheckRequest is the body for POST /system/storage-engine-check.
type StorageCheckRequest struct {
Provider string `json:"provider"` // "minio" or "cos"
MinIO *types.MinIOEngineConfig `json:"minio,omitempty"`
COS *types.COSEngineConfig `json:"cos,omitempty"`
}
// StorageCheckResponse is the response for a single-engine connectivity check.
type StorageCheckResponse struct {
OK bool `json:"ok"`
Message string `json:"message"`
}
// CheckStorageEngine tests connectivity for a single storage engine using the provided config.
// @Summary 测试存储引擎连通性
// @Description 使用当前填写的参数测试 MinIO/COS 连通性,不保存配置
// @Tags 系统
// @Accept json
// @Produce json
// @Param body body StorageCheckRequest true "存储引擎配置"
// @Success 200 {object} StorageCheckResponse
// @Router /system/storage-engine-check [post]
func (h *SystemHandler) CheckStorageEngine(c *gin.Context) {
ctx := logger.CloneContext(c.Request.Context())
var req StorageCheckRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(400, gin.H{"code": 1, "msg": "请求体格式错误"})
return
}
switch req.Provider {
case "minio":
h.checkMinio(c, ctx, req.MinIO)
case "cos":
h.checkCOS(c, ctx, req.COS)
default:
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: true, Message: "本地存储无需检测"}})
}
}
func (h *SystemHandler) checkMinio(c *gin.Context, ctx context.Context, cfg *types.MinIOEngineConfig) {
if cfg == nil {
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "未提供 MinIO 配置"}})
return
}
endpoint, accessKeyID, secretAccessKey := cfg.Endpoint, cfg.AccessKeyID, cfg.SecretAccessKey
if cfg.Mode != "remote" {
endpoint = os.Getenv("MINIO_ENDPOINT")
accessKeyID = os.Getenv("MINIO_ACCESS_KEY_ID")
secretAccessKey = os.Getenv("MINIO_SECRET_ACCESS_KEY")
}
if endpoint == "" || accessKeyID == "" || secretAccessKey == "" {
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "Endpoint、Access Key、Secret Key 不能为空"}})
return
}
if cfg.Mode == "remote" {
if blocked, reason := isBlockedStorageEndpoint(endpoint); blocked {
logger.Warnf(ctx, "Storage check: MinIO endpoint blocked by SSRF protection", "endpoint", endpoint)
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: reason}})
return
}
}
err := file.CheckMinioConnectivity(ctx, endpoint, accessKeyID, secretAccessKey, cfg.BucketName, cfg.UseSSL)
if err != nil {
logger.Error(ctx, "Storage check: MinIO connectivity failed", "error", err)
errMsg := err.Error()
if strings.Contains(errMsg, "does not exist") {
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: fmt.Sprintf("Bucket「%s」不存在", cfg.BucketName)}})
return
}
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: sanitizeStorageCheckError(err)}})
return
}
msg := "连接成功"
if cfg.BucketName != "" {
msg = fmt.Sprintf("连接成功,Bucket「%s」已确认存在", cfg.BucketName)
}
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: true, Message: msg}})
}
func (h *SystemHandler) checkCOS(c *gin.Context, ctx context.Context, cfg *types.COSEngineConfig) {
if cfg == nil {
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "未提供 COS 配置"}})
return
}
if cfg.SecretID == "" || cfg.SecretKey == "" || cfg.Region == "" || cfg.BucketName == "" {
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "Secret ID、Secret Key、Region、Bucket 名称不能为空"}})
return
}
if !cosFieldPattern.MatchString(cfg.Region) {
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "Region 格式不正确,仅允许字母、数字、点、连字符"}})
return
}
if !cosFieldPattern.MatchString(cfg.BucketName) {
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "Bucket 名称格式不正确,仅允许字母、数字、点、连字符"}})
return
}
err := file.CheckCosConnectivity(ctx, cfg.BucketName, cfg.Region, cfg.SecretID, cfg.SecretKey)
if err != nil {
logger.Errorf(ctx, "Storage check: COS connectivity failed, bucket: %s, error: %v", cfg.BucketName, err)
errMsg := err.Error()
if strings.Contains(errMsg, "403") {
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: "认证失败,请检查 Secret ID / Secret Key 是否正确"}})
return
}
if strings.Contains(errMsg, "404") || strings.Contains(errMsg, "NoSuchBucket") {
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: fmt.Sprintf("Bucket「%s」不存在,请检查名称和 Region", cfg.BucketName)}})
return
}
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: sanitizeStorageCheckError(err)}})
return
}
c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: true, Message: fmt.Sprintf("连接成功,Bucket「%s」已确认存在", cfg.BucketName)}})
}
+116
View File
@@ -599,6 +599,12 @@ func (h *TenantHandler) GetTenantKV(c *gin.Context) {
case "prompt-templates":
h.GetPromptTemplates(c)
return
case "parser-engine-config":
h.GetTenantParserEngineConfig(c)
return
case "storage-engine-config":
h.GetTenantStorageEngineConfig(c)
return
default:
logger.Info(ctx, "KV key not supported", "key", key)
c.Error(errors.NewBadRequestError("unsupported key"))
@@ -633,6 +639,12 @@ func (h *TenantHandler) UpdateTenantKV(c *gin.Context) {
case "conversation-config":
h.updateTenantConversationInternal(c)
return
case "parser-engine-config":
h.updateTenantParserEngineConfigInternal(c)
return
case "storage-engine-config":
h.updateTenantStorageEngineConfigInternal(c)
return
default:
logger.Info(ctx, "KV key not supported", "key", key)
c.Error(errors.NewBadRequestError("unsupported key"))
@@ -713,6 +725,110 @@ func (h *TenantHandler) GetTenantWebSearchConfig(c *gin.Context) {
})
}
// GetTenantParserEngineConfig returns the tenant's parser engine config (MinerU endpoint, API key, etc.).
func (h *TenantHandler) GetTenantParserEngineConfig(c *gin.Context) {
ctx := c.Request.Context()
tenant := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
if tenant == nil {
logger.Error(ctx, "Tenant is empty")
c.Error(errors.NewBadRequestError("Tenant is empty"))
return
}
data := tenant.ParserEngineConfig
if data == nil {
data = &types.ParserEngineConfig{}
}
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": data,
})
}
// updateTenantParserEngineConfigInternal updates the tenant's parser engine config.
func (h *TenantHandler) updateTenantParserEngineConfigInternal(c *gin.Context) {
ctx := c.Request.Context()
var cfg types.ParserEngineConfig
if err := c.ShouldBindJSON(&cfg); err != nil {
logger.Error(ctx, "Failed to parse request parameters", err)
c.Error(errors.NewValidationError("Invalid request data").WithDetails(err.Error()))
return
}
tenant := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
if tenant == nil {
logger.Error(ctx, "Tenant is empty")
c.Error(errors.NewBadRequestError("Tenant is empty"))
return
}
tenant.ParserEngineConfig = &cfg
updatedTenant, err := h.service.UpdateTenant(ctx, tenant)
if err != nil {
if appErr, ok := errors.IsAppError(err); ok {
c.Error(appErr)
} else {
logger.ErrorWithFields(ctx, err, nil)
c.Error(errors.NewInternalServerError("Failed to update tenant parser engine config").WithDetails(err.Error()))
}
return
}
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": updatedTenant.ParserEngineConfig,
"message": "解析引擎配置已更新",
})
}
// GetTenantStorageEngineConfig returns the tenant's storage engine config (Local, MinIO, COS parameters).
func (h *TenantHandler) GetTenantStorageEngineConfig(c *gin.Context) {
ctx := c.Request.Context()
tenant := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
if tenant == nil {
logger.Error(ctx, "Tenant is empty")
c.Error(errors.NewBadRequestError("Tenant is empty"))
return
}
data := tenant.StorageEngineConfig
if data == nil {
data = &types.StorageEngineConfig{}
}
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": data,
})
}
// updateTenantStorageEngineConfigInternal updates the tenant's storage engine config.
func (h *TenantHandler) updateTenantStorageEngineConfigInternal(c *gin.Context) {
ctx := c.Request.Context()
var cfg types.StorageEngineConfig
if err := c.ShouldBindJSON(&cfg); err != nil {
logger.Error(ctx, "Failed to parse request parameters", err)
c.Error(errors.NewValidationError("Invalid request data").WithDetails(err.Error()))
return
}
tenant := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
if tenant == nil {
logger.Error(ctx, "Tenant is empty")
c.Error(errors.NewBadRequestError("Tenant is empty"))
return
}
tenant.StorageEngineConfig = &cfg
updatedTenant, err := h.service.UpdateTenant(ctx, tenant)
if err != nil {
if appErr, ok := errors.IsAppError(err); ok {
c.Error(appErr)
} else {
logger.ErrorWithFields(ctx, err, nil)
c.Error(errors.NewInternalServerError("Failed to update tenant storage engine config").WithDetails(err.Error()))
}
return
}
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": updatedTenant.StorageEngineConfig,
"message": "存储引擎配置已更新",
})
}
func (h *TenantHandler) buildDefaultConversationConfig() *types.ConversationConfig {
return &types.ConversationConfig{
Prompt: h.config.Conversation.Summary.Prompt,