feat: introduce WeKnora Lite edition with lightweight configuration and deployment

- Added a new `.env.lite.example` file for the Lite version, providing a minimal configuration template.
- Updated `.env.example` to remove deprecated variables and include new Docreader settings.
- Enhanced Docker configurations to support the Lite version, including a new Dockerfile for the Docreader service.
- Introduced a Makefile target for building and running the Lite version, along with packaging capabilities.
- Created GitHub workflows for building and releasing Lite binaries, including Homebrew formula support.
- Implemented a new service file for managing the Lite version as a system service.

This update enables a streamlined, single-binary deployment of WeKnora, reducing external dependencies and simplifying setup.
This commit is contained in:
wizardchen
2026-02-28 13:22:07 +08:00
committed by lyingbug
parent 5241dbc39e
commit 397689d2f3
140 changed files with 13365 additions and 5413 deletions
+80
View File
@@ -0,0 +1,80 @@
package types
// ReadRequest is the unified transport-agnostic request for document reading.
// Set FileContent for file mode, URL for URL mode.
type ReadRequest struct {
FileContent []byte
FileName string
FileType string
URL string
Title string
ParserEngine string
RequestID string
ParserEngineOverrides map[string]string
ImageStorage map[string]string // storage provider config (provider, endpoint, bucket, ...)
}
// ReadResult is the transport-agnostic result of document reading.
type ReadResult struct {
MarkdownContent string
ImageRefs []ImageRef
ImageDirPath string
Metadata map[string]string
Error string
}
// ImageRef represents an image reference extracted from the document.
type ImageRef struct {
Filename string
OriginalRef string
MimeType string
StorageKey string
ImageData []byte // inline image bytes (universal fallback for cross-machine deployments)
}
// ParserEngineInfo describes a registered parser engine.
type ParserEngineInfo struct {
Name string
Description string
FileTypes []string
Available bool
UnavailableReason string
}
// --- Internal types used by chunking pipeline ---
type DocParserStorageConfig struct {
Provider string
Region string
BucketName string
AccessKeyID string
SecretAccessKey string
AppID string
PathPrefix string
Endpoint string
}
type DocParserVLMConfig struct {
ModelName string
BaseURL string
APIKey string
InterfaceType string
}
type ParsedChunk struct {
Content string
Seq int
Start int
End int
Images []ParsedImage
ChunkID string // populated by processChunks with the actual DB UUID
}
type ParsedImage struct {
URL string
Caption string
OCRText string
OriginalURL string
Start int
End int
}
+13
View File
@@ -11,6 +11,7 @@ const (
TypeKBDelete = "kb:delete" // 知识库删除任务
TypeKnowledgeListDelete = "knowledge:list_delete" // 批量删除知识任务
TypeDataTableSummary = "datatable:summary" // 表格摘要任务
TypeImageMultimodal = "image:multimodal" // 图片多模态处理任务(OCR + VLM Caption)
)
// ExtractChunkPayload represents the extract chunk task payload
@@ -97,6 +98,18 @@ type KnowledgeListDeletePayload struct {
KnowledgeIDs []string `json:"knowledge_ids"`
}
// ImageMultimodalPayload represents the image multimodal processing task payload.
type ImageMultimodalPayload struct {
TenantID uint64 `json:"tenant_id"`
KnowledgeID string `json:"knowledge_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
ChunkID string `json:"chunk_id"` // parent text chunk
ImageURL string `json:"image_url"` // serving URL (/files/images/xxx.png)
ImageLocalPath string `json:"image_local_path"` // absolute path on disk
EnableOCR bool `json:"enable_ocr"`
EnableCaption bool `json:"enable_caption"`
}
// KBCloneTaskStatus represents the status of a knowledge base clone task
type KBCloneTaskStatus string
@@ -0,0 +1,25 @@
package interfaces
import (
"context"
"github.com/Tencent/WeKnora/internal/types"
)
// DocReader is the core interface for reading documents into markdown.
type DocReader interface {
Read(ctx context.Context, req *types.ReadRequest) (*types.ReadResult, error)
}
// DocumentReader extends DocReader with transport lifecycle management
// and remote engine discovery. Used by gRPC/HTTP clients that talk to
// the Python docreader service.
type DocumentReader interface {
DocReader
Reconnect(addr string) error
IsConnected() bool
// ListEngines queries the remote docreader for its registered parser engines.
// Returns engines the remote service supports, allowing auto-discovery of
// newly added engines without Go code changes.
ListEngines(ctx context.Context, overrides map[string]string) ([]types.ParserEngineInfo, error)
}
+3
View File
@@ -9,6 +9,9 @@ import (
// FileService is the interface for file services.
// FileService provides methods to save, retrieve, and delete files.
type FileService interface {
// CheckConnectivity verifies that the storage backend is reachable and
// properly configured (e.g. bucket exists, credentials valid).
CheckConnectivity(ctx context.Context) error
// SaveFile saves a file.
SaveFile(ctx context.Context, file *multipart.FileHeader, tenantID uint64, knowledgeID string) (string, error)
// SaveBytes saves bytes data to a file and returns the file path.
+3
View File
@@ -6,6 +6,7 @@ import (
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/models/embedding"
"github.com/Tencent/WeKnora/internal/models/rerank"
"github.com/Tencent/WeKnora/internal/models/vlm"
"github.com/Tencent/WeKnora/internal/types"
)
@@ -29,6 +30,8 @@ type ModelService interface {
GetRerankModel(ctx context.Context, modelId string) (rerank.Reranker, error)
// GetChatModel gets a chat model
GetChatModel(ctx context.Context, modelId string) (chat.Chat, error)
// GetVLMModel gets a vision language model
GetVLMModel(ctx context.Context, modelId string) (vlm.VLM, error)
}
// ModelRepository defines the model repository interface
@@ -0,0 +1,9 @@
package interfaces
import "github.com/hibiken/asynq"
// TaskEnqueuer abstracts task enqueueing. *asynq.Client satisfies this interface.
// For Lite mode (no Redis), a synchronous implementation dispatches tasks inline.
type TaskEnqueuer interface {
Enqueue(task *asynq.Task, opts ...asynq.Option) (*asynq.TaskInfo, error)
}
+23
View File
@@ -95,6 +95,12 @@ type KnowledgeBaseConfig struct {
FAQConfig *FAQConfig `yaml:"faq_config" json:"faq_config"`
}
// ParserEngineRule maps a set of file types to a specific parser engine.
type ParserEngineRule struct {
FileTypes []string `yaml:"file_types" json:"file_types"`
Engine string `yaml:"engine" json:"engine"`
}
// ChunkingConfig represents the document splitting configuration
type ChunkingConfig struct {
// Chunk size
@@ -105,6 +111,23 @@ type ChunkingConfig struct {
Separators []string `yaml:"separators" json:"separators"`
// EnableMultimodal (deprecated, kept for backward compatibility with old data)
EnableMultimodal bool `yaml:"enable_multimodal,omitempty" json:"enable_multimodal,omitempty"`
// ParserEngineRules configures which parser engine to use for each file type.
// When empty, the builtin engine is used for all types.
ParserEngineRules []ParserEngineRule `yaml:"parser_engine_rules,omitempty" json:"parser_engine_rules,omitempty"`
}
// ResolveParserEngine returns the engine name for the given file type
// based on the configured rules. Returns empty string (builtin) when
// no rule matches.
func (c ChunkingConfig) ResolveParserEngine(fileType string) string {
for _, rule := range c.ParserEngineRules {
for _, ft := range rule.FileTypes {
if ft == fileType {
return rule.Engine
}
}
}
return ""
}
// COSConfig represents the COS configuration
+1
View File
@@ -11,6 +11,7 @@ const (
ElasticFaissRetrieverEngineType RetrieverEngineType = "elasticfaiss"
QdrantRetrieverEngineType RetrieverEngineType = "qdrant"
MilvusRetrieverEngineType RetrieverEngineType = "milvus"
SQLiteRetrieverEngineType RetrieverEngineType = "sqlite"
)
// RetrieverType represents the type of retriever
+3
View File
@@ -126,6 +126,9 @@ type SearchResult struct {
// MatchedContent is the actual content that was matched in vector search
// For FAQ: this is the matched question text (standard or similar question)
MatchedContent string `json:"matched_content,omitempty"`
// KnowledgeBaseID is the ID of the knowledge base this result belongs to
KnowledgeBaseID string `json:"knowledge_base_id,omitempty"`
}
// SearchParams represents the search parameters
+160
View File
@@ -3,6 +3,7 @@ package types
import (
"database/sql/driver"
"encoding/json"
"fmt"
"os"
"strings"
"time"
@@ -31,6 +32,10 @@ var retrieverEngineMapping = map[string][]RetrieverEngineParams{
{RetrieverType: VectorRetrieverType, RetrieverEngineType: MilvusRetrieverEngineType},
{RetrieverType: KeywordsRetrieverType, RetrieverEngineType: MilvusRetrieverEngineType},
},
"sqlite": {
{RetrieverType: KeywordsRetrieverType, RetrieverEngineType: SQLiteRetrieverEngineType},
{RetrieverType: VectorRetrieverType, RetrieverEngineType: SQLiteRetrieverEngineType},
},
}
// GetRetrieverEngineMapping returns the retriever engine mapping
@@ -89,6 +94,10 @@ type Tenant struct {
// Deprecated: ConversationConfig is deprecated, use CustomAgent (builtin-quick-answer) config instead.
// This field is kept for backward compatibility and will be removed in future versions.
ConversationConfig *ConversationConfig `yaml:"conversation_config" json:"conversation_config" gorm:"type:jsonb"`
// Parser engine config overrides (MinerU endpoint, API key, etc.). Used when parsing documents; overrides env.
ParserEngineConfig *ParserEngineConfig `yaml:"parser_engine_config" json:"parser_engine_config" gorm:"type:jsonb"`
// Storage engine config: parameters for Local, MinIO, COS. Used for document/file storage and docreader.
StorageEngineConfig *StorageEngineConfig `yaml:"storage_engine_config" json:"storage_engine_config" gorm:"type:jsonb"`
// Creation time
CreatedAt time.Time `yaml:"created_at" json:"created_at"`
// Last updated time
@@ -189,3 +198,154 @@ func (c *ConversationConfig) Scan(value interface{}) error {
}
return json.Unmarshal(b, c)
}
// ParserEngineConfig holds tenant-level overrides for document parser engines (e.g. MinerU endpoint, API key).
// These values take precedence over environment variables when parsing documents.
type ParserEngineConfig struct {
DocReaderAddr string `json:"docreader_addr"` // 文档解析服务地址
MinerUEndpoint string `json:"mineru_endpoint"` // MinerU 自建服务端点
MinerUAPIKey string `json:"mineru_api_key"` // MinerU 云 API Key
MinerUAPIBaseURL string `json:"mineru_api_base_url"` // MinerU 云 API Base URL(可选)
// MinerU 自建解析参数
MinerUModel string `json:"mineru_model,omitempty"` // backend: pipeline, vlm-*, hybrid-*
MinerUEnableFormula *bool `json:"mineru_enable_formula,omitempty"`
MinerUEnableTable *bool `json:"mineru_enable_table,omitempty"`
MinerUEnableOCR *bool `json:"mineru_enable_ocr,omitempty"`
MinerULanguage string `json:"mineru_language,omitempty"`
// MinerU 云 API 解析参数
MinerUCloudModel string `json:"mineru_cloud_model,omitempty"` // model_version: pipeline, vlm, MinerU-HTML
MinerUCloudEnableFormula *bool `json:"mineru_cloud_enable_formula,omitempty"`
MinerUCloudEnableTable *bool `json:"mineru_cloud_enable_table,omitempty"`
MinerUCloudEnableOCR *bool `json:"mineru_cloud_enable_ocr,omitempty"`
MinerUCloudLanguage string `json:"mineru_cloud_language,omitempty"`
}
// ToOverridesMap returns a map suitable for ParserEngineOverrides in parse requests.
// Keys are snake_case (mineru_endpoint, mineru_api_key, mineru_api_base_url).
func (c *ParserEngineConfig) ToOverridesMap() map[string]string {
if c == nil {
return nil
}
m := make(map[string]string)
if c.MinerUEndpoint != "" {
m["mineru_endpoint"] = c.MinerUEndpoint
}
if c.MinerUAPIKey != "" {
m["mineru_api_key"] = c.MinerUAPIKey
}
if c.MinerUAPIBaseURL != "" {
m["mineru_api_base_url"] = c.MinerUAPIBaseURL
}
if c.MinerUModel != "" {
m["mineru_model"] = c.MinerUModel
}
if c.MinerUEnableFormula != nil {
m["mineru_enable_formula"] = fmt.Sprintf("%v", *c.MinerUEnableFormula)
}
if c.MinerUEnableTable != nil {
m["mineru_enable_table"] = fmt.Sprintf("%v", *c.MinerUEnableTable)
}
if c.MinerUEnableOCR != nil {
m["mineru_enable_ocr"] = fmt.Sprintf("%v", *c.MinerUEnableOCR)
}
if c.MinerULanguage != "" {
m["mineru_language"] = c.MinerULanguage
}
if c.MinerUCloudModel != "" {
m["mineru_cloud_model"] = c.MinerUCloudModel
}
if c.MinerUCloudEnableFormula != nil {
m["mineru_cloud_enable_formula"] = fmt.Sprintf("%v", *c.MinerUCloudEnableFormula)
}
if c.MinerUCloudEnableTable != nil {
m["mineru_cloud_enable_table"] = fmt.Sprintf("%v", *c.MinerUCloudEnableTable)
}
if c.MinerUCloudEnableOCR != nil {
m["mineru_cloud_enable_ocr"] = fmt.Sprintf("%v", *c.MinerUCloudEnableOCR)
}
if c.MinerUCloudLanguage != "" {
m["mineru_cloud_language"] = c.MinerUCloudLanguage
}
if len(m) == 0 {
return nil
}
return m
}
// Value implements the driver.Valuer interface for ParserEngineConfig
func (c *ParserEngineConfig) Value() (driver.Value, error) {
if c == nil {
return nil, nil
}
return json.Marshal(c)
}
// Scan implements the sql.Scanner interface for ParserEngineConfig
func (c *ParserEngineConfig) Scan(value interface{}) error {
if value == nil {
return nil
}
b, ok := value.([]byte)
if !ok {
return nil
}
return json.Unmarshal(b, c)
}
// StorageEngineConfig holds tenant-level storage engine parameters for Local, MinIO, and COS.
// Knowledge bases select which provider to use; parameters are read from here.
type StorageEngineConfig struct {
DefaultProvider string `json:"default_provider"` // "local", "minio", "cos"
Local *LocalEngineConfig `json:"local,omitempty"`
MinIO *MinIOEngineConfig `json:"minio,omitempty"`
COS *COSEngineConfig `json:"cos,omitempty"`
}
// LocalEngineConfig is for local file system storage (single-machine deployment only).
type LocalEngineConfig struct {
PathPrefix string `json:"path_prefix"`
}
// MinIOEngineConfig is for MinIO/S3-compatible object storage.
// Mode "docker" uses env vars for endpoint/credentials; "remote" uses the fields below.
type MinIOEngineConfig struct {
Mode string `json:"mode"` // "docker" or "remote"
Endpoint string `json:"endpoint"`
AccessKeyID string `json:"access_key_id"`
SecretAccessKey string `json:"secret_access_key"`
BucketName string `json:"bucket_name"`
UseSSL bool `json:"use_ssl"`
PathPrefix string `json:"path_prefix"`
}
// COSEngineConfig is for Tencent Cloud COS.
type COSEngineConfig struct {
SecretID string `json:"secret_id"`
SecretKey string `json:"secret_key"`
Region string `json:"region"`
BucketName string `json:"bucket_name"`
AppID string `json:"app_id"`
PathPrefix string `json:"path_prefix"`
}
// Value implements the driver.Valuer interface for StorageEngineConfig
func (c *StorageEngineConfig) Value() (driver.Value, error) {
if c == nil {
return nil, nil
}
return json.Marshal(c)
}
// Scan implements the sql.Scanner interface for StorageEngineConfig
func (c *StorageEngineConfig) Scan(value interface{}) error {
if value == nil {
return nil
}
b, ok := value.([]byte)
if !ok {
return nil
}
return json.Unmarshal(b, c)
}