feat: introduce WeKnora Lite edition with lightweight configuration and deployment
- Added a new `.env.lite.example` file for the Lite version, providing a minimal configuration template. - Updated `.env.example` to remove deprecated variables and include new Docreader settings. - Enhanced Docker configurations to support the Lite version, including a new Dockerfile for the Docreader service. - Introduced a Makefile target for building and running the Lite version, along with packaging capabilities. - Created GitHub workflows for building and releasing Lite binaries, including Homebrew formula support. - Implemented a new service file for managing the Lite version as a system service. This update enables a streamlined, single-binary deployment of WeKnora, reducing external dependencies and simplifying setup.
This commit is contained in:
@@ -0,0 +1,80 @@
|
||||
package types
|
||||
|
||||
// ReadRequest is the unified transport-agnostic request for document reading.
|
||||
// Set FileContent for file mode, URL for URL mode.
|
||||
type ReadRequest struct {
|
||||
FileContent []byte
|
||||
FileName string
|
||||
FileType string
|
||||
URL string
|
||||
Title string
|
||||
ParserEngine string
|
||||
RequestID string
|
||||
ParserEngineOverrides map[string]string
|
||||
ImageStorage map[string]string // storage provider config (provider, endpoint, bucket, ...)
|
||||
}
|
||||
|
||||
// ReadResult is the transport-agnostic result of document reading.
|
||||
type ReadResult struct {
|
||||
MarkdownContent string
|
||||
ImageRefs []ImageRef
|
||||
ImageDirPath string
|
||||
Metadata map[string]string
|
||||
Error string
|
||||
}
|
||||
|
||||
// ImageRef represents an image reference extracted from the document.
|
||||
type ImageRef struct {
|
||||
Filename string
|
||||
OriginalRef string
|
||||
MimeType string
|
||||
StorageKey string
|
||||
ImageData []byte // inline image bytes (universal fallback for cross-machine deployments)
|
||||
}
|
||||
|
||||
// ParserEngineInfo describes a registered parser engine.
|
||||
type ParserEngineInfo struct {
|
||||
Name string
|
||||
Description string
|
||||
FileTypes []string
|
||||
Available bool
|
||||
UnavailableReason string
|
||||
}
|
||||
|
||||
// --- Internal types used by chunking pipeline ---
|
||||
|
||||
type DocParserStorageConfig struct {
|
||||
Provider string
|
||||
Region string
|
||||
BucketName string
|
||||
AccessKeyID string
|
||||
SecretAccessKey string
|
||||
AppID string
|
||||
PathPrefix string
|
||||
Endpoint string
|
||||
}
|
||||
|
||||
type DocParserVLMConfig struct {
|
||||
ModelName string
|
||||
BaseURL string
|
||||
APIKey string
|
||||
InterfaceType string
|
||||
}
|
||||
|
||||
type ParsedChunk struct {
|
||||
Content string
|
||||
Seq int
|
||||
Start int
|
||||
End int
|
||||
Images []ParsedImage
|
||||
ChunkID string // populated by processChunks with the actual DB UUID
|
||||
}
|
||||
|
||||
type ParsedImage struct {
|
||||
URL string
|
||||
Caption string
|
||||
OCRText string
|
||||
OriginalURL string
|
||||
Start int
|
||||
End int
|
||||
}
|
||||
@@ -11,6 +11,7 @@ const (
|
||||
TypeKBDelete = "kb:delete" // 知识库删除任务
|
||||
TypeKnowledgeListDelete = "knowledge:list_delete" // 批量删除知识任务
|
||||
TypeDataTableSummary = "datatable:summary" // 表格摘要任务
|
||||
TypeImageMultimodal = "image:multimodal" // 图片多模态处理任务(OCR + VLM Caption)
|
||||
)
|
||||
|
||||
// ExtractChunkPayload represents the extract chunk task payload
|
||||
@@ -97,6 +98,18 @@ type KnowledgeListDeletePayload struct {
|
||||
KnowledgeIDs []string `json:"knowledge_ids"`
|
||||
}
|
||||
|
||||
// ImageMultimodalPayload represents the image multimodal processing task payload.
|
||||
type ImageMultimodalPayload struct {
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
KnowledgeID string `json:"knowledge_id"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||||
ChunkID string `json:"chunk_id"` // parent text chunk
|
||||
ImageURL string `json:"image_url"` // serving URL (/files/images/xxx.png)
|
||||
ImageLocalPath string `json:"image_local_path"` // absolute path on disk
|
||||
EnableOCR bool `json:"enable_ocr"`
|
||||
EnableCaption bool `json:"enable_caption"`
|
||||
}
|
||||
|
||||
// KBCloneTaskStatus represents the status of a knowledge base clone task
|
||||
type KBCloneTaskStatus string
|
||||
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
package interfaces
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// DocReader is the core interface for reading documents into markdown.
|
||||
type DocReader interface {
|
||||
Read(ctx context.Context, req *types.ReadRequest) (*types.ReadResult, error)
|
||||
}
|
||||
|
||||
// DocumentReader extends DocReader with transport lifecycle management
|
||||
// and remote engine discovery. Used by gRPC/HTTP clients that talk to
|
||||
// the Python docreader service.
|
||||
type DocumentReader interface {
|
||||
DocReader
|
||||
Reconnect(addr string) error
|
||||
IsConnected() bool
|
||||
// ListEngines queries the remote docreader for its registered parser engines.
|
||||
// Returns engines the remote service supports, allowing auto-discovery of
|
||||
// newly added engines without Go code changes.
|
||||
ListEngines(ctx context.Context, overrides map[string]string) ([]types.ParserEngineInfo, error)
|
||||
}
|
||||
@@ -9,6 +9,9 @@ import (
|
||||
// FileService is the interface for file services.
|
||||
// FileService provides methods to save, retrieve, and delete files.
|
||||
type FileService interface {
|
||||
// CheckConnectivity verifies that the storage backend is reachable and
|
||||
// properly configured (e.g. bucket exists, credentials valid).
|
||||
CheckConnectivity(ctx context.Context) error
|
||||
// SaveFile saves a file.
|
||||
SaveFile(ctx context.Context, file *multipart.FileHeader, tenantID uint64, knowledgeID string) (string, error)
|
||||
// SaveBytes saves bytes data to a file and returns the file path.
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/models/chat"
|
||||
"github.com/Tencent/WeKnora/internal/models/embedding"
|
||||
"github.com/Tencent/WeKnora/internal/models/rerank"
|
||||
"github.com/Tencent/WeKnora/internal/models/vlm"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
@@ -29,6 +30,8 @@ type ModelService interface {
|
||||
GetRerankModel(ctx context.Context, modelId string) (rerank.Reranker, error)
|
||||
// GetChatModel gets a chat model
|
||||
GetChatModel(ctx context.Context, modelId string) (chat.Chat, error)
|
||||
// GetVLMModel gets a vision language model
|
||||
GetVLMModel(ctx context.Context, modelId string) (vlm.VLM, error)
|
||||
}
|
||||
|
||||
// ModelRepository defines the model repository interface
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
package interfaces
|
||||
|
||||
import "github.com/hibiken/asynq"
|
||||
|
||||
// TaskEnqueuer abstracts task enqueueing. *asynq.Client satisfies this interface.
|
||||
// For Lite mode (no Redis), a synchronous implementation dispatches tasks inline.
|
||||
type TaskEnqueuer interface {
|
||||
Enqueue(task *asynq.Task, opts ...asynq.Option) (*asynq.TaskInfo, error)
|
||||
}
|
||||
@@ -95,6 +95,12 @@ type KnowledgeBaseConfig struct {
|
||||
FAQConfig *FAQConfig `yaml:"faq_config" json:"faq_config"`
|
||||
}
|
||||
|
||||
// ParserEngineRule maps a set of file types to a specific parser engine.
|
||||
type ParserEngineRule struct {
|
||||
FileTypes []string `yaml:"file_types" json:"file_types"`
|
||||
Engine string `yaml:"engine" json:"engine"`
|
||||
}
|
||||
|
||||
// ChunkingConfig represents the document splitting configuration
|
||||
type ChunkingConfig struct {
|
||||
// Chunk size
|
||||
@@ -105,6 +111,23 @@ type ChunkingConfig struct {
|
||||
Separators []string `yaml:"separators" json:"separators"`
|
||||
// EnableMultimodal (deprecated, kept for backward compatibility with old data)
|
||||
EnableMultimodal bool `yaml:"enable_multimodal,omitempty" json:"enable_multimodal,omitempty"`
|
||||
// ParserEngineRules configures which parser engine to use for each file type.
|
||||
// When empty, the builtin engine is used for all types.
|
||||
ParserEngineRules []ParserEngineRule `yaml:"parser_engine_rules,omitempty" json:"parser_engine_rules,omitempty"`
|
||||
}
|
||||
|
||||
// ResolveParserEngine returns the engine name for the given file type
|
||||
// based on the configured rules. Returns empty string (builtin) when
|
||||
// no rule matches.
|
||||
func (c ChunkingConfig) ResolveParserEngine(fileType string) string {
|
||||
for _, rule := range c.ParserEngineRules {
|
||||
for _, ft := range rule.FileTypes {
|
||||
if ft == fileType {
|
||||
return rule.Engine
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// COSConfig represents the COS configuration
|
||||
|
||||
@@ -11,6 +11,7 @@ const (
|
||||
ElasticFaissRetrieverEngineType RetrieverEngineType = "elasticfaiss"
|
||||
QdrantRetrieverEngineType RetrieverEngineType = "qdrant"
|
||||
MilvusRetrieverEngineType RetrieverEngineType = "milvus"
|
||||
SQLiteRetrieverEngineType RetrieverEngineType = "sqlite"
|
||||
)
|
||||
|
||||
// RetrieverType represents the type of retriever
|
||||
|
||||
@@ -126,6 +126,9 @@ type SearchResult struct {
|
||||
// MatchedContent is the actual content that was matched in vector search
|
||||
// For FAQ: this is the matched question text (standard or similar question)
|
||||
MatchedContent string `json:"matched_content,omitempty"`
|
||||
|
||||
// KnowledgeBaseID is the ID of the knowledge base this result belongs to
|
||||
KnowledgeBaseID string `json:"knowledge_base_id,omitempty"`
|
||||
}
|
||||
|
||||
// SearchParams represents the search parameters
|
||||
|
||||
@@ -3,6 +3,7 @@ package types
|
||||
import (
|
||||
"database/sql/driver"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -31,6 +32,10 @@ var retrieverEngineMapping = map[string][]RetrieverEngineParams{
|
||||
{RetrieverType: VectorRetrieverType, RetrieverEngineType: MilvusRetrieverEngineType},
|
||||
{RetrieverType: KeywordsRetrieverType, RetrieverEngineType: MilvusRetrieverEngineType},
|
||||
},
|
||||
"sqlite": {
|
||||
{RetrieverType: KeywordsRetrieverType, RetrieverEngineType: SQLiteRetrieverEngineType},
|
||||
{RetrieverType: VectorRetrieverType, RetrieverEngineType: SQLiteRetrieverEngineType},
|
||||
},
|
||||
}
|
||||
|
||||
// GetRetrieverEngineMapping returns the retriever engine mapping
|
||||
@@ -89,6 +94,10 @@ type Tenant struct {
|
||||
// Deprecated: ConversationConfig is deprecated, use CustomAgent (builtin-quick-answer) config instead.
|
||||
// This field is kept for backward compatibility and will be removed in future versions.
|
||||
ConversationConfig *ConversationConfig `yaml:"conversation_config" json:"conversation_config" gorm:"type:jsonb"`
|
||||
// Parser engine config overrides (MinerU endpoint, API key, etc.). Used when parsing documents; overrides env.
|
||||
ParserEngineConfig *ParserEngineConfig `yaml:"parser_engine_config" json:"parser_engine_config" gorm:"type:jsonb"`
|
||||
// Storage engine config: parameters for Local, MinIO, COS. Used for document/file storage and docreader.
|
||||
StorageEngineConfig *StorageEngineConfig `yaml:"storage_engine_config" json:"storage_engine_config" gorm:"type:jsonb"`
|
||||
// Creation time
|
||||
CreatedAt time.Time `yaml:"created_at" json:"created_at"`
|
||||
// Last updated time
|
||||
@@ -189,3 +198,154 @@ func (c *ConversationConfig) Scan(value interface{}) error {
|
||||
}
|
||||
return json.Unmarshal(b, c)
|
||||
}
|
||||
|
||||
// ParserEngineConfig holds tenant-level overrides for document parser engines (e.g. MinerU endpoint, API key).
|
||||
// These values take precedence over environment variables when parsing documents.
|
||||
type ParserEngineConfig struct {
|
||||
DocReaderAddr string `json:"docreader_addr"` // 文档解析服务地址
|
||||
MinerUEndpoint string `json:"mineru_endpoint"` // MinerU 自建服务端点
|
||||
MinerUAPIKey string `json:"mineru_api_key"` // MinerU 云 API Key
|
||||
MinerUAPIBaseURL string `json:"mineru_api_base_url"` // MinerU 云 API Base URL(可选)
|
||||
|
||||
// MinerU 自建解析参数
|
||||
MinerUModel string `json:"mineru_model,omitempty"` // backend: pipeline, vlm-*, hybrid-*
|
||||
MinerUEnableFormula *bool `json:"mineru_enable_formula,omitempty"`
|
||||
MinerUEnableTable *bool `json:"mineru_enable_table,omitempty"`
|
||||
MinerUEnableOCR *bool `json:"mineru_enable_ocr,omitempty"`
|
||||
MinerULanguage string `json:"mineru_language,omitempty"`
|
||||
|
||||
// MinerU 云 API 解析参数
|
||||
MinerUCloudModel string `json:"mineru_cloud_model,omitempty"` // model_version: pipeline, vlm, MinerU-HTML
|
||||
MinerUCloudEnableFormula *bool `json:"mineru_cloud_enable_formula,omitempty"`
|
||||
MinerUCloudEnableTable *bool `json:"mineru_cloud_enable_table,omitempty"`
|
||||
MinerUCloudEnableOCR *bool `json:"mineru_cloud_enable_ocr,omitempty"`
|
||||
MinerUCloudLanguage string `json:"mineru_cloud_language,omitempty"`
|
||||
}
|
||||
|
||||
// ToOverridesMap returns a map suitable for ParserEngineOverrides in parse requests.
|
||||
// Keys are snake_case (mineru_endpoint, mineru_api_key, mineru_api_base_url).
|
||||
func (c *ParserEngineConfig) ToOverridesMap() map[string]string {
|
||||
if c == nil {
|
||||
return nil
|
||||
}
|
||||
m := make(map[string]string)
|
||||
if c.MinerUEndpoint != "" {
|
||||
m["mineru_endpoint"] = c.MinerUEndpoint
|
||||
}
|
||||
if c.MinerUAPIKey != "" {
|
||||
m["mineru_api_key"] = c.MinerUAPIKey
|
||||
}
|
||||
if c.MinerUAPIBaseURL != "" {
|
||||
m["mineru_api_base_url"] = c.MinerUAPIBaseURL
|
||||
}
|
||||
if c.MinerUModel != "" {
|
||||
m["mineru_model"] = c.MinerUModel
|
||||
}
|
||||
if c.MinerUEnableFormula != nil {
|
||||
m["mineru_enable_formula"] = fmt.Sprintf("%v", *c.MinerUEnableFormula)
|
||||
}
|
||||
if c.MinerUEnableTable != nil {
|
||||
m["mineru_enable_table"] = fmt.Sprintf("%v", *c.MinerUEnableTable)
|
||||
}
|
||||
if c.MinerUEnableOCR != nil {
|
||||
m["mineru_enable_ocr"] = fmt.Sprintf("%v", *c.MinerUEnableOCR)
|
||||
}
|
||||
if c.MinerULanguage != "" {
|
||||
m["mineru_language"] = c.MinerULanguage
|
||||
}
|
||||
if c.MinerUCloudModel != "" {
|
||||
m["mineru_cloud_model"] = c.MinerUCloudModel
|
||||
}
|
||||
if c.MinerUCloudEnableFormula != nil {
|
||||
m["mineru_cloud_enable_formula"] = fmt.Sprintf("%v", *c.MinerUCloudEnableFormula)
|
||||
}
|
||||
if c.MinerUCloudEnableTable != nil {
|
||||
m["mineru_cloud_enable_table"] = fmt.Sprintf("%v", *c.MinerUCloudEnableTable)
|
||||
}
|
||||
if c.MinerUCloudEnableOCR != nil {
|
||||
m["mineru_cloud_enable_ocr"] = fmt.Sprintf("%v", *c.MinerUCloudEnableOCR)
|
||||
}
|
||||
if c.MinerUCloudLanguage != "" {
|
||||
m["mineru_cloud_language"] = c.MinerUCloudLanguage
|
||||
}
|
||||
if len(m) == 0 {
|
||||
return nil
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// Value implements the driver.Valuer interface for ParserEngineConfig
|
||||
func (c *ParserEngineConfig) Value() (driver.Value, error) {
|
||||
if c == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return json.Marshal(c)
|
||||
}
|
||||
|
||||
// Scan implements the sql.Scanner interface for ParserEngineConfig
|
||||
func (c *ParserEngineConfig) Scan(value interface{}) error {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
b, ok := value.([]byte)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return json.Unmarshal(b, c)
|
||||
}
|
||||
|
||||
// StorageEngineConfig holds tenant-level storage engine parameters for Local, MinIO, and COS.
|
||||
// Knowledge bases select which provider to use; parameters are read from here.
|
||||
type StorageEngineConfig struct {
|
||||
DefaultProvider string `json:"default_provider"` // "local", "minio", "cos"
|
||||
Local *LocalEngineConfig `json:"local,omitempty"`
|
||||
MinIO *MinIOEngineConfig `json:"minio,omitempty"`
|
||||
COS *COSEngineConfig `json:"cos,omitempty"`
|
||||
}
|
||||
|
||||
// LocalEngineConfig is for local file system storage (single-machine deployment only).
|
||||
type LocalEngineConfig struct {
|
||||
PathPrefix string `json:"path_prefix"`
|
||||
}
|
||||
|
||||
// MinIOEngineConfig is for MinIO/S3-compatible object storage.
|
||||
// Mode "docker" uses env vars for endpoint/credentials; "remote" uses the fields below.
|
||||
type MinIOEngineConfig struct {
|
||||
Mode string `json:"mode"` // "docker" or "remote"
|
||||
Endpoint string `json:"endpoint"`
|
||||
AccessKeyID string `json:"access_key_id"`
|
||||
SecretAccessKey string `json:"secret_access_key"`
|
||||
BucketName string `json:"bucket_name"`
|
||||
UseSSL bool `json:"use_ssl"`
|
||||
PathPrefix string `json:"path_prefix"`
|
||||
}
|
||||
|
||||
// COSEngineConfig is for Tencent Cloud COS.
|
||||
type COSEngineConfig struct {
|
||||
SecretID string `json:"secret_id"`
|
||||
SecretKey string `json:"secret_key"`
|
||||
Region string `json:"region"`
|
||||
BucketName string `json:"bucket_name"`
|
||||
AppID string `json:"app_id"`
|
||||
PathPrefix string `json:"path_prefix"`
|
||||
}
|
||||
|
||||
// Value implements the driver.Valuer interface for StorageEngineConfig
|
||||
func (c *StorageEngineConfig) Value() (driver.Value, error) {
|
||||
if c == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return json.Marshal(c)
|
||||
}
|
||||
|
||||
// Scan implements the sql.Scanner interface for StorageEngineConfig
|
||||
func (c *StorageEngineConfig) Scan(value interface{}) error {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
b, ok := value.([]byte)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return json.Unmarshal(b, c)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user